opalmixer.h

Go to the documentation of this file.
00001 /*
00002  * opalmixer.h
00003  *
00004  * OPAL media mixers
00005  *
00006  * Open Phone Abstraction Library (OPAL)
00007  * Formally known as the Open H323 project.
00008  *
00009  * Copyright (C) 2007 Post Increment
00010  *
00011  * The contents of this file are subject to the Mozilla Public License
00012  * Version 1.0 (the "License"); you may not use this file except in
00013  * compliance with the License. You may obtain a copy of the License at
00014  * http://www.mozilla.org/MPL/
00015  *
00016  * Software distributed under the License is distributed on an "AS IS"
00017  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
00018  * the License for the specific language governing rights and limitations
00019  * under the License.
00020  *
00021  * The Original Code is Open Phone Abstraction Library.
00022  *
00023  * The Initial Developer of the Original Code is Post Increment
00024  *
00025  * Contributor(s): Craig Southeren (craigs@postincrement.com)
00026  *                 Robert Jongbloed (robertj@voxlucida.com.au)
00027  *
00028  * $Revision: 27149 $
00029  * $Author: rjongbloed $
00030  * $Date: 2012-03-07 18:32:36 -0600 (Wed, 07 Mar 2012) $
00031  */
00032 
00033 
00034 #ifndef OPAL_OPAL_OPALMIXER_H
00035 #define OPAL_OPAL_OPALMIXER_H
00036 
00037 #ifndef _PTLIB_H
00038 #include <ptlib.h>
00039 #endif
00040 
00041 #include <opal/buildopts.h>
00042 
00043 #include <queue>
00044 
00045 #include <opal/localep.h>
00046 #include <codec/vidcodec.h>
00047 #include <ptclib/threadpool.h>
00048 
00049 
00050 class RTP_DataFrame;
00051 class OpalJitterBuffer;
00052 class OpalMixerConnection;
00053 
00054 
00055 //#define OPAL_MIXER_AUDIO_DEBUG 1
00056 
00057 
00058 #define OPAL_OPT_LISTEN_ONLY "Listen-Only" 
00059 
00060 
00062 
00072 class OpalBaseMixer
00073 {
00074   public:
00075     OpalBaseMixer(
00076       bool pushThread,    
00077       unsigned periodMS,  
00078       unsigned periodTS   
00079     );
00080 
00081     virtual ~OpalBaseMixer();
00082 
00083     typedef PString Key_T;
00084 
00087     virtual bool AddStream(
00088       const Key_T & key   
00089     );
00090 
00093     virtual void RemoveStream(
00094       const Key_T & key   
00095     );
00096 
00099     virtual void RemoveAllStreams();
00100 
00106     virtual bool WriteStream(
00107       const Key_T & key,          
00108       const RTP_DataFrame & input 
00109     );
00110 
00120     virtual RTP_DataFrame * ReadMixed();
00121     virtual bool ReadMixed(RTP_DataFrame & mixed);
00122 
00133     virtual bool OnMixed(
00134       RTP_DataFrame * & mixed   
00135     );
00136 
00140     void StartPushThread();
00141 
00146     void StopPushThread(bool lock = true);
00147 
00150     unsigned GetPeriodTS() const { return m_periodTS; }
00151 
00152   protected:
00153     struct Stream {
00154       virtual ~Stream() { }
00155       virtual void QueuePacket(const RTP_DataFrame & rtp) = 0;
00156       queue<RTP_DataFrame> m_queue;
00157     };
00158     typedef std::map<Key_T, Stream *> StreamMap_T;
00159 
00160     virtual Stream * CreateStream() = 0;
00161     virtual bool MixStreams(RTP_DataFrame & frame) = 0;
00162     virtual size_t GetOutputSize() const = 0;
00163 
00164     virtual bool OnPush();
00165     void PushThreadMain();
00166 
00167     bool      m_pushThread;      // true if to use a thread to push data out
00168     unsigned  m_periodMS;        // Mixing interval in milliseconds
00169     unsigned  m_periodTS;        // Mixing interval in timestamp units
00170 
00171     StreamMap_T     m_inputStreams;     // Map of key to stream for input RTP frame queues
00172     unsigned        m_outputTimestamp;  // RTP timestamp for output data
00173     RTP_DataFrame * m_pushFrame;        // Cached frame for pushing RTP
00174     PThread *       m_workerThread;     // reader thread handle
00175     bool            m_threadRunning;    // used to stop reader thread
00176     PMutex          m_mutex;            // mutex for list of streams and thread handle
00177 };
00178 
00180 
00189 class OpalAudioMixer : public OpalBaseMixer
00190 {
00191   public:
00192     OpalAudioMixer(
00193       bool stereo = false,    
00194       unsigned sampleRate = OpalMediaFormat::AudioClockRate, 
00195       bool pushThread = true, 
00196       unsigned period = 10    
00197     );
00198 
00199     ~OpalAudioMixer() { StopPushThread(); }
00200 
00203     virtual void RemoveStream(
00204       const Key_T & key   
00205     );
00206 
00209     virtual void RemoveAllStreams();
00210 
00213     bool IsStereo() const { return m_stereo; }
00214 
00217     unsigned GetSampleRate() const { return m_sampleRate; }
00218 
00225     bool SetSampleRate(
00226       unsigned rate   
00227     );
00228 
00235     bool SetJitterBufferSize(
00236       const Key_T & key,       
00237       unsigned minJitterDelay, 
00238       unsigned maxJitterDelay  
00239     );
00240 
00241   protected:
00242     struct AudioStream : public Stream
00243     {
00244       AudioStream(OpalAudioMixer & mixer);
00245       ~AudioStream();
00246 
00247       virtual void QueuePacket(const RTP_DataFrame & rtp);
00248       const short * GetAudioDataPtr();
00249 
00250       OpalAudioMixer   & m_mixer;
00251       OpalJitterBuffer * m_jitter;
00252       unsigned           m_nextTimestamp;
00253       PShortArray        m_cacheSamples;
00254       size_t             m_samplesUsed;
00255     };
00256 
00257     virtual Stream * CreateStream();
00258     virtual bool MixStreams(RTP_DataFrame & frame);
00259     virtual size_t GetOutputSize() const;
00260 
00261     void PreMixStreams();
00262     void MixStereo(RTP_DataFrame & frame);
00263     void MixAdditive(RTP_DataFrame & frame, const short * audioToSubtract);
00264 
00265   protected:
00266     bool     m_stereo;
00267     unsigned m_sampleRate;
00268 
00269     AudioStream    * m_left;
00270     AudioStream    * m_right;
00271     std::vector<int> m_mixedAudio;
00272 };
00273 
00274 
00276 
00277 #if OPAL_VIDEO
00278 
00285 class OpalVideoMixer : public OpalBaseMixer
00286 {
00287   public:
00288     enum Styles {
00289       eSideBySideLetterbox, 
00293       eSideBySideScaled,    
00297       eStackedPillarbox,    
00301       eStackedScaled,       
00305       eGrid,                
00307     };
00308 
00309     OpalVideoMixer(
00310       Styles style,           
00311       unsigned width,         
00312       unsigned height,        
00313       unsigned rate = 15,     
00314       bool pushThread = true  
00315     );
00316 
00317     ~OpalVideoMixer() { StopPushThread(); }
00318 
00321     unsigned GetFrameWidth() const { return m_width; }
00322 
00325     unsigned GetFrameHeight() const { return m_height; }
00326 
00329     unsigned GetFrameRate() const { return 1000/m_periodMS; }
00330 
00334     bool SetFrameRate(
00335       unsigned rate   // New frames per second.
00336     );
00337 
00341     bool SetFrameSize(
00342       unsigned width,   
00343       unsigned height   
00344     );
00345 
00346   protected:
00347     struct VideoStream : public Stream
00348     {
00349       VideoStream(OpalVideoMixer & mixer);
00350       virtual void QueuePacket(const RTP_DataFrame & rtp);
00351       void InsertVideoFrame(unsigned x, unsigned y, unsigned w, unsigned h);
00352 
00353       OpalVideoMixer & m_mixer;
00354     };
00355 
00356     friend struct VideoStream;
00357 
00358     virtual Stream * CreateStream();
00359     virtual bool MixStreams(RTP_DataFrame & frame);
00360     virtual size_t GetOutputSize() const;
00361 
00362   protected:
00363     Styles     m_style;
00364     unsigned   m_width, m_height;
00365     BYTE       m_bgFillRed,m_bgFillGreen,m_bgFillBlue;
00366 
00367     PBYTEArray m_frameStore;
00368     size_t     m_lastStreamCount;
00369 };
00370 
00371 #endif // OPAL_VIDEO
00372 
00373 
00375 
00376 
00381 struct OpalMixerNodeInfo
00382 {
00383   OpalMixerNodeInfo(const char * name = NULL)
00384     : m_name(name)
00385     , m_listenOnly(false)
00386     , m_sampleRate(OpalMediaFormat::AudioClockRate)
00387 #if OPAL_VIDEO
00388     , m_audioOnly(false)
00389     , m_style(OpalVideoMixer::eGrid)
00390     , m_width(PVideoFrameInfo::CIFWidth)
00391     , m_height(PVideoFrameInfo::CIFHeight)
00392     , m_rate(15)
00393 #endif
00394     , m_mediaPassThru(false)
00395   { }
00396 
00397   virtual ~OpalMixerNodeInfo() { }
00398 
00399   virtual OpalMixerNodeInfo * Clone() const { return new OpalMixerNodeInfo(*this); }
00400 
00401   PString  m_name;                
00402   bool     m_listenOnly;          
00403   unsigned m_sampleRate;          
00404 #if OPAL_VIDEO
00405   bool     m_audioOnly;           
00406   OpalVideoMixer::Styles m_style; 
00407   unsigned m_width;               
00408   unsigned m_height;              
00409   unsigned m_rate;                
00410 #endif
00411   bool     m_mediaPassThru;       
00413 };
00414 
00415 
00417 
00418 class OpalMixerNode;
00419 
00420 
00425 class OpalMixerNodeManager : public PObject
00426 {
00427     PCLASSINFO(OpalMixerNodeManager, PObject);
00428   public:
00433     OpalMixerNodeManager();
00434 
00438         virtual ~OpalMixerNodeManager();
00439 
00442     virtual void ShutDown();
00443 
00449     virtual PBoolean GarbageCollection();
00451 
00460     virtual OpalMixerNode * CreateNode(
00461       OpalMixerNodeInfo * info 
00462     );
00463 
00469     virtual PSafePtr<OpalMixerNode> AddNode(
00470       OpalMixerNodeInfo * info 
00471     );
00472 
00475     void AddNode(OpalMixerNode * node);
00476 
00480     PSafePtr<OpalMixerNode> GetFirstNode(
00481       PSafetyMode mode = PSafeReference 
00482     ) const { return PSafePtr<OpalMixerNode>(m_nodesByUID, mode); }
00483 
00487     virtual PSafePtr<OpalMixerNode> FindNode(
00488       const PString & name,             
00489       PSafetyMode mode = PSafeReference 
00490     );
00491 
00496     virtual void RemoveNode(
00497       OpalMixerNode & node
00498     );
00499 
00502     void AddNodeName(
00503       PString name,        
00504       OpalMixerNode * node 
00505     );
00506 
00509     void RemoveNodeName(
00510       PString name        
00511     );
00512 
00516     void RemoveNodeNames(
00517       PStringList names   
00518     );
00519 
00522     void QueueUserInput(
00523       const PSafePtr<OpalMixerNode> & node,     
00524       const OpalMixerConnection * connection,   
00525       const PString & value                     
00526     );
00528 
00529   protected:
00530     PSafeDictionary<PGloballyUniqueID, OpalMixerNode> m_nodesByUID;
00531     PDictionary<PString, OpalMixerNode>               m_nodesByName;
00532 
00533     struct UserInput {
00534       UserInput(
00535         const PSafePtr<OpalMixerNode> & node,
00536         const OpalMixerConnection * connection,
00537         const PString & value
00538       ) : m_node(node)
00539         , m_connection(connection)
00540         , m_value(value)
00541       { }
00542 
00543       PSafePtr<OpalMixerNode> m_node;
00544       const OpalMixerConnection * m_connection;
00545       PString m_value;
00546 
00547       void Work();
00548     };
00549     PQueuedThreadPool<UserInput> m_userInputPool;
00550 };
00551 
00552 
00554 
00559 class OpalMixerEndPoint : public OpalLocalEndPoint
00560 {
00561     PCLASSINFO(OpalMixerEndPoint, OpalLocalEndPoint);
00562   public:
00567     OpalMixerEndPoint(
00568       OpalManager & manager,  
00569       const char * prefix     
00570     );
00571 
00574     ~OpalMixerEndPoint();
00575 
00580     virtual void ShutDown();
00582 
00595     virtual OpalMediaFormatList GetMediaFormats() const;
00596 
00626     virtual PSafePtr<OpalConnection> MakeConnection(
00627       OpalCall & call,           
00628       const PString & party,     
00629       void * userData = NULL,    
00630       unsigned options = 0,      
00631       OpalConnection::StringOptions * stringOptions = NULL 
00632     );
00633 
00638     virtual PBoolean GarbageCollection();
00640 
00649     PSafePtr<OpalMixerConnection> GetMixerConnectionWithLock(
00650       const PString & token,     
00651       PSafetyMode mode = PSafeReadWrite 
00652     ) { return GetConnectionWithLockAs<OpalMixerConnection>(token, mode); }
00653 
00657     virtual OpalMixerConnection * CreateConnection(
00658       PSafePtr<OpalMixerNode> node, 
00659       OpalCall & call,              
00660       void * userData,              
00661       unsigned options,             
00662       OpalConnection::StringOptions * stringOptions 
00663     );
00665 
00673     PSafePtr<OpalMixerNode> AddNode(
00674       OpalMixerNodeInfo * info 
00675     );
00676 
00683     virtual OpalMixerNode * CreateNode(
00684       OpalMixerNodeInfo * info 
00685     );
00686 
00690     PSafePtr<OpalMixerNode> GetFirstNode(
00691       PSafetyMode mode = PSafeReference 
00692     ) const { return m_nodeManager.GetFirstNode(mode); }
00693 
00697     PSafePtr<OpalMixerNode> FindNode(
00698       const PString & name,             
00699       PSafetyMode mode = PSafeReference 
00700     ) { return m_nodeManager.FindNode(name, mode); }
00701 
00706     void RemoveNode(
00707       OpalMixerNode & node 
00708     ) { m_nodeManager.RemoveNode(node); }
00710 
00725     void SetAdHocNodeInfo(
00726       const OpalMixerNodeInfo & info
00727     );
00728     void SetAdHocNodeInfo(
00729       OpalMixerNodeInfo * info
00730     );
00731 
00743     OpalMixerNodeInfo * GetAdHocNodeInfo() { return m_adHocNodeInfo; }
00744 
00747     const OpalMixerNodeManager & GetNodeManager() const { return m_nodeManager; }
00748           OpalMixerNodeManager & GetNodeManager()       { return m_nodeManager; }
00750 
00751   protected:
00752     OpalMixerNodeInfo  * m_adHocNodeInfo;
00753     OpalMixerNodeManager m_nodeManager;
00754 };
00755 
00756 
00758 
00761 class OpalMixerConnection : public OpalLocalConnection
00762 {
00763     PCLASSINFO(OpalMixerConnection, OpalLocalConnection);
00764   public:
00769     OpalMixerConnection(
00770       PSafePtr<OpalMixerNode> node, 
00771       OpalCall & call,              
00772       OpalMixerEndPoint & endpoint, 
00773       void * userData,              
00774       unsigned options = 0,         
00775       OpalConnection::StringOptions * stringOptions = NULL 
00776     );
00777 
00780     ~OpalMixerConnection();
00782 
00802     virtual void OnReleased();
00803 
00810     virtual OpalMediaFormatList GetMediaFormats() const;
00811 
00826     virtual OpalMediaStream * CreateMediaStream(
00827       const OpalMediaFormat & mediaFormat, 
00828       unsigned sessionID,                  
00829       PBoolean isSource                    
00830     );
00831 
00834     virtual void OnStartMediaPatch(
00835       OpalMediaPatch & patch    
00836     );
00837 
00839     virtual void OnApplyStringOptions();
00840 
00847     virtual PBoolean SendUserInputString(
00848       const PString & value                   
00849     );
00850 
00867     virtual PBoolean SendUserInputTone(
00868       char tone,        
00869       unsigned duration = 0  
00870     );
00872 
00877     void SetListenOnly(
00878       bool listenOnly   
00879     );
00880 
00883     bool GetListenOnly() const { return m_listenOnly; }
00884 
00887     PSafePtr<OpalMixerNode> GetNode() const { return m_node; }
00889 
00890   protected:
00891     OpalMixerEndPoint     & m_endpoint;
00892     PSafePtr<OpalMixerNode> m_node;
00893     bool                    m_listenOnly;
00894 };
00895 
00896 
00900 class OpalMixerMediaStream : public OpalMediaStream
00901 {
00902     PCLASSINFO(OpalMixerMediaStream, OpalMediaStream);
00903   public:
00908     OpalMixerMediaStream(
00909       OpalConnection & conn,               
00910       const OpalMediaFormat & mediaFormat, 
00911       unsigned sessionID,                  
00912       bool isSource,                       
00913       PSafePtr<OpalMixerNode> node,        
00914       bool listenOnly                      
00915     );
00916 
00919     ~OpalMixerMediaStream();
00921 
00926     virtual PBoolean Open();
00927 
00933     virtual PBoolean WritePacket(
00934       RTP_DataFrame & packet
00935     );
00936 
00940     virtual PBoolean IsSynchronous() const;
00941 
00952     virtual PBoolean RequiresPatchThread() const;
00953 
00961     virtual bool EnableJitterBuffer(bool enab = true) const;
00963 
00968     PSafePtr<OpalMixerNode> GetNode() { return m_node; }
00970 
00971   protected:
00972     virtual void InternalClose();
00973 
00974     PSafePtr<OpalMixerNode> m_node;
00975     bool m_listenOnly;
00976 #if OPAL_VIDEO
00977     bool m_video;
00978 #endif
00979 };
00980 
00981 
00985 class OpalMixerNode : public PSafeObject
00986 {
00987     PCLASSINFO(OpalMixerNode, PSafeObject);
00988   public:
00993     OpalMixerNode(
00994       OpalMixerNodeManager & manager, 
00995       OpalMixerNodeInfo * info        
00996     );
00997     OpalMixerNode(
00998       OpalMixerEndPoint & endpoint,   
00999       OpalMixerNodeInfo * info        
01000     );
01001 
01004     ~OpalMixerNode();
01005 
01010     void ShutDown();
01012 
01019     void PrintOn(
01020       ostream & strm    
01021     ) const;
01023 
01028     void AttachConnection(
01029       OpalConnection * connection  
01030     );
01031 
01034     void DetachConnection(
01035       OpalConnection * connection  
01036     );
01037 
01040     bool AttachStream(
01041       OpalMixerMediaStream * stream     
01042     );
01043 
01046     void DetachStream(
01047       OpalMixerMediaStream * stream     
01048     );
01049 
01052     void UseMediaPassThrough(
01053       unsigned sessionID,                 
01054       OpalConnection * connection = NULL  
01055     );
01056 
01063     bool SetJitterBufferSize(
01064       const OpalBaseMixer::Key_T & key, 
01065       unsigned minJitterDelay,          
01066       unsigned maxJitterDelay           
01067     ) { return m_audioMixer.SetJitterBufferSize(key, minJitterDelay, maxJitterDelay); }
01068 
01071     bool WriteAudio(
01072       const OpalBaseMixer::Key_T & key, 
01073       const RTP_DataFrame & input       
01074     ) { return m_audioMixer.WriteStream(key, input); }
01075 
01076 #if OPAL_VIDEO
01077 
01079     bool WriteVideo(
01080       const OpalBaseMixer::Key_T & key, 
01081       const RTP_DataFrame & input       
01082     ) { return m_videoMixer.WriteStream(key, input); }
01083 #endif // OPAL_VIDEO
01084 
01087     virtual void BroadcastUserInput(
01088       const OpalConnection * connection,      
01089       const PString & value                   
01090     );
01092 
01097     const PGloballyUniqueID & GetGUID() const { return m_guid; }
01098 
01101     const PStringList & GetNames() const { return m_names; }
01102 
01105     void AddName(
01106       const PString & name
01107     );
01108 
01111     void RemoveName(
01112       const PString & name
01113     );
01114 
01120     PINDEX GetConnectionCount() const { return m_connections.GetSize(); }
01121 
01124     template <class Subclass>
01125     PSafePtr<Subclass> GetFirstConnectionAs(
01126       PSafetyMode mode = PSafeReference
01127     ) const { return PSafePtr<Subclass>(m_connections, mode); }
01128 
01131     PSafePtr<OpalConnection> GetFirstConnection(
01132       PSafetyMode mode = PSafeReference
01133     ) const { return GetFirstConnectionAs<OpalConnection>(mode); }
01134 
01137     const OpalMixerNodeInfo & GetNodeInfo() { return *m_info; }
01138 
01141     const PTime & GetCreationTime() const { return m_creationTime; }
01143 
01144   protected:
01145     void Construct();
01146 
01147     OpalMixerNodeManager & m_manager;
01148     PGloballyUniqueID      m_guid;
01149     PStringList            m_names;
01150     OpalMixerNodeInfo    * m_info;
01151     PTime                  m_creationTime;
01152 
01153     PSafeList<OpalConnection> m_connections;
01154 
01155     struct MediaMixer
01156     {
01157       MediaMixer();
01158 
01159       PSafeList<OpalMixerMediaStream> m_outputStreams;
01160     };
01161 
01162     struct AudioMixer : public OpalAudioMixer, public MediaMixer
01163     {
01164       AudioMixer(const OpalMixerNodeInfo & info);
01165       ~AudioMixer();
01166 
01167       virtual bool OnPush();
01168 
01169       struct CachedAudio {
01170         CachedAudio();
01171         ~CachedAudio();
01172         enum { Collecting, Collected, Completed } m_state;
01173         RTP_DataFrame    m_raw;
01174         RTP_DataFrame    m_encoded;
01175         OpalTranscoder * m_transcoder;
01176       };
01177       std::map<PString, CachedAudio> m_cache;
01178 
01179       void PushOne(
01180         PSafePtr<OpalMixerMediaStream> & stream,
01181         CachedAudio & cache,
01182         const short * audioToSubtract
01183       );
01184 #ifdef OPAL_MIXER_AUDIO_DEBUG
01185       class PAudioMixerDebug * m_audioDebug;
01186 #endif
01187     };
01188     AudioMixer m_audioMixer;
01189 
01190 #if OPAL_VIDEO
01191     struct VideoMixer : public OpalVideoMixer, public MediaMixer
01192     {
01193       VideoMixer(const OpalMixerNodeInfo & info);
01194       ~VideoMixer();
01195 
01196       virtual bool OnMixed(RTP_DataFrame * & output);
01197 
01198       PDictionary<PString, OpalTranscoder> m_transcoders;
01199     };
01200     VideoMixer m_videoMixer;
01201 #endif // OPAL_VIDEO
01202 };
01203 
01204 
01205 #endif // OPAL_OPAL_OPAL_MIXER
01206 
01207 

Generated on 14 Aug 2013 for OPAL by  doxygen 1.4.7