opalmixer.h

Go to the documentation of this file.
00001 /*
00002  * opalmixer.h
00003  *
00004  * OPAL media mixers
00005  *
00006  * Open Phone Abstraction Library (OPAL)
00007  * Formally known as the Open H323 project.
00008  *
00009  * Copyright (C) 2007 Post Increment
00010  *
00011  * The contents of this file are subject to the Mozilla Public License
00012  * Version 1.0 (the "License"); you may not use this file except in
00013  * compliance with the License. You may obtain a copy of the License at
00014  * http://www.mozilla.org/MPL/
00015  *
00016  * Software distributed under the License is distributed on an "AS IS"
00017  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
00018  * the License for the specific language governing rights and limitations
00019  * under the License.
00020  *
00021  * The Original Code is Open Phone Abstraction Library.
00022  *
00023  * The Initial Developer of the Original Code is Post Increment
00024  *
00025  * Contributor(s): Craig Southeren (craigs@postincrement.com)
00026  *                 Robert Jongbloed (robertj@voxlucida.com.au)
00027  *
00028  * $Revision: 29867 $
00029  * $Author: rjongbloed $
00030  * $Date: 2013-06-02 22:06:44 -0500 (Sun, 02 Jun 2013) $
00031  */
00032 
00033 
00034 #ifndef OPAL_OPAL_OPALMIXER_H
00035 #define OPAL_OPAL_OPALMIXER_H
00036 
00037 #ifndef _PTLIB_H
00038 #include <ptlib.h>
00039 #endif
00040 
00041 #include <opal/buildopts.h>
00042 
00043 #if OPAL_HAS_MIXER
00044 
00045 #include <queue>
00046 
00047 #include <ep/localep.h>
00048 #include <codec/vidcodec.h>
00049 
00050 
00051 class RTP_DataFrame;
00052 class OpalJitterBuffer;
00053 class OpalMixerConnection;
00054 
00055 
00056 //#define OPAL_MIXER_AUDIO_DEBUG 1
00057 
00058 
00059 #define OPAL_OPT_LISTEN_ONLY "Listen-Only"      
00060 #define OPAL_OPT_CONF_OWNER  "Conference-Owner" 
00061 
00062 
00064 
00074 class OpalBaseMixer : public PObject
00075 {
00076   public:
00077     OpalBaseMixer(
00078       bool pushThread,    
00079       unsigned periodMS,  
00080       unsigned periodTS   
00081     );
00082 
00083     virtual ~OpalBaseMixer();
00084 
00085     typedef PString Key_T;
00086 
00089     virtual bool AddStream(
00090       const Key_T & key   
00091     );
00092 
00095     virtual void RemoveStream(
00096       const Key_T & key   
00097     );
00098 
00101     virtual void RemoveAllStreams();
00102 
00108     virtual bool WriteStream(
00109       const Key_T & key,          
00110       const RTP_DataFrame & input 
00111     );
00112 
00122     virtual RTP_DataFrame * ReadMixed();
00123     virtual bool ReadMixed(RTP_DataFrame & mixed);
00124 
00135     virtual bool OnMixed(
00136       RTP_DataFrame * & mixed   
00137     );
00138 
00142     void StartPushThread();
00143 
00148     void StopPushThread(bool lock = true);
00149 
00152     unsigned GetPeriodTS() const { return m_periodTS; }
00153 
00154   protected:
00155     struct Stream : public PObject {
00156       virtual ~Stream() { }
00157       virtual void QueuePacket(const RTP_DataFrame & rtp) = 0;
00158       std::queue<RTP_DataFrame> m_queue;
00159     };
00160     typedef std::map<Key_T, Stream *> StreamMap_T;
00161 
00162     virtual Stream * CreateStream() = 0;
00163     virtual bool MixStreams(RTP_DataFrame & frame) = 0;
00164     virtual size_t GetOutputSize() const = 0;
00165 
00166     virtual bool OnPush();
00167     void PushThreadMain();
00168 
00169     bool      m_pushThread;      // true if to use a thread to push data out
00170     unsigned  m_periodMS;        // Mixing interval in milliseconds
00171     unsigned  m_periodTS;        // Mixing interval in timestamp units
00172 
00173     StreamMap_T     m_inputStreams;     // Map of key to stream for input RTP frame queues
00174     unsigned        m_outputTimestamp;  // RTP timestamp for output data
00175     RTP_DataFrame * m_pushFrame;        // Cached frame for pushing RTP
00176     PThread *       m_workerThread;     // reader thread handle
00177     bool            m_threadRunning;    // used to stop reader thread
00178     PMutex          m_mutex;            // mutex for list of streams and thread handle
00179 };
00180 
00182 
00191 class OpalAudioMixer : public OpalBaseMixer
00192 {
00193   public:
00194     OpalAudioMixer(
00195       bool stereo = false,    
00196       unsigned sampleRate = OpalMediaFormat::AudioClockRate, 
00197       bool pushThread = true, 
00198       unsigned period = 10    
00199     );
00200 
00201     ~OpalAudioMixer() { StopPushThread(); }
00202 
00205     virtual void RemoveStream(
00206       const Key_T & key   
00207     );
00208 
00211     virtual void RemoveAllStreams();
00212 
00215     bool IsStereo() const { return m_stereo; }
00216 
00219     unsigned GetSampleRate() const { return m_sampleRate; }
00220 
00227     bool SetSampleRate(
00228       unsigned rate   
00229     );
00230 
00237     bool SetJitterBufferSize(
00238       const Key_T & key,                    
00239       const OpalJitterBuffer::Init & init   
00240     );
00241 
00242   protected:
00243     struct AudioStream : public Stream
00244     {
00245       AudioStream(OpalAudioMixer & mixer);
00246       ~AudioStream();
00247 
00248       virtual void QueuePacket(const RTP_DataFrame & rtp);
00249       const short * GetAudioDataPtr();
00250 
00251       OpalAudioMixer   & m_mixer;
00252       OpalJitterBuffer * m_jitter;
00253       unsigned           m_nextTimestamp;
00254       PShortArray        m_cacheSamples;
00255       size_t             m_samplesUsed;
00256     };
00257 
00258     virtual Stream * CreateStream();
00259     virtual bool MixStreams(RTP_DataFrame & frame);
00260     virtual size_t GetOutputSize() const;
00261 
00262     void PreMixStreams();
00263     void MixStereo(RTP_DataFrame & frame);
00264     void MixAdditive(RTP_DataFrame & frame, const short * audioToSubtract);
00265 
00266   protected:
00267     bool     m_stereo;
00268     unsigned m_sampleRate;
00269 
00270     AudioStream    * m_left;
00271     AudioStream    * m_right;
00272     std::vector<int> m_mixedAudio;
00273 };
00274 
00275 
00277 
00278 #if OPAL_VIDEO
00279 
00286 class OpalVideoMixer : public OpalBaseMixer
00287 {
00288   public:
00289     enum Styles {
00290       eSideBySideLetterbox, 
00294       eSideBySideScaled,    
00298       eStackedPillarbox,    
00302       eStackedScaled,       
00306       eGrid,                
00308     };
00309 
00310     OpalVideoMixer(
00311       Styles style,           
00312       unsigned width,         
00313       unsigned height,        
00314       unsigned rate = 15,     
00315       bool pushThread = true  
00316     );
00317 
00318     ~OpalVideoMixer() { StopPushThread(); }
00319 
00322     unsigned GetFrameWidth() const { return m_width; }
00323 
00326     unsigned GetFrameHeight() const { return m_height; }
00327 
00330     unsigned GetFrameRate() const { return 1000/m_periodMS; }
00331 
00335     bool SetFrameRate(
00336       unsigned rate   // New frames per second.
00337     );
00338 
00342     bool SetFrameSize(
00343       unsigned width,   
00344       unsigned height   
00345     );
00346 
00347   protected:
00348     struct VideoStream : public Stream
00349     {
00350       VideoStream(OpalVideoMixer & mixer);
00351       virtual void QueuePacket(const RTP_DataFrame & rtp);
00352       void InsertVideoFrame(unsigned x, unsigned y, unsigned w, unsigned h);
00353 
00354       OpalVideoMixer & m_mixer;
00355     };
00356 
00357     friend struct VideoStream;
00358 
00359     virtual Stream * CreateStream();
00360     virtual bool MixStreams(RTP_DataFrame & frame);
00361     virtual size_t GetOutputSize() const;
00362 
00363   protected:
00364     Styles     m_style;
00365     unsigned   m_width, m_height;
00366     BYTE       m_bgFillRed,m_bgFillGreen,m_bgFillBlue;
00367 
00368     PBYTEArray m_frameStore;
00369     size_t     m_lastStreamCount;
00370 };
00371 
00372 #endif // OPAL_VIDEO
00373 
00374 
00376 
00377 
00382 struct OpalMixerNodeInfo
00383 {
00384   OpalMixerNodeInfo(const char * name = NULL)
00385     : m_name(name)
00386     , m_closeOnEmpty(false)
00387     , m_listenOnly(false)
00388     , m_sampleRate(OpalMediaFormat::AudioClockRate)
00389 #if OPAL_VIDEO
00390     , m_audioOnly(false)
00391     , m_style(OpalVideoMixer::eGrid)
00392     , m_width(PVideoFrameInfo::CIFWidth)
00393     , m_height(PVideoFrameInfo::CIFHeight)
00394     , m_rate(15)
00395 #endif
00396     , m_mediaPassThru(false)
00397   { }
00398 
00399   virtual ~OpalMixerNodeInfo() { }
00400 
00401   virtual OpalMixerNodeInfo * Clone() const { return new OpalMixerNodeInfo(*this); }
00402 
00403   PString  m_name;                
00404   bool     m_closeOnEmpty;        
00405   bool     m_listenOnly;          
00406   unsigned m_sampleRate;          
00407 #if OPAL_VIDEO
00408   bool     m_audioOnly;           
00409   OpalVideoMixer::Styles m_style; 
00410   unsigned m_width;               
00411   unsigned m_height;              
00412   unsigned m_rate;                
00413 #endif
00414   bool     m_mediaPassThru;       
00417   PString m_displayText;          
00418   PString m_subject;              
00419   PString m_notes;                
00420   PString m_keywords;             
00421 };
00422 
00423 
00425 
00426 class OpalMixerNode;
00427 
00428 
00433 class OpalMixerNodeManager
00434 {
00435   public:
00440     OpalMixerNodeManager(OpalManager & manager);
00441 
00445     virtual ~OpalMixerNodeManager();
00446 
00449     virtual void ShutDown();
00450 
00456     virtual PBoolean GarbageCollection();
00458 
00467     virtual OpalMixerNode * CreateNode(
00468       OpalMixerNodeInfo * info 
00469     );
00470 
00476     virtual PSafePtr<OpalMixerNode> AddNode(
00477       OpalMixerNodeInfo * info 
00478     );
00479 
00482     void AddNode(OpalMixerNode * node);
00483 
00487     PSafePtr<OpalMixerNode> GetFirstNode(
00488       PSafetyMode mode = PSafeReference 
00489     ) const { return PSafePtr<OpalMixerNode>(m_nodesByUID, mode); }
00490 
00494     virtual PSafePtr<OpalMixerNode> FindNode(
00495       const PString & name,             
00496       PSafetyMode mode = PSafeReference 
00497     );
00498 
00503     virtual void RemoveNode(
00504       OpalMixerNode & node
00505     );
00506 
00509     void AddNodeName(
00510       PString name,        
00511       OpalMixerNode * node 
00512     );
00513 
00516     void RemoveNodeName(
00517       PString name        
00518     );
00519 
00523     void RemoveNodeNames(
00524       const PStringSet & names   
00525     );
00526 
00529     virtual PString CreateInternalURI(
00530       const PGloballyUniqueID & guid
00531     );
00532 
00537     virtual void OnNodeStatusChanged(
00538       const OpalMixerNode & node,            
00539       OpalConferenceState::ChangeType change 
00540     );
00541 
00543     OpalManager & GetManager() const { return m_manager; }
00545 
00546   protected:
00547     OpalManager & m_manager;
00548 
00549     PSafeDictionary<PGloballyUniqueID, OpalMixerNode> m_nodesByUID;
00550     PSafeDictionary<PString, OpalMixerNode>           m_nodesByName;
00551 };
00552 
00553 
00555 
00560 class OpalMixerEndPoint : public OpalLocalEndPoint, public OpalMixerNodeManager
00561 {
00562     PCLASSINFO(OpalMixerEndPoint, OpalLocalEndPoint);
00563   public:
00568     OpalMixerEndPoint(
00569       OpalManager & manager,  
00570       const char * prefix     
00571     );
00572 
00575     ~OpalMixerEndPoint();
00576 
00581     virtual void ShutDown();
00583 
00596     virtual OpalMediaFormatList GetMediaFormats() const;
00597 
00625     virtual PSafePtr<OpalConnection> MakeConnection(
00626       OpalCall & call,           
00627       const PString & party,     
00628       void * userData = NULL,    
00629       unsigned options = 0,      
00630       OpalConnection::StringOptions * stringOptions = NULL 
00631     );
00632 
00648     virtual bool GetConferenceStates(
00649       OpalConferenceStates & states,          
00650       const PString & name = PString::Empty() 
00651     ) const;
00652 
00657     virtual PBoolean GarbageCollection();
00659 
00668     PSafePtr<OpalMixerConnection> GetMixerConnectionWithLock(
00669       const PString & token,     
00670       PSafetyMode mode = PSafeReadWrite 
00671     ) { return GetConnectionWithLockAs<OpalMixerConnection>(token, mode); }
00672 
00676     virtual OpalMixerConnection * CreateConnection(
00677       PSafePtr<OpalMixerNode> node, 
00678       OpalCall & call,              
00679       void * userData,              
00680       unsigned options,             
00681       OpalConnection::StringOptions * stringOptions 
00682     );
00683 
00687     virtual OpalMixerNodeInfo * FindNodeInfo(
00688       const PString & name
00689     );
00691 
00706     void SetAdHocNodeInfo(
00707       const OpalMixerNodeInfo & info
00708     );
00709     void SetAdHocNodeInfo(
00710       OpalMixerNodeInfo * info
00711     );
00712 
00724     OpalMixerNodeInfo * GetAdHocNodeInfo() { return m_adHocNodeInfo; }
00725 
00743     void SetFactoryNodeInfo(
00744       const OpalMixerNodeInfo & info
00745     );
00746     void SetFactoryNodeInfo(
00747       OpalMixerNodeInfo * info
00748     );
00749 
00754     virtual PString GetNewFactoryName();
00755 
00757     OpalManager & GetManager() const { return OpalLocalEndPoint::GetManager(); }
00759 
00760   protected:
00761     virtual PString CreateInternalURI(const PGloballyUniqueID & guid);
00762     virtual void OnNodeStatusChanged(const OpalMixerNode & node, OpalConferenceState::ChangeType change);
00763 
00764     OpalMixerNodeInfo  * m_adHocNodeInfo;
00765     OpalMixerNodeInfo  * m_factoryNodeInfo;
00766     PMutex               m_infoMutex; // For above two fields
00767     PAtomicInteger       m_factoryIndex;
00768 };
00769 
00770 
00772 
00775 class OpalMixerConnection : public OpalLocalConnection
00776 {
00777     PCLASSINFO(OpalMixerConnection, OpalLocalConnection);
00778   public:
00783     OpalMixerConnection(
00784       PSafePtr<OpalMixerNode> node, 
00785       OpalCall & call,              
00786       OpalMixerEndPoint & endpoint, 
00787       void * userData,              
00788       unsigned options = 0,         
00789       OpalConnection::StringOptions * stringOptions = NULL 
00790     );
00791 
00794     ~OpalMixerConnection();
00796 
00816     virtual void OnReleased();
00817 
00824     virtual OpalMediaFormatList GetMediaFormats() const;
00825 
00840     virtual OpalMediaStream * CreateMediaStream(
00841       const OpalMediaFormat & mediaFormat, 
00842       unsigned sessionID,                  
00843       PBoolean isSource                    
00844     );
00845 
00848     virtual void OnStartMediaPatch(
00849       OpalMediaPatch & patch    
00850     );
00851 
00853     virtual void OnApplyStringOptions();
00854 
00861     virtual PBoolean SendUserInputString(
00862       const PString & value                   
00863     );
00864 
00881     virtual PBoolean SendUserInputTone(
00882       char tone,        
00883       unsigned duration = 0  
00884     );
00885 
00898     virtual bool GetConferenceState(
00899       OpalConferenceState * state  
00900     ) const;
00902 
00907     void SetListenOnly(
00908       bool listenOnly   
00909     );
00910 
00913     bool GetListenOnly() const { return m_listenOnly; }
00914 
00917     PSafePtr<OpalMixerNode> GetNode() const { return m_node; }
00919 
00920   protected:
00921     OpalMixerEndPoint     & m_endpoint;
00922     PSafePtr<OpalMixerNode> m_node;
00923     bool                    m_listenOnly;
00924 };
00925 
00926 
00930 class OpalMixerMediaStream : public OpalMediaStream
00931 {
00932     PCLASSINFO(OpalMixerMediaStream, OpalMediaStream);
00933   public:
00938     OpalMixerMediaStream(
00939       OpalConnection & conn,               
00940       const OpalMediaFormat & mediaFormat, 
00941       unsigned sessionID,                  
00942       bool isSource,                       
00943       PSafePtr<OpalMixerNode> node,        
00944       bool listenOnly                      
00945     );
00946 
00949     ~OpalMixerMediaStream();
00951 
00956     virtual PBoolean Open();
00957 
00963     virtual PBoolean WritePacket(
00964       RTP_DataFrame & packet
00965     );
00966 
00970     virtual PBoolean IsSynchronous() const;
00971 
00982     virtual PBoolean RequiresPatchThread() const;
00984 
00989     PSafePtr<OpalMixerNode> GetNode() { return m_node; }
00991 
00992   protected:
00993     virtual void InternalClose();
00994     virtual bool InternalSetJitterBuffer(const OpalJitterBuffer::Init & init) const;
00995 
00996     PSafePtr<OpalMixerNode> m_node;
00997     bool m_listenOnly;
00998 #if OPAL_VIDEO
00999     bool m_video;
01000 #endif
01001 };
01002 
01003 
01007 class OpalMixerNode : public PSafeObject
01008 {
01009     PCLASSINFO(OpalMixerNode, PSafeObject);
01010   public:
01015     OpalMixerNode(
01016       OpalMixerNodeManager & manager, 
01017       OpalMixerNodeInfo * info        
01018     );
01019 
01022     ~OpalMixerNode();
01023 
01028     void ShutDown();
01030 
01037     void PrintOn(
01038       ostream & strm    
01039     ) const;
01041 
01046     virtual void AttachConnection(
01047       OpalConnection * connection  
01048     );
01049 
01052     virtual void DetachConnection(
01053       OpalConnection * connection  
01054     );
01055 
01058     virtual bool AttachStream(
01059       OpalMixerMediaStream * stream     
01060     );
01061 
01064     virtual void DetachStream(
01065       OpalMixerMediaStream * stream     
01066     );
01067 
01070     virtual void UseMediaPassThrough(
01071       unsigned sessionID,                 
01072       OpalConnection * connection = NULL  
01073     );
01074 
01081     bool SetJitterBufferSize(
01082       const OpalBaseMixer::Key_T & key,     
01083       const OpalJitterBuffer::Init & init   
01084     ) { return m_audioMixer.SetJitterBufferSize(key, init); }
01085 
01088     bool WriteAudio(
01089       const OpalBaseMixer::Key_T & key, 
01090       const RTP_DataFrame & input       
01091     ) { return m_audioMixer.WriteStream(key, input); }
01092 
01093 #if OPAL_VIDEO
01094 
01096     bool WriteVideo(
01097       const OpalBaseMixer::Key_T & key, 
01098       const RTP_DataFrame & input       
01099     ) { return m_videoMixer.WriteStream(key, input); }
01100 #endif // OPAL_VIDEO
01101 
01104     virtual void BroadcastUserInput(
01105       const OpalConnection * connection,      
01106       const PString & value                   
01107     );
01108 
01121     virtual void GetConferenceState(
01122       OpalConferenceState & state  
01123     ) const;
01125 
01130     const PGloballyUniqueID & GetGUID() const { return m_guid; }
01131 
01134     const PStringSet & GetNames() const { return m_names; }
01135 
01138     void AddName(
01139       const PString & name
01140     );
01141 
01144     void RemoveName(
01145       const PString & name
01146     );
01147 
01153     PINDEX GetConnectionCount() const { return m_connections.GetSize(); }
01154 
01157     template <class Subclass>
01158     PSafePtr<Subclass> GetFirstConnectionAs(
01159       PSafetyMode mode = PSafeReference
01160     ) const { return PSafePtr<Subclass>(m_connections, mode); }
01161 
01164     PSafePtr<OpalConnection> GetFirstConnection(
01165       PSafetyMode mode = PSafeReference
01166     ) const { return GetFirstConnectionAs<OpalConnection>(mode); }
01167 
01170     const OpalMixerNodeInfo & GetNodeInfo() { return *m_info; }
01171 
01174     const PTime & GetCreationTime() const { return m_creationTime; }
01175 
01182     void SetOwnerConnection(
01183       const PString & connectionIdentifier
01184     ) { m_ownerConnection = connectionIdentifier; }
01186 
01187   protected:
01188     OpalMixerNodeManager & m_manager;
01189     PGloballyUniqueID      m_guid;
01190     PStringSet             m_names;
01191     OpalMixerNodeInfo    * m_info;
01192     PTime                  m_creationTime;
01193     PAtomicBoolean         m_shuttingDown;
01194 
01195     PSafeList<OpalConnection> m_connections;
01196     PString                   m_ownerConnection;
01197 
01198     struct MediaMixer
01199     {
01200       MediaMixer();
01201       void CloseOne(const PSafePtr<OpalMixerMediaStream> & stream);
01202 
01203       PSafeList<OpalMixerMediaStream> m_outputStreams;
01204     };
01205 
01206     struct AudioMixer : public OpalAudioMixer, public MediaMixer
01207     {
01208       AudioMixer(const OpalMixerNodeInfo & info);
01209       ~AudioMixer();
01210 
01211       virtual bool OnPush();
01212 
01213       struct CachedAudio {
01214         CachedAudio();
01215         ~CachedAudio();
01216         enum { Collecting, Collected, Completed } m_state;
01217         RTP_DataFrame    m_raw;
01218         RTP_DataFrame    m_encoded;
01219         OpalTranscoder * m_transcoder;
01220       };
01221       std::map<PString, CachedAudio> m_cache;
01222 
01223       void PushOne(
01224         PSafePtr<OpalMixerMediaStream> & stream,
01225         CachedAudio & cache,
01226         const short * audioToSubtract
01227       );
01228 #ifdef OPAL_MIXER_AUDIO_DEBUG
01229       class PAudioMixerDebug * m_audioDebug;
01230 #endif
01231     };
01232     AudioMixer m_audioMixer;
01233 
01234 #if OPAL_VIDEO
01235     struct VideoMixer : public OpalVideoMixer, public MediaMixer
01236     {
01237       VideoMixer(const OpalMixerNodeInfo & info);
01238       ~VideoMixer();
01239 
01240       virtual bool OnMixed(RTP_DataFrame * & output);
01241 
01242       PDictionary<PString, OpalTranscoder> m_transcoders;
01243     };
01244     VideoMixer m_videoMixer;
01245 #endif // OPAL_VIDEO
01246 };
01247 
01248 
01249 #endif // OPAL_HAS_MIXER
01250 
01251 #endif // OPAL_OPAL_OPAL_MIXER
01252 
01253 

Generated on 21 Jun 2013 for OPAL by  doxygen 1.4.7