# opalmixer.h

Go to the documentation of this file.

    00001 /*
    00002  * opalmixer.h
    00003  *
    00004  * OPAL media mixers
    00005  *
    00006  * Open Phone Abstraction Library (OPAL)
    00007  * Formally known as the Open H323 project.
    00008  *
    00009  * Copyright (C) 2007 Post Increment
    00010  *
    00011  * The contents of this file are subject to the Mozilla Public License
    00012  * Version 1.0 (the "License"); you may not use this file except in
    00013  * compliance with the License. You may obtain a copy of the License at
    00014  * http://www.mozilla.org/MPL/
    00015  *
    00016  * Software distributed under the License is distributed on an "AS IS"
    00017  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
    00018  * the License for the specific language governing rights and limitations
    00019  * under the License.
    00020  *
    00021  * The Original Code is Open Phone Abstraction Library.
    00022  *
    00023  * The Initial Developer of the Original Code is Post Increment
    00024  *
    00025  * Contributor(s): Craig Southeren (craigs@postincrement.com)
    00026  *                 Robert Jongbloed (robertj@voxlucida.com.au)
    00027  *
    00028  * $Revision: 29867 $
    00029  * $Author: rjongbloed $
    00030  * $Date: 2013-06-02 22:06:44 -0500 (Sun, 02 Jun 2013) $
    00031  */
    00032
    00033
    00034 #ifndef OPAL_OPAL_OPALMIXER_H
    00035 #define OPAL_OPAL_OPALMIXER_H
    00036
    00037 #ifndef _PTLIB_H
    00038 #include <ptlib.h>
    00039 #endif
    00040
    00041 #include <opal/buildopts.h>
    00042
    00043 #if OPAL_HAS_MIXER
    00044
    00045 #include <queue>
    00046
    00047 #include <ep/localep.h>
    00048 #include <codec/vidcodec.h>
    00049
    00050
    00051 class RTP_DataFrame;
    00052 class OpalJitterBuffer;
    00053 class OpalMixerConnection;
    00054
    00055
    00056 //#define OPAL_MIXER_AUDIO_DEBUG 1
    00057
    00058
    00059 #define OPAL_OPT_LISTEN_ONLY "Listen-Only"
    00060 #define OPAL_OPT_CONF_OWNER  "Conference-Owner"
    00061
    00062
    00064
    00074 class OpalBaseMixer : public PObject
    00075 {
    00076   public:
    00077     OpalBaseMixer(
    00078       bool pushThread,
    00079       unsigned periodMS,
    00080       unsigned periodTS
    00081     );
    00082
    00083     virtual ~OpalBaseMixer();
    00084
    00085     typedef PString Key_T;
    00086
    00089     virtual bool AddStream(
    00090       const Key_T & key
    00091     );
    00092
    00095     virtual void RemoveStream(
    00096       const Key_T & key
    00097     );
    00098
    00101     virtual void RemoveAllStreams();
    00102
    00108     virtual bool WriteStream(
    00109       const Key_T & key,
    00110       const RTP_DataFrame & input
    00111     );
    00112
    00122     virtual RTP_DataFrame * ReadMixed();
    00123     virtual bool ReadMixed(RTP_DataFrame & mixed);
    00124
    00135     virtual bool OnMixed(
    00136       RTP_DataFrame * & mixed
    00137     );
    00138
    00142     void StartPushThread();
    00143
    00148     void StopPushThread(bool lock = true);
    00149
    00152     unsigned GetPeriodTS() const { return m_periodTS; }
    00153
    00154   protected:
    00155     struct Stream : public PObject {
    00156       virtual ~Stream() { }
    00157       virtual void QueuePacket(const RTP_DataFrame & rtp) = 0;
    00158       std::queue<RTP_DataFrame> m_queue;
    00159     };
    00160     typedef std::map<Key_T, Stream *> StreamMap_T;
    00161
    00162     virtual Stream * CreateStream() = 0;
    00163     virtual bool MixStreams(RTP_DataFrame & frame) = 0;
    00164     virtual size_t GetOutputSize() const = 0;
    00165
    00166     virtual bool OnPush();
    00167     void PushThreadMain();
    00168
    00169     bool      m_pushThread;      // true if to use a thread to push data out
    00170     unsigned  m_periodMS;        // Mixing interval in milliseconds
    00171     unsigned  m_periodTS;        // Mixing interval in timestamp units
    00172
    00173     StreamMap_T     m_inputStreams;     // Map of key to stream for input RTP frame queues
    00174     unsigned        m_outputTimestamp;  // RTP timestamp for output data
    00175     RTP_DataFrame * m_pushFrame;        // Cached frame for pushing RTP
    00176     PThread *       m_workerThread;     // reader thread handle
    00177     bool            m_threadRunning;    // used to stop reader thread
    00178     PMutex          m_mutex;            // mutex for list of streams and thread handle
    00179 };
    00180
    00182
    00191 class OpalAudioMixer : public OpalBaseMixer
    00192 {
    00193   public:
    00194     OpalAudioMixer(
    00195       bool stereo = false,
    00196       unsigned sampleRate = OpalMediaFormat::AudioClockRate,
    00197       bool pushThread = true,
    00198       unsigned period = 10
    00199     );
    00200
    00201     ~OpalAudioMixer() { StopPushThread(); }
    00202
    00205     virtual void RemoveStream(
    00206       const Key_T & key
    00207     );
    00208
    00211     virtual void RemoveAllStreams();
    00212
    00215     bool IsStereo() const { return m_stereo; }
    00216
    00219     unsigned GetSampleRate() const { return m_sampleRate; }
    00220
    00227     bool SetSampleRate(
    00228       unsigned rate
    00229     );
    00230
    00237     bool SetJitterBufferSize(
    00238       const Key_T & key,
    00239       const OpalJitterBuffer::Init & init
    00240     );
    00241
    00242   protected:
    00243     struct AudioStream : public Stream
    00244     {
    00245       AudioStream(OpalAudioMixer & mixer);
    00246       ~AudioStream();
    00247
    00248       virtual void QueuePacket(const RTP_DataFrame & rtp);
    00249       const short * GetAudioDataPtr();
    00250
    00251       OpalAudioMixer   & m_mixer;
    00252       OpalJitterBuffer * m_jitter;
    00253       unsigned           m_nextTimestamp;
    00254       PShortArray        m_cacheSamples;
    00255       size_t             m_samplesUsed;
    00256     };
    00257
    00258     virtual Stream * CreateStream();
    00259     virtual bool MixStreams(RTP_DataFrame & frame);
    00260     virtual size_t GetOutputSize() const;
    00261
    00262     void PreMixStreams();
    00263     void MixStereo(RTP_DataFrame & frame);
    00264     void MixAdditive(RTP_DataFrame & frame, const short * audioToSubtract);
    00265
    00266   protected:
    00267     bool     m_stereo;
    00268     unsigned m_sampleRate;
    00269
    00270     AudioStream    * m_left;
    00271     AudioStream    * m_right;
    00272     std::vector<int> m_mixedAudio;
    00273 };
    00274
    00275
    00277
    00278 #if OPAL_VIDEO
    00279
    00286 class OpalVideoMixer : public OpalBaseMixer
    00287 {
    00288   public:
    00289     enum Styles {
    00290       eSideBySideLetterbox,
    00294       eSideBySideScaled,
    00298       eStackedPillarbox,
    00302       eStackedScaled,
    00306       eGrid,
    00308     };
    00309
    00310     OpalVideoMixer(
    00311       Styles style,
    00312       unsigned width,
    00313       unsigned height,
    00314       unsigned rate = 15,
    00315       bool pushThread = true
    00316     );
    00317
    00318     ~OpalVideoMixer() { StopPushThread(); }
    00319
    00322     unsigned GetFrameWidth() const { return m_width; }
    00323
    00326     unsigned GetFrameHeight() const { return m_height; }
    00327
    00330     unsigned GetFrameRate() const { return 1000/m_periodMS; }
    00331
    00335     bool SetFrameRate(
    00336       unsigned rate   // New frames per second.
    00337     );
    00338
    00342     bool SetFrameSize(
    00343       unsigned width,
    00344       unsigned height
    00345     );
    00346
    00347   protected:
    00348     struct VideoStream : public Stream
    00349     {
    00350       VideoStream(OpalVideoMixer & mixer);
    00351       virtual void QueuePacket(const RTP_DataFrame & rtp);
    00352       void InsertVideoFrame(unsigned x, unsigned y, unsigned w, unsigned h);
    00353
    00354       OpalVideoMixer & m_mixer;
    00355     };
    00356
    00357     friend struct VideoStream;
    00358
    00359     virtual Stream * CreateStream();
    00360     virtual bool MixStreams(RTP_DataFrame & frame);
    00361     virtual size_t GetOutputSize() const;
    00362
    00363   protected:
    00364     Styles     m_style;
    00365     unsigned   m_width, m_height;
    00366     BYTE       m_bgFillRed,m_bgFillGreen,m_bgFillBlue;
    00367
    00368     PBYTEArray m_frameStore;
    00369     size_t     m_lastStreamCount;
    00370 };
    00371
    00372 #endif // OPAL_VIDEO
    00373
    00374
    00376
    00377
    00382 struct OpalMixerNodeInfo
    00383 {
    00384   OpalMixerNodeInfo(const char * name = NULL)
    00385     : m_name(name)
    00386     , m_closeOnEmpty(false)
    00387     , m_listenOnly(false)
    00388     , m_sampleRate(OpalMediaFormat::AudioClockRate)
    00389 #if OPAL_VIDEO
    00390     , m_audioOnly(false)
    00391     , m_style(OpalVideoMixer::eGrid)
    00392     , m_width(PVideoFrameInfo::CIFWidth)
    00393     , m_height(PVideoFrameInfo::CIFHeight)
    00394     , m_rate(15)
    00395 #endif
    00396     , m_mediaPassThru(false)
    00397   { }
    00398
    00399   virtual ~OpalMixerNodeInfo() { }
    00400
    00401   virtual OpalMixerNodeInfo * Clone() const { return new OpalMixerNodeInfo(*this); }
    00402
    00403   PString  m_name;
    00404   bool     m_closeOnEmpty;
    00405   bool     m_listenOnly;
    00406   unsigned m_sampleRate;
    00407 #if OPAL_VIDEO
    00408   bool     m_audioOnly;
    00409   OpalVideoMixer::Styles m_style;
    00410   unsigned m_width;
    00411   unsigned m_height;
    00412   unsigned m_rate;
    00413 #endif
    00414   bool     m_mediaPassThru;
    00417   PString m_displayText;
    00418   PString m_subject;
    00419   PString m_notes;
    00420   PString m_keywords;
    00421 };
    00422
    00423
    00425
    00426 class OpalMixerNode;
    00427
    00428
    00433 class OpalMixerNodeManager
    00434 {
    00435   public:
    00440     OpalMixerNodeManager(OpalManager & manager);
    00441
    00445     virtual ~OpalMixerNodeManager();
    00446
    00449     virtual void ShutDown();
    00450
    00456     virtual PBoolean GarbageCollection();
    00458
    00467     virtual OpalMixerNode * CreateNode(
    00468       OpalMixerNodeInfo * info
    00469     );
    00470
    00476     virtual PSafePtr<OpalMixerNode> AddNode(
    00477       OpalMixerNodeInfo * info
    00478     );
    00479
    00482     void AddNode(OpalMixerNode * node);
    00483
    00487     PSafePtr<OpalMixerNode> GetFirstNode(
    00488       PSafetyMode mode = PSafeReference
    00489     ) const { return PSafePtr<OpalMixerNode>(m_nodesByUID, mode); }
    00490
    00494     virtual PSafePtr<OpalMixerNode> FindNode(
    00495       const PString & name,
    00496       PSafetyMode mode = PSafeReference
    00497     );
    00498
    00503     virtual void RemoveNode(
    00504       OpalMixerNode & node
    00505     );
    00506
    00509     void AddNodeName(
    00510       PString name,
    00511       OpalMixerNode * node
    00512     );
    00513
    00516     void RemoveNodeName(
    00517       PString name
    00518     );
    00519
    00523     void RemoveNodeNames(
    00524       const PStringSet & names
    00525     );
    00526
    00529     virtual PString CreateInternalURI(
    00530       const PGloballyUniqueID & guid
    00531     );
    00532
    00537     virtual void OnNodeStatusChanged(
    00538       const OpalMixerNode & node,
    00539       OpalConferenceState::ChangeType change
    00540     );
    00541
    00543     OpalManager & GetManager() const { return m_manager; }
    00545
    00546   protected:
    00547     OpalManager & m_manager;
    00548
    00549     PSafeDictionary<PGloballyUniqueID, OpalMixerNode> m_nodesByUID;
    00550     PSafeDictionary<PString, OpalMixerNode>           m_nodesByName;
    00551 };
    00552
    00553
    00555
    00560 class OpalMixerEndPoint : public OpalLocalEndPoint, public OpalMixerNodeManager
    00561 {
    00562     PCLASSINFO(OpalMixerEndPoint, OpalLocalEndPoint);
    00563   public:
    00568     OpalMixerEndPoint(
    00569       OpalManager & manager,
    00570       const char * prefix
    00571     );
    00572
    00575     ~OpalMixerEndPoint();
    00576
    00581     virtual void ShutDown();
    00583
    00596     virtual OpalMediaFormatList GetMediaFormats() const;
    00597
    00625     virtual PSafePtr<OpalConnection> MakeConnection(
    00626       OpalCall & call,
    00627       const PString & party,
    00628       void * userData = NULL,
    00629       unsigned options = 0,
    00630       OpalConnection::StringOptions * stringOptions = NULL
    00631     );
    00632
    00648     virtual bool GetConferenceStates(
    00649       OpalConferenceStates & states,
    00650       const PString & name = PString::Empty()
    00651     ) const;
    00652
    00657     virtual PBoolean GarbageCollection();
    00659
    00668     PSafePtr<OpalMixerConnection> GetMixerConnectionWithLock(
    00669       const PString & token,
    00670       PSafetyMode mode = PSafeReadWrite
    00671     ) { return GetConnectionWithLockAs<OpalMixerConnection>(token, mode); }
    00672
    00676     virtual OpalMixerConnection * CreateConnection(
    00677       PSafePtr<OpalMixerNode> node,
    00678       OpalCall & call,
    00679       void * userData,
    00680       unsigned options,
    00681       OpalConnection::StringOptions * stringOptions
    00682     );
    00683
    00687     virtual OpalMixerNodeInfo * FindNodeInfo(
    00688       const PString & name
    00689     );
    00691
    00706     void SetAdHocNodeInfo(
    00707       const OpalMixerNodeInfo & info
    00708     );
    00709     void SetAdHocNodeInfo(
    00710       OpalMixerNodeInfo * info
    00711     );
    00712
    00724     OpalMixerNodeInfo * GetAdHocNodeInfo() { return m_adHocNodeInfo; }
    00725
    00743     void SetFactoryNodeInfo(
    00744       const OpalMixerNodeInfo & info
    00745     );
    00746     void SetFactoryNodeInfo(
    00747       OpalMixerNodeInfo * info
    00748     );
    00749
    00754     virtual PString GetNewFactoryName();
    00755
    00757     OpalManager & GetManager() const { return OpalLocalEndPoint::GetManager(); }
    00759
    00760   protected:
    00761     virtual PString CreateInternalURI(const PGloballyUniqueID & guid);
    00762     virtual void OnNodeStatusChanged(const OpalMixerNode & node, OpalConferenceState::ChangeType change);
    00763
    00764     OpalMixerNodeInfo  * m_adHocNodeInfo;
    00765     OpalMixerNodeInfo  * m_factoryNodeInfo;
    00766     PMutex               m_infoMutex; // For above two fields
    00767     PAtomicInteger       m_factoryIndex;
    00768 };
    00769
    00770
    00772
    00775 class OpalMixerConnection : public OpalLocalConnection
    00776 {
    00777     PCLASSINFO(OpalMixerConnection, OpalLocalConnection);
    00778   public:
    00783     OpalMixerConnection(
    00784       PSafePtr<OpalMixerNode> node,
    00785       OpalCall & call,
    00786       OpalMixerEndPoint & endpoint,
    00787       void * userData,
    00788       unsigned options = 0,
    00789       OpalConnection::StringOptions * stringOptions = NULL
    00790     );
    00791
    00794     ~OpalMixerConnection();
    00796
    00816     virtual void OnReleased();
    00817
    00824     virtual OpalMediaFormatList GetMediaFormats() const;
    00825
    00840     virtual OpalMediaStream * CreateMediaStream(
    00841       const OpalMediaFormat & mediaFormat,
    00842       unsigned sessionID,
    00843       PBoolean isSource
    00844     );
    00845
    00848     virtual void OnStartMediaPatch(
    00849       OpalMediaPatch & patch
    00850     );
    00851
    00853     virtual void OnApplyStringOptions();
    00854
    00861     virtual PBoolean SendUserInputString(
    00862       const PString & value
    00863     );
    00864
    00881     virtual PBoolean SendUserInputTone(
    00882       char tone,
    00883       unsigned duration = 0
    00884     );
    00885
    00898     virtual bool GetConferenceState(
    00899       OpalConferenceState * state
    00900     ) const;
    00902
    00907     void SetListenOnly(
    00908       bool listenOnly
    00909     );
    00910
    00913     bool GetListenOnly() const { return m_listenOnly; }
    00914
    00917     PSafePtr<OpalMixerNode> GetNode() const { return m_node; }
    00919
    00920   protected:
    00921     OpalMixerEndPoint     & m_endpoint;
    00922     PSafePtr<OpalMixerNode> m_node;
    00923     bool                    m_listenOnly;
    00924 };
    00925
    00926
    00930 class OpalMixerMediaStream : public OpalMediaStream
    00931 {
    00932     PCLASSINFO(OpalMixerMediaStream, OpalMediaStream);
    00933   public:
    00938     OpalMixerMediaStream(
    00939       OpalConnection & conn,
    00940       const OpalMediaFormat & mediaFormat,
    00941       unsigned sessionID,
    00942       bool isSource,
    00943       PSafePtr<OpalMixerNode> node,
    00944       bool listenOnly
    00945     );
    00946
    00949     ~OpalMixerMediaStream();
    00951
    00956     virtual PBoolean Open();
    00957
    00963     virtual PBoolean WritePacket(
    00964       RTP_DataFrame & packet
    00965     );
    00966
    00970     virtual PBoolean IsSynchronous() const;
    00971
    00982     virtual PBoolean RequiresPatchThread() const;
    00984
    00989     PSafePtr<OpalMixerNode> GetNode() { return m_node; }
    00991
    00992   protected:
    00993     virtual void InternalClose();
    00994     virtual bool InternalSetJitterBuffer(const OpalJitterBuffer::Init & init) const;
    00995
    00996     PSafePtr<OpalMixerNode> m_node;
    00997     bool m_listenOnly;
    00998 #if OPAL_VIDEO
    00999     bool m_video;
    01000 #endif
    01001 };
    01002
    01003
    01007 class OpalMixerNode : public PSafeObject
    01008 {
    01009     PCLASSINFO(OpalMixerNode, PSafeObject);
    01010   public:
    01015     OpalMixerNode(
    01016       OpalMixerNodeManager & manager,
    01017       OpalMixerNodeInfo * info
    01018     );
    01019
    01022     ~OpalMixerNode();
    01023
    01028     void ShutDown();
    01030
    01037     void PrintOn(
    01038       ostream & strm
    01039     ) const;
    01041
    01046     virtual void AttachConnection(
    01047       OpalConnection * connection
    01048     );
    01049
    01052     virtual void DetachConnection(
    01053       OpalConnection * connection
    01054     );
    01055
    01058     virtual bool AttachStream(
    01059       OpalMixerMediaStream * stream
    01060     );
    01061
    01064     virtual void DetachStream(
    01065       OpalMixerMediaStream * stream
    01066     );
    01067
    01070     virtual void UseMediaPassThrough(
    01071       unsigned sessionID,
    01072       OpalConnection * connection = NULL
    01073     );
    01074
    01081     bool SetJitterBufferSize(
    01082       const OpalBaseMixer::Key_T & key,
    01083       const OpalJitterBuffer::Init & init
    01084     ) { return m_audioMixer.SetJitterBufferSize(key, init); }
    01085
    01088     bool WriteAudio(
    01089       const OpalBaseMixer::Key_T & key,
    01090       const RTP_DataFrame & input
    01091     ) { return m_audioMixer.WriteStream(key, input); }
    01092
    01093 #if OPAL_VIDEO
    01094
    01096     bool WriteVideo(
    01097       const OpalBaseMixer::Key_T & key,
    01098       const RTP_DataFrame & input
    01099     ) { return m_videoMixer.WriteStream(key, input); }
    01100 #endif // OPAL_VIDEO
    01101
    01104     virtual void BroadcastUserInput(
    01105       const OpalConnection * connection,
    01106       const PString & value
    01107     );
    01108
    01121     virtual void GetConferenceState(
    01122       OpalConferenceState & state
    01123     ) const;
    01125
    01130     const PGloballyUniqueID & GetGUID() const { return m_guid; }
    01131
    01134     const PStringSet & GetNames() const { return m_names; }
    01135
    01138     void AddName(
    01139       const PString & name
    01140     );
    01141
    01144     void RemoveName(
    01145       const PString & name
    01146     );
    01147
    01153     PINDEX GetConnectionCount() const { return m_connections.GetSize(); }
    01154
    01157     template <class Subclass>
    01158     PSafePtr<Subclass> GetFirstConnectionAs(
    01159       PSafetyMode mode = PSafeReference
    01160     ) const { return PSafePtr<Subclass>(m_connections, mode); }
    01161
    01164     PSafePtr<OpalConnection> GetFirstConnection(
    01165       PSafetyMode mode = PSafeReference
    01166     ) const { return GetFirstConnectionAs<OpalConnection>(mode); }
    01167
    01170     const OpalMixerNodeInfo & GetNodeInfo() { return *m_info; }
    01171
    01174     const PTime & GetCreationTime() const { return m_creationTime; }
    01175
    01182     void SetOwnerConnection(
    01183       const PString & connectionIdentifier
    01184     ) { m_ownerConnection = connectionIdentifier; }
    01186
    01187   protected:
    01188     OpalMixerNodeManager & m_manager;
    01189     PGloballyUniqueID      m_guid;
    01190     PStringSet             m_names;
    01191     OpalMixerNodeInfo    * m_info;
    01192     PTime                  m_creationTime;
    01193     PAtomicBoolean         m_shuttingDown;
    01194
    01195     PSafeList<OpalConnection> m_connections;
    01196     PString                   m_ownerConnection;
    01197
    01198     struct MediaMixer
    01199     {
    01200       MediaMixer();
    01201       void CloseOne(const PSafePtr<OpalMixerMediaStream> & stream);
    01202
    01203       PSafeList<OpalMixerMediaStream> m_outputStreams;
    01204     };
    01205
    01206     struct AudioMixer : public OpalAudioMixer, public MediaMixer
    01207     {
    01208       AudioMixer(const OpalMixerNodeInfo & info);
    01209       ~AudioMixer();
    01210
    01211       virtual bool OnPush();
    01212
    01213       struct CachedAudio {
    01214         CachedAudio();
    01215         ~CachedAudio();
    01216         enum { Collecting, Collected, Completed } m_state;
    01217         RTP_DataFrame    m_raw;
    01218         RTP_DataFrame    m_encoded;
    01219         OpalTranscoder * m_transcoder;
    01220       };
    01221       std::map<PString, CachedAudio> m_cache;
    01222
    01223       void PushOne(
    01224         PSafePtr<OpalMixerMediaStream> & stream,
    01225         CachedAudio & cache,
    01226         const short * audioToSubtract
    01227       );
    01228 #ifdef OPAL_MIXER_AUDIO_DEBUG
    01229       class PAudioMixerDebug * m_audioDebug;
    01230 #endif
    01231     };
    01232     AudioMixer m_audioMixer;
    01233
    01234 #if OPAL_VIDEO
    01235     struct VideoMixer : public OpalVideoMixer, public MediaMixer
    01236     {
    01237       VideoMixer(const OpalMixerNodeInfo & info);
    01238       ~VideoMixer();
    01239
    01240       virtual bool OnMixed(RTP_DataFrame * & output);
    01241
    01242       PDictionary<PString, OpalTranscoder> m_transcoders;
    01243     };
    01244     VideoMixer m_videoMixer;
    01245 #endif // OPAL_VIDEO
    01246 };
    01247
    01248
    01249 #endif // OPAL_HAS_MIXER
    01250
    01251 #endif // OPAL_OPAL_OPAL_MIXER
    01252
    01253