# opalmixer.h

Go to the documentation of this file.

    00001 /*
    00002  * opalmixer.h
    00003  *
    00004  * OPAL media mixers
    00005  *
    00006  * Open Phone Abstraction Library (OPAL)
    00007  * Formally known as the Open H323 project.
    00008  *
    00009  * Copyright (C) 2007 Post Increment
    00010  *
    00011  * The contents of this file are subject to the Mozilla Public License
    00012  * Version 1.0 (the "License"); you may not use this file except in
    00013  * compliance with the License. You may obtain a copy of the License at
    00014  * http://www.mozilla.org/MPL/
    00015  *
    00016  * Software distributed under the License is distributed on an "AS IS"
    00017  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
    00018  * the License for the specific language governing rights and limitations
    00019  * under the License.
    00020  *
    00021  * The Original Code is Open Phone Abstraction Library.
    00022  *
    00023  * The Initial Developer of the Original Code is Post Increment
    00024  *
    00025  * Contributor(s): Craig Southeren (craigs@postincrement.com)
    00026  *                 Robert Jongbloed (robertj@voxlucida.com.au)
    00027  *
    00028  * $Revision: 27149 $
    00029  * $Author: rjongbloed $
    00030  * $Date: 2012-03-07 18:32:36 -0600 (Wed, 07 Mar 2012) $
    00031  */
    00032
    00033
    00034 #ifndef OPAL_OPAL_OPALMIXER_H
    00035 #define OPAL_OPAL_OPALMIXER_H
    00036
    00037 #ifndef _PTLIB_H
    00038 #include <ptlib.h>
    00039 #endif
    00040
    00041 #include <opal/buildopts.h>
    00042
    00043 #include <queue>
    00044
    00045 #include <opal/localep.h>
    00046 #include <codec/vidcodec.h>
    00047 #include <ptclib/threadpool.h>
    00048
    00049
    00050 class RTP_DataFrame;
    00051 class OpalJitterBuffer;
    00052 class OpalMixerConnection;
    00053
    00054
    00055 //#define OPAL_MIXER_AUDIO_DEBUG 1
    00056
    00057
    00058 #define OPAL_OPT_LISTEN_ONLY "Listen-Only"
    00059
    00060
    00062
    00072 class OpalBaseMixer
    00073 {
    00074   public:
    00075     OpalBaseMixer(
    00076       bool pushThread,
    00077       unsigned periodMS,
    00078       unsigned periodTS
    00079     );
    00080
    00081     virtual ~OpalBaseMixer();
    00082
    00083     typedef PString Key_T;
    00084
    00087     virtual bool AddStream(
    00088       const Key_T & key
    00089     );
    00090
    00093     virtual void RemoveStream(
    00094       const Key_T & key
    00095     );
    00096
    00099     virtual void RemoveAllStreams();
    00100
    00106     virtual bool WriteStream(
    00107       const Key_T & key,
    00108       const RTP_DataFrame & input
    00109     );
    00110
    00120     virtual RTP_DataFrame * ReadMixed();
    00121     virtual bool ReadMixed(RTP_DataFrame & mixed);
    00122
    00133     virtual bool OnMixed(
    00134       RTP_DataFrame * & mixed
    00135     );
    00136
    00140     void StartPushThread();
    00141
    00146     void StopPushThread(bool lock = true);
    00147
    00150     unsigned GetPeriodTS() const { return m_periodTS; }
    00151
    00152   protected:
    00153     struct Stream {
    00154       virtual ~Stream() { }
    00155       virtual void QueuePacket(const RTP_DataFrame & rtp) = 0;
    00156       queue<RTP_DataFrame> m_queue;
    00157     };
    00158     typedef std::map<Key_T, Stream *> StreamMap_T;
    00159
    00160     virtual Stream * CreateStream() = 0;
    00161     virtual bool MixStreams(RTP_DataFrame & frame) = 0;
    00162     virtual size_t GetOutputSize() const = 0;
    00163
    00164     virtual bool OnPush();
    00165     void PushThreadMain();
    00166
    00167     bool      m_pushThread;      // true if to use a thread to push data out
    00168     unsigned  m_periodMS;        // Mixing interval in milliseconds
    00169     unsigned  m_periodTS;        // Mixing interval in timestamp units
    00170
    00171     StreamMap_T     m_inputStreams;     // Map of key to stream for input RTP frame queues
    00172     unsigned        m_outputTimestamp;  // RTP timestamp for output data
    00173     RTP_DataFrame * m_pushFrame;        // Cached frame for pushing RTP
    00174     PThread *       m_workerThread;     // reader thread handle
    00175     bool            m_threadRunning;    // used to stop reader thread
    00176     PMutex          m_mutex;            // mutex for list of streams and thread handle
    00177 };
    00178
    00180
    00189 class OpalAudioMixer : public OpalBaseMixer
    00190 {
    00191   public:
    00192     OpalAudioMixer(
    00193       bool stereo = false,
    00194       unsigned sampleRate = OpalMediaFormat::AudioClockRate,
    00195       bool pushThread = true,
    00196       unsigned period = 10
    00197     );
    00198
    00199     ~OpalAudioMixer() { StopPushThread(); }
    00200
    00203     virtual void RemoveStream(
    00204       const Key_T & key
    00205     );
    00206
    00209     virtual void RemoveAllStreams();
    00210
    00213     bool IsStereo() const { return m_stereo; }
    00214
    00217     unsigned GetSampleRate() const { return m_sampleRate; }
    00218
    00225     bool SetSampleRate(
    00226       unsigned rate
    00227     );
    00228
    00235     bool SetJitterBufferSize(
    00236       const Key_T & key,
    00237       unsigned minJitterDelay,
    00238       unsigned maxJitterDelay
    00239     );
    00240
    00241   protected:
    00242     struct AudioStream : public Stream
    00243     {
    00244       AudioStream(OpalAudioMixer & mixer);
    00245       ~AudioStream();
    00246
    00247       virtual void QueuePacket(const RTP_DataFrame & rtp);
    00248       const short * GetAudioDataPtr();
    00249
    00250       OpalAudioMixer   & m_mixer;
    00251       OpalJitterBuffer * m_jitter;
    00252       unsigned           m_nextTimestamp;
    00253       PShortArray        m_cacheSamples;
    00254       size_t             m_samplesUsed;
    00255     };
    00256
    00257     virtual Stream * CreateStream();
    00258     virtual bool MixStreams(RTP_DataFrame & frame);
    00259     virtual size_t GetOutputSize() const;
    00260
    00261     void PreMixStreams();
    00262     void MixStereo(RTP_DataFrame & frame);
    00263     void MixAdditive(RTP_DataFrame & frame, const short * audioToSubtract);
    00264
    00265   protected:
    00266     bool     m_stereo;
    00267     unsigned m_sampleRate;
    00268
    00269     AudioStream    * m_left;
    00270     AudioStream    * m_right;
    00271     std::vector<int> m_mixedAudio;
    00272 };
    00273
    00274
    00276
    00277 #if OPAL_VIDEO
    00278
    00285 class OpalVideoMixer : public OpalBaseMixer
    00286 {
    00287   public:
    00288     enum Styles {
    00289       eSideBySideLetterbox,
    00293       eSideBySideScaled,
    00297       eStackedPillarbox,
    00301       eStackedScaled,
    00305       eGrid,
    00307     };
    00308
    00309     OpalVideoMixer(
    00310       Styles style,
    00311       unsigned width,
    00312       unsigned height,
    00313       unsigned rate = 15,
    00314       bool pushThread = true
    00315     );
    00316
    00317     ~OpalVideoMixer() { StopPushThread(); }
    00318
    00321     unsigned GetFrameWidth() const { return m_width; }
    00322
    00325     unsigned GetFrameHeight() const { return m_height; }
    00326
    00329     unsigned GetFrameRate() const { return 1000/m_periodMS; }
    00330
    00334     bool SetFrameRate(
    00335       unsigned rate   // New frames per second.
    00336     );
    00337
    00341     bool SetFrameSize(
    00342       unsigned width,
    00343       unsigned height
    00344     );
    00345
    00346   protected:
    00347     struct VideoStream : public Stream
    00348     {
    00349       VideoStream(OpalVideoMixer & mixer);
    00350       virtual void QueuePacket(const RTP_DataFrame & rtp);
    00351       void InsertVideoFrame(unsigned x, unsigned y, unsigned w, unsigned h);
    00352
    00353       OpalVideoMixer & m_mixer;
    00354     };
    00355
    00356     friend struct VideoStream;
    00357
    00358     virtual Stream * CreateStream();
    00359     virtual bool MixStreams(RTP_DataFrame & frame);
    00360     virtual size_t GetOutputSize() const;
    00361
    00362   protected:
    00363     Styles     m_style;
    00364     unsigned   m_width, m_height;
    00365     BYTE       m_bgFillRed,m_bgFillGreen,m_bgFillBlue;
    00366
    00367     PBYTEArray m_frameStore;
    00368     size_t     m_lastStreamCount;
    00369 };
    00370
    00371 #endif // OPAL_VIDEO
    00372
    00373
    00375
    00376
    00381 struct OpalMixerNodeInfo
    00382 {
    00383   OpalMixerNodeInfo(const char * name = NULL)
    00384     : m_name(name)
    00385     , m_listenOnly(false)
    00386     , m_sampleRate(OpalMediaFormat::AudioClockRate)
    00387 #if OPAL_VIDEO
    00388     , m_audioOnly(false)
    00389     , m_style(OpalVideoMixer::eGrid)
    00390     , m_width(PVideoFrameInfo::CIFWidth)
    00391     , m_height(PVideoFrameInfo::CIFHeight)
    00392     , m_rate(15)
    00393 #endif
    00394     , m_mediaPassThru(false)
    00395   { }
    00396
    00397   virtual ~OpalMixerNodeInfo() { }
    00398
    00399   virtual OpalMixerNodeInfo * Clone() const { return new OpalMixerNodeInfo(*this); }
    00400
    00401   PString  m_name;
    00402   bool     m_listenOnly;
    00403   unsigned m_sampleRate;
    00404 #if OPAL_VIDEO
    00405   bool     m_audioOnly;
    00406   OpalVideoMixer::Styles m_style;
    00407   unsigned m_width;
    00408   unsigned m_height;
    00409   unsigned m_rate;
    00410 #endif
    00411   bool     m_mediaPassThru;
    00413 };
    00414
    00415
    00417
    00418 class OpalMixerNode;
    00419
    00420
    00425 class OpalMixerNodeManager : public PObject
    00426 {
    00427     PCLASSINFO(OpalMixerNodeManager, PObject);
    00428   public:
    00433     OpalMixerNodeManager();
    00434
    00438         virtual ~OpalMixerNodeManager();
    00439
    00442     virtual void ShutDown();
    00443
    00449     virtual PBoolean GarbageCollection();
    00451
    00460     virtual OpalMixerNode * CreateNode(
    00461       OpalMixerNodeInfo * info
    00462     );
    00463
    00469     virtual PSafePtr<OpalMixerNode> AddNode(
    00470       OpalMixerNodeInfo * info
    00471     );
    00472
    00475     void AddNode(OpalMixerNode * node);
    00476
    00480     PSafePtr<OpalMixerNode> GetFirstNode(
    00481       PSafetyMode mode = PSafeReference
    00482     ) const { return PSafePtr<OpalMixerNode>(m_nodesByUID, mode); }
    00483
    00487     virtual PSafePtr<OpalMixerNode> FindNode(
    00488       const PString & name,
    00489       PSafetyMode mode = PSafeReference
    00490     );
    00491
    00496     virtual void RemoveNode(
    00497       OpalMixerNode & node
    00498     );
    00499
    00502     void AddNodeName(
    00503       PString name,
    00504       OpalMixerNode * node
    00505     );
    00506
    00509     void RemoveNodeName(
    00510       PString name
    00511     );
    00512
    00516     void RemoveNodeNames(
    00517       PStringList names
    00518     );
    00519
    00522     void QueueUserInput(
    00523       const PSafePtr<OpalMixerNode> & node,
    00524       const OpalMixerConnection * connection,
    00525       const PString & value
    00526     );
    00528
    00529   protected:
    00530     PSafeDictionary<PGloballyUniqueID, OpalMixerNode> m_nodesByUID;
    00531     PDictionary<PString, OpalMixerNode>               m_nodesByName;
    00532
    00533     struct UserInput {
    00534       UserInput(
    00535         const PSafePtr<OpalMixerNode> & node,
    00536         const OpalMixerConnection * connection,
    00537         const PString & value
    00538       ) : m_node(node)
    00539         , m_connection(connection)
    00540         , m_value(value)
    00541       { }
    00542
    00543       PSafePtr<OpalMixerNode> m_node;
    00544       const OpalMixerConnection * m_connection;
    00545       PString m_value;
    00546
    00547       void Work();
    00548     };
    00549     PQueuedThreadPool<UserInput> m_userInputPool;
    00550 };
    00551
    00552
    00554
    00559 class OpalMixerEndPoint : public OpalLocalEndPoint
    00560 {
    00561     PCLASSINFO(OpalMixerEndPoint, OpalLocalEndPoint);
    00562   public:
    00567     OpalMixerEndPoint(
    00568       OpalManager & manager,
    00569       const char * prefix
    00570     );
    00571
    00574     ~OpalMixerEndPoint();
    00575
    00580     virtual void ShutDown();
    00582
    00595     virtual OpalMediaFormatList GetMediaFormats() const;
    00596
    00626     virtual PSafePtr<OpalConnection> MakeConnection(
    00627       OpalCall & call,
    00628       const PString & party,
    00629       void * userData = NULL,
    00630       unsigned options = 0,
    00631       OpalConnection::StringOptions * stringOptions = NULL
    00632     );
    00633
    00638     virtual PBoolean GarbageCollection();
    00640
    00649     PSafePtr<OpalMixerConnection> GetMixerConnectionWithLock(
    00650       const PString & token,
    00651       PSafetyMode mode = PSafeReadWrite
    00652     ) { return GetConnectionWithLockAs<OpalMixerConnection>(token, mode); }
    00653
    00657     virtual OpalMixerConnection * CreateConnection(
    00658       PSafePtr<OpalMixerNode> node,
    00659       OpalCall & call,
    00660       void * userData,
    00661       unsigned options,
    00662       OpalConnection::StringOptions * stringOptions
    00663     );
    00665
    00673     PSafePtr<OpalMixerNode> AddNode(
    00674       OpalMixerNodeInfo * info
    00675     );
    00676
    00683     virtual OpalMixerNode * CreateNode(
    00684       OpalMixerNodeInfo * info
    00685     );
    00686
    00690     PSafePtr<OpalMixerNode> GetFirstNode(
    00691       PSafetyMode mode = PSafeReference
    00692     ) const { return m_nodeManager.GetFirstNode(mode); }
    00693
    00697     PSafePtr<OpalMixerNode> FindNode(
    00698       const PString & name,
    00699       PSafetyMode mode = PSafeReference
    00700     ) { return m_nodeManager.FindNode(name, mode); }
    00701
    00706     void RemoveNode(
    00707       OpalMixerNode & node
    00708     ) { m_nodeManager.RemoveNode(node); }
    00710
    00725     void SetAdHocNodeInfo(
    00726       const OpalMixerNodeInfo & info
    00727     );
    00728     void SetAdHocNodeInfo(
    00729       OpalMixerNodeInfo * info
    00730     );
    00731
    00743     OpalMixerNodeInfo * GetAdHocNodeInfo() { return m_adHocNodeInfo; }
    00744
    00747     const OpalMixerNodeManager & GetNodeManager() const { return m_nodeManager; }
    00748           OpalMixerNodeManager & GetNodeManager()       { return m_nodeManager; }
    00750
    00751   protected:
    00752     OpalMixerNodeInfo  * m_adHocNodeInfo;
    00753     OpalMixerNodeManager m_nodeManager;
    00754 };
    00755
    00756
    00758
    00761 class OpalMixerConnection : public OpalLocalConnection
    00762 {
    00763     PCLASSINFO(OpalMixerConnection, OpalLocalConnection);
    00764   public:
    00769     OpalMixerConnection(
    00770       PSafePtr<OpalMixerNode> node,
    00771       OpalCall & call,
    00772       OpalMixerEndPoint & endpoint,
    00773       void * userData,
    00774       unsigned options = 0,
    00775       OpalConnection::StringOptions * stringOptions = NULL
    00776     );
    00777
    00780     ~OpalMixerConnection();
    00782
    00802     virtual void OnReleased();
    00803
    00810     virtual OpalMediaFormatList GetMediaFormats() const;
    00811
    00826     virtual OpalMediaStream * CreateMediaStream(
    00827       const OpalMediaFormat & mediaFormat,
    00828       unsigned sessionID,
    00829       PBoolean isSource
    00830     );
    00831
    00834     virtual void OnStartMediaPatch(
    00835       OpalMediaPatch & patch
    00836     );
    00837
    00839     virtual void OnApplyStringOptions();
    00840
    00847     virtual PBoolean SendUserInputString(
    00848       const PString & value
    00849     );
    00850
    00867     virtual PBoolean SendUserInputTone(
    00868       char tone,
    00869       unsigned duration = 0
    00870     );
    00872
    00877     void SetListenOnly(
    00878       bool listenOnly
    00879     );
    00880
    00883     bool GetListenOnly() const { return m_listenOnly; }
    00884
    00887     PSafePtr<OpalMixerNode> GetNode() const { return m_node; }
    00889
    00890   protected:
    00891     OpalMixerEndPoint     & m_endpoint;
    00892     PSafePtr<OpalMixerNode> m_node;
    00893     bool                    m_listenOnly;
    00894 };
    00895
    00896
    00900 class OpalMixerMediaStream : public OpalMediaStream
    00901 {
    00902     PCLASSINFO(OpalMixerMediaStream, OpalMediaStream);
    00903   public:
    00908     OpalMixerMediaStream(
    00909       OpalConnection & conn,
    00910       const OpalMediaFormat & mediaFormat,
    00911       unsigned sessionID,
    00912       bool isSource,
    00913       PSafePtr<OpalMixerNode> node,
    00914       bool listenOnly
    00915     );
    00916
    00919     ~OpalMixerMediaStream();
    00921
    00926     virtual PBoolean Open();
    00927
    00933     virtual PBoolean WritePacket(
    00934       RTP_DataFrame & packet
    00935     );
    00936
    00940     virtual PBoolean IsSynchronous() const;
    00941
    00952     virtual PBoolean RequiresPatchThread() const;
    00953
    00961     virtual bool EnableJitterBuffer(bool enab = true) const;
    00963
    00968     PSafePtr<OpalMixerNode> GetNode() { return m_node; }
    00970
    00971   protected:
    00972     virtual void InternalClose();
    00973
    00974     PSafePtr<OpalMixerNode> m_node;
    00975     bool m_listenOnly;
    00976 #if OPAL_VIDEO
    00977     bool m_video;
    00978 #endif
    00979 };
    00980
    00981
    00985 class OpalMixerNode : public PSafeObject
    00986 {
    00987     PCLASSINFO(OpalMixerNode, PSafeObject);
    00988   public:
    00993     OpalMixerNode(
    00994       OpalMixerNodeManager & manager,
    00995       OpalMixerNodeInfo * info
    00996     );
    00997     OpalMixerNode(
    00998       OpalMixerEndPoint & endpoint,
    00999       OpalMixerNodeInfo * info
    01000     );
    01001
    01004     ~OpalMixerNode();
    01005
    01010     void ShutDown();
    01012
    01019     void PrintOn(
    01020       ostream & strm
    01021     ) const;
    01023
    01028     void AttachConnection(
    01029       OpalConnection * connection
    01030     );
    01031
    01034     void DetachConnection(
    01035       OpalConnection * connection
    01036     );
    01037
    01040     bool AttachStream(
    01041       OpalMixerMediaStream * stream
    01042     );
    01043
    01046     void DetachStream(
    01047       OpalMixerMediaStream * stream
    01048     );
    01049
    01052     void UseMediaPassThrough(
    01053       unsigned sessionID,
    01054       OpalConnection * connection = NULL
    01055     );
    01056
    01063     bool SetJitterBufferSize(
    01064       const OpalBaseMixer::Key_T & key,
    01065       unsigned minJitterDelay,
    01066       unsigned maxJitterDelay
    01067     ) { return m_audioMixer.SetJitterBufferSize(key, minJitterDelay, maxJitterDelay); }
    01068
    01071     bool WriteAudio(
    01072       const OpalBaseMixer::Key_T & key,
    01073       const RTP_DataFrame & input
    01074     ) { return m_audioMixer.WriteStream(key, input); }
    01075
    01076 #if OPAL_VIDEO
    01077
    01079     bool WriteVideo(
    01080       const OpalBaseMixer::Key_T & key,
    01081       const RTP_DataFrame & input
    01082     ) { return m_videoMixer.WriteStream(key, input); }
    01083 #endif // OPAL_VIDEO
    01084
    01087     virtual void BroadcastUserInput(
    01088       const OpalConnection * connection,
    01089       const PString & value
    01090     );
    01092
    01097     const PGloballyUniqueID & GetGUID() const { return m_guid; }
    01098
    01101     const PStringList & GetNames() const { return m_names; }
    01102
    01105     void AddName(
    01106       const PString & name
    01107     );
    01108
    01111     void RemoveName(
    01112       const PString & name
    01113     );
    01114
    01120     PINDEX GetConnectionCount() const { return m_connections.GetSize(); }
    01121
    01124     template <class Subclass>
    01125     PSafePtr<Subclass> GetFirstConnectionAs(
    01126       PSafetyMode mode = PSafeReference
    01127     ) const { return PSafePtr<Subclass>(m_connections, mode); }
    01128
    01131     PSafePtr<OpalConnection> GetFirstConnection(
    01132       PSafetyMode mode = PSafeReference
    01133     ) const { return GetFirstConnectionAs<OpalConnection>(mode); }
    01134
    01137     const OpalMixerNodeInfo & GetNodeInfo() { return *m_info; }
    01138
    01141     const PTime & GetCreationTime() const { return m_creationTime; }
    01143
    01144   protected:
    01145     void Construct();
    01146
    01147     OpalMixerNodeManager & m_manager;
    01148     PGloballyUniqueID      m_guid;
    01149     PStringList            m_names;
    01150     OpalMixerNodeInfo    * m_info;
    01151     PTime                  m_creationTime;
    01152
    01153     PSafeList<OpalConnection> m_connections;
    01154
    01155     struct MediaMixer
    01156     {
    01157       MediaMixer();
    01158
    01159       PSafeList<OpalMixerMediaStream> m_outputStreams;
    01160     };
    01161
    01162     struct AudioMixer : public OpalAudioMixer, public MediaMixer
    01163     {
    01164       AudioMixer(const OpalMixerNodeInfo & info);
    01165       ~AudioMixer();
    01166
    01167       virtual bool OnPush();
    01168
    01169       struct CachedAudio {
    01170         CachedAudio();
    01171         ~CachedAudio();
    01172         enum { Collecting, Collected, Completed } m_state;
    01173         RTP_DataFrame    m_raw;
    01174         RTP_DataFrame    m_encoded;
    01175         OpalTranscoder * m_transcoder;
    01176       };
    01177       std::map<PString, CachedAudio> m_cache;
    01178
    01179       void PushOne(
    01180         PSafePtr<OpalMixerMediaStream> & stream,
    01181         CachedAudio & cache,
    01182         const short * audioToSubtract
    01183       );
    01184 #ifdef OPAL_MIXER_AUDIO_DEBUG
    01185       class PAudioMixerDebug * m_audioDebug;
    01186 #endif
    01187     };
    01188     AudioMixer m_audioMixer;
    01189
    01190 #if OPAL_VIDEO
    01191     struct VideoMixer : public OpalVideoMixer, public MediaMixer
    01192     {
    01193       VideoMixer(const OpalMixerNodeInfo & info);
    01194       ~VideoMixer();
    01195
    01196       virtual bool OnMixed(RTP_DataFrame * & output);
    01197
    01198       PDictionary<PString, OpalTranscoder> m_transcoders;
    01199     };
    01200     VideoMixer m_videoMixer;
    01201 #endif // OPAL_VIDEO
    01202 };
    01203
    01204
    01205 #endif // OPAL_OPAL_OPAL_MIXER
    01206
    01207