AgoraMediaBase.h

Go to the documentation of this file.

//  Agora Engine SDK
//
//  Created by Sting Feng in 2017-11.
//  Copyright (c) 2017 Agora.io. All rights reserved.
 
#pragma once  // NOLINT(build/header_guard)
 
#include <cstring>
#include <stdint.h>
#include <limits>
#include <stddef.h>
 
#ifndef OPTIONAL_ENUM_SIZE_T
#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
#define OPTIONAL_ENUM_SIZE_T enum : size_t
#else
#define OPTIONAL_ENUM_SIZE_T enum
#endif
#endif
 
#if !defined(__APPLE__)
#define __deprecated
#endif
 
namespace agora {
namespace rtc {
 
typedef unsigned int uid_t;
typedef unsigned int track_id_t;
typedef unsigned int conn_id_t;
typedef unsigned int video_track_id_t;
 
static const unsigned int INVALID_TRACK_ID = 0xffffffff;
static const unsigned int DEFAULT_CONNECTION_ID = 0;
static const unsigned int DUMMY_CONNECTION_ID = (std::numeric_limits<unsigned int>::max)();
 
struct EncodedVideoFrameInfo;
struct ExtensionContext {
    bool isValid;
    uid_t uid;
    const char *providerName;
    const char *extensionName;
    ExtensionContext():isValid(false), uid(0), providerName(NULL), extensionName(NULL) {}
};
 
 
enum VIDEO_SOURCE_TYPE {
  VIDEO_SOURCE_CAMERA_PRIMARY = 0,
  VIDEO_SOURCE_CAMERA = VIDEO_SOURCE_CAMERA_PRIMARY,
  VIDEO_SOURCE_CAMERA_SECONDARY = 1,
  VIDEO_SOURCE_SCREEN_PRIMARY = 2,
  VIDEO_SOURCE_SCREEN = VIDEO_SOURCE_SCREEN_PRIMARY,
  VIDEO_SOURCE_SCREEN_SECONDARY = 3,
  VIDEO_SOURCE_CUSTOM = 4,
  VIDEO_SOURCE_MEDIA_PLAYER = 5,
  VIDEO_SOURCE_RTC_IMAGE_PNG = 6,
  VIDEO_SOURCE_RTC_IMAGE_JPEG = 7,
  VIDEO_SOURCE_RTC_IMAGE_GIF = 8,
  VIDEO_SOURCE_REMOTE = 9,
  VIDEO_SOURCE_TRANSCODED = 10,
 
  VIDEO_SOURCE_CAMERA_THIRD = 11,
  VIDEO_SOURCE_CAMERA_FOURTH = 12,
  VIDEO_SOURCE_SCREEN_THIRD = 13,
  VIDEO_SOURCE_SCREEN_FOURTH = 14,
  VIDEO_SOURCE_SPEECH_DRIVEN = 15,
 
  VIDEO_SOURCE_UNKNOWN = 100
};
 
enum AUDIO_SOURCE_TYPE {
  AUDIO_SOURCE_MICROPHONE = 0,
  AUDIO_SOURCE_CUSTOM = 1,
  AUDIO_SOURCE_MEDIA_PLAYER = 2,
  AUDIO_SOURCE_LOOPBACK_RECORDING = 3,
  AUDIO_SOURCE_MIXED_STREAM = 4,
  AUDIO_SOURCE_REMOTE_USER = 5,
  AUDIO_SOURCE_REMOTE_CHANNEL = 6,
 
  AUDIO_SOURCE_UNKNOWN = 100
};
 
enum AudioRoute {
  ROUTE_DEFAULT = -1,
  ROUTE_HEADSET = 0,
  ROUTE_EARPIECE = 1,
  ROUTE_HEADSETNOMIC = 2,
  ROUTE_SPEAKERPHONE = 3,
  ROUTE_LOUDSPEAKER = 4,
  ROUTE_BLUETOOTH_DEVICE_HFP = 5,
  ROUTE_USB = 6,
  ROUTE_HDMI = 7,
  ROUTE_DISPLAYPORT = 8,
  ROUTE_AIRPLAY = 9,
  ROUTE_BLUETOOTH_DEVICE_A2DP = 10,
};
 
enum BYTES_PER_SAMPLE {
  TWO_BYTES_PER_SAMPLE = 2,
};
 
struct AudioParameters {
  int sample_rate;
  size_t channels;
  size_t frames_per_buffer;
 
  AudioParameters() : sample_rate(0), channels(0), frames_per_buffer(0) {}
};
 
enum RAW_AUDIO_FRAME_OP_MODE_TYPE {
  RAW_AUDIO_FRAME_OP_MODE_READ_ONLY = 0,
 
  RAW_AUDIO_FRAME_OP_MODE_READ_WRITE = 2,
};
 
}  // namespace rtc
 
namespace media {
enum MEDIA_SOURCE_TYPE {
  AUDIO_PLAYOUT_SOURCE = 0,
  AUDIO_RECORDING_SOURCE = 1,
  PRIMARY_CAMERA_SOURCE = 2,
  SECONDARY_CAMERA_SOURCE = 3,
  PRIMARY_SCREEN_SOURCE = 4,
  SECONDARY_SCREEN_SOURCE = 5,
  CUSTOM_VIDEO_SOURCE = 6,
  MEDIA_PLAYER_SOURCE = 7,
  RTC_IMAGE_PNG_SOURCE = 8,
  RTC_IMAGE_JPEG_SOURCE = 9,
  RTC_IMAGE_GIF_SOURCE = 10,
  REMOTE_VIDEO_SOURCE = 11,
  TRANSCODED_VIDEO_SOURCE = 12,
  SPEECH_DRIVEN_VIDEO_SOURCE = 13,
  UNKNOWN_MEDIA_SOURCE = 100
};
 
namespace base {
 
typedef void* view_t;
 
typedef const char* user_id_t;
 
static const uint8_t kMaxCodecNameLength = 50;
 
struct PacketOptions {
  uint32_t timestamp;
  // Audio level indication.
  uint8_t audioLevelIndication;
  PacketOptions() : timestamp(0), audioLevelIndication(127) {}
};
 
struct AudioEncodedFrameInfo {
  uint64_t sendTs;
  uint8_t codec;
  AudioEncodedFrameInfo() : sendTs(0), codec(0) {}
};
 
struct AudioPcmFrame {
  OPTIONAL_ENUM_SIZE_T{
      // Stereo, 32 kHz, 60 ms (2 * 32 * 60)
      kMaxDataSizeSamples = 3840,
      kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t),
  };
 
  int64_t capture_timestamp;
  size_t samples_per_channel_;
  int sample_rate_hz_;
  size_t num_channels_;
  int audio_track_number_;
  rtc::BYTES_PER_SAMPLE bytes_per_sample;
  int16_t data_[kMaxDataSizeSamples];
  
  bool is_stereo_;
 
  AudioPcmFrame& operator=(const AudioPcmFrame& src) {
    if (this == &src) {
      return *this;
    }
 
    capture_timestamp = src.capture_timestamp;
    samples_per_channel_ = src.samples_per_channel_;
    sample_rate_hz_ = src.sample_rate_hz_;
    bytes_per_sample = src.bytes_per_sample;
    num_channels_ = src.num_channels_;
    is_stereo_ = src.is_stereo_;
    this->audio_track_number_ = src.audio_track_number_;
 
    size_t length = src.samples_per_channel_ * src.num_channels_;
    if (length > kMaxDataSizeSamples) {
      length = kMaxDataSizeSamples;
    }
 
    memcpy(data_, src.data_, length * sizeof(int16_t));
 
    return *this;
  }
 
  AudioPcmFrame()
      : capture_timestamp(0),
        samples_per_channel_(0),
        sample_rate_hz_(0),
        num_channels_(0),
        audio_track_number_(0),
        bytes_per_sample(rtc::TWO_BYTES_PER_SAMPLE),
        is_stereo_(false) {
    memset(data_, 0, sizeof(data_));
  }
 
  AudioPcmFrame(const AudioPcmFrame& src)
      : capture_timestamp(src.capture_timestamp),
        samples_per_channel_(src.samples_per_channel_),
        sample_rate_hz_(src.sample_rate_hz_),
        num_channels_(src.num_channels_),
        audio_track_number_(src.audio_track_number_),
        bytes_per_sample(src.bytes_per_sample),
        is_stereo_(src.is_stereo_) {
    size_t length = src.samples_per_channel_ * src.num_channels_;
    if (length > kMaxDataSizeSamples) {
      length = kMaxDataSizeSamples;
    }
 
    memcpy(this->data_, src.data_, length * sizeof(int16_t));
  }
};
 
enum AUDIO_DUAL_MONO_MODE {
  AUDIO_DUAL_MONO_STEREO = 0,
  AUDIO_DUAL_MONO_L = 1,
  AUDIO_DUAL_MONO_R = 2,
  AUDIO_DUAL_MONO_MIX = 3
};
 
enum VIDEO_PIXEL_FORMAT {
  VIDEO_PIXEL_DEFAULT = 0,
  VIDEO_PIXEL_I420 = 1,
  VIDEO_PIXEL_BGRA = 2,
  VIDEO_PIXEL_NV21 = 3,
  VIDEO_PIXEL_RGBA = 4,
  VIDEO_PIXEL_NV12 = 8,
  VIDEO_TEXTURE_2D = 10,
  VIDEO_TEXTURE_OES = 11,
  /*
  12: pixel format for iOS CVPixelBuffer NV12
  */
  VIDEO_CVPIXEL_NV12 = 12,
  /*
  13: pixel format for iOS CVPixelBuffer I420
  */
  VIDEO_CVPIXEL_I420 = 13,
  /*
  14: pixel format for iOS CVPixelBuffer BGRA
  */
  VIDEO_CVPIXEL_BGRA = 14,
  VIDEO_CVPIXEL_P010 = 15,
  VIDEO_PIXEL_I422 = 16,
  VIDEO_TEXTURE_ID3D11TEXTURE2D = 17,
  VIDEO_PIXEL_I010 = 18,
};
 
enum RENDER_MODE_TYPE {
  RENDER_MODE_HIDDEN = 1,
  RENDER_MODE_FIT = 2,
  RENDER_MODE_ADAPTIVE __deprecated = 3,
};
 
enum CAMERA_VIDEO_SOURCE_TYPE {
  CAMERA_SOURCE_FRONT = 0,
  CAMERA_SOURCE_BACK = 1,
  VIDEO_SOURCE_UNSPECIFIED = 2,
};
 
class IVideoFrameMetaInfo {
 public:
  enum META_INFO_KEY {
    KEY_FACE_CAPTURE = 0,
  };
  virtual ~IVideoFrameMetaInfo(){};
  virtual const char* getMetaInfoStr(META_INFO_KEY key) const = 0;
};
 
struct ColorSpace {
  enum PrimaryID {
    // The indices are equal to the values specified in T-REC H.273 Table 2.
    PRIMARYID_BT709 = 1,
    PRIMARYID_UNSPECIFIED = 2,
    PRIMARYID_BT470M = 4,
    PRIMARYID_BT470BG = 5,
    PRIMARYID_SMPTE170M = 6,  // Identical to BT601
    PRIMARYID_SMPTE240M = 7,
    PRIMARYID_FILM = 8,
    PRIMARYID_BT2020 = 9,
    PRIMARYID_SMPTEST428 = 10,
    PRIMARYID_SMPTEST431 = 11,
    PRIMARYID_SMPTEST432 = 12,
    PRIMARYID_JEDECP22 = 22,  // Identical to EBU3213-E
  };
 
  enum RangeID {
    // The indices are equal to the values specified at
    // https://www.webmproject.org/docs/container/#colour for the element Range.
    RANGEID_INVALID = 0,
    // Limited Rec. 709 color range with RGB values ranging from 16 to 235.
    RANGEID_LIMITED = 1,
    // Full RGB color range with RGB valees from 0 to 255.
    RANGEID_FULL = 2,
    // Range is defined by MatrixCoefficients/TransferCharacteristics.
    RANGEID_DERIVED = 3,
  };
 
  enum MatrixID {
    // The indices are equal to the values specified in T-REC H.273 Table 4.
    MATRIXID_RGB = 0,
    MATRIXID_BT709 = 1,
    MATRIXID_UNSPECIFIED = 2,
    MATRIXID_FCC = 4,
    MATRIXID_BT470BG = 5,
    MATRIXID_SMPTE170M = 6,
    MATRIXID_SMPTE240M = 7,
    MATRIXID_YCOCG = 8,
    MATRIXID_BT2020_NCL = 9,
    MATRIXID_BT2020_CL = 10,
    MATRIXID_SMPTE2085 = 11,
    MATRIXID_CDNCLS = 12,
    MATRIXID_CDCLS = 13,
    MATRIXID_BT2100_ICTCP = 14,
  };
 
  enum TransferID {
    // The indices are equal to the values specified in T-REC H.273 Table 3.
    TRANSFERID_BT709 = 1,
    TRANSFERID_UNSPECIFIED = 2,
    TRANSFERID_GAMMA22 = 4,
    TRANSFERID_GAMMA28 = 5,
    TRANSFERID_SMPTE170M = 6,
    TRANSFERID_SMPTE240M = 7,
    TRANSFERID_LINEAR = 8,
    TRANSFERID_LOG = 9,
    TRANSFERID_LOG_SQRT = 10,
    TRANSFERID_IEC61966_2_4 = 11,
    TRANSFERID_BT1361_ECG = 12,
    TRANSFERID_IEC61966_2_1 = 13,
    TRANSFERID_BT2020_10 = 14,
    TRANSFERID_BT2020_12 = 15,
    TRANSFERID_SMPTEST2084 = 16,
    TRANSFERID_SMPTEST428 = 17,
    TRANSFERID_ARIB_STD_B67 = 18,
  };
 
  PrimaryID primaries;
  TransferID transfer;
  MatrixID matrix;
  RangeID range;
 
  ColorSpace()
      : primaries(PRIMARYID_UNSPECIFIED), transfer(TRANSFERID_UNSPECIFIED),
        matrix(MATRIXID_UNSPECIFIED), range(RANGEID_INVALID) {}
 
  bool validate() const {
    return primaries != PRIMARYID_UNSPECIFIED || transfer != TRANSFERID_UNSPECIFIED ||
           matrix != MATRIXID_UNSPECIFIED ||
           range != RANGEID_INVALID;
  }
};
 
struct Hdr10MetadataInfo {
  uint16_t redPrimaryX;
  uint16_t redPrimaryY;
  uint16_t greenPrimaryX;
  uint16_t greenPrimaryY;
  uint16_t bluePrimaryX;
  uint16_t bluePrimaryY;
  uint16_t whitePointX;
  uint16_t whitePointY;
  unsigned int maxMasteringLuminance;
  unsigned int minMasteringLuminance;
  uint16_t maxContentLightLevel;
  uint16_t maxFrameAverageLightLevel;
 
  Hdr10MetadataInfo()
      : redPrimaryX(0),
        redPrimaryY(0),
        greenPrimaryX(0),
        greenPrimaryY(0),
        bluePrimaryX(0),
        bluePrimaryY(0),
        whitePointX(0),
        whitePointY(0),
        maxMasteringLuminance(0),
        minMasteringLuminance(0),
        maxContentLightLevel(0),
        maxFrameAverageLightLevel(0){}
 
  bool validate() const {
    return maxContentLightLevel >= 0 && maxContentLightLevel <= 20000 &&
           maxFrameAverageLightLevel >= 0 &&
           maxFrameAverageLightLevel <= 20000;
  }
};
 
enum ALPHA_STITCH_MODE {
  NO_ALPHA_STITCH = 0,
  ALPHA_STITCH_UP = 1,
  ALPHA_STITCH_BELOW = 2,
  ALPHA_STITCH_LEFT = 3,
  ALPHA_STITCH_RIGHT = 4,
};
 
 
struct ExternalVideoFrame {
  ExternalVideoFrame()
      : type(VIDEO_BUFFER_RAW_DATA),
        format(VIDEO_PIXEL_DEFAULT),
        buffer(NULL),
        stride(0),
        height(0),
        cropLeft(0),
        cropTop(0),
        cropRight(0),
        cropBottom(0),
        rotation(0),
        timestamp(0),
        eglContext(NULL),
        eglType(EGL_CONTEXT10),
        textureId(0),
        fenceObject(0),
        metadataBuffer(NULL),
        metadataSize(0),
        alphaBuffer(NULL),
        fillAlphaBuffer(false),
        alphaStitchMode(NO_ALPHA_STITCH),
        d3d11Texture2d(NULL),
        textureSliceIndex(0){}
 
  enum EGL_CONTEXT_TYPE {
    EGL_CONTEXT10 = 0,
    EGL_CONTEXT14 = 1,
  };
 
  enum VIDEO_BUFFER_TYPE {
    VIDEO_BUFFER_RAW_DATA = 1,
    VIDEO_BUFFER_ARRAY = 2,
    VIDEO_BUFFER_TEXTURE = 3,
  };
 
  VIDEO_BUFFER_TYPE type;
  VIDEO_PIXEL_FORMAT format;
 
  void* buffer;
  int stride;
  int height;
  int cropLeft;
  int cropTop;
  int cropRight;
  int cropBottom;
  int rotation;
  long long timestamp;
  void* eglContext;
  EGL_CONTEXT_TYPE eglType;
  int textureId;
  long long fenceObject; 
  float matrix[16];
  uint8_t* metadataBuffer;
  int metadataSize;
  uint8_t* alphaBuffer;
  bool fillAlphaBuffer;
  ALPHA_STITCH_MODE alphaStitchMode;
 
  void *d3d11Texture2d;
 
  int textureSliceIndex;
 
  Hdr10MetadataInfo hdr10MetadataInfo;
 
  ColorSpace colorSpace;
};
 
struct VideoFrame {
  VideoFrame():
  type(VIDEO_PIXEL_DEFAULT),
  width(0),
  height(0),
  yStride(0),
  uStride(0),
  vStride(0),
  yBuffer(NULL),
  uBuffer(NULL),
  vBuffer(NULL),
  rotation(0),
  renderTimeMs(0),
  avsync_type(0),
  metadata_buffer(NULL),
  metadata_size(0),
  sharedContext(0),
  textureId(0),
  d3d11Texture2d(NULL),
  alphaBuffer(NULL),
  alphaStitchMode(NO_ALPHA_STITCH),
  pixelBuffer(NULL),
  metaInfo(NULL){
    memset(matrix, 0, sizeof(matrix));
  }
  VIDEO_PIXEL_FORMAT type;
  int width;
  int height;
  int yStride;
  int uStride;
  int vStride;
  uint8_t* yBuffer;
  uint8_t* uBuffer;
  uint8_t* vBuffer;
  int rotation;
  int64_t renderTimeMs;
  int avsync_type;
  uint8_t* metadata_buffer;
  int metadata_size;
  void* sharedContext;
  int textureId;
  void* d3d11Texture2d;
  float matrix[16];
  uint8_t* alphaBuffer;
  ALPHA_STITCH_MODE alphaStitchMode;
  void* pixelBuffer;
  IVideoFrameMetaInfo* metaInfo;
 
  Hdr10MetadataInfo hdr10MetadataInfo;
 
  ColorSpace colorSpace;
};
 
class IVideoFrameObserver {
 public:
  virtual void onFrame(const VideoFrame* frame) = 0;
  virtual ~IVideoFrameObserver() {}
  virtual bool isExternal() { return true; }
  virtual VIDEO_PIXEL_FORMAT getVideoFormatPreference() { return VIDEO_PIXEL_DEFAULT; }
};
 
enum MEDIA_PLAYER_SOURCE_TYPE {
  MEDIA_PLAYER_SOURCE_DEFAULT,
  MEDIA_PLAYER_SOURCE_FULL_FEATURED,
  MEDIA_PLAYER_SOURCE_SIMPLE,
};
 
enum VIDEO_MODULE_POSITION {
  POSITION_POST_CAPTURER = 1 << 0,
  POSITION_PRE_RENDERER = 1 << 1,
  POSITION_PRE_ENCODER = 1 << 2,
  POSITION_POST_CAPTURER_ORIGIN = 1 << 3,
};
 
}  // namespace base
 
#define MAX_CONTENT_INSPECT_MODULE_COUNT 32
enum CONTENT_INSPECT_RESULT {
  CONTENT_INSPECT_NEUTRAL = 1,
  CONTENT_INSPECT_SEXY = 2,
  CONTENT_INSPECT_PORN = 3,
};
 
enum CONTENT_INSPECT_TYPE {
  CONTENT_INSPECT_INVALID = 0,
  CONTENT_INSPECT_MODERATION __deprecated = 1,
  CONTENT_INSPECT_SUPERVISION = 2,
  CONTENT_INSPECT_IMAGE_MODERATION = 3
};
 
struct ContentInspectModule {
  CONTENT_INSPECT_TYPE type;
  unsigned int interval;
  base::VIDEO_MODULE_POSITION position;
  ContentInspectModule() {
    type = CONTENT_INSPECT_INVALID;
    interval = 0;
    position = base::POSITION_PRE_ENCODER;
  }
};
struct ContentInspectConfig {
  const char* extraInfo;
  const char* serverConfig;
  ContentInspectModule modules[MAX_CONTENT_INSPECT_MODULE_COUNT];
  int moduleCount;
  ContentInspectConfig& operator=(const ContentInspectConfig& rth) {
    extraInfo = rth.extraInfo;
    serverConfig = rth.serverConfig;
    moduleCount = rth.moduleCount;
    memcpy(&modules, &rth.modules, MAX_CONTENT_INSPECT_MODULE_COUNT * sizeof(ContentInspectModule));
    return *this;
  }
  ContentInspectConfig() : extraInfo(NULL), serverConfig(NULL), moduleCount(0) {}
};
struct SnapshotConfig {
  const char* filePath;
 
  media::base::VIDEO_MODULE_POSITION position;
  SnapshotConfig() :filePath(NULL), position(media::base::POSITION_PRE_ENCODER) {}
};
 
class IAudioPcmFrameSink {
 public:
  virtual void onFrame(agora::media::base::AudioPcmFrame* frame) = 0;
  virtual ~IAudioPcmFrameSink() {}
};
 
class IAudioFrameObserverBase {
 public:
  enum AUDIO_FRAME_TYPE {
    FRAME_TYPE_PCM16 = 0,
  };
  enum { MAX_HANDLE_TIME_CNT = 10 };
  struct AudioFrame {
    AUDIO_FRAME_TYPE type;
    int samplesPerChannel;
    agora::rtc::BYTES_PER_SAMPLE bytesPerSample;
    int channels;
    int samplesPerSec;
    void* buffer;
    int64_t renderTimeMs;
    int avsync_type;
    int64_t presentationMs;
    int audioTrackNumber;
    uint32_t rtpTimestamp;
 
    AudioFrame()
        : type(FRAME_TYPE_PCM16),
          samplesPerChannel(0),
          bytesPerSample(rtc::TWO_BYTES_PER_SAMPLE),
          channels(0),
          samplesPerSec(0),
          buffer(NULL),
          renderTimeMs(0),
          avsync_type(0),
          presentationMs(0),
          audioTrackNumber(0),
          rtpTimestamp(0) {}
  };
 
  enum AUDIO_FRAME_POSITION {
    AUDIO_FRAME_POSITION_NONE = 0x0000,
    AUDIO_FRAME_POSITION_PLAYBACK = 0x0001,
    AUDIO_FRAME_POSITION_RECORD = 0x0002,
    AUDIO_FRAME_POSITION_MIXED = 0x0004,
    AUDIO_FRAME_POSITION_BEFORE_MIXING = 0x0008,
    AUDIO_FRAME_POSITION_EAR_MONITORING = 0x0010,
  };
 
  struct AudioParams {
    int sample_rate;
 
    int channels;
 
    rtc::RAW_AUDIO_FRAME_OP_MODE_TYPE mode;
 
    int samples_per_call;
 
    AudioParams()
        : sample_rate(0),
          channels(0),
          mode(rtc::RAW_AUDIO_FRAME_OP_MODE_READ_ONLY),
          samples_per_call(0) {}
    AudioParams(int samplerate, int channel, rtc::RAW_AUDIO_FRAME_OP_MODE_TYPE type,
                int samplesPerCall)
        : sample_rate(samplerate),
          channels(channel),
          mode(type),
          samples_per_call(samplesPerCall) {}
  };
 
 public:
  virtual ~IAudioFrameObserverBase() {}
 
  virtual bool onRecordAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
  virtual bool onPlaybackAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
  virtual bool onMixedAudioFrame(const char* channelId, AudioFrame& audioFrame) = 0;
  virtual bool onEarMonitoringAudioFrame(AudioFrame& audioFrame) = 0;
  virtual bool onPlaybackAudioFrameBeforeMixing(const char* channelId, base::user_id_t userId,
                                                AudioFrame& audioFrame) {
    (void)channelId;
    (void)userId;
    (void)audioFrame;
    return true;
  }
 
  virtual int getObservedAudioFramePosition() = 0;
 
  virtual AudioParams getPlaybackAudioParams() = 0;
 
  virtual AudioParams getRecordAudioParams() = 0;
 
  virtual AudioParams getMixedAudioParams() = 0;
 
  virtual AudioParams getEarMonitoringAudioParams() = 0;
};
 
class IAudioFrameObserver : public IAudioFrameObserverBase {
 public:
  using IAudioFrameObserverBase::onPlaybackAudioFrameBeforeMixing;
  virtual bool onPlaybackAudioFrameBeforeMixing(const char* channelId, rtc::uid_t uid,
                                                AudioFrame& audioFrame) = 0;
};
 
struct AudioSpectrumData {
  const float* audioSpectrumData;
  int dataLength;
 
  AudioSpectrumData() : audioSpectrumData(NULL), dataLength(0) {}
  AudioSpectrumData(const float* data, int length) : audioSpectrumData(data), dataLength(length) {}
};
 
struct UserAudioSpectrumInfo {
  agora::rtc::uid_t uid;
  struct AudioSpectrumData spectrumData;
 
  UserAudioSpectrumInfo() : uid(0) {}
 
  UserAudioSpectrumInfo(agora::rtc::uid_t uid, const float* data, int length)
      : uid(uid), spectrumData(data, length) {}
};
 
class IAudioSpectrumObserver {
 public:
  virtual ~IAudioSpectrumObserver() {}
 
  virtual bool onLocalAudioSpectrum(const AudioSpectrumData& data) = 0;
  virtual bool onRemoteAudioSpectrum(const UserAudioSpectrumInfo* spectrums,
                                     unsigned int spectrumNumber) = 0;
};
 
class IVideoEncodedFrameObserver {
 public:
  virtual bool onEncodedVideoFrameReceived(
      const char* channelId, rtc::uid_t uid, const uint8_t* imageBuffer, size_t length,
      const rtc::EncodedVideoFrameInfo& videoEncodedFrameInfo) = 0;
 
  virtual ~IVideoEncodedFrameObserver() {}
};
 
class IVideoFrameObserver {
 public:
  typedef media::base::VideoFrame VideoFrame;
  enum VIDEO_FRAME_PROCESS_MODE {
    PROCESS_MODE_READ_ONLY,  // Observer works as a pure renderer and will not modify the original
                             // frame.
    PROCESS_MODE_READ_WRITE,  // Observer works as a filter that will process the video frame and
                              // affect the following frame processing in SDK.
  };
 
 public:
  virtual ~IVideoFrameObserver() {}
 
  virtual bool onCaptureVideoFrame(agora::rtc::VIDEO_SOURCE_TYPE sourceType,
                                   VideoFrame& videoFrame) = 0;
 
  virtual bool onPreEncodeVideoFrame(agora::rtc::VIDEO_SOURCE_TYPE sourceType,
                                     VideoFrame& videoFrame) = 0;
 
  virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0;
 
  virtual bool onRenderVideoFrame(const char* channelId, rtc::uid_t remoteUid,
                                  VideoFrame& videoFrame) = 0;
 
  virtual bool onTranscodedVideoFrame(VideoFrame& videoFrame) = 0;
 
  virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode() { return PROCESS_MODE_READ_ONLY; }
 
  virtual base::VIDEO_PIXEL_FORMAT getVideoFormatPreference() { return base::VIDEO_PIXEL_DEFAULT; }
 
  virtual bool getRotationApplied() { return false; }
 
  virtual bool getMirrorApplied() { return false; }
 
  virtual uint32_t getObservedFramePosition() {
    return base::POSITION_POST_CAPTURER | base::POSITION_PRE_RENDERER;
  }
 
  virtual bool isExternal() { return true; }
};
 
enum EXTERNAL_VIDEO_SOURCE_TYPE {
  VIDEO_FRAME = 0,
  ENCODED_VIDEO_FRAME,
};
 
enum MediaRecorderContainerFormat {
  FORMAT_MP4 = 1,
};
enum MediaRecorderStreamType {
  STREAM_TYPE_AUDIO = 0x01,
  STREAM_TYPE_VIDEO = 0x02,
  STREAM_TYPE_BOTH = STREAM_TYPE_AUDIO | STREAM_TYPE_VIDEO,
};
enum RecorderState {
  RECORDER_STATE_ERROR = -1,
  RECORDER_STATE_START = 2,
  RECORDER_STATE_STOP = 3,
};
enum RecorderReasonCode {
  RECORDER_REASON_NONE = 0,
  RECORDER_REASON_WRITE_FAILED = 1,
  RECORDER_REASON_NO_STREAM = 2,
  RECORDER_REASON_OVER_MAX_DURATION = 3,
  RECORDER_REASON_CONFIG_CHANGED = 4,
};
struct MediaRecorderConfiguration {
  const char* storagePath;
  MediaRecorderContainerFormat containerFormat;
  MediaRecorderStreamType streamType;
  int maxDurationMs;
  int recorderInfoUpdateInterval;
  int width;
  int height;
  int fps;
  int sample_rate;
  int channel_num;
  agora::rtc::VIDEO_SOURCE_TYPE videoSourceType;
 
  MediaRecorderConfiguration()
      : storagePath(NULL),
        containerFormat(FORMAT_MP4),
        streamType(STREAM_TYPE_BOTH),
        maxDurationMs(120000),
        recorderInfoUpdateInterval(0),
        width(1280),
        height(720),
        fps(30),
        sample_rate(48000),
        channel_num(1),
        videoSourceType(rtc::VIDEO_SOURCE_CAMERA_PRIMARY) {}
  MediaRecorderConfiguration(const char* path, MediaRecorderContainerFormat format,
                             MediaRecorderStreamType type, int duration, int interval)
      : storagePath(path),
        containerFormat(format),
        streamType(type),
        maxDurationMs(duration),
        recorderInfoUpdateInterval(interval),
        width(1280),
        height(720),
        fps(30),
        sample_rate(48000),
        channel_num(1),
        videoSourceType(rtc::VIDEO_SOURCE_CAMERA_PRIMARY) {}
};
 
class IFaceInfoObserver {
 public:
  virtual bool onFaceInfo(const char* outFaceInfo) = 0;
 
  virtual ~IFaceInfoObserver() {}
};
 
struct RecorderInfo {
  const char* fileName;
  unsigned int durationMs;
  unsigned int fileSize;
 
  RecorderInfo() : fileName(NULL), durationMs(0), fileSize(0) {}
  RecorderInfo(const char* name, unsigned int dur, unsigned int size)
      : fileName(name), durationMs(dur), fileSize(size) {}
};
 
class IMediaRecorderObserver {
 public:
  virtual void onRecorderStateChanged(const char* channelId, rtc::uid_t uid, RecorderState state,
                                      RecorderReasonCode reason) = 0;
  virtual void onRecorderInfoUpdated(const char* channelId, rtc::uid_t uid,
                                     const RecorderInfo& info) = 0;
 
  virtual ~IMediaRecorderObserver() {}
};
 
}  // namespace media
}  // namespace agora