此文我自己看,所以乱七八糟没有整理,别人看不懂我不管😂
src/audio/audio_state.cc
void AudioState::AddSendingStream(webrtc::AudioSendStream* stream,
int sample_rate_hz,
size_t num_channels) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
auto& properties = sending_streams_[stream];
properties.sample_rate_hz = sample_rate_hz;
properties.num_channels = num_channels;
UpdateAudioTransportWithSendingStreams();
// Make sure recording is initialized; start recording if enabled.
auto* adm = config_.audio_device_module.get();
if (!adm->Recording()) {
if (adm->InitRecording() == 0) {
if (recording_enabled_) {
adm->StartRecording();
}
} else {
RTC_DLOG_F(LS_ERROR) << "Failed to initialize recording.";
}
}
}
src/media/engine/webrtcvoiceengine.cc
void WebRtcVoiceEngine::Init() {
...
#if defined(WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE)
// No ADM supplied? Create a default one.
if (!adm_) {
adm_ = webrtc::AudioDeviceModule::Create(
webrtc::AudioDeviceModule::kPlatformDefaultAudio);
}
#endif // WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE
RTC_CHECK(adm());
webrtc::adm_helpers::Init(adm());
webrtc::apm_helpers::Init(apm());
// Set up AudioState.
{
webrtc::AudioState::Config config;
if (audio_mixer_) {
config.audio_mixer = audio_mixer_;
} else {
config.audio_mixer = webrtc::AudioMixerImpl::Create();
}
config.audio_processing = apm_;
config.audio_device_module = adm_;
audio_state_ = webrtc::AudioState::Create(config);
}
// Connect the ADM to our audio path.
adm()->RegisterAudioCallback(audio_state()->audio_transport());
// Save the default AGC configuration settings. This must happen before
// calling ApplyOptions or the default will be overwritten.
default_agc_config_ = webrtc::apm_helpers::GetAgcConfig(apm());
// Set default engine options.
{
AudioOptions options;
options.echo_cancellation = true;
options.auto_gain_control = true;
options.noise_suppression = true;
options.highpass_filter = true;
options.stereo_swapping = false;
options.audio_jitter_buffer_max_packets = 50;
options.audio_jitter_buffer_fast_accelerate = false;
options.audio_jitter_buffer_min_delay_ms = 0;
options.typing_detection = true;
options.experimental_agc = false;
options.extended_filter_aec = false;
options.delay_agnostic_aec = false;
options.experimental_ns = false;
options.residual_echo_detector = true;
bool error = ApplyOptions(options);
RTC_DCHECK(error);
}
initialized_ = true;
}
WebRtcVoiceEngine
类必须定义了HAVE_WEBRTC_VOICE
宏才会被用到。
adm_
可以在WebRtcVoiceEngine
的构造函数中由外部创建传进来,当adm_
为空时且定义了WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE
宏时才使用内置的方法创建adm_
。
AudioLayer
的类型:
enum AudioLayer {
kPlatformDefaultAudio = 0,
kWindowsCoreAudio,
kWindowsCoreAudio2, // experimental
kLinuxAlsaAudio,
kLinuxPulseAudio,
kAndroidJavaAudio,
kAndroidOpenSLESAudio,
kAndroidJavaInputAndOpenSLESOutputAudio,
kAndroidAAudioAudio,
kAndroidJavaInputAndAAudioOutputAudio,
kDummyAudio,
};
AudioDeviceModule::Create
创建AudioDeviceModuleImpl
:
rtc::scoped_refptr<AudioDeviceModule> AudioDeviceModule::Create(
const AudioLayer audio_layer) {
RTC_LOG(INFO) << __FUNCTION__;
return AudioDeviceModule::CreateForTest(audio_layer);
}
rtc::scoped_refptr<AudioDeviceModuleForTest> AudioDeviceModule::CreateForTest(
const AudioLayer audio_layer) {
RTC_LOG(INFO) << __FUNCTION__;
// The "AudioDeviceModule::kWindowsCoreAudio2" audio layer has its own
// dedicated factory method which should be used instead.
if (audio_layer == AudioDeviceModule::kWindowsCoreAudio2) {
RTC_LOG(LS_ERROR) << "Use the CreateWindowsCoreAudioAudioDeviceModule() "
"factory method instead for this option.";
return nullptr;
}
// Create the generic reference counted (platform independent) implementation.
rtc::scoped_refptr<AudioDeviceModuleImpl> audioDevice(
new rtc::RefCountedObject<AudioDeviceModuleImpl>(audio_layer));
// Ensure that the current platform is supported.
if (audioDevice->CheckPlatform() == -1) {
return nullptr;
}
// Create the platform-dependent implementation.
if (audioDevice->CreatePlatformSpecificObjects() == -1) {
return nullptr;
}
// Ensure that the generic audio buffer can communicate with the platform
// specific parts.
if (audioDevice->AttachAudioBuffer() == -1) {
return nullptr;
}
return audioDevice;
}
src/modules/audio_device/audio_device_impl.cc
int32_t AudioDeviceModuleImpl::StartRecording() {
RTC_LOG(INFO) << __FUNCTION__;
CHECKinitialized_();
if (Recording()) {
return 0;
}
audio_device_buffer_.StartRecording();
int32_t result = audio_device_->StartRecording();
RTC_LOG(INFO) << "output: " << result;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.StartRecordingSuccess",
static_cast<int>(result == 0));
return result;
}
audio_device_
在各个平台创建不同的实现类,iOS平台是AudioDeviceIOS
,但实际在iOS上的AudioDevice并不是这样创建出来的,因为iOS上adm_
对象是上层创建再传进来的,所以这里的webrtc::AudioDeviceModule::Create
并不会调用,下面解析adm_
的创建。
WebRtcVoiceEngine的创建
回过头来看看WebRtcVoiceEngine
是怎么创建的。
src/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm
- (instancetype)
initWithNativeAudioEncoderFactory:... {
...
std::unique_ptr<cricket::MediaEngineInterface> media_engine =
cricket::WebRtcMediaEngineFactory::Create(audioDeviceModule,
audioEncoderFactory,
audioDecoderFactory,
std::move(videoEncoderFactory),
std::move(videoDecoderFactory),
nullptr, // audio mixer
audioProcessingModule);
...
return self;
#endif
}
- (rtc::scoped_refptr<webrtc::AudioDeviceModule>)audioDeviceModule {
#if defined(WEBRTC_IOS)
return webrtc::CreateAudioDeviceModule();
#else
return nullptr;
#endif
}
RTCPeerConnectionFactory
在initWithNativeAudioEncoderFactory
时调用了WebRtcMediaEngineFactory::Create
。
这里的audioDeviceModule
在iOS平台由webrtc::CreateAudioDeviceModule();
创建,mac平台为nullptr
由内部创建。
src/sdk/objc/native/api/audio_device_module.mm
namespace webrtc {
rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule() {
RTC_LOG(INFO) << __FUNCTION__;
#if defined(WEBRTC_IOS)
return new rtc::RefCountedObject<ios_adm::AudioDeviceModuleIOS>();
#else
RTC_LOG(LERROR)
<< "current platform is not supported => this module will self destruct!";
return nullptr;
#endif
}
}
可以看出iOS版创建AudioDeviceModuleIOS
,在其Init
方法中创建AudioDeviceIOS
。
src/sdk/objc/native/src/audio/audio_device_module_ios.mm
int32_t AudioDeviceModuleIOS::Init() {
RTC_LOG(INFO) << __FUNCTION__;
if (initialized_)
return 0;
audio_device_buffer_.reset(new webrtc::AudioDeviceBuffer());
audio_device_.reset(new ios_adm::AudioDeviceIOS());
RTC_CHECK(audio_device_);
this->AttachAudioBuffer();
AudioDeviceGeneric::InitStatus status = audio_device_->Init();
RTC_HISTOGRAM_ENUMERATION(
"WebRTC.Audio.InitializationResult", static_cast<int>(status),
static_cast<int>(AudioDeviceGeneric::InitStatus::NUM_STATUSES));
if (status != AudioDeviceGeneric::InitStatus::OK) {
RTC_LOG(LS_ERROR) << "Audio device initialization failed.";
return -1;
}
initialized_ = true;
return 0;
}
同时将AudioDeviceIOS
跟AudioDeviceBuffer
关联,录制的音频数据写入AudioDeviceBuffer
。
不过并不是直接将录制的音频直接写入AudioDeviceBuffer
,中间经过FineAudioBuffer
转了一手。
FineAudioBuffer
将采集到的音频转换并且按10ms的帧率传递给AudioDeviceBuffer
。
FineAudioBuffer
不仅用于采集音频时也可用于播放音频。
src/sdk/objc/native/src/audio/audio_device_ios.mm
fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_));
系统音频录制与回调
iOS音频录制采集使用AudioUnit,实现类VoiceProcessingAudioUnit
,代码位于src/sdk/objc/native/src/audio/voice_processing_audio_unit.mm
。
系统采集到的音频数据回调到AudioDeviceIOS::OnDeliverRecordedData
:
src/sdk/objc/native/src/audio/audio_device_ios.mm
OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 bus_number,
UInt32 num_frames,
AudioBufferList* /* io_data */) {
RTC_DCHECK_RUN_ON(&io_thread_checker_);
OSStatus result = noErr;
// Simply return if recording is not enabled.
if (!rtc::AtomicOps::AcquireLoad(&recording_)) return result;
// Set the size of our own audio buffer and clear it first to avoid copying
// in combination with potential reallocations.
// On real iOS devices, the size will only be set once (at first callback).
record_audio_buffer_.Clear();
record_audio_buffer_.SetSize(num_frames);
// Allocate AudioBuffers to be used as storage for the received audio.
// The AudioBufferList structure works as a placeholder for the
// AudioBuffer structure, which holds a pointer to the actual data buffer
// in |record_audio_buffer_|. Recorded audio will be rendered into this memory
// at each input callback when calling AudioUnitRender().
AudioBufferList audio_buffer_list;
audio_buffer_list.mNumberBuffers = 1;
AudioBuffer* audio_buffer = &audio_buffer_list.mBuffers[0];
audio_buffer->mNumberChannels = record_parameters_.channels();
audio_buffer->mDataByteSize =
record_audio_buffer_.size() * VoiceProcessingAudioUnit::kBytesPerSample;
audio_buffer->mData = reinterpret_cast<int8_t*>(record_audio_buffer_.data());
// Obtain the recorded audio samples by initiating a rendering cycle.
// Since it happens on the input bus, the |io_data| parameter is a reference
// to the preallocated audio buffer list that the audio unit renders into.
// We can make the audio unit provide a buffer instead in io_data, but we
// currently just use our own.
// TODO(henrika): should error handling be improved?
result = audio_unit_->Render(flags, time_stamp, bus_number, num_frames, &audio_buffer_list);
if (result != noErr) {
RTCLogError(@"Failed to render audio.");
return result;
}
// Get a pointer to the recorded audio and send it to the WebRTC ADB.
// Use the FineAudioBuffer instance to convert between native buffer size
// and the 10ms buffer size used by WebRTC.
fine_audio_buffer_->DeliverRecordedData(record_audio_buffer_, kFixedRecordDelayEstimate);
return noErr;
}
从上面代码可以看到,使用了自己的缓冲区AudioBufferList
,真正的缓冲指针是record_audio_buffer_
,然后调用audio_unit_->Render
,使用AudioUnitRender
将数据写入到WebRTC提供的数据缓冲区中:
src/sdk/objc/native/src/audio/voice_processing_audio_unit.mm
OSStatus VoiceProcessingAudioUnit::Render(AudioUnitRenderActionFlags* flags,
const AudioTimeStamp* time_stamp,
UInt32 output_bus_number,
UInt32 num_frames,
AudioBufferList* io_data) {
RTC_DCHECK(vpio_unit_) << "Init() not called.";
OSStatus result = AudioUnitRender(vpio_unit_, flags, time_stamp,
output_bus_number, num_frames, io_data);
if (result != noErr) {
RTCLogError(@"Failed to render audio unit. Error=%ld", (long)result);
}
return result;
}
之后将record_audio_buffer_
传递给FineAudioBuffer
进行处理。
FineAudioBuffer
src/modules/audio_device/fine_audio_buffer.cc
void FineAudioBuffer::DeliverRecordedData(
rtc::ArrayView<const int16_t> audio_buffer,
int record_delay_ms) {
RTC_DCHECK(IsReadyForRecord());
// Always append new data and grow the buffer when needed.
record_buffer_.AppendData(audio_buffer.data(), audio_buffer.size());
// Consume samples from buffer in chunks of 10ms until there is not
// enough data left. The number of remaining samples in the cache is given by
// the new size of the internal |record_buffer_|.
const size_t num_elements_10ms =
record_channels_ * record_samples_per_channel_10ms_;
while (record_buffer_.size() >= num_elements_10ms) {
audio_device_buffer_->SetRecordedBuffer(record_buffer_.data(),
record_samples_per_channel_10ms_);
audio_device_buffer_->SetVQEData(playout_delay_ms_, record_delay_ms);
audio_device_buffer_->DeliverRecordedData();
memmove(record_buffer_.data(), record_buffer_.data() + num_elements_10ms,
(record_buffer_.size() - num_elements_10ms) * sizeof(int16_t));
record_buffer_.SetSize(record_buffer_.size() - num_elements_10ms);
}
}
FineAudioBuffer
收到系统音频数据时追加到自己的record_buffer_缓冲区中,同时将缓冲区中所有10ms数据传递给AudioDeviceBuffer
。
src/modules/audio_device/audio_device_buffer.cc
int32_t AudioDeviceBuffer::DeliverRecordedData() {
if (!audio_transport_cb_) {
RTC_LOG(LS_WARNING) << "Invalid audio transport";
return 0;
}
const size_t frames = rec_buffer_.size() / rec_channels_;
const size_t bytes_per_frame = rec_channels_ * sizeof(int16_t);
uint32_t new_mic_level_dummy = 0;
uint32_t total_delay_ms = play_delay_ms_ + rec_delay_ms_;
int32_t res = audio_transport_cb_->RecordedDataIsAvailable(
rec_buffer_.data(), frames, bytes_per_frame, rec_channels_,
rec_sample_rate_, total_delay_ms, 0, 0, typing_status_,
new_mic_level_dummy);
if (res == -1) {
RTC_LOG(LS_ERROR) << "RecordedDataIsAvailable() failed";
}
return 0;
}
AudioDeviceBuffer
收到数据后回调给AudioTransport::RecordedDataIsAvailable
。
AudioTransport
由AudioDeviceBuffer::RegisterAudioCallback
注册:
void WebRtcVoiceEngine::Init() {
...
src/media/engine/webrtcvoiceengine.cc
void WebRtcVoiceEngine::Init() {
...
// Connect the ADM to our audio path.
adm()->RegisterAudioCallback(audio_state()->audio_transport());
...
}
audio_state()
获取 的是audio_state_
对象。audio_state_
的创建:
audio_state_ = webrtc::AudioState::Create(config);
src/audio/audio_state.cc
AudioState::AudioState(const AudioState::Config& config)
: config_(config),
audio_transport_(config_.audio_mixer, config_.audio_processing.get()) {
process_thread_checker_.DetachFromThread();
RTC_DCHECK(config_.audio_mixer);
RTC_DCHECK(config_.audio_device_module);
}
audio_transport_
是AudioTransportImpl
实例。
AudioTransportImpl
AudioTransportImpl
收到采集的音频数据后做了很多事情:
src/audio/audio_transport_impl.cc
// Not used in Chromium. Process captured audio and distribute to all sending
// streams, and try to do this at the lowest possible sample rate.
int32_t AudioTransportImpl::RecordedDataIsAvailable(
const void* audio_data,
const size_t number_of_frames,
const size_t bytes_per_sample,
const size_t number_of_channels,
const uint32_t sample_rate,
const uint32_t audio_delay_milliseconds,
const int32_t /*clock_drift*/,
const uint32_t /*volume*/,
const bool key_pressed,
uint32_t& /*new_mic_volume*/) { // NOLINT: to avoid changing APIs
RTC_DCHECK(audio_data);
RTC_DCHECK_GE(number_of_channels, 1);
RTC_DCHECK_LE(number_of_channels, 2);
RTC_DCHECK_EQ(2 * number_of_channels, bytes_per_sample);
RTC_DCHECK_GE(sample_rate, AudioProcessing::NativeRate::kSampleRate8kHz);
// 100 = 1 second / data duration (10 ms).
RTC_DCHECK_EQ(number_of_frames * 100, sample_rate);
RTC_DCHECK_LE(bytes_per_sample * number_of_frames * number_of_channels,
AudioFrame::kMaxDataSizeBytes);
int send_sample_rate_hz = 0;
size_t send_num_channels = 0;
bool swap_stereo_channels = false;
{
rtc::CritScope lock(&capture_lock_);
send_sample_rate_hz = send_sample_rate_hz_;
send_num_channels = send_num_channels_;
swap_stereo_channels = swap_stereo_channels_;
}
std::unique_ptr<AudioFrame> audio_frame(new AudioFrame());
InitializeCaptureFrame(sample_rate, send_sample_rate_hz, number_of_channels,
send_num_channels, audio_frame.get());
voe::RemixAndResample(static_cast<const int16_t*>(audio_data),
number_of_frames, number_of_channels, sample_rate,
&capture_resampler_, audio_frame.get());
ProcessCaptureFrame(audio_delay_milliseconds, key_pressed,
swap_stereo_channels, audio_processing_,
audio_frame.get());
// Typing detection (utilizes the APM/VAD decision). We let the VAD determine
// if we're using this feature or not.
// TODO(solenberg): is_enabled() takes a lock. Work around that.
bool typing_detected = false;
if (audio_processing_->voice_detection()->is_enabled()) {
if (audio_frame->vad_activity_ != AudioFrame::kVadUnknown) {
bool vad_active = audio_frame->vad_activity_ == AudioFrame::kVadActive;
typing_detected = typing_detection_.Process(key_pressed, vad_active);
}
}
// Measure audio level of speech after all processing.
double sample_duration = static_cast<double>(number_of_frames) / sample_rate;
audio_level_.ComputeLevel(*audio_frame.get(), sample_duration);
// Copy frame and push to each sending stream. The copy is required since an
// encoding task will be posted internally to each stream.
{
rtc::CritScope lock(&capture_lock_);
typing_noise_detected_ = typing_detected;
RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0);
if (!sending_streams_.empty()) {
auto it = sending_streams_.begin();
while (++it != sending_streams_.end()) {
std::unique_ptr<AudioFrame> audio_frame_copy(new AudioFrame());
audio_frame_copy->CopyFrom(*audio_frame.get());
(*it)->SendAudioData(std::move(audio_frame_copy));
}
// Send the original frame to the first stream w/o copying.
(*sending_streams_.begin())->SendAudioData(std::move(audio_frame));
}
}
return 0;
}
voe::RemixAndResample
先判断源声道数是否大于目标声道数,然后大于则降低到目标声道数,然后重采样转换到目标采样率。
最后源声道如果为单声道且目标声道为双声道则将音频数据转换到双声道。
经过重采样和声道转换后进行一下步更多的音频处理:
ProcessCaptureFrame(audio_delay_milliseconds, key_pressed,
swap_stereo_channels, audio_processing_,
audio_frame.get());
audio_processing_
为config_.audio_processing
,也是WebRtcVoiceEngine
的rtc::scoped_refptr<webrtc::AudioProcessing> apm_
。
RTCPeerConnectionFactory
中创建了(rtc::scoped_refptr<webrtc::AudioProcessing>)audioProcessingModule
if (!audioProcessingModule) audioProcessingModule = webrtc::AudioProcessingBuilder().Create();
所以audio_processing_
由webrtc::AudioProcessingBuilder().Create();
创建。
src/modules/audio_processing/audio_processing_impl.cc
AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
config, std::move(capture_post_processing_),
std::move(render_pre_processing_), std::move(echo_control_factory_),
std::move(echo_detector_), std::move(capture_analyzer_));
if (apm->Initialize() != AudioProcessing::kNoError) {
delete apm;
apm = nullptr;
}
return apm;
}
AudioProcessingImpl
集合了众多音频处理模块。
ProcessCaptureFrame
调用了AudioProcessingImpl::ProcessStream(AudioFrame* frame)
src/modules/audio_processing/audio_processing_impl.cc
int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
...
capture_.capture_audio->DeinterleaveFrom(frame);
RETURN_ON_ERR(ProcessCaptureStreamLocked());
capture_.capture_audio->InterleaveTo(
frame, submodule_states_.CaptureMultiBandProcessingActive() ||
submodule_states_.CaptureFullBandProcessingActive());
DeinterleaveFrom
将音频按声道逐行扫描分离到capture_.capture_audio
中。
ProcessCaptureStreamLocked()
进行真正的音频处理,详细的过程参考另一篇文章WebRTC采集音频后的音频处理算法。
最后使用InterleaveTo
将分离的音频再合并回来。
typing_detection_.Process
检测处理键盘打字声音。
audio_level_.ComputeLevel
计算音量。
最后SendAudioData
阶段,先遍历sending_streams_
除了第一个AudioSendStream
,新建AudioFrame
拷贝audio_frame
数据,这里必须要拷贝,因为每个AudioSendStream
都独立编码处理音频帧,而第一个AudioSendStream
不需要拷贝数据直接将audio_frame
提交给其处理。
ProcessCaptureStreamLocked音频处理
单独写了一篇文章:WebRTC采集音频后的音频处理算法
AudioSendStream
sending_streams_
中存储的是AudioSendStream
对象,AudioSendStream
由WebRtcVoiceMediaChannel::WebRtcAudioSendStream
创建并持有:
src/media/engine/webrtcvoiceengine.cc
WebRtcAudioSendStream(...){
...
stream_ = call_->CreateAudioSendStream(config_);
...
}
WebRtcAudioSendStream
在WebRtcVoiceMediaChannel::AddSendStream
中创建。
AddSendStream
被调用的流程如下:
iOS:
RTCPeerConnection::setLocalDescription
->
c++:
PeerConnection::SetLocalDescription
->
PeerConnection::ApplyLocalDescription
->
PeerConnection::UpdateSessionState
->
PeerConnection::PushdownMediaDescription
->
BaseChannel::SetLocalContent
->
VoiceChannel::SetLocalContent_w
->
BaseChannel::UpdateLocalStreams_w
->
WebRtcVoiceMediaChannel::AddSendStream
AudioSendStream::SendAudioData
真正发送是由voe::CreateChannelSend
创建的ChannelSend
调用ProcessAndEncodeAudio
.
void ChannelSend::ProcessAndEncodeAudio(
std::unique_ptr<AudioFrame> audio_frame) {
RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
// Avoid posting any new tasks if sending was already stopped in StopSend().
rtc::CritScope cs(&encoder_queue_lock_);
if (!encoder_queue_is_active_) {
return;
}
// Profile time between when the audio frame is added to the task queue and
// when the task is actually executed.
audio_frame->UpdateProfileTimeStamp();
encoder_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(
new ProcessAndEncodeAudioTask(std::move(audio_frame), this)));
}
这儿先说一个比较主要的对象:Call
。上面的encoder_queue_
由其成员RtpTransportControllerSendInterface
创建:transport_send_ptr_->GetWorkerQueue()
。Call
最初由PeerConnectionFactory
创建:
src/pc/peerconnectionfactory.cc
rtc::scoped_refptr<PeerConnectionInterface>
PeerConnectionFactory::CreatePeerConnection(
const PeerConnectionInterface::RTCConfiguration& configuration,
PeerConnectionDependencies dependencies) {
RTC_DCHECK(signaling_thread_->IsCurrent());
// Set internal defaults if optional dependencies are not set.
if (!dependencies.cert_generator) {
dependencies.cert_generator =
absl::make_unique<rtc::RTCCertificateGenerator>(signaling_thread_,
network_thread_);
}
if (!dependencies.allocator) {
network_thread_->Invoke<void>(RTC_FROM_HERE, [this, &configuration,
&dependencies]() {
dependencies.allocator = absl::make_unique<cricket::BasicPortAllocator>(
default_network_manager_.get(), default_socket_factory_.get(),
configuration.turn_customizer);
});
}
// TODO(zstein): Once chromium injects its own AsyncResolverFactory, set
// |dependencies.async_resolver_factory| to a new
// |rtc::BasicAsyncResolverFactory| if no factory is provided.
network_thread_->Invoke<void>(
RTC_FROM_HERE,
rtc::Bind(&cricket::PortAllocator::SetNetworkIgnoreMask,
dependencies.allocator.get(), options_.network_ignore_mask));
std::unique_ptr<RtcEventLog> event_log =
worker_thread_->Invoke<std::unique_ptr<RtcEventLog>>(
RTC_FROM_HERE,
rtc::Bind(&PeerConnectionFactory::CreateRtcEventLog_w, this));
std::unique_ptr<Call> call = worker_thread_->Invoke<std::unique_ptr<Call>>(
RTC_FROM_HERE,
rtc::Bind(&PeerConnectionFactory::CreateCall_w, this, event_log.get()));
rtc::scoped_refptr<PeerConnection> pc(
new rtc::RefCountedObject<PeerConnection>(this, std::move(event_log),
std::move(call)));
ActionsBeforeInitializeForTesting(pc);
if (!pc->Initialize(configuration, std::move(dependencies))) {
return nullptr;
}
return PeerConnectionProxy::Create(signaling_thread(), pc);
}
成员 RtpTransportControllerSendInterface *transport_send_ptr_
的创建:
Call* Call::Create(const Call::Config& config) {
return new internal::Call(
config, absl::make_unique<RtpTransportControllerSend>(
Clock::GetRealTimeClock(), config.event_log,
config.network_controller_factory, config.bitrate_config));
}
src/call/rtp_transport_controller_send.cc
:
RtpTransportControllerSend
的构造函数中创建了线程:process_thread_(ProcessThread::Create("SendControllerThread"))
。
并且在process_thread_
中注册了两个Module:
process_thread_->RegisterModule(&pacer_, RTC_FROM_HERE);
process_thread_->RegisterModule(send_side_cc_.get(), RTC_FROM_HERE);
process_thread_:
process_thread_
的实现类是ProcessThreadImpl
,线程由平台线程类rtc::PlatformThread
创建。
线程执行时循环回调:ProcessThreadImpl::Process()
。
ProcessThreadImpl::Process()
遍历所有注册的module,每个moudule检测自己的定时器,当时间到达时执行module的Process
方法。
同时ProcessThreadImpl::Process()
还取出自己queue_
中的所有task逐个执行task->Run()
,执行完成后删除task。
然后根据modules的需等待的最小时间使用wake_up_.Wait
挂起线程,等待超时或者WakeUp
,PostTask
,RegisterModule
,Stop
方法唤醒。
TaskQueue:
RtpTransportControllerSend
的成员:rtc::TaskQueue task_queue_
在每个平台都有不同的实现:
iOS平台由src/rtc_base/task_queue_gcd.cc
实现,根据平台不同还有几个版本:
src/rtc_base/task_queue_win.cc
src/rtc_base/task_queue_libevent.cc
src/rtc_base/task_queue_stdlib.cc
总的来说,TaskQueue
自己维护一个线程来处理自己队列中的任务。
iOS中直接使用gcd实现,内部的queue_
是串行队列: queue_(dispatch_queue_create(queue_name, DISPATCH_QUEUE_SERIAL))
,PostTask
时调用dispatch_async_f
进行异步排队处理。
encoder_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(
new ProcessAndEncodeAudioTask(std::move(audio_frame), this)));
当音频数据采集且处理过后,将AudioFrame
扔到一个TaskQueue
中逐个进行处理,处理方法为ProcessAndEncodeAudioOnTaskQueue
:
src/audio/channel_send.cc
void ChannelSend::ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input) {
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK_GT(audio_input->samples_per_channel_, 0);
RTC_DCHECK_LE(audio_input->num_channels_, 2);
// Measure time between when the audio frame is added to the task queue and
// when the task is actually executed. Goal is to keep track of unwanted
// extra latency added by the task queue.
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Audio.EncodingTaskQueueLatencyMs",
audio_input->ElapsedProfileTimeMs());
bool is_muted = InputMute();
AudioFrameOperations::Mute(audio_input, previous_frame_muted_, is_muted);
if (_includeAudioLevelIndication) {
size_t length =
audio_input->samples_per_channel_ * audio_input->num_channels_;
RTC_CHECK_LE(length, AudioFrame::kMaxDataSizeBytes);
if (is_muted && previous_frame_muted_) {
rms_level_.AnalyzeMuted(length);
} else {
rms_level_.Analyze(
rtc::ArrayView<const int16_t>(audio_input->data(), length));
}
}
previous_frame_muted_ = is_muted;
// Add 10ms of raw (PCM) audio data to the encoder @ 32kHz.
// The ACM resamples internally.
audio_input->timestamp_ = _timeStamp;
// This call will trigger AudioPacketizationCallback::SendData if encoding
// is done and payload is ready for packetization and transmission.
// Otherwise, it will return without invoking the callback.
if (audio_coding_->Add10MsData(*audio_input) < 0) {
RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed.";
return;
}
_timeStamp += static_cast<uint32_t>(audio_input->samples_per_channel_);
}
audio_coding_
是音频编码器,当编码完成后回调ChannelSend::SendData
:
int32_t ChannelSend::SendData(FrameType frameType,
uint8_t payloadType,
uint32_t timeStamp,
const uint8_t* payloadData,
size_t payloadSize,
const RTPFragmentationHeader* fragmentation) {
RTC_DCHECK_RUN_ON(encoder_queue_);
rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize);
if (media_transport() != nullptr) {
return SendMediaTransportAudio(frameType, payloadType, timeStamp, payload,
fragmentation);
} else {
return SendRtpAudio(frameType, payloadType, timeStamp, payload,
fragmentation);
}
}
追寻media_transport() 的创建至最初,发现是由PeerConnectionInterface::use_media_transport决定:
src/api/peerconnectioninterface.h
PeerConnectionInterface::
// If MediaTransportFactory is provided in PeerConnectionFactory, this flag
// informs PeerConnection that it should use the MediaTransportInterface.
// It's invalid to set it to |true| if the MediaTransportFactory wasn't
// provided.
bool use_media_transport = false;
WebRTC默认使用use_media_transport = false
创建PeerConnection
,所以在默认设置下media_transport()
为null,因此ChannelSend::SendData
执行SendRtpAudio
。
SendRtpAudio
将编码后的数据进行RTP打包,经SRTP加密后发送。
_rtpRtcpModule
的实现类:
src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc
: ModuleRtpRtcpImpl
。