前言
上一篇文章的丢帧是依据编码后的码率和目标码率来决定丢帧，
而本文介绍的丢帧依据是目标帧率。
由此可对丢帧策略分类如下：
- 编码后的码率和目标码率来决定丢帧
- 目标帧率决定丢帧
调用流程
前处理模块调用丢帧处理
ViEEncoder::DeliverFrame
->int32_t VideoProcessingModuleImpl::PreprocessFrame
->int32_t VPMFramePreprocessor::PreprocessFrame
PreprocessFrame前处理先进行丢帧处理，然后进行重采样处理。丢帧处理中，每来一帧先进行输入帧率统计，然后进行丢帧判断。输入帧率统计和上一篇码率决定丢帧输入丢帧统计算法一样。
int32_t VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame,
I420VideoFrame** processed_frame) {
if (frame.IsZeroSize()) {
return VPM_PARAMETER_ERROR;
}
vd_->UpdateIncomingframe_rate(); //更新输入帧率
if (vd_->DropFrame()) {//判断是否丢帧
return 1; // drop 1 frame
}
// Resizing incoming frame if needed. Otherwise, remains NULL.
// We are not allowed to resample the input frame (must make a copy of it).
*processed_frame = NULL;
if (spatial_resampler_->ApplyResample(frame.width(), frame.height())) {
int32_t ret = spatial_resampler_->ResampleFrame(frame, &resampled_frame_);
if (ret != VPM_OK) return ret;
*processed_frame = &resampled_frame_;
}
// Perform content analysis on the frame to be encoded.
if (enable_ca_) {
// Compute new metrics every |kSkipFramesCA| frames, starting with
// the first frame.
if (frame_cnt_ % kSkipFrameCA == 0) {
if (*processed_frame == NULL) {
content_metrics_ = ca_->ComputeContentMetrics(frame);
} else {
content_metrics_ = ca_->ComputeContentMetrics(resampled_frame_);
}
}
++frame_cnt_;
}
return VPM_OK;
}```
##输入帧率统计
每来一帧都将此时的时间作为样本，记录在滑动窗口为kFrameCountHistory_size的incoming_frame_times_数组中。并进行ProcessIncomingframe_rate输入帧率的计算。
void VPMVideoDecimator::UpdateIncomingframe_rate() {
int64_t now = TickTime::MillisecondTimestamp();
if (incoming_frame_times_[0] == 0) {
// First no shift.
} else {
// Shift.
for (int i = kFrameCountHistory_size - 2; i >= 0; i--) {
incoming_frame_times_[i+1] = incoming_frame_times_[i];
}
}
incoming_frame_times_[0] = now;
ProcessIncomingframe_rate(now);
}```
统计最多不超过2秒钟的样本，计算输入帧率。
void VPMVideoDecimator::ProcessIncomingframe_rate(int64_t now) {
int32_t num = 0;
int32_t nrOfFrames = 0;
for (num = 1; num < (kFrameCountHistory_size - 1); num++) {
// Don't use data older than 2sec.
if (incoming_frame_times_[num] <= 0 ||
now - incoming_frame_times_[num] > kFrameHistoryWindowMs) {
break;
} else {
nrOfFrames++;
}
}
if (num > 1) {
int64_t diff = now - incoming_frame_times_[num-1];
incoming_frame_rate_ = 1.0;
if (diff > 0) {
incoming_frame_rate_ = nrOfFrames * 1000.0f / static_cast<float>(diff);
}
} else {
incoming_frame_rate_ = static_cast<float>(nrOfFrames);
}
}```
##目标帧率丢帧核心
一、当2 * overshoot < (int32_t) incomingframe_rate，即输入帧率大于目标帧率，小于2倍目标帧率的情况下。具体细节看注释，这里假设incomingframe_rate=20，target_frame_rate_=15。具体就是实现均匀丢帧。
bool VPMVideoDecimator::DropFrame() {
if (!enable_temporal_decimation_) return false;
if (incoming_frame_rate_ <= 0) return false;
const uint32_t incomingframe_rate =
static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
if (target_frame_rate_ == 0) return true;
bool drop = false;
if (incomingframe_rate > target_frame_rate_) {//输入帧率大于目标帧率
int32_t overshoot =
overshoot_modifier_ + (incomingframe_rate - target_frame_rate_);//20-15=5，超出帧率
if (overshoot < 0) {
overshoot = 0;
overshoot_modifier_ = 0;
}
if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) {//2*5<20,即20<2*15，输入帧率小于2倍目标帧率
if (drop_count_) { // Just got here so drop to be sure.
drop_count_ = 0;
return true;
}
const uint32_t dropVar = incomingframe_rate / overshoot;//20/5=4，丢帧比率
if (keep_count_ >= dropVar) {//均匀丢帧
drop = true;
overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;//-（20%5）/3=0，修正overshoot
keep_count_ = 1;//重置保留帧数
} else {
keep_count_++;//保留帧数
}
} else {
keep_count_ = 0;
const uint32_t dropVar = overshoot / target_frame_rate_;
if (drop_count_ < dropVar) {
drop = true;
drop_count_++;
} else {
overshoot_modifier_ = overshoot % target_frame_rate_;
drop = false;
drop_count_ = 0;
}
}
}
return drop;
}```
二、当2 * overshoot >=(int32_t) incomingframe_rate时，即输入帧率大于等于2倍目标帧率，此时，均匀丢帧每次丢1帧以上，具体丢帧方法和上一部分略有区别。
bool VPMVideoDecimator::DropFrame() {
if (!enable_temporal_decimation_) return false;
if (incoming_frame_rate_ <= 0) return false;
const uint32_t incomingframe_rate =
static_cast<uint32_t>(incoming_frame_rate_ + 0.5f);
if (target_frame_rate_ == 0) return true;
bool drop = false;
if (incomingframe_rate > target_frame_rate_) {//输入帧率大于目标帧率
int32_t overshoot =
overshoot_modifier_ + (incomingframe_rate - target_frame_rate_); //30-10=20，超出帧率
if (overshoot < 0) {
overshoot = 0;
overshoot_modifier_ = 0;
}
if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) { //2*20>30，即30>2*10，输入帧率大于2倍目标帧率
if (drop_count_) { // Just got here so drop to be sure.
drop_count_ = 0;
return true;
}
const uint32_t dropVar = incomingframe_rate / overshoot;
if (keep_count_ >= dropVar) {
drop = true;
overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;
keep_count_ = 1;
} else {
keep_count_++;
}
} else {
keep_count_ = 0;
const uint32_t dropVar = overshoot / target_frame_rate_; //20/10=2，丢帧比率
if (drop_count_ < dropVar) {//一次丢1帧以上
drop = true;
drop_count_++;
} else {
overshoot_modifier_ = overshoot % target_frame_rate_; //20%10=0，overshoot修正
drop = false;
drop_count_ = 0;
}
}
}
return drop;
}```