AE曲线变速实现主要是依据时间重映射方式实现，支持渐变直线和贝塞尔曲线两种类型
1、时间重映射代码逻辑
AE并不是完全按照变速曲线来计算位置的，在出现速度为负数情况并且计算到第0帧后还存在一段时间负数区间，这种情况下，在后面的正速区间AE会内部补偿延后一段时间才开始按照速度曲线取值，造成了效果和速度曲线对不上的情况，这种情况暂时不予处理
// "Animates" time based on the layer's "tm" property.
class TimeRemapper final : public AnimatablePropertyContainer {
public:
TimeRemapper(const skjson::ObjectValue& jtm, const AnimationBuilder* abuilder, float scale)
: fScale(scale) {
CalculateRemapPoint(&jtm);
this->bind(*abuilder, &jtm, fT);
}
void CalculateRemapPoint(const skjson::ObjectValue* jprop) {
const skjson::ArrayValue* jkfs = (*jprop)["k"];
if (jkfs && jkfs->size() > 0) {
remap_point.reserve(jkfs->size());
for (int i = 0; i < jkfs->size(); ++i) {
const skjson::ObjectValue* jkf = (*jkfs)[i];
if (!jkf) {
continue;
}
float t;
if (!skottie::Parse<float>((*jkf)["t"], &t)) {
continue;
}
remap_point.push_back(t);
}
}
}
/**
* 新增重映射点检测，当正好在重映射点的时候，将这一帧速度设置为0，防止曲线速度出现定格样式速度直接
* 计算为整数最大值
*/
bool HitRemapPosition(int index) {
for (int i = 0; i < remap_point.size(); ++i) {
if (index == remap_point[i])
return true;
}
return false;
}
float t() const { return fT * fScale; }
private:
void onSync() override {
// nothing to sync - we just track t
}
const float fScale;
std::vector<int> remap_point;
ScalarValue fT = 0;
};
2、视频变速
视频变速只需要知道当前变速后的时间，不需要关心曲线速度
float calculateRemaperTime(float t) {
fRemapper->seek(t);
float after = fRemapper->t();
return after * fTimeScale;
}
StateChanged onSeek(float t) override {
...
float remap_time = calculateRemaperTime(t);
...
}
3音频变速
音频变速实现采样SoundTouch开源框架，优点是支持实时变速
音频变速需要知道当前曲线的实时速度取值，根据时间重映射后前后两帧参数计算当前速度
float before = t;
audio_remapper_->seek(t);
float after = audio_remapper_->t();
float speed = (after - last_t2) / (before - last_t1);
last_t2 = after;
last_t1 = before;
知道音频速度值还需要知道什么时候开始变速，目前按照帧对齐方式，通过计算视频速度区间，然后按照音频和视频一帧时间间隔比例调整音频速度区间
if (abs(speed - last_speed) > 0.01) {
// 遇到重映射插值点（突变点）速度设为0
if (hit_remap_pos) {
speed = 0;
}
SpeedInfo speedInfo;
speedInfo.start_index = i - ip;
speedInfo.speed = speed;
speed_deqeue_.push_back(speedInfo);
last_speed = speed;
}
...
...
调整音频速度区间
for (int i = 0; i < speed_deqeue_.size(); ++i) {
SpeedInfo *speedInfo = &speed_deqeue_[i];
// 44100HZ采样一帧音频时长time_scale=1024 / 44100，视频帧率默认25，基本上两帧音频对应一帧视频，
// 然后乘以当前速度，按照原来开始和结束帧区间缩放得到音频速度区间
float ratio = 1.0 / time_scale / video_framerate_ * abs(speedInfo->speed);
if (abs(speedInfo->speed) < 1e-6) {
ratio = 1.0;
}
int start = speedInfo->start_index;
int end = speedInfo->end_index;
int area = 0;
if (i < speed_deqeue_.size() - 1) {
end = speed_deqeue_[i + 1].start_index;
area = (end - start) * ratio;
} else {
area = last_area * ratio;
}
speedInfo->start_index = last_end_index;
speedInfo->end_index = speedInfo->start_index + area;
last_end_index = speedInfo->end_index;
...
}
模版中如果有速度为负数情况，音频需要在解码到速度区间为负数之前先缓存一段时间音频，缓存音频区间结束帧位置是倒序区间开始帧位置
if (speedInfo->speed < 0) {
if (!in_reverse_area) {
reverse_start_index_ = speedInfo->start_index;
}
in_reverse_area = true;
} else {
if (in_reverse_area) {
reverse_end_index_ = speedInfo->start_index;
if (reverse_start_index_ >= audio_start_index) {
//倒序区间
areverse_area_map_.insert(std::pair<int, int>(reverse_start_index_ - audio_start_index,
reverse_end_index_ - audio_start_index));
}
}
in_reverse_area = false;
}
void SetSpeedInfo(std::deque<SpeedInfo> speed_deque, std::map<int, int> areverse_area) {
this->speed_deque_ = speed_deque;
this->areverse_area_ = areverse_area;
std::map<int, int>::iterator iter;
for(iter = areverse_area_.begin(); iter != areverse_area_.end(); iter++) {
int start_index = iter->first;
int end_index = iter->second;
int reverse_frame_cnt = end_index - start_index;
//缓存音频区间结束帧位置是倒序区间开始帧位置
areverse_cache_area_.insert(std::pair<int, int>(std::max(0, start_index - reverse_frame_cnt), start_index));
}
}
最后还需要剔除部分模版开始一段时间速度一直是0的特殊情况
//剔除开始一段时间速度为0的区间
std::deque<SpeedInfo>::iterator iterator = speed_deqeue_.begin();
for (;iterator != speed_deqeue_.end();) {
if(iterator->start_index < audio_start_index)
iterator = speed_deqeue_.erase(iterator);
else {
iterator->start_index -= audio_start_index;
iterator->end_index -= audio_start_index;
iterator++;
}
}
4、SoundTouch音频处理
SoundTouch内部也是采用FIFO机制，因此put一次数据，需要循环多次取出音频数据，在输入原始音频数据之前要确保音频数据足够1024字节大小，否则可能会有噪音出现
short* response = new short[samplesCount * 2];
sound_touch_->putSamples(request, samplesCount);
int nsample = 0;
do {
nsample = sound_touch_->receiveSamples(response, samplesCount);
AddSamplesToFifo(audio_fifo_s16_, (uint8_t**)&response, nsample);
} while (nsample != 0);
delete[] response;
delete[] request;
5、开发过程中遇到问题汇总
1、一些模版特殊情况处理，比如模版开始一段时间速度一直为0，重映射插值点速度值计算问题
2、音频seek后更新当前音频帧位置和速度区间为负数情况清空缓存后重新填充缓存问题
3、音频开源库SoundTouch使用过程中发现丢帧和效果有噪音问题
