项目介绍
image-20220325112028641.png
本项目主要实现功能实现一个屏幕录制器，从显卡中抓屏，从mic中获取pcm音频数据，
将其编码封装成一个MP4文件，以当前日期存放到指定路径中
git地址：https://gitee.com/lisiwen945/av_codec/tree/master/screen_recode
主要用到的技术：
directx3d：从显卡中抓屏，抓取的数据是bgra格式
ffmpeg：pcm编码成aac数据，音频重采样，视频像素转换，视频编码 ，音视频封装成mp4
qt：屏幕录制器界面开发，事件响应，音频采集
image-20220325114400254.png
音频raw数据获取
// 设置音频格式
QAudioFormat fmt;
fmt.setSampleRate(m_sampleRate);
fmt.setChannelCount(m_channels);
fmt.setSampleSize(16); // 采样精度
fmt.setSampleType(QAudioFormat::UnSignedInt);
fmt.setByteOrder(QAudioFormat::LittleEndian);
fmt.setCodec("audio/pcm");
// 创建音频录制设备（测试不插耳机会失败，电脑自带的录音设备好像不行）
m_input = new QAudioInput(fmt);
// 开始录制
m_io = m_input->start();
LynPcmData pcmData;
int size = 1024 * 2 * 2; // 每次读1024个样本，采样精度是16（两个字节），双声道
std::unique_lock<std::mutex> lock(m_mux);
pcmData.data = (unsigned char *)malloc(size);
int readedSize = 0;
while (readedSize < size && !m_isExit)
{
int br = m_input->bytesReady();
if (br < 1024) {
usleep(100);
continue;
}
int s = 1024;
s = s < (size - readedSize) ? s : size - readedSize;
int len = m_io->read((char *)pcmData.data + readedSize, s);
readedSize += len;
}
pcmData.size = readedSize;
音频重采样
// 因为aac编码中要求输入的pcm数据是float类型，而且要求是平面的（左右声道分开存放），所以需要重采样
m_asc = swr_alloc_set_opts(m_asc,
m_actx->channel_layout,m_actx->sample_fmt,m_actx->sample_rate, // 输出格式
av_get_default_channel_layout(2), AV_SAMPLE_FMT_S16, 44100, // 输入格式
0,0);
int ret = swr_init(m_asc);
// 构造输出信息
m_pcm = av_frame_alloc();
m_pcm->format = m_actx->sample_fmt;
m_pcm->channels = m_actx->channels;
m_pcm->channel_layout = m_actx->channel_layout;
m_pcm->nb_samples = pcmData->size / 4; // 样本数量
int ret = av_frame_get_buffer(m_pcm, 0);
// 输入数据
const uint8_t *data[AV_NUM_DATA_POINTERS] = {0};
data[0] = (uint8_t *)pcmData->data;
// 开始转换
int len = swr_convert(m_asc, m_pcm->data, pcmData->size / 4,
data, pcmData->size / 4);
视频raw数据获取
//1 创建directx3d对象
static IDirect3D9 *d3d = NULL;
if (!d3d)
{
d3d = Direct3DCreate9(D3D_SDK_VERSION);
}
if (!d3d) return;
//2 创建显卡的设备对象
static IDirect3DDevice9 *device = NULL;
if (!device)
{
D3DPRESENT_PARAMETERS pa;
ZeroMemory(&pa, sizeof(pa));
pa.Windowed = true;
pa.Flags = D3DPRESENTFLAG_LOCKABLE_BACKBUFFER;
pa.SwapEffect = D3DSWAPEFFECT_DISCARD;
pa.hDeviceWindow = GetDesktopWindow();
d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, 0,
D3DCREATE_HARDWARE_VERTEXPROCESSING, &pa, &device
);
}
if (!device) {
return;
}
//3创建离屏表面
int w = GetSystemMetrics(SM_CXSCREEN);
int h = GetSystemMetrics(SM_CYSCREEN);
static IDirect3DSurface9 *sur = NULL;
if (!sur)
{
device->CreateOffscreenPlainSurface(w, h,
D3DFMT_A8R8G8B8, D3DPOOL_SCRATCH, &sur, 0);
}
if (!sur)return;
//4 抓屏
device->GetFrontBufferData(0, sur);
//5 取出数据
D3DLOCKED_RECT rect;
ZeroMemory(&rect, sizeof(rect));
if (sur->LockRect(&rect, 0, 0) != S_OK)
{
return;
}
frame.w = w;
frame.h = h;
frame.dataSize = w * h * 4;
frame.data = malloc(frame.dataSize);
memcpy(frame.data , rect.pBits, frame.dataSize);
sur->UnlockRect();
视频像素转换
m_vsc = sws_getCachedContext(m_vsc,
frame->w, frame->h, AV_PIX_FMT_BGRA, // 输入信息
m_outWidth, m_outHeight, AV_PIX_FMT_YUV420P, // 输出信息
SWS_BICUBIC,
NULL,NULL,NULL);
m_yuv = av_frame_alloc();
m_yuv->format = AV_PIX_FMT_YUV420P;
m_yuv->width = m_outWidth;
m_yuv->height = m_outHeight;
m_yuv->pts = 0;
int ret = av_frame_get_buffer(m_yuv, 32);
if (ret != 0) {
std::cout << "av_frame_get_buffer failed!" << std::endl;
return false;
}
// rgb to yuv
uint8_t *indata[AV_NUM_DATA_POINTERS] = { 0 };
indata[0] = (uint8_t *)frame->data;
int insize[AV_NUM_DATA_POINTERS] = { 0 };
insize[0] = frame->w * 4;
int hight = sws_scale(m_vsc, indata, insize, 0, frame->h,
m_yuv->data, m_yuv->linesize);
CHECK_ERROR(hight <= 0, ret, "error");
音视频编码
// 获取视频解码器
const AVCodec *videoCodec = avcodec_find_encoder(AV_CODEC_ID_H264);
// 创建上下文
m_vctx = avcodec_alloc_context3(videoCodec);
//比特率，压缩后每秒大小
m_vctx->bit_rate = 4000000;
m_vctx->width = m_outWidth;
m_vctx->height = m_outHeight;
//时间基数
m_vctx->time_base = { 1, 10 };
m_vctx->framerate = { 10, 1 };
//画面组大小，多少帧一个关键帧
m_vctx->gop_size = 10;
m_vctx->max_b_frames = 0;
m_vctx->pix_fmt = AV_PIX_FMT_YUV420P;
m_vctx->codec_id = AV_CODEC_ID_H264;
av_opt_set(m_vctx->priv_data, "preset", "superfast", 0);
m_vctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
// 打开视频编码器
int ret = avcodec_open2(m_vctx, videoCodec, NULL);
m_videoStream = avformat_new_stream(m_ctx, NULL);
m_videoStream->codecpar->codec_tag = 0;
avcodec_parameters_from_context(m_videoStream->codecpar, m_vctx);
// 音频编码器
const AVCodec *audioCodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
m_actx = avcodec_alloc_context3(audioCodec);
// 设置参数
m_actx->bit_rate = 64000;
m_actx->sample_rate = 44100;
m_actx->sample_fmt = AV_SAMPLE_FMT_FLTP;
m_actx->channels = 2;
m_actx->channel_layout = av_get_default_channel_layout(2);
m_actx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
// 打开设备
ret = avcodec_open2(m_actx, audioCodec, NULL);
m_audioStream->codecpar->codec_tag = 0;
avcodec_parameters_from_context(m_audioStream->codecpar, m_actx);
// 后面都是往解码器灌入frame数据，获取编码后的包
avcodec_send_frame()
avcodec_receive_packet()
音视频pts同步
pkt->pts = m_apts;
pkt->dts = pkt->pts;
// 音频pts根据样本数和采样率和时间基数确定
m_apts += av_rescale_q(m_pcm->nb_samples, { 1, m_actx->sample_rate}, m_actx->time_base);
m_yuv->pts = m_vpts++;
// 视频pts 根据时间基数确定，时间基数和帧率相关
av_packet_rescale_ts(p, m_vctx->time_base, m_videoStream->time_base);
MP4封装
// 封装文件输出上下文
avformat_alloc_output_context2(&m_ctx, NULL, NULL, fileName.c_str());
av_dump_format(m_ctx, 0, m_fileName.c_str(), 1);
// 打开io
ret = avio_open(&m_ctx->pb, m_fileName.c_str(), AVIO_FLAG_WRITE);
// 写入封装头
ret = avformat_write_header(m_ctx, NULL);
// 将包写入文件
av_interleaved_write_frame(m_ctx, pkt);
// 写入尾部信息
av_write_trailer(m_ctx)
注意事项
- 音频编码和视频编码在不同的线程中，需要考虑多线程同步的问题
- 视频的帧率控制需要考虑屏幕抓屏和编码的时间，不能简单的用sleep控制，得用定时器
- 要注意编解码packet和frame的释放时机
- 编码器内部会有缓存，在结束时理论上应该将frame设置成NULL冲刷缓冲区，因为是视频录制结束缺个几帧影响不大
