在我们的项目中，iOS端视频播放使用的是第三方开源框架ijkplayer，在硬解码的时候，解码出的数据格式是NV12，而Unity只能渲染RGB格式，所以需要做转换。
解决方案：
在iOS端使用ffmpeg的函数将NV12格式转换为YUV420P格式，将YUV三个分量分别传入Unity中，然后在Unity中将YUV420P转换为RGB并渲染。
具体实现：
- NV12转换为YUV420P
//获取到NV12数据的两个指针，一个指针指向Y数据，一个指针指向UV数据
UInt8 *bufferPtrY = (UInt8 *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer,0);
UInt8 *bufferPtrUV = (UInt8 *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer,1);
if (!bufferPtr || !bufferPtr1) {
printf("!bufferPtrY || !bufferPtrUV\n");
return;
}
size_t width = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
size_t height = CVPixelBufferGetHeightOfPlane(imageBuffer,0);
AVFrame *pFrame=*dest_frame;
if (pFrame == NULL)
{
pFrame = av_frame_alloc();
av_image_alloc(pFrame->data, pFrame->linesize, (int)width, (int)height, AV_PIX_FMT_YUV420P,1);
*dest_frame = pFrame;
}
static int sws_flags = SWS_FAST_BILINEAR;
uint8_t * srcSlice[8] ={bufferPtrY, bufferPtrUV, NULL, NULL,NULL,NULL,NULL,NULL};
int srcStride[8] = {(int)width, (int)width, 0, 0, 0,0,0,0};
if (img_convert_ctx == NULL)
{
img_convert_ctx = sws_getContext(
(int)width,
(int)height,
AV_PIX_FMT_NV12,
(int)width,
overlay->h,
AV_PIX_FMT_YUV420P,
sws_flags,
NULL, NULL, NULL);
}
int ret = sws_scale(
img_convert_ctx,
srcSlice,
srcStride,
0,
overlay->h,
pFrame->data,
pFrame->linesize);
此时pFrame变量即包含YUV420P格式的图像数据。
- 使用OpenGL将数据拷贝至显存
GLuint tex[3];
// 如果tex[0]为0，那么代表所有tex还没有初始化
if (0 == tex[0]) {
// 初始化创建所有纹理
for (int i = 0; i < 3; i++)
{
glGenTextures(1, &tex[i]);
glBindTexture(GL_TEXTURE_2D, tex[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
}
glBindTexture(GL_TEXTURE_2D, tex[0]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, w, h, 0, GL_ALPHA, GL_UNSIGNED_BYTE, pData0);
glBindTexture(GL_TEXTURE_2D, tex[1]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, w/2, h/2, 0, GL_ALPHA, GL_UNSIGNED_BYTE, pData1);
glBindTexture(GL_TEXTURE_2D, tex[2]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, w/2, h/2, 0, GL_ALPHA, GL_UNSIGNED_BYTE, pData2);
// 通过消息发送给Unity
UnitySendMessage(VRController::Instance()->GetRenderCallbackGoName(), VRController::Instance()->GetRenderCallbackFuncName(), [[NSString stringWithFormat:@"%ld|%ld|%ld",tex[0], tex[1], tex[2]] UTF8String]);
其中pData0、pData1、pData2分别对应YUV三个方向的分量。
- Unity获取显存中的图像数据
public void display(string texString) {
string[] tex = texString.Split ('|');
IntPtr nativeTex1 = (IntPtr)long.Parse (tex [0]);
IntPtr nativeTex2 = (IntPtr)long.Parse (tex [1]);
IntPtr nativeTex3 = (IntPtr)long.Parse (tex [2]);
if (_videoTextureY == null) {
_videoTextureY = Texture2D.CreateExternalTexture ((int)videoLineW, (int)videoH, TextureFormat.Alpha8,false, false, (IntPtr)nativeTex1);
_videoTextureY.filterMode = FilterMode.Bilinear;
_videoTextureY.wrapMode = TextureWrapMode.Clamp;
}
if (_videoTextureU == null) {
_videoTextureU = Texture2D.CreateExternalTexture ((int)videoLineW / 2, (int)videoH / 2, TextureFormat.Alpha8,false, false, (IntPtr)nativeTex2);
_videoTextureU.filterMode = FilterMode.Point;
_videoTextureU.wrapMode = TextureWrapMode.Clamp;
}
if (_videoTextureV == null) {
_videoTextureV = Texture2D.CreateExternalTexture ((int)videoLineW / 2, (int)videoH / 2, TextureFormat.Alpha8,false, false, (IntPtr)nativeTex3);
_videoTextureV.filterMode = FilterMode.Point;
_videoTextureV.wrapMode = TextureWrapMode.Clamp;
}
_videoTextureY.UpdateExternalTexture ((IntPtr)nativeTex1);
_videoTextureU.UpdateExternalTexture ((IntPtr)nativeTex2);
_videoTextureV.UpdateExternalTexture ((IntPtr)nativeTex3);
}
- 在shader中将YUV格式转换为RGB格式
fixed4 yuv;
yuv.x = tex2D(_MainTexY, i.uv).a;
yuv.y = tex2D(_MainTexU, i.uv).a - 0.5;
yuv.z = tex2D(_MainTexV, i.uv).a - 0.5;
c.r = yuv.x + 1.403*yuv.z;
c.g = yuv.x - 0.344*yuv.y - 0.714*yuv.z;
c.b = yuv.x + 1.770*yuv.y;
