虹软人脸识别SDK在网络摄像头中的实际应用

目前在人脸识别领域中，网络摄像头的使用很普遍，但接入网络摄像头和人脸识别SDK有一定门槛，在此篇中介绍过虹软人脸识别SDK的接入流程，本文着重介绍网络摄像头获取视频流并处理的流程(红色框内)，以下内容仅供参考。

市面上目前有很多款网络摄像头，以海康摄像头为例。海康SDK包含很多接口，接入有一定难度，这里只介绍获取视频帧相关的接口。
1.海康SDK接入基本流程
a.初始化并登录验证

    NET_DVR_Init();
    NET_DVR_DEVICEINFO_V30 struDeviceInfo = { 0 };
    long lUserID = NET_DVR_Login_V30(m_cameraIp, m_cameraPort,
        m_cameraUser, m_cameraPwd, &struDeviceInfo);
    if (lUserID < 0)
    {
        NET_DVR_Cleanup();
        return false;
    }

b.创建线程并注册回调函数

    thread videoThread(&HCNetCamera::getCameraPreview, this);
    videoThread.detach();

    bool HCNetCamera::getCameraPreview()
    {
        NET_DVR_CLIENTINFO ClientInfo;
        ClientInfo.lChannel = 1; //Channel number 设备通道号
        ClientInfo.hPlayWnd = NULL;  //窗口为空，设备SDK不解码只取流
        ClientInfo.lLinkMode = 0;    //Main Stream
        ClientInfo.sMultiCastIP = NULL;
        //预览取流
        g_realHandle = NET_DVR_RealPlay_V30(g_cameraUserId, &ClientInfo, fRealDataCallBack, NULL, TRUE);
        if (g_realHandle < 0)
        {
            qDebug() << "NET_DVR_RealPlay_V30 failed! Error number: " << NET_DVR_GetLastError();
            return false;
        }
        return true;
    }

　　

c.使用回调接口，获取实时的视频帧数据

    void CALLBACK fRealDataCallBack(LONG lRealHandle, DWORD dwDataType, BYTE *pBuffer, DWORD dwBufSize, void *pUser)
    {
        UNREFERENCED_PARAMETER(lRealHandle);
        UNREFERENCED_PARAMETER(pUser);

        DWORD dRet = 0;
        BOOL inData = FALSE;

        switch (dwDataType)
        {
        case NET_DVR_SYSHEAD:
            if (g_cameraPort >= 0)
            {
                break; //同一路码流不需要多次调用开流接口
            }
            if (!PlayM4_GetPort(&g_cameraPort))
            {
                break;
            }
            if (!PlayM4_OpenStream(g_cameraPort, pBuffer, dwBufSize, 1024 * 1024))
            {
                dRet = PlayM4_GetLastError(g_cameraPort);
                break;
            }
            //设置解码回调函数
            if (!PlayM4_SetDecCallBack(g_cameraPort, DecCBFun))
            {
                dRet = PlayM4_GetLastError(g_cameraPort);
                break;
            }
            //打开视频解码
            if (!PlayM4_Play(g_cameraPort, NULL))
            {
                dRet = PlayM4_GetLastError(g_cameraPort);
                break;
            }
            dRet = PlayM4_GetLastError(g_cameraPort);
            break;

        case NET_DVR_STREAMDATA:       //视频流数据
        default:
            inData = PlayM4_InputData(g_cameraPort, pBuffer, dwBufSize);
            while (!inData)
            {
                Sleep(10);
                inData = PlayM4_InputData(g_cameraPort, pBuffer, dwBufSize);
                dRet = PlayM4_GetLastError(g_cameraPort);
                OutputDebugString(L"PlayM4_InputData failed \n");
            }
            break;
        }
    }

    //解码回调 视频为YUV数据(YV12)
    void CALLBACK DecCBFun(long port, char * pBuf, long nSize, FRAME_INFO * pFrameInfo, long nReserved1, long nReserved2)
    {
        UNREFERENCED_PARAMETER(nReserved1);
        UNREFERENCED_PARAMETER(nReserved2);
        UNREFERENCED_PARAMETER(nSize);
        UNREFERENCED_PARAMETER(port);

        //图像格式转换
        if (pFrameInfo->nType == T_YV12)
        {
            {
                lock_guard<mutex> locker(g_CameraMutex);

                Utils_ns::ImageUtils_ns::YV12ToBGR24_FFMPEG((unsigned char*)pBuf, (unsigned char*)g_curRGBImage->imageData,
                    pFrameInfo->nWidth, pFrameInfo->nHeight);//得到全部RGB图像
            }
        }
    }

　　

d.应用层获取视频帧，这里为了简化操作，只获取了当前帧；大家也可以使用线程安全队列来处理

    int HCNetCamera::getFrame(Mat& image)
    {
        lock_guard<mutex> locker(g_CameraMutex);
        if (g_curRGBImage && g_curRGBImage->imageData)
        {
            image = g_curRGBImage;
            return 0;
        }
        return -1;
    }

    //以下是线程函数的一部分，主要是取帧，然后进行人脸检测
    {
        lock_guard<std::mutex> locker(g_CameraMutex);
        int ret = m_camera->getFrame(curFrame);
        if (ret == -1)
        {
            continue;
        }
    }

    ftProcessor->faceDetect(curFrame);

　　

2.基本图像格式转换
       目前虹软SDK支持以下几种图像数据格式：

在实际开发过程中一般使用opencv，opencv默认的图像数据格式是BGR24，而我使用的海康摄像头视频编码格式是H264，视频帧数据格式是YV12，因此需要将YV12转换为BGR24 ，同时也会说明下怎么转换为虹软SDK支持的其它格式，主要参考了[2]，以下的代码仅供参考。
a.YV12 To BGR24

void yv12ToBGR24(unsigned char* yv12, unsigned  char* bgr24, int width, int height)
{
unsigned char* y_yv12 = yv12;
unsigned char* v_yv12 = yv12 + width*height;
unsigned char* u_yv12 = yv12 + width*height + width*height / 4;

unsigned char* b = bgr24;
unsigned char* g = bgr24 + 1;
unsigned char* r = bgr24 + 2;

int yIndex, uIndex, vIndex;
for (int i = 0; i < height; ++i)
{
    for (int j = 0; j < width; ++j)
    {
        yIndex = i * width + j;
        vIndex = (i / 2) * (width / 2) + (j / 2);
        uIndex = vIndex;

        *b = (unsigned char)(y_yv12[yIndex] + 1.732446 * (u_yv12[vIndex] - 128));
        *g = (unsigned char)(y_yv12[yIndex] - 0.698001 * (u_yv12[uIndex] - 128) - 0.703125 * (v_yv12[vIndex] - 128));
        *r = (unsigned char)(y_yv12[yIndex] + 1.370705 * (v_yv12[uIndex] - 128));

        b += 3;
        g += 3;
        r += 3;
    }
}
}

　　

b.YV12 To I420

void yv12ToI420(unsigned char yv12, unsigned char i420, int width, int height)
{

unsigned char* y_yv12 = yv12;
unsigned char* v_yv12 = yv12 + width*height;
unsigned char* u_yv12 = yv12 + width*height + width*height / 4;

unsigned char* y_i420 = i420;
unsigned char* u_i420 = i420 + width*height;
unsigned char* v_i420 = i420 + width*height + width*height / 4;

memcpy(i420, yv12, width*height);
memcpy(v_i420, v_yv12, width*height / 4);
memcpy(u_i420, u_yv12, width*height / 4);
}

　　

c.YV12 To NV21

void yv12ToNV21(unsigned char yv12, unsigned char nv21, int width, int height)
{

unsigned char* y_yv12 = yv12;
unsigned char* v_yv12 = yv12 + width*height;
unsigned char* u_yv12 = yv12 + width*height + width*height / 4;

unsigned char* y_nv21 = nv21;
unsigned char* v_nv21 = nv21 + width*height;
unsigned char* u_nv21 = nv21 + width*height + 1;

memcpy(nv21, yv12, width*height);

for (int i = 0; i < width*height / 4; ++i)
{
    *v_nv21 = *v_yv12;
    *u_nv21 = *u_yv12;

    v_nv21 += 2;
    u_nv21 += 2;

    ++v_yv12;
    ++u_yv12;
}
}

　　

d.YV12 To NV12

void yv12ToNV12(unsigned char yv12, unsigned char nv12, int width, int height)
{

unsigned char* y_yv12 = yv12;
unsigned char* v_yv12 = yv12 + width*height;
unsigned char* u_yv12 = yv12 + width*height + width*height / 4;

unsigned char* y_nv12 = nv12;
unsigned char* u_nv12 = nv12 + width*height;
unsigned char* v_nv12 = nv12 + width*height + 1;

memcpy(nv12, yv12, width*height);

for (int i = 0; i < width*height / 4; ++i)
{
    *v_nv12 = *v_yv12;
    *u_nv12 = *u_yv12;

    v_nv12 += 2;
    u_nv12 += 2;

    ++v_yv12;
    ++u_yv12;
}
}

　　

e.YV12 To YUYV

void yv12ToYUYV(unsigned char yv12, unsigned char yuyv, int width, int height)
{

unsigned char* y_yv12 = yv12;
unsigned char* v_yv12 = yv12 + width*height;
unsigned char* u_yv12 = yv12 + width*height + width*height / 4;

unsigned char* y_yuyv = yuyv;
unsigned char* u_yuyv = yuyv + 1;
unsigned char* v_yuyv = yuyv + 3;

for (int i = 0; i < width; ++i)
{
    for (int j = 0; j < height; ++j)
    {
        *y_yuyv = *y_yv12;

        y_yuyv += 2;
        ++y_yv12;
    }
}

for (int j = 0; j < height / 2; ++j)
{
    for (int i = 0; i < width / 2; ++i)
    {
        *u_yuyv = *u_yv12;
        *(u_yuyv + width * 2) = *u_yv12;
        u_yuyv += 4;
        ++u_yv12;

        *v_yuyv = *v_yv12;
        *(v_yuyv + width * 2) = *v_yv12;
        v_yuyv += 4;
        ++v_yv12;
    }
    u_yuyv += width * 2;
    v_yuyv += width * 2;
}
}

　

虹软免费的sdk下载： https://ai.arcsoft.com.cn/third/mobile.html?cnblogs

　

参考：
[1] 虹软AI 人脸识别SDK接入 — 性能优化篇（多线程）
[2] 图像实战 - 图像格式转换
[3] 微软官方解析图像格式
[4]一文读懂 YUV 的采样与格式