多路视频播放优化之——播放渲染优化

渲染方式

         目前视频播放渲染总体上分为CPU渲染和GPU渲染。为了使视频监控客户端可以播放多路视频，优先使用GPU渲染播放界面更好。GPU渲染在这里使用OpenGL库。

Qt下使用OpenGL

Qt需要实现的步骤

1.实现一个继承自QOpenGLWidget的窗口类；

2.实现关键的接口

virtual void initializeGL();//初始化OpenGL相关资源

virtual void resizeGL(int w, int h);//窗口大小变化更新

virtual void paintGL();//绘制每一帧数据

OpenGL需要实现的步骤

1.初始化顶点数组对象和纹理数组对象；

        glVertexAttribPointer(VER_ATTR_VER, 2, GL_FLOAT, GL_FALSE, 0, vertices);
        glEnableVertexAttribArray(VER_ATTR_VER);

        glVertexAttribPointer(VER_ATTR_TEX, 2, GL_FLOAT, GL_FALSE, 0, textures);
        glEnableVertexAttribArray(VER_ATTR_TEX)

2.创建顶点着色器和片元着色器对象；

        GLuint vs = glCreateShader(GL_VERTEX_SHADER);
        GLuint fs = glCreateShader(GL_FRAGMENT_SHADER);

3.使用GLSL语言，实现顶点着色器和片元着色器；

        顶点着色器代码如下：

char szVS[] = "				\
    attribute vec4 verIn;		\
    attribute vec2 texIn;		\
    varying vec2 texOut;		\
                                \
    void main(){				\
        gl_Position = verIn;	\
        texOut = texIn;			\
    }							\
    ";
  

        片元着色器代码如下:

char szFS[] = "				\
    varying vec2 texOut;		\
    uniform sampler2D tex_y;	\
    uniform sampler2D tex_u;	\
    uniform sampler2D tex_v;	\
                                \
    void main(){				\
        vec3 yuv;				\
        vec3 rgb;				\
        yuv.x = texture2D(tex_y, texOut).r;			\
        yuv.y = texture2D(tex_u, texOut).r - 0.5;	\
        yuv.z = texture2D(tex_v, texOut).r - 0.5;	\
        rgb = mat3( 1,       1,         1,			\
            0,       -0.39465,  2.03211,			\
            1.13983, -0.58060,  0) * yuv;			\
        gl_FragColor = vec4(rgb, 1);				\
    }												\
    ";

4.将顶点着色器和片元着色器分别绑定顶点着色器对象和片元着色器对象；

    顶点着色器绑定：

    const GLchar* pszVS = szVS;
    GLint len = strlen(szVS);
    glShaderSource(vs, 1, (const GLchar**)&pszVS, &len);

   片元着色器绑定:

    const GLchar* pszFS = szFS;
    len = strlen(szFS);
    glShaderSource(fs, 1, (const GLchar**)&pszFS, &len);

5.编译顶点着色器和片元着色器；

    glCompileShader(vs);
    glCompileShader(fs);

6.创建着色器程序对象；

        prog_yuv = glCreateProgram();

7.将顶点着色器和片元着色器附加到着色器程序对象上；

    glAttachShader(prog_yuv, vs);
    glAttachShader(prog_yuv, fs);

 

8.绑定着色器程序中的全局变量；

    glBindAttribLocation(prog_yuv, VER_ATTR_VER, "verIn");
    glBindAttribLocation(prog_yuv, VER_ATTR_TEX, "texIn");

9.将附加的顶点着色器和片元着色器链接成可执行程序；

    glLinkProgram(prog_yuv);

10.判断可执行程序是否有效；

   glValidateProgram(prog_yuv);

11.获取着色器中的Y，U ，V 分量；

    texUniformY = glGetUniformLocation(prog_yuv, "tex_y");
    texUniformU = glGetUniformLocation(prog_yuv, "tex_u");
    texUniformV = glGetUniformLocation(prog_yuv, "tex_v");

12.初始化YUV纹理；

    glGenTextures(3, tex_yuv);
    for (int i = 0; i < 3; ++i) {
        glBindTexture(GL_TEXTURE_2D, tex_yuv[i]);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    }

        

13.从队列中获取一帧一帧数据，执行界面渲染。

实现的关键代码

初始化OpenGL资源

void GLWidget::initializeGL() {
    if (!s_glew_init.test_and_set()) {
        if (glewInit() != 0) {
            s_glew_init.clear();
            qFatal("glewInit failed");
            return;
        }
    }
    initVAO();
    loadYUVShader();
    initYUV();
}

窗口变化

void GLWidget::resizeGL(int w, int h) {
    glViewport(0,0,w,h);
    setAspectRatio(aspect_ratio);
}

绘制实现

void GLWnd::paintGL() {
    calcFPS();
    GLWidget::paintGL();

    if (last_frame.isNull()) {

    }
    else {
        drawFrame(&last_frame);
        if (draw_time) {
            drawTime();
        }
        if (draw_fps) {
            drawFPS();
        }
        if (draw_resolution) {
            drawResolution();
        }
    }
}

绘制YUV并显示

void GLWidget::drawYUV(Frame* pFrame) {
    assert(pFrame->type == PIX_FMT_IYUV || pFrame->type == PIX_FMT_YV12);

    int w = pFrame->w;
    int h = pFrame->h;
    int y_size = w*h;
    GLubyte* y = (GLubyte*)pFrame->buf.base;
    GLubyte* u = y + y_size;
    GLubyte* v = u + (y_size>>2);
    if (pFrame->type == PIX_FMT_YV12) {
        GLubyte* tmp = u;
        u = v;
        v = tmp;
    }

    glUseProgram(prog_yuv);

    glActiveTexture(GL_TEXTURE0);
    glBindTexture(GL_TEXTURE_2D, tex_yuv[0]);
    glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, w, h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, y);
    glUniform1i(texUniformY, 0);

    glActiveTexture(GL_TEXTURE1);
    glBindTexture(GL_TEXTURE_2D, tex_yuv[1]);
    glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, w/2, h/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, u);
    glUniform1i(texUniformU, 1);

    glActiveTexture(GL_TEXTURE2);
    glBindTexture(GL_TEXTURE_2D, tex_yuv[2]);
    glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, w/2, h/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, v);
    glUniform1i(texUniformV, 2);

    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);

    glUseProgram(0);
}