OPENGL + FFMPEG 实现全景播放器

OPENGL + FFMPEG 实现全景播放器

  • 加载并编译着色器，用于处理顶点和片段着色。
  • 创建一个球体模型，用于映射视频帧。
  • 实现渲染帧的功能，将解码后的帧数据传递给GPU并绘制在球体上

Video 视频处理 编解码

闪屏问题

每次渲染新帧时，屏幕可能会出现闪屏现象，这可能是渲染的帧没有正确地显示或者上一帧的内容在新帧显示之前被清空。启用了VSync（垂直同步）。VSync会使得帧刷新率与显示器的刷新率同步，避免在图像未完全刷新时就进行绘制，从而减少闪屏和撕裂的情况。

双缓存问题

启用双缓冲机制。这会使得渲染完成的一帧不会立即显示，而是先保存在一个缓冲区中，直到当前帧全部渲染完成后再交换缓冲区，避免屏幕上出现不完整的图像。

1
glfwWindowHint(GLFW_DOUBLEBUFFER, GL_TRUE);

多线程处理，其中一个线程专门负责视频解码和帧数据准备，而主线程只负责渲染。这种方法有效地避免了主线程因为解码而阻塞导致的渲染延迟，从而消除了闪屏问题。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
// 解码线程，负责解码视频帧
void decodeThread() {
    while (keepRunning) {
        if (videoDecoder.decodeFrame()) {
            // 将解码后的帧放入渲染队列
            std::unique_lock<std::mutex> lock(mutex);
            frameQueue.push(videoDecoder.getFrameData());
            lock.unlock();
        }
    }
}

// 渲染线程，负责渲染已经解码的帧
void renderThread() {
    while (keepRunning) {
        if (!frameQueue.empty()) {
            std::unique_lock<std::mutex> lock(mutex);
            uint8_t* frameData = frameQueue.front();
            frameQueue.pop();
            lock.unlock();

            // 渲染帧
            renderer.renderFrame(frameData, width, height);
        }
    }
}

清除颜色和深度缓存

1
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

避免深度遮挡

1
2
3
4
5
// 禁用深度写入
glDepthMask(GL_FALSE);
glBindVertexArray(VAO);
glDrawElements(GL_TRIANGLES, 6 * (50 - 1) * (50 - 1), GL_UNSIGNED_INT, 0);
glDepthMask(GL_TRUE);

帧同步处理

1
2
3
4
5
6
7
8
9
// 计算每帧的播放时间
double frameDelay = 1.0 / getFrameRate();

// 根据当前时间判断是否需要渲染下一帧
double currentTime = glfwGetTime();
if (currentTime - lastFrameTime >= frameDelay) {
    lastFrameTime = currentTime;
    // 继续渲染新帧
}

渲染帧部分

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
void OpenGLRenderer::renderFrame(uint8_t* frameData, int frameWidth, int frameHeight) {
    // 将视频帧数据上传到纹理
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, frameWidth, frameHeight, 0, GL_RGB, GL_UNSIGNED_BYTE, frameData);

    // 清除颜色和深度缓冲
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    // 禁用深度写入
    glDepthMask(GL_FALSE);
    glBindVertexArray(VAO);

    // 绘制球体
    glDrawElements(GL_TRIANGLES, 6 * (50 - 1) * (50 - 1), GL_UNSIGNED_INT, 0);
    glDepthMask(GL_TRUE);

    // 交换缓冲区，显示当前帧
    glfwSwapBuffers(window);
}

球体创建与纹理绑定

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
// 球体创建和绑定
void OpenGLRenderer::createSphere(float radius, unsigned int rings, unsigned int sectors) {
    std::vector<float> vertices;  // 顶点
    std::vector<unsigned int> indices;  // 索引
    float const R = 1.0f / (static_cast<float>(rings) - 1);
    float const S = 1.0f / (static_cast<float)(sectors) - 1);
    int r, s;

    // 生成顶点数据
    vertices.resize(rings * sectors * 5);  
    std::vector<float>::iterator v = vertices.begin();
    for (r = 0; r < rings; r++) for (s = 0; s < sectors; s++) {
        float const y = sin(-M_PI_2 + M_PI * r * R);
        float const x = cos(2 * M_PI * s * S) * sin(M_PI * r * R);
        float const z = sin(2 * M_PI * s * S) * sin(M_PI * r * R);
		// 顶点
        *v++ = x * radius;  
        *v++ = y * radius;  
        *v++ = z * radius;  
		// 纹理
        *v++ = s * S;  
        *v++ = 1 - r * R;  
    }

    // 生成索引数据
    indices.resize(rings * sectors * 6);  // 每个面有两个三角形，每个三角形有3个顶点
    std::vector<unsigned int>::iterator i = indices.begin();
    for (r = 0; r < rings - 1; r++) for (s = 0; s < sectors - 1; s++) {
        *i++ = r * sectors + s;
        *i++ = r * sectors + (s + 1);
        *i++ = (r + 1) * sectors + (s + 1);
        *i++ = (r + 1) * sectors + (s + 1);
        *i++ = (r + 1) * sectors + s;
        *i++ = r * sectors + s;
    }

    // 绑定
    glGenVertexArrays(1, &VAO);
    glGenBuffers(1, &VBO);
    glGenBuffers(1, &EBO);

    glBindVertexArray(VAO);

    // 上传顶点数据到缓存
    glBindBuffer(GL_ARRAY_BUFFER, VBO);
    glBufferData(GL_ARRAY_BUFFER, vertices.size() * sizeof(float), &vertices[0], GL_STATIC_DRAW);
    // 上传索引数据
    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
    glBufferData(GL_ELEMENT_ARRAY_BUFFER, indices.size() * sizeof(unsigned int), &indices[0], GL_STATIC_DRAW);

    // 设置顶点属性指针，描述顶点数据的结构
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void*)0);  // 顶点位置
    glEnableVertexAttribArray(0);

    glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(float), (void*)(3 * sizeof(float)));  // 纹理坐标
    glEnableVertexAttribArray(1);

    // 生成并绑定纹理对象
    glGenTextures(1, &textureID);
    glBindTexture(GL_TEXTURE_2D, textureID);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
}

VideoDecoder.h

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
// 视频解码器类，解码视频帧并转换为RGB格式
class VideoDecoder {
public:
    VideoDecoder(const std::string& filePath); 
    ~VideoDecoder();  

    bool initialize();  
    bool decodeFrame();  // 解码下一帧视频
    uint8_t* getFrameData();  // 获取当前帧的RGB数据
    int getWidth() const;  
    int getHeight() const;  
    double getFrameRate() const; 
    void printVideoInfo() const;  

private:
    std::string filePath;  
    AVFormatContext* pFormatCtx;  // 格式上下文，用于处理容器格式
    AVCodecContext* pCodecCtx;  // 编解码器上下文，用于处理编解码器相关信息
    AVFrame* pFrame;  // 存储解码后的原始帧
    AVFrame* pFrameRGB;  // 存储转换后的RGB帧
    struct SwsContext* swsCtx;  // 用于图像格式转换的上下文
    uint8_t* buffer;  // 存储RGB帧的缓冲区
    int videoStream; 
};

编译着色器 连接着色器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
// 编译着色器
GLuint OpenGLRenderer::compileShader(GLenum type, const std::string& source) {
    GLuint shader = glCreateShader(type);
    const char* src = source.c_str();
    glShaderSource(shader, 1, &src, nullptr);
    glCompileShader(shader);

    int success;
    glGetShaderiv(shader, GL_COMPILE_STATUS, &success);
    if (!success) {
        char infoLog[512];
        glGetShaderInfoLog(shader, 512, nullptr, infoLog);
        std::cerr << "Shader Compilation Error:\n" << infoLog << std::endl;
    }
    return shader;
}

// 链接着色器程序
GLuint OpenGLRenderer::linkProgram(GLuint vertexShader, GLuint fragmentShader) {
    GLuint program = glCreateProgram();
    glAttachShader(program, vertexShader);
    glAttachShader(program, fragmentShader);
    glLinkProgram(program);

    int success;
    glGetProgramiv(program, GL_LINK_STATUS, &success);
    if (!success) {
        char infoLog[512];
        glGetProgramInfoLog(program, 512, nullptr, infoLog);
        std::cerr << "Program Linking Error:\n" << infoLog << std::endl;
    }
    return program;
}

VideoDecoder.cpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#include "VideoDecoder.h"
#include <iostream>

// 构造函数：初始化成员变量，接收视频文件路径
VideoDecoder::VideoDecoder(const std::string& filePath)
    : filePath(filePath), pFormatCtx(nullptr), pCodecCtx(nullptr), pFrame(nullptr),
    pFrameRGB(nullptr), swsCtx(nullptr), buffer(nullptr), videoStream(-1) {}


VideoDecoder::~VideoDecoder() {
    av_free(buffer); 
    av_frame_free(&pFrameRGB);  
    av_frame_free(&pFrame);  
    avcodec_free_context(&pCodecCtx);  
    avformat_close_input(&pFormatCtx);  
    sws_freeContext(swsCtx); 
}

// 编解码器
bool VideoDecoder::initialize() {
    // 分配格式上下文
    pFormatCtx = avformat_alloc_context();
    // 打开视频文件
    if (avformat_open_input(&pFormatCtx, filePath.c_str(), nullptr, nullptr) != 0) {
        return false;
    }

    // 查找视频流
    if (avformat_find_stream_info(pFormatCtx, nullptr) < 0) {
        return false;
    }
    for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
        if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            videoStream = i;
            break;
        }
    }
    if (videoStream == -1) {
        return false;
    }

    // 获取视频流编解码参数
    AVCodecParameters* pCodecPar = pFormatCtx->streams[videoStream]->codecpar;
    // 查找解码器
    const AVCodec* pCodec = avcodec_find_decoder(pCodecPar->codec_id);
    if (pCodec == nullptr) {
        return false;
    }

    // 分配编解码器上下文
    pCodecCtx = avcodec_alloc_context3(pCodec);
    if (!pCodecCtx) {
        std::cerr << "error1" << std::endl;
        return false;
    }

    // 将编解码参数拷贝到上下文
    if (avcodec_parameters_to_context(pCodecCtx, pCodecPar) < 0) {
        std::cerr << "error2" << std::endl;
        return false;
    }

    // 打开编解码器
    if (avcodec_open2(pCodecCtx, pCodec, nullptr) < 0) {
        std::cerr << "error3" << std::endl;
        return false;
    }

    // 分配帧
    pFrame = av_frame_alloc();
    pFrameRGB = av_frame_alloc();
    if (pFrameRGB == nullptr || pFrame == nullptr) {
        std::cerr << "分配帧失败" << std::endl;
        return false;
    }

    // 分配缓冲区
    int numBytes = av_image_get_buffer_size(AV_PIX_FMT_RGB24, pCodecCtx->width, pCodecCtx->height, 32);
    buffer = (uint8_t*)av_malloc(numBytes * sizeof(uint8_t));
    av_image_fill_arrays(pFrameRGB->data, pFrameRGB->linesize, buffer, AV_PIX_FMT_RGB24, pCodecCtx->width, pCodecCtx->height, 1);

    // 设置转换为RGB格式
    swsCtx = sws_getContext(
        pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt,
        pCodecCtx->width, pCodecCtx->height, AV_PIX_FMT_RGB24,
        SWS_BILINEAR, nullptr, nullptr, nullptr
    );

    return true;
}

// 解码下一帧视频
bool VideoDecoder::decodeFrame() {
    AVPacket packet;
    // 读取视频包
    if (av_read_frame(pFormatCtx, &packet) >= 0) {
        if (packet.stream_index == videoStream) {
            int ret = avcodec_send_packet(pCodecCtx, &packet);
            if (ret >= 0) {
                ret = avcodec_receive_frame(pCodecCtx, pFrame);
                av_packet_unref(&packet);
                if (ret >= 0) {
                    // 将解码后的原始帧转换为RGB格式
                    sws_scale(
                        swsCtx,
                        (uint8_t const* const*)pFrame->data,
                        pFrame->linesize,
                        0,
                        pCodecCtx->height,
                        pFrameRGB->data,
                        pFrameRGB->linesize
                    );
                    return true;
                }
            }
        }
        av_packet_unref(&packet);
    }
    return false;
}

void VideoDecoder::printVideoInfo() const {
    std::cout << "Video Information:" << std::endl;
    std::cout << "Format: " << pFormatCtx->iformat->long_name << std::endl;
    std::cout << "Duration: " << pFormatCtx->duration / AV_TIME_BASE << " seconds" << std::endl;
    std::cout << "Width: " << pCodecCtx->width << std::endl;
    std::cout << "Height: " << pCodecCtx->height << std::endl;
    std::cout << "Frame Rate: " << av_q2d(pFormatCtx->streams[videoStream]->avg_frame_rate) << " fps" << std::endl;
    std::cout << "Codec: " << pCodecCtx->codec->long_name << std::endl;
}

// 当前帧的RGB数据
uint8_t* VideoDecoder::getFrameData() {
    return pFrameRGB->data[0];
}

int VideoDecoder::getWidth() const {
    return pCodecCtx->width;
}

int VideoDecoder::getHeight() const {
    return pCodecCtx->height;
}

// 帧率
double VideoDecoder::getFrameRate() const {
    return av_q2d(pFormatCtx->streams[videoStream]->avg_frame_rate);
}

Input 处理鼠标和窗口事件

Input.h

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
// 输入管理器类，负责处理输入相关的逻辑，包括鼠标和键盘输入。
class InputManager {
public:
    InputManager(); 
    void setWindow(GLFWwindow* window); 
    void updateCamera(float& yaw, float& pitch, glm::vec3& cameraFront);  // 更新摄像机的朝向
    void processInput(GLFWwindow* window, glm::vec3& cameraPos, const glm::vec3& cameraFront, const glm::vec3& cameraUp, float deltaTime);  // 处理键盘输入，更新摄像机位置

private:
    GLFWwindow* window;  
    double lastX, lastY;  
    bool dragging;  
    float sensitivity;  // 鼠标灵敏度
    float speed;  // 摄像机移动速度

    static void mouseCallback(GLFWwindow* window, double xpos, double ypos);  // 静态鼠标移动回调函数
    static void mouseButtonCallback(GLFWwindow* window, int button, int action, int mods);  // 静态鼠标按钮回调函数
};

Input.cpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// 构造
InputManager::InputManager()
    : window(nullptr), lastX(640), lastY(360), dragging(false), sensitivity(0.2f), speed(2.5f) {}

void InputManager::setWindow(GLFWwindow* window) {
    this->window = window;
    glfwSetCursorPosCallback(window, mouseCallback);
    glfwSetMouseButtonCallback(window, mouseButtonCallback);
}

// 更新摄像机的视角
void InputManager::updateCamera(float& yaw, float& pitch, glm::vec3& cameraFront) {
    if (dragging) { 
        double xpos, ypos;
        glfwGetCursorPos(window, &xpos, &ypos);

        // 鼠标偏移量
        float xoffset = static_cast<float>(xpos - lastX);
        float yoffset = static_cast<float>(lastY - ypos);

        lastX = xpos;
        lastY = ypos;

        xoffset *= sensitivity;
        yoffset *= sensitivity;

        yaw += xoffset;
        pitch += yoffset;

        if (pitch > 89.0f)
            pitch = 89.0f;
        if (pitch < -89.0f)
            pitch = -89.0f;

        // 重新根据yaw、pitch算摄像机的前向向量
        glm::vec3 front;
        front.x = cos(glm::radians(yaw)) * cos(glm::radians(pitch));
        front.y = sin(glm::radians(pitch));
        front.z = sin(glm::radians(yaw)) * cos(glm::radians(pitch));
        cameraFront = glm::normalize(front);
    }
}

// 摄像机移动
void InputManager::processInput(GLFWwindow* window, glm::vec3& cameraPos, const glm::vec3& cameraFront, const glm::vec3& cameraUp, float deltaTime) {
    float velocity = speed * deltaTime;  
    if (glfwGetKey(window, GLFW_KEY_W) == GLFW_PRESS)
        cameraPos += cameraFront * velocity;  
    if (glfwGetKey(window, GLFW_KEY_S) == GLFW_PRESS)
        cameraPos -= cameraFront * velocity;  
    if (glfwGetKey(window, GLFW_KEY_A) == GLFW_PRESS)
        cameraPos -= glm::normalize(glm::cross(cameraFront, cameraUp)) * velocity;   A
    if (glfwGetKey(window, GLFW_KEY_D) == GLFW_PRESS)
        cameraPos += glm::normalize(glm::cross(cameraFront, cameraUp)) * velocity;
}

// 更新鼠标位置
void InputManager::mouseCallback(GLFWwindow* window, double xpos, double ypos) {
    InputManager* inputManager = static_cast<InputManager*>(glfwGetWindowUserPointer(window));
    if (inputManager && inputManager->dragging) {  
        inputManager->lastX = xpos;
        inputManager->lastY = ypos;
    }
}

// 鼠标按钮回调
void InputManager::mouseButtonCallback(GLFWwindow* window, int button, int action, int mods) {
    InputManager* inputManager = static_cast<InputManager*>(glfwGetWindowUserPointer(window));
    if (inputManager) {  
        if (button == GLFW_MOUSE_BUTTON_LEFT) {  // 左键触发
            if (action == GLFW_PRESS) {
                inputManager->dragging = true;  
                glfwGetCursorPos(window, &inputManager->lastX, &inputManager->lastY); 
            }
            else if (action == GLFW_RELEASE) {
                inputManager->dragging = false;  
            }
        }
    }
}