使用ffmpeg和sdl2进行卡顿渲染

Question

使用下面的代码，我得到了影片文件的卡顿渲染效果。有趣的是，当用ffmpeg转储信息时，它说它有25fps和00:01:32.90的持续时间；但是当我自己计算帧和时间时，它给出了大约252秒的时间，我猜接收帧和发送包的代码(int cap(vid v))会多次绘制相同的帧。但我看不出出了什么问题？

//PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/:/usr/lib64/pkgconfig/   --> add path to PKF_Config search path
//export PKG_CONFIG_PATH --> export PKG_Config search path to become visible for gcc
//gcc ffmpeg_capture_fl.c -Wall -pedantic -fPIC `pkg-config --cflags --libs libavdevice libavformat libavcodec libavutil libavdevice libavfilter libswscale libswresample sdl2`


#include <libavdevice/avdevice.h>
#include <libavutil/opt.h>
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
#include <stdio.h>
#include <stdlib.h>
#include <libavutil/rational.h>
#include <sys/time.h>
#include <unistd.h>


typedef struct vid{
AVFormatContext *inc;
AVInputFormat *iformat;
AVCodecContext  *pCodecCtx;
AVCodec         *pCodec;
AVFrame         *pFrame;
int videoStream;} vid;

typedef struct sws{
struct SwsContext *ctx;
uint8_t **buffer;
int *linesize;
} sws;


vid cap_init_fl(char *fl);
int cap(vid v);
void cap_close(vid v);

sws init_swsctx(int width, int height, enum AVPixelFormat pix_fmt, int new_width, int new_height, enum AVPixelFormat new_pxf);
void conv_pxlformat(sws scale, uint8_t **src_data, int *src_linesize, int height);
void free_sws(sws inp);

#include <SDL2/SDL.h>

typedef struct sdl_window{
    SDL_Window *window;
    SDL_Renderer *renderer;
    SDL_Texture *texture;
    SDL_Event *event;
    int width;
    int height;
    int pitch;
    uint32_t sdl_pxl_frmt;
    }sdl_window;

sdl_window init_windowBGR24_ffmpg(int width, int height);
int render_on_texture_update(sdl_window wow, uint8_t *data);
void close_window(sdl_window wow);


vid cap_init_fl(char *fl){
    vid v = {NULL, NULL, NULL, NULL, NULL, -1};
    int i;
    
    av_register_all();
    avdevice_register_all();

    if( 0 > avformat_open_input( &(v.inc), fl , v.iformat, NULL)) {
        printf("Input device could not been opened\n");
        cap_close(v);
        exit(1);
        }

    if(avformat_find_stream_info(v.inc, NULL)<0){
        printf("Stream information could not been found.\n");
        cap_close(v);
        exit(2);
    }

    // Dump information about file onto standard error
    av_dump_format(v.inc, 0, fl, 0);

    // Find the first video stream
    v.videoStream=-1;
    for(i=0; i<v.inc->nb_streams; i++){
      if(v.inc->streams[i]->codecpar->codec_type==AVMEDIA_TYPE_VIDEO) {
        v.videoStream=i;
        break;
      }}

    if(v.videoStream==-1){
        printf("Could not find video stream.\n");
        cap_close(v);
        exit(3);
        }

    // Find the decoder for the video stream
    v.pCodec=avcodec_find_decoder(v.inc->streams[v.videoStream]->codecpar->codec_id);
        if(v.pCodec==NULL) {
          printf("Unsupported codec!\n");
        cap_close(v);
        exit(4);
           // Codec not found
        }

    

    // Get a pointer to the codec context for the video stream
    
    if((v.pCodecCtx=avcodec_alloc_context3(NULL)) == NULL){
        printf("Could not allocate codec context\n");
        cap_close(v);
        exit(10);}

    avcodec_parameters_to_context (v.pCodecCtx, v.inc->streams[v.videoStream]->codecpar);       
    
    // Open codec
    if(avcodec_open2(v.pCodecCtx, v.pCodec, NULL)<0){
        printf("Could not open codec");
        cap_close(v);
        exit(5);
        }
        
    
    // Allocate video frame
    v.pFrame=av_frame_alloc();
    if(v.pFrame==NULL){
        printf("Could not allocate AVframe");
        cap_close(v);
        exit(6);}

    
    return v;
}



int cap(vid v){
    int errorCodeRF, errorCodeSP, errorCodeRecFR;
    AVPacket pkt;
        
    if((errorCodeRF = av_read_frame(v.inc, &pkt))  >= 0){
        
        if (pkt.stream_index == v.videoStream) {
            
            errorCodeSP = avcodec_send_packet(v.pCodecCtx, &pkt);
            
            if (errorCodeSP >= 0 || errorCodeSP == AVERROR(EAGAIN)){
                
                errorCodeRecFR = avcodec_receive_frame(v.pCodecCtx, v.pFrame);
                
                if (errorCodeRecFR < 0){ 
                    av_packet_unref(&pkt);
                    return errorCodeRecFR;
                    }
                else{
                    av_packet_unref(&pkt);
                    return 0;
                    }
                
                }
            else{
                 av_packet_unref(&pkt);
                 return errorCodeSP;}
                
        }}
                
    else{
        return errorCodeRF;}
        return 1;
        }
    
    

void cap_close(vid v){
    if(v.pFrame != NULL) av_free(v.pFrame);
    avcodec_close(v.pCodecCtx);
    avformat_close_input(&(v.inc));
    if(v.inc != NULL) avformat_free_context(v.inc);
    
    v.inc = NULL;
    v.iformat = NULL;
    v.pCodecCtx = NULL;
    v.pCodec = NULL;
    v.pFrame = NULL;
    v.videoStream=-1;}



sws init_swsctx(int width, int height, enum AVPixelFormat pix_fmt, int new_width, int new_height, enum AVPixelFormat new_pxf){

int nwidth, nheight;
sws scale;

scale.buffer = (uint8_t **) malloc(4 * sizeof(uint8_t *));
scale.linesize = (int *) malloc(4 * sizeof(int));


nwidth = (new_width <= 0) ?  width : new_width;
nheight = (new_height <= 0) ?  height : new_height;

av_image_alloc(scale.buffer, scale.linesize, nwidth, nheight, new_pxf, 1);
scale.ctx = sws_getContext(width, height, pix_fmt, nwidth, nheight, new_pxf, SWS_BILINEAR, NULL, NULL, NULL);

if(scale.ctx==NULL){
    printf("Could not allocate SWS-Context\n");
    av_freep(&(scale.buffer)[0]);
    free(scale.buffer);
    free(scale.linesize);
    exit(12);
    }
            
return scale;}


void conv_pxlformat(sws scale, uint8_t **src_data, int *src_linesize, int height){  
    sws_scale(scale.ctx, (const uint8_t **) src_data, src_linesize, 0, height, scale.buffer, scale.linesize);
    }


void free_sws(sws inp){
    av_freep(&(inp.buffer)[0]);
    free(inp.buffer);
    free(inp.linesize); 
}


sdl_window init_windowBGR24_ffmpg(int width, int height){

 sdl_window wow;
    
 if (SDL_Init(SDL_INIT_VIDEO) < 0) {
    printf("Couldn't initialize SDL in function: create_sdl_window(...)\n");
    exit(7);}
    
 wow.window = SDL_CreateWindow("SDL_CreateTexture",
                        SDL_WINDOWPOS_UNDEFINED,
                        SDL_WINDOWPOS_UNDEFINED,
                        width, height,
                        SDL_WINDOW_RESIZABLE);

 wow.renderer = SDL_CreateRenderer(wow.window, -1, SDL_RENDERER_ACCELERATED);
 wow.texture = SDL_CreateTexture(wow.renderer, SDL_PIXELFORMAT_BGR24, SDL_TEXTUREACCESS_STREAMING, width, height);
 wow.width = width;
 wow.height = height;
 wow.pitch = width * 3;  //only true for 3 byte / 24bit packed formats like bgr24
 wow.sdl_pxl_frmt = SDL_PIXELFORMAT_BGR24;
 SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); 
 SDL_SetHint(SDL_HINT_RENDER_VSYNC, "enable"); 
 SDL_RenderSetLogicalSize(wow.renderer, wow.width, wow.height);
 return wow;
}


int render_on_texture_update(sdl_window wow, uint8_t *data){
    
    if (SDL_UpdateTexture(wow.texture, NULL, data, wow.pitch)< 0){
        printf("SDL_render_on_texture_update_failed: %s\n", SDL_GetError());
        return -1;
        }
    SDL_RenderClear(wow.renderer);
    SDL_RenderCopy(wow.renderer, wow.texture, NULL, NULL);
    SDL_RenderPresent(wow.renderer);

    return 0; 
}


void close_window(sdl_window wow){
    SDL_DestroyRenderer(wow.renderer);
    SDL_Quit();
}




int main(){
    int n, vid_error;
    long int time_per_frame_usec, duration_usec;
    vid v;
    sdl_window wow;
    struct timeval tval_start, tval_start1, tval_end, tval_duration;
    sws scale;
    SDL_Event event;
   
    vid_error = AVERROR(EAGAIN);
    v = cap_init_fl("mw_maze_test.mp4"); 
    
    while(vid_error == AVERROR(EAGAIN)){
        vid_error =cap(v);}
        
    if(vid_error < 0){
        printf("Could not read from Capture\n");
        cap_close(v);
        return 0;
        }

    wow = init_windowBGR24_ffmpg((v.pCodecCtx)->width, (v.pCodecCtx)->height);
    scale = init_swsctx((v.pCodecCtx)->width, (v.pCodecCtx)->height, (v.pCodecCtx)->pix_fmt, 0, 0, AV_PIX_FMT_BGR24);
        
    time_per_frame_usec =  ((long int)((v.inc)->streams[v.videoStream]->avg_frame_rate.den) *  1000000 / (long int) ((v.inc)->streams[v.videoStream]->avg_frame_rate.num));
    
    printf("Time per frame: %ld\n", time_per_frame_usec);
    n = 0;
    
    gettimeofday(&tval_start, NULL);
    gettimeofday(&tval_start1, NULL);
    
    while ((vid_error =cap(v)) >= 0) {
                
                if (SDL_PollEvent(&event) != 0) {
                        if (event.type == SDL_QUIT)
                        break;}
                    
                conv_pxlformat(scale, (v.pFrame)->data, (v.pFrame)->linesize, (v.pCodecCtx)->height);
                gettimeofday(&tval_end, NULL);
                timersub(&tval_end, &tval_start, &tval_duration);
                duration_usec = (long int)tval_duration.tv_sec * 1000000 + (long int)tval_duration.tv_usec;
                
                while(duration_usec < time_per_frame_usec) {
                gettimeofday(&tval_end, NULL);
                timersub(&tval_end, &tval_start, &tval_duration);
                duration_usec = (long int)tval_duration.tv_sec * 1000000 + (long int)tval_duration.tv_usec;
                    }
                
                gettimeofday(&tval_start, NULL);
                render_on_texture_update(wow, *(scale.buffer)); 
                n++;
        }
  
    gettimeofday(&tval_end, NULL);
    timersub(&tval_end, &tval_start1, &tval_duration);
    duration_usec = (long int)tval_duration.tv_sec * 1000000 + (long int)tval_duration.tv_usec;
    
    printf("Total time and frames; %ld %i\n", duration_usec, n);
    
    if(vid_error == AVERROR(EAGAIN)){
                    printf("AVERROR(EAGAIN) occured\n)");
                    }   
                
    
    sws_freeContext(scale.ctx);
    free_sws(scale);
    close_window(wow);
    cap_close(v); 
    return 0;   
}

the output is:

Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'mw_maze_test.mp4':
  Metadata:
    major_brand     : mp42
    minor_version   : 0
    compatible_brands: isommp42
    creation_time   : 2014-02-14T21:09:52.000000Z
  Duration: 00:01:32.90, start: 0.000000, bitrate: 347 kb/s
    Stream #0:0(und): Video: h264 (Constrained Baseline) (avc1 / 0x31637661), yuv420p, 384x288 [SAR 1:1 DAR 4:3], 249 kb/s, 25 fps, 25 tbr, 25 tbn, 50 tbc (default)
    Metadata:
      handler_name    : VideoHandler
    Stream #0:1(und): Audio: aac (LC) (mp4a / 0x6134706D), 44100 Hz, stereo, fltp, 96 kb/s (default)
    Metadata:
      creation_time   : 2014-02-14T21:09:53.000000Z
      handler_name    : IsoMedia File Produced by Google, 5-11-2011
Time per frame: 40000
Total time and frames; 252881576 6322

Answer 1

发帖一分钟后我就明白了。问题是，当帧不是视频帧(因此是音频帧)时，(int cap(vid v))返回1。main中的while循环在每次从cap >= 0返回时运行，因此也为音频帧运行，因此需要将main循环更改为仅在从视频帧返回时运行：

while ((vid_error =cap(v)) >= 0) {
                if(vid_error == 0){
                // code as before
        }}...