ffplay(2.0.1)中的音视频同步

 1 static void refresh_loop_wait_event(VideoState *is, SDL_Event *event) {
 2     double remaining_time = 0.0;
 3      SDL_PumpEvents();/*不停的循环内部更新消息*/
 4     while (!SDL_PeepEvents(event, 1, SDL_GETEVENT, SDL_ALLEVENTS)) {/*check the event queue for messages*/
 5         if (!cursor_hidden && av_gettime() - cursor_last_shown > CURSOR_HIDE_DELAY) {
 6             SDL_ShowCursor(0);
 7             cursor_hidden = 1;
 8         }
 9         if (remaining_time > 0.0)
10             av_usleep((int64_t)(remaining_time * 1000000.0));/*使用这个函数来休眠,取代之前版本中的定时器*/
11         remaining_time = REFRESH_RATE;/*10ms*/
12         if (is->show_mode != SHOW_MODE_NONE && (!is->paused || is->force_refresh))
13             video_refresh(is, &remaining_time);
14         SDL_PumpEvents();
15     }
16 }

  1 /* called to display each frame */
  2 static void video_refresh(void *opaque, double *remaining_time)
  3 {
  4     VideoState *is = opaque;
  5     VideoPicture *vp;
  6     double time;
  7 
  8     SubPicture *sp, *sp2;
  9 
 10     if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)/*如果用外部时钟同步的话*/
 11         check_external_clock_speed(is);
 12 
 13     if (!display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
 14         time = av_gettime() / 1000000.0;
 15         if (is->force_refresh || is->last_vis_time + rdftspeed < time) {/*强制刷新视频*/
 16             video_display(is);
 17             is->last_vis_time = time;/*记录本次的时间*/
 18         }
 19         *remaining_time = FFMIN(*remaining_time, is->last_vis_time + rdftspeed - time);
 20     }
 21 
 22     if (is->video_st) {
 23         int redisplay = 0;
 24         if (is->force_refresh)
 25             redisplay = pictq_prev_picture(is);
 26 retry:
 27         if (is->pictq_size == 0) {/*如果缓冲区没有数据*/
 28             SDL_LockMutex(is->pictq_mutex);
 29             if (is->frame_last_dropped_pts != AV_NOPTS_VALUE && is->frame_last_dropped_pts > is->frame_last_pts) {
 30                 update_video_pts(is, is->frame_last_dropped_pts, is->frame_last_dropped_pos, is->frame_last_dropped_serial);
 31                 is->frame_last_dropped_pts = AV_NOPTS_VALUE;
 32             }
 33             SDL_UnlockMutex(is->pictq_mutex);
 34             // nothing to do, no picture to display in the queue
 35         } else {
 36             double last_duration, duration, delay;
 37             /* dequeue the picture */
 38             vp = &is->pictq[is->pictq_rindex];
 39 
 40             if (vp->serial != is->videoq.serial) {
 41                 pictq_next_picture(is);
 42                 redisplay = 0;
 43                 goto retry;
 44             }
 45 
 46             if (is->paused)
 47                 goto display;
 48 
 49             /* compute nominal last_duration *//*通过计算当前要显示的帧和上一帧pts的差来预测当期帧显示时间---预测--->下一帧的到来时间*/
 50             last_duration = vp->pts - is->frame_last_pts;/*计算上一帧的显示时间(名义上)*/
 51             if (!isnan(last_duration) && last_duration > 0 && last_duration < is->max_frame_duration) {/*判断上一帧显示的时间是否在范围内*/
 52                 /* if duration of the last frame was sane, update last_duration in video state */
 53                 is->frame_last_duration = last_duration;/*更新一帧的持续显示时间*/
 54             }
 55             if (redisplay)
 56                 delay = 0.0;
 57             else
 58                 delay = compute_target_delay(is->frame_last_duration, is);/*通过上一帧的情况来预测本次的情况,这样可以得到下一帧的到来时间*/
 59 
 60             time= av_gettime()/1000000.0;
 61             if (time < is->frame_timer + delay && !redisplay) {/**/
 62                 *remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
 63                 return;
 64             }
 65 
 66             is->frame_timer += delay;
 67             if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)
 68                 is->frame_timer = time;
 69 
 70             SDL_LockMutex(is->pictq_mutex);
 71             if (!redisplay && !isnan(vp->pts))
 72                 update_video_pts(is, vp->pts, vp->pos, vp->serial);/*更新当前帧pts和pos*/
 73             SDL_UnlockMutex(is->pictq_mutex);
 74 
 75             if (is->pictq_size > 1) {/*如果缓冲中帧数比较多的时候,例如下一帧也已经到了*/
 76                 VideoPicture *nextvp = &is->pictq[(is->pictq_rindex + 1) % VIDEO_PICTURE_QUEUE_SIZE];
 77                 duration = nextvp->pts - vp->pts;/*这个时候,应该用已经在缓存中的下一帧pts-当前pts来真实计算当前持续显示时间*/
 78                 if(!is->step && (redisplay || framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration){/*如果延迟时间超过一帧了,就采取丢掉当前帧*/
 79                     if (!redisplay)
 80                         is->frame_drops_late++;
 81                     pictq_next_picture(is);/*采取丢帧策略,丢弃迟来的帧,取下一帧*/
 82                     redisplay = 0;
 83                     goto retry;
 84                 }
 85             }
 86 
 87             if (is->subtitle_st) {
 88                     while (is->subpq_size > 0) {
 89                         sp = &is->subpq[is->subpq_rindex];
 90 
 91                         if (is->subpq_size > 1)
 92                             sp2 = &is->subpq[(is->subpq_rindex + 1) % SUBPICTURE_QUEUE_SIZE];
 93                         else
 94                             sp2 = NULL;
 95 
 96                         if (sp->serial != is->subtitleq.serial
 97                                 || (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))
 98                                 || (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000))))
 99                         {
100                             free_subpicture(sp);
101 
102                             /* update queue size and signal for next picture */
103                             if (++is->subpq_rindex == SUBPICTURE_QUEUE_SIZE)
104                                 is->subpq_rindex = 0;
105 
106                             SDL_LockMutex(is->subpq_mutex);
107                             is->subpq_size--;
108                             SDL_CondSignal(is->subpq_cond);
109                             SDL_UnlockMutex(is->subpq_mutex);
110                         } else {
111                             break;
112                         }
113                     }
114             }
115 
116 display:
117             /* display picture */
118             if (!display_disable && is->show_mode == SHOW_MODE_VIDEO)
119                 video_display(is);
120 
121             pictq_next_picture(is);
122 
123             if (is->step && !is->paused)
124                 stream_toggle_pause(is);
125         }
126     }
127     is->force_refresh = 0;
128     if (show_status) {/*显示状态*/
129         static int64_t last_time;
130         int64_t cur_time;
131         int aqsize, vqsize, sqsize;
132         double av_diff;
133 
134         cur_time = av_gettime();
135         if (!last_time || (cur_time - last_time) >= 30000) {
136             aqsize = 0;
137             vqsize = 0;
138             sqsize = 0;
139             if (is->audio_st)
140                 aqsize = is->audioq.size;
141             if (is->video_st)
142                 vqsize = is->videoq.size;
143             if (is->subtitle_st)
144                 sqsize = is->subtitleq.size;
145             av_diff = 0;
146             if (is->audio_st && is->video_st)
147                 av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);
148             else if (is->video_st)
149                 av_diff = get_master_clock(is) - get_clock(&is->vidclk);
150             else if (is->audio_st)
151                 av_diff = get_master_clock(is) - get_clock(&is->audclk);
152             av_log(NULL, AV_LOG_INFO,
153                    "%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64"   \r",
154                    get_master_clock(is),
155                    (is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : "   ")),
156                    av_diff,
157                    is->frame_drops_early + is->frame_drops_late,
158                    aqsize / 1024,
159                    vqsize / 1024,
160                    sqsize,
161                    is->video_st ? is->video_st->codec->pts_correction_num_faulty_dts : 0,
162                    is->video_st ? is->video_st->codec->pts_correction_num_faulty_pts : 0);
163             fflush(stdout);
164             last_time = cur_time;
165         }
166     }
167 }

 1 static double compute_target_delay(double delay, VideoState *is)
 2 {
 3     double sync_threshold, diff;
 4 
 5     /* update delay to follow master synchronisation source */
 6     if (get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER) {
 7         /* if video is slave, we try to correct big delays by
 8            duplicating or deleting a frame *//*我们通过复制和删除一帧来纠正大的延时*/
 9         diff = get_clock(&is->vidclk) - get_master_clock(is);
10 
11         /* skip or repeat frame. We take into account the
12            delay to compute the threshold. I still don‘t know
13            if it is the best guess */
14         sync_threshold = FFMAX(AV_SYNC_THRESHOLD_MIN, FFMIN(AV_SYNC_THRESHOLD_MAX, delay));
15         if (!isnan(diff) && fabs(diff) < is->max_frame_duration) {
16             if (diff <= -sync_threshold)/*当前视频帧落后于主时钟源*/
17             {
18                 delay = FFMAX(0, delay + diff);
19             }
20             else if (diff >= sync_threshold && delay > AV_SYNC_FRAMEDUP_THRESHOLD)/*视频帧超前,但If a frame duration is longer than this, it will not be duplicated to compensate AV sync*/
21             {    /*大概意思是:本来当视频帧超前的时候,
22                 我们应该要选择重复该帧或者下面的2倍延时(即加重延时的策略),
23                 但因为该帧的显示时间大于显示更新门槛,
24                 所以这个时候不应该以该帧做同步*/
25                 delay = delay + diff;
26             }
27             else if (diff >= sync_threshold)
28             {
29                 delay = 2 * delay;/*采取加倍延时*/
30             }
31         }
32     }
33 
34     av_dlog(NULL, "video: delay=%0.3f A-V=%f\n",
35             delay, -diff);
36 
37     return delay;
38 }