1.time-base转换，整体框架
先看关于time-base变换的整体框架。
av_read_frame就是读取packet的过程，这时使用的time_base是AVStream->time_base，经过解码器后，这时使用的time_base还是AVStream->time_base，根据是否要过滤器需求，如果有这个需求，使用av_buffersrc_add_frame插入帧，经过过滤器处理完后，就会经过buffersink和abuffersink部分，使用av_buffer_get_frame_flags，这时使用的time_base变化为enc->time_base。编码完后，到了mux_timebase，这时使用的time_base变化为enc->st->time_base。最终就写数据到av_interleaved_write_frame，写到文件或网络流中。
2.源码剖析
音视频解码
// This does not quite work like avcodec_decode_audio4/avcodec_decode_video2.
// There is the following difference: if you got a frame, you must call
// it again with pkt=NULL. pkt==NULL is treated differently from pkt->size==0
// (pkt==NULL means get more output, pkt->size==0 is a flush/drain packet)
static int decode(AVCodecContext *avctx, AVFrame *frame, int *got_frame, AVPacket *pkt)
{
int ret;
*got_frame = 0;
if (pkt) {
ret = avcodec_send_packet(avctx, pkt);
// In particular, we don't expect AVERROR(EAGAIN), because we read all
// decoded frames with avcodec_receive_frame() until done.
if (ret < 0 && ret != AVERROR_EOF)
return ret;
}
ret = avcodec_receive_frame(avctx, frame);
if (ret < 0 && ret != AVERROR(EAGAIN))
return ret;
if (ret >= 0)
*got_frame = 1;
return 0;
}发送解码帧到过滤器中
static int send_frame_to_filters(InputStream *ist, AVFrame *decoded_frame)
{
int i, ret;
AVFrame *f;
av_assert1(ist->nb_filters > 0); /* ensure ret is initialized */
for (i = 0; i < ist->nb_filters; i++) { // 可能接入多个filter，目前我们只看一个的
if (i < ist->nb_filters - 1) {
f = ist->filter_frame;
ret = av_frame_ref(f, decoded_frame);
if (ret < 0)
break;
} else
f = decoded_frame; // 发送到filter
// ist->filters[0]->graph->graph->filters[n]
ret = ifilter_send_frame(ist->filters[i], f);
if (ret == AVERROR_EOF)
ret = 0; /* ignore */
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR,
"Failed to inject frame into filter network: %s\n", av_err2str(ret));
break;
}
}
return ret;
}发送解码帧给各个过滤器
/**
做一些参数的检查，包括过滤器链表有没有初始化，没有的化先初始化，找到处理该包的第一个过滤器，
第一个过滤器一定是buffersrc， ffmpeg自带的，由这个过滤器传给自己ffmpeg 参数带的第一个过滤器，
这相当于一个入口
* @brief ifilter_send_frame
* @param ifilter
* @param frame
* @return
*/
static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
{
FilterGraph *fg = ifilter->graph;
int need_reinit, ret, i;
/* determine if the parameters for this input changed */
need_reinit = ifilter->format != frame->format;
switch (ifilter->ist->st->codecpar->codec_type) {
case AVMEDIA_TYPE_AUDIO:
need_reinit |= ifilter->sample_rate != frame->sample_rate ||
ifilter->channels != frame->channels ||
ifilter->channel_layout != frame->channel_layout;
break;
case AVMEDIA_TYPE_VIDEO:
need_reinit |= ifilter->width != frame->width ||
ifilter->height != frame->height;
break;
}
if (!ifilter->ist->reinit_filters && fg->graph)
need_reinit = 0;
if (!!ifilter->hw_frames_ctx != !!frame->hw_frames_ctx ||
(ifilter->hw_frames_ctx && ifilter->hw_frames_ctx->data != frame->hw_frames_ctx->data))
need_reinit = 1;
if (need_reinit) { // 初始化filter
ret = ifilter_parameters_from_frame(ifilter, frame);
if (ret < 0)
return ret;
}
/* (re)init the graph if possible, otherwise buffer the frame and return */
if (need_reinit || !fg->graph) {
for (i = 0; i < fg->nb_inputs; i++) {
if (!ifilter_has_all_input_formats(fg)) {
AVFrame *tmp = av_frame_clone(frame);
if (!tmp)
return AVERROR(ENOMEM);
av_frame_unref(frame);
if (!av_fifo_space(ifilter->frame_queue)) {
ret = av_fifo_realloc2(ifilter->frame_queue, 2 * av_fifo_size(ifilter->frame_queue));
if (ret < 0) {
av_frame_free(&tmp);
return ret;
}
}
av_fifo_generic_write(ifilter->frame_queue, &tmp, sizeof(tmp), NULL);
return 0;
}
}
ret = reap_filters(1);
if (ret < 0 && ret != AVERROR_EOF) {
av_log(NULL, AV_LOG_ERROR, "Error while filtering: %s\n", av_err2str(ret));
return ret;
}
ret = configure_filtergraph(fg);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error reinitializing filters!\n");
return ret;
}
}
//frame数据的控制权交给了av_buffersrc_add_frame_flags，然后外部的frame被reset
ret = av_buffersrc_add_frame_flags(ifilter->filter, frame, AV_BUFFERSRC_FLAG_PUSH);
if (ret < 0) {
if (ret != AVERROR_EOF)
av_log(NULL, AV_LOG_ERROR, "Error while filtering: %s\n", av_err2str(ret));
return ret;
}
return 0;
}配置视频和过滤器之间的关系，比如帧率，time-base等，然后再是连接其它视频过滤器。
// video filter的输入buffer，和audio的abuffer区分开来
static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
AVFilterInOut *in)
{
AVFilterContext *last_filter;
const AVFilter *buffer_filt = avfilter_get_by_name("buffer");
InputStream *ist = ifilter->ist;
InputFile *f = input_files[ist->file_index];
//关于time-base的修正
AVRational tb = ist->framerate.num ? av_inv_q(ist->framerate) :
ist->st->time_base;
AVRational fr = ist->framerate;
AVRational sar;
AVBPrint args;
char name[255];
int ret, pad_idx = 0;
int64_t tsoffset = 0;
AVBufferSrcParameters *par = av_buffersrc_parameters_alloc();
if (!par)
return AVERROR(ENOMEM);
memset(par, 0, sizeof(*par));
par->format = AV_PIX_FMT_NONE;
if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
av_log(NULL, AV_LOG_ERROR, "Cannot connect video filter to audio input\n");
ret = AVERROR(EINVAL);
goto fail;
}
if (!fr.num)
fr = av_guess_frame_rate(input_files[ist->file_index]->ctx, ist->st, NULL);
sar = ifilter->sample_aspect_ratio;
if(!sar.den)
sar = (AVRational){0,1};
av_bprint_init(&args, 0, AV_BPRINT_SIZE_AUTOMATIC);
av_bprintf(&args,
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:"
"pixel_aspect=%d/%d:sws_param=flags=%d",
ifilter->width, ifilter->height, ifilter->format,
tb.num, tb.den, sar.num, sar.den,
SWS_BILINEAR + ((ist->dec_ctx->flags&AV_CODEC_FLAG_BITEXACT) ? SWS_BITEXACT:0));
if (fr.num && fr.den)
av_bprintf(&args, ":frame_rate=%d/%d", fr.num, fr.den);
snprintf(name, sizeof(name), "graph %d input from stream %d:%d", fg->index,
ist->file_index, ist->st->index);
if ((ret = avfilter_graph_create_filter(&ifilter->filter, buffer_filt, name,
args.str, NULL, fg->graph)) < 0)
goto fail;
par->hw_frames_ctx = ifilter->hw_frames_ctx;
ret = av_buffersrc_parameters_set(ifilter->filter, par);
if (ret < 0)
goto fail;
av_freep(&par);
last_filter = ifilter->filter;
if (ist->autorotate) {
double theta = get_rotation(ist->st);
if (fabs(theta - 90) < 1.0) {
ret = insert_filter(&last_filter, &pad_idx, "transpose", "clock");
} else if (fabs(theta - 180) < 1.0) {
ret = insert_filter(&last_filter, &pad_idx, "hflip", NULL);
if (ret < 0)
return ret;
ret = insert_filter(&last_filter, &pad_idx, "vflip", NULL);
} else if (fabs(theta - 270) < 1.0) {
ret = insert_filter(&last_filter, &pad_idx, "transpose", "cclock");
} else if (fabs(theta) > 1.0) {
char rotate_buf[64];
snprintf(rotate_buf, sizeof(rotate_buf), "%f*PI/180", theta);
ret = insert_filter(&last_filter, &pad_idx, "rotate", rotate_buf);
}
if (ret < 0)
return ret;
}
if (do_deinterlace) {
AVFilterContext *yadif;
snprintf(name, sizeof(name), "deinterlace_in_%d_%d",
ist->file_index, ist->st->index);
if ((ret = avfilter_graph_create_filter(&yadif,
avfilter_get_by_name("yadif"),
name, "", NULL,
fg->graph)) < 0)
return ret;
if ((ret = avfilter_link(last_filter, 0, yadif, 0)) < 0)
return ret;
last_filter = yadif;
}
snprintf(name, sizeof(name), "trim_in_%d_%d",
ist->file_index, ist->st->index);
if (copy_ts) {
tsoffset = f->start_time == AV_NOPTS_VALUE ? 0 : f->start_time;
if (!start_at_zero && f->ctx->start_time != AV_NOPTS_VALUE)
tsoffset += f->ctx->start_time;
}
ret = insert_trim(((f->start_time == AV_NOPTS_VALUE) || !f->accurate_seek) ?
AV_NOPTS_VALUE : tsoffset, f->recording_time,
&last_filter, &pad_idx, name);
if (ret < 0)
return ret;
if ((ret = avfilter_link(last_filter, 0, in->filter_ctx, in->pad_idx)) < 0)
return ret;
return 0;
fail:
av_freep(&par);
return ret;
}解码音频
static int decode_audio(InputStream *ist, AVPacket *pkt, int *got_output,
int *decode_failed)
{
AVFrame *decoded_frame;
AVCodecContext *avctx = ist->dec_ctx;
int ret, err = 0;
AVRational decoded_frame_tb;
if (!ist->decoded_frame && !(ist->decoded_frame = av_frame_alloc()))
return AVERROR(ENOMEM);
if (!ist->filter_frame && !(ist->filter_frame = av_frame_alloc()))
return AVERROR(ENOMEM);
decoded_frame = ist->decoded_frame;
update_benchmark(NULL);
ret = decode(avctx, decoded_frame, got_output, pkt);
update_benchmark("decode_audio %d.%d", ist->file_index, ist->st->index);
if (ret < 0)
*decode_failed = 1;
if (ret >= 0 && avctx->sample_rate <= 0) {
av_log(avctx, AV_LOG_ERROR, "Sample rate %d invalid\n", avctx->sample_rate);
ret = AVERROR_INVALIDDATA;
}
if (ret != AVERROR_EOF)
check_decode_result(ist, got_output, ret);
if (!*got_output || ret < 0)
return ret;
ist->samples_decoded += decoded_frame->nb_samples; // 计算samples数量
ist->frames_decoded++; // 计算帧数量
/* increment next_dts to use for the case where the input stream does not
have timestamps or there are multiple frames in the packet */
//音视频时间转换微妙
ist->next_pts += ((int64_t)AV_TIME_BASE * decoded_frame->nb_samples) /
avctx->sample_rate; // 计算下一帧pts
ist->next_dts += ((int64_t)AV_TIME_BASE * decoded_frame->nb_samples) /
avctx->sample_rate; // 计算下一帧dts
if (decoded_frame->pts != AV_NOPTS_VALUE) {
decoded_frame_tb = ist->st->time_base;
} else if (pkt && pkt->pts != AV_NOPTS_VALUE) {
decoded_frame->pts = pkt->pts;
decoded_frame_tb = ist->st->time_base;
}else {
decoded_frame->pts = ist->dts;
decoded_frame_tb = AV_TIME_BASE_Q;
}
if (decoded_frame->pts != AV_NOPTS_VALUE)
decoded_frame->pts = av_rescale_delta(decoded_frame_tb, decoded_frame->pts,
(AVRational){1, avctx->sample_rate}, decoded_frame->nb_samples, &ist->filter_in_rescale_delta_last,
(AVRational){1, avctx->sample_rate});
ist->nb_samples = decoded_frame->nb_samples;
err = send_frame_to_filters(ist, decoded_frame); // 先把帧发送到filter
av_frame_unref(ist->filter_frame);
av_frame_unref(decoded_frame);
return err < 0 ? err : ret;
}从graph输出口获取filter数据，获取过滤器输出后的数据(包含音视频)，并转换time_base为enc->time_base。经过过滤器的数据，也要保证音视频同步。
/**
* Get and encode new output from any of the filtergraphs, without causing
* activity.
*
* @return 0 for success, <0 for severe errors
*/
static int reap_filters(int flush)
{
AVFrame *filtered_frame = NULL;
int i;
/* Reap all buffers present in the buffer sinks */
for (i = 0; i < nb_output_streams; i++) {
OutputStream *ost = output_streams[i];
OutputFile *of = output_files[ost->file_index];
AVFilterContext *filter;
AVCodecContext *enc = ost->enc_ctx;
int ret = 0;
//对应的stream copy的时候没有filter
if (!ost->filter || !ost->filter->graph->graph)
continue; // 还没有创建graph
filter = ost->filter->filter;
if (!ost->initialized) { // 只初始化一次
char error[1024] = "";
ret = init_output_stream(ost, error, sizeof(error));
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error initializing output stream %d:%d -- %s\n",
ost->file_index, ost->index, error);
exit_program(1);
}
}
if (!ost->filtered_frame && !(ost->filtered_frame = av_frame_alloc())) {
return AVERROR(ENOMEM);
}
filtered_frame = ost->filtered_frame;
while (1) {
double float_pts = AV_NOPTS_VALUE; // this is identical to filtered_frame.pts but with higher precision
ret = av_buffersink_get_frame_flags(filter, filtered_frame, // 从graph输出口获取filter数据
AV_BUFFERSINK_FLAG_NO_REQUEST);
if (ret < 0) {
if (ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) {
av_log(NULL, AV_LOG_WARNING,
"Error in av_buffersink_get_frame_flags(): %s\n", av_err2str(ret));
} else if (flush && ret == AVERROR_EOF) {
if (av_buffersink_get_type(filter) == AVMEDIA_TYPE_VIDEO)
//视频导出
do_video_out(of, ost, NULL, AV_NOPTS_VALUE);
}
break;
}
if (ost->finished) {
av_frame_unref(filtered_frame);
continue;
}
if (filtered_frame->pts != AV_NOPTS_VALUE) {
int64_t start_time = (of->start_time == AV_NOPTS_VALUE) ? 0 : of->start_time;
AVRational filter_tb = av_buffersink_get_time_base(filter); // filter out的time_base
AVRational tb = enc->time_base; // 编码器的time_base
int extra_bits = av_clip(29 - av_log2(tb.den), 0, 16);
tb.den <<= extra_bits;
float_pts =
av_rescale_q(filtered_frame->pts, filter_tb, tb) -
av_rescale_q(start_time, AV_TIME_BASE_Q, tb); // 转成统一的timebase对比
float_pts /= 1 << extra_bits;
// avoid exact midoints to reduce the chance of rounding differences, this can be removed in case the fps code is changed to work with integers
float_pts += FFSIGN(float_pts) * 1.0 / (1<<17);
filtered_frame->pts =
av_rescale_q(filtered_frame->pts, filter_tb, enc->time_base) -
av_rescale_q(start_time, AV_TIME_BASE_Q, enc->time_base);
}
switch (av_buffersink_get_type(filter)) {
case AVMEDIA_TYPE_VIDEO:
if (!ost->frame_aspect_ratio.num)
enc->sample_aspect_ratio = filtered_frame->sample_aspect_ratio;
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "filter -> pts:%s pts_time:%s exact:%f time_base:%d/%d\n",
av_ts2str(filtered_frame->pts), av_ts2timestr(filtered_frame->pts, &enc->time_base),
float_pts,
enc->time_base.num, enc->time_base.den);
}
//获取视频过滤器输出，输出也要保证音视频同步
do_video_out(of, ost, filtered_frame, float_pts);
break;
case AVMEDIA_TYPE_AUDIO:
if (!(enc->codec->capabilities & AV_CODEC_CAP_PARAM_CHANGE) &&
enc->channels != filtered_frame->channels) {
av_log(NULL, AV_LOG_ERROR,
"Audio filter graph output is not normalized and encoder does not support parameter changes\n");
break;
}
//获取音频过滤器输出，输出也要保证音视频同步
do_audio_out(of, ost, filtered_frame); // 音频
break;
default:
// TODO support subtitle filters
av_assert0(0);
}
av_frame_unref(filtered_frame);
}
}
return 0;
}经过过滤器输出后，就需要编码。
static void do_video_out(OutputFile *of,
OutputStream *ost,
AVFrame *next_picture,
double sync_ipts)
{
int ret, format_video_sync;
AVPacket pkt;
AVCodecContext *enc = ost->enc_ctx;
AVCodecParameters *mux_par = ost->st->codecpar;
AVRational frame_rate;
int nb_frames, nb0_frames, i;
double delta, delta0;
double duration = 0;
int frame_size = 0;
InputStream *ist = NULL;
AVFilterContext *filter = ost->filter->filter;
if (ost->source_index >= 0)
ist = input_streams[ost->source_index];
frame_rate = av_buffersink_get_frame_rate(filter);
if (frame_rate.num > 0 && frame_rate.den > 0)
duration = 1/(av_q2d(frame_rate) * av_q2d(enc->time_base));
if(ist && ist->st->start_time != AV_NOPTS_VALUE && ist->st->first_dts != AV_NOPTS_VALUE && ost->frame_rate.num)
duration = FFMIN(duration, 1/(av_q2d(ost->frame_rate) * av_q2d(enc->time_base)));
if (!ost->filters_script &&
!ost->filters &&
(nb_filtergraphs == 0 || !filtergraphs[0]->graph_desc) &&
next_picture &&
ist &&
lrintf(next_picture->pkt_duration * av_q2d(ist->st->time_base) / av_q2d(enc->time_base)) > 0) {
duration = lrintf(next_picture->pkt_duration * av_q2d(ist->st->time_base) / av_q2d(enc->time_base));
}
if (!next_picture) {
//end, flushing
nb0_frames = nb_frames = mid_pred(ost->last_nb0_frames[0],
ost->last_nb0_frames[1],
ost->last_nb0_frames[2]);
} else {
delta0 = sync_ipts - ost->sync_opts; // delta0 is the "drift" between the input frame (next_picture) and where it would fall in the output.
delta = delta0 + duration;
/* by default, we output a single frame */
nb0_frames = 0; // tracks the number of times the PREVIOUS frame should be duplicated, mostly for variable framerate (VFR)
nb_frames = 1;
format_video_sync = video_sync_method;
if (format_video_sync == VSYNC_AUTO) {
if(!strcmp(of->ctx->oformat->name, "avi")) {
format_video_sync = VSYNC_VFR;
} else
format_video_sync = (of->ctx->oformat->flags & AVFMT_VARIABLE_FPS) ? ((of->ctx->oformat->flags & AVFMT_NOTIMESTAMPS) ? VSYNC_PASSTHROUGH : VSYNC_VFR) : VSYNC_CFR;
if ( ist
&& format_video_sync == VSYNC_CFR
&& input_files[ist->file_index]->ctx->nb_streams == 1
&& input_files[ist->file_index]->input_ts_offset == 0) {
format_video_sync = VSYNC_VSCFR;
}
if (format_video_sync == VSYNC_CFR && copy_ts) {
format_video_sync = VSYNC_VSCFR;
}
}
ost->is_cfr = (format_video_sync == VSYNC_CFR || format_video_sync == VSYNC_VSCFR);
if (delta0 < 0 &&
delta > 0 &&
format_video_sync != VSYNC_PASSTHROUGH &&
format_video_sync != VSYNC_DROP) {
if (delta0 < -0.6) {
av_log(NULL, AV_LOG_VERBOSE, "Past duration %f too large\n", -delta0);
} else
av_log(NULL, AV_LOG_DEBUG, "Clipping frame in rate conversion by %f\n", -delta0);
sync_ipts = ost->sync_opts;
duration += delta0;
delta0 = 0;
}
switch (format_video_sync) {
case VSYNC_VSCFR:
if (ost->frame_number == 0 && delta0 >= 0.5) {
av_log(NULL, AV_LOG_DEBUG, "Not duplicating %d initial frames\n", (int)lrintf(delta0));
delta = duration;
delta0 = 0;
ost->sync_opts = lrint(sync_ipts);
}
case VSYNC_CFR:
// FIXME set to 0.5 after we fix some dts/pts bugs like in avidec.c
if (frame_drop_threshold && delta < frame_drop_threshold && ost->frame_number) {
nb_frames = 0;
} else if (delta < -1.1)
nb_frames = 0;
else if (delta > 1.1) {
nb_frames = lrintf(delta);
if (delta0 > 1.1)
nb0_frames = lrintf(delta0 - 0.6);
}
break;
case VSYNC_VFR:
if (delta <= -0.6)
nb_frames = 0;
else if (delta > 0.6)
ost->sync_opts = lrint(sync_ipts);
break;
case VSYNC_DROP:
case VSYNC_PASSTHROUGH:
ost->sync_opts = lrint(sync_ipts);
break;
default:
av_assert0(0);
}
}
nb_frames = FFMIN(nb_frames, ost->max_frames - ost->frame_number);
nb0_frames = FFMIN(nb0_frames, nb_frames);
memmove(ost->last_nb0_frames + 1,
ost->last_nb0_frames,
sizeof(ost->last_nb0_frames[0]) * (FF_ARRAY_ELEMS(ost->last_nb0_frames) - 1));
ost->last_nb0_frames[0] = nb0_frames;
if (nb0_frames == 0 && ost->last_dropped) {
nb_frames_drop++;
av_log(NULL, AV_LOG_VERBOSE,
"*** dropping frame %d from stream %d at ts %"PRId64"\n",
ost->frame_number, ost->st->index, ost->last_frame->pts);
}
if (nb_frames > (nb0_frames && ost->last_dropped) + (nb_frames > nb0_frames)) {
if (nb_frames > dts_error_threshold * 30) {
av_log(NULL, AV_LOG_ERROR, "%d frame duplication too large, skipping\n", nb_frames - 1);
nb_frames_drop++;
return;
}
nb_frames_dup += nb_frames - (nb0_frames && ost->last_dropped) - (nb_frames > nb0_frames);
av_log(NULL, AV_LOG_VERBOSE, "*** %d dup!\n", nb_frames - 1);
if (nb_frames_dup > dup_warning) {
av_log(NULL, AV_LOG_WARNING, "More than %d frames duplicated\n", dup_warning);
dup_warning *= 10;
}
}
ost->last_dropped = nb_frames == nb0_frames && next_picture;
/* duplicates frame if needed */
for (i = 0; i < nb_frames; i++) {
AVFrame *in_picture;
int forced_keyframe = 0;
double pts_time;
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
if (i < nb0_frames && ost->last_frame) {
in_picture = ost->last_frame;
} else
in_picture = next_picture;
if (!in_picture)
return;
in_picture->pts = ost->sync_opts;
if (!check_recording_time(ost))
return;
if (enc->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME) &&
ost->top_field_first >= 0)
in_picture->top_field_first = !!ost->top_field_first;
if (in_picture->interlaced_frame) {
if (enc->codec->id == AV_CODEC_ID_MJPEG)
mux_par->field_order = in_picture->top_field_first ? AV_FIELD_TT:AV_FIELD_BB;
else
mux_par->field_order = in_picture->top_field_first ? AV_FIELD_TB:AV_FIELD_BT;
} else
mux_par->field_order = AV_FIELD_PROGRESSIVE;
in_picture->quality = enc->global_quality;
in_picture->pict_type = 0;
if (ost->forced_kf_ref_pts == AV_NOPTS_VALUE &&
in_picture->pts != AV_NOPTS_VALUE)
ost->forced_kf_ref_pts = in_picture->pts;
pts_time = in_picture->pts != AV_NOPTS_VALUE ?
(in_picture->pts - ost->forced_kf_ref_pts) * av_q2d(enc->time_base) : NAN;
if (ost->forced_kf_index < ost->forced_kf_count &&
in_picture->pts >= ost->forced_kf_pts[ost->forced_kf_index]) {
ost->forced_kf_index++;
forced_keyframe = 1;
} else if (ost->forced_keyframes_pexpr) {
double res;
ost->forced_keyframes_expr_const_values[FKF_T] = pts_time;
res = av_expr_eval(ost->forced_keyframes_pexpr,
ost->forced_keyframes_expr_const_values, NULL);
ff_dlog(NULL, "force_key_frame: n:%f n_forced:%f prev_forced_n:%f t:%f prev_forced_t:%f -> res:%f\n",
ost->forced_keyframes_expr_const_values[FKF_N],
ost->forced_keyframes_expr_const_values[FKF_N_FORCED],
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_N],
ost->forced_keyframes_expr_const_values[FKF_T],
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_T],
res);
if (res) {
forced_keyframe = 1;
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_N] =
ost->forced_keyframes_expr_const_values[FKF_N];
ost->forced_keyframes_expr_const_values[FKF_PREV_FORCED_T] =
ost->forced_keyframes_expr_const_values[FKF_T];
ost->forced_keyframes_expr_const_values[FKF_N_FORCED] += 1;
}
ost->forced_keyframes_expr_const_values[FKF_N] += 1;
} else if ( ost->forced_keyframes
&& !strncmp(ost->forced_keyframes, "source", 6)
&& in_picture->key_frame==1) {
forced_keyframe = 1;
}
if (forced_keyframe) {
in_picture->pict_type = AV_PICTURE_TYPE_I;
av_log(NULL, AV_LOG_DEBUG, "Forced keyframe at time %f\n", pts_time);
}
update_benchmark(NULL);
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "encoder <- type:video "
"frame_pts:%s frame_pts_time:%s time_base:%d/%d\n",
av_ts2str(in_picture->pts), av_ts2timestr(in_picture->pts, &enc->time_base),
enc->time_base.num, enc->time_base.den);
}
ost->frames_encoded++;
//发送frame数据编码
ret = avcodec_send_frame(enc, in_picture);
if (ret < 0)
goto error;
// Make sure Closed Captions will not be duplicated
av_frame_remove_side_data(in_picture, AV_FRAME_DATA_A53_CC);
while (1) {
//获取编码后的数据
ret = avcodec_receive_packet(enc, &pkt);
update_benchmark("encode_video %d.%d", ost->file_index, ost->index);
if (ret == AVERROR(EAGAIN))
break;
if (ret < 0)
goto error;
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "encoder -> type:video "
"pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s\n",
av_ts2str(pkt.pts), av_ts2timestr(pkt.pts, &enc->time_base),
av_ts2str(pkt.dts), av_ts2timestr(pkt.dts, &enc->time_base));
}
if (pkt.pts == AV_NOPTS_VALUE && !(enc->codec->capabilities & AV_CODEC_CAP_DELAY))
pkt.pts = ost->sync_opts;
//也要转换为封装模块下的mux_timebase
av_packet_rescale_ts(&pkt, enc->time_base, ost->mux_timebase);
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "encoder -> type:video "
"pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s\n",
av_ts2str(pkt.pts), av_ts2timestr(pkt.pts, &ost->mux_timebase),
av_ts2str(pkt.dts), av_ts2timestr(pkt.dts, &ost->mux_timebase));
}
frame_size = pkt.size;
//输出到文件
output_packet(of, &pkt, ost, 0);
/* if two pass, output log */
if (ost->logfile && enc->stats_out) {
fprintf(ost->logfile, "%s", enc->stats_out);
}
}
ost->sync_opts++;
/*
* For video, number of frames in == number of packets out.
* But there may be reordering, so we can't throw away frames on encoder
* flush, we need to limit them here, before they go into encoder.
*/
ost->frame_number++;
if (vstats_filename && frame_size)
do_video_stats(ost, frame_size);
}
if (!ost->last_frame)
ost->last_frame = av_frame_alloc();
av_frame_unref(ost->last_frame);
if (next_picture && ost->last_frame)
av_frame_ref(ost->last_frame, next_picture);
else
av_frame_free(&ost->last_frame);
return;
error:
av_log(NULL, AV_LOG_FATAL, "Video encoding failed\n");
exit_program(1);
}输出到文件，或发送网络流。
/**
* @brief output_packet 输出packet
* @param of 属于哪个输出文件
* @param pkt 输出包
* @param ost 属于哪个输出流
* @param eof 是否结束
*/
static void output_packet(OutputFile *of, AVPacket *pkt,
OutputStream *ost, int eof)
{
int ret = 0;
/* apply the output bitstream filters, if any */
if (ost->nb_bitstream_filters) { // 需要bitstream filters
int idx;
ret = av_bsf_send_packet(ost->bsf_ctx[0], eof ? NULL : pkt);
if (ret < 0)
goto finish;
eof = 0;
idx = 1;
while (idx) {
/* get a packet from the previous filter up the chain */
ret = av_bsf_receive_packet(ost->bsf_ctx[idx - 1], pkt);
if (ret == AVERROR(EAGAIN)) {
ret = 0;
idx--;
continue;
} else if (ret == AVERROR_EOF) {
eof = 1;
} else if (ret < 0)
goto finish;
/* send it to the next filter down the chain or to the muxer */
if (idx < ost->nb_bitstream_filters) {
ret = av_bsf_send_packet(ost->bsf_ctx[idx], eof ? NULL : pkt);
if (ret < 0)
goto finish;
idx++;
eof = 0;
} else if (eof)
goto finish;
else
//正真输出到具体文件或网络流
write_packet(of, pkt, ost, 0);
}
} else if (!eof)
write_packet(of, pkt, ost, 0);
finish:
if (ret < 0 && ret != AVERROR_EOF) {
av_log(NULL, AV_LOG_ERROR, "Error applying bitstream filters to an output "
"packet for stream #%d:%d.\n", ost->file_index, ost->index);
if(exit_on_error)
exit_program(1);
}
}正真输出到具体文件或网络流。
static void write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int unqueue)
{
AVFormatContext *s = of->ctx;
AVStream *st = ost->st;
int ret;
/*
* Audio encoders may split the packets -- #frames in != #packets out.
* But there is no reordering, so we can limit the number of output packets
* by simply dropping them here.
* Counting encoded video frames needs to be done separately because of
* reordering, see do_video_out().
* Do not count the packet when unqueued because it has been counted when queued.
*/
if (!(st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && ost->encoding_needed) && !unqueue) {
if (ost->frame_number >= ost->max_frames) {
av_packet_unref(pkt);
return;
}
ost->frame_number++;
}
if (!of->header_written) { // 头部还没有写入
AVPacket tmp_pkt = {0};
/* the muxer is not initialized yet, buffer the packet */
if (!av_fifo_space(ost->muxing_queue)) {
int new_size = FFMIN(2 * av_fifo_size(ost->muxing_queue),
ost->max_muxing_queue_size);
if (new_size <= av_fifo_size(ost->muxing_queue)) {
av_log(NULL, AV_LOG_ERROR,
"Too many packets buffered for output stream %d:%d.\n",
ost->file_index, ost->st->index);
exit_program(1);
}
ret = av_fifo_realloc2(ost->muxing_queue, new_size);
if (ret < 0)
exit_program(1);
}
ret = av_packet_make_refcounted(pkt);
if (ret < 0)
exit_program(1);
av_packet_move_ref(&tmp_pkt, pkt);
av_fifo_generic_write(ost->muxing_queue, &tmp_pkt, sizeof(tmp_pkt), NULL);
return;
}
if ((st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && video_sync_method == VSYNC_DROP) ||
(st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_sync_method < 0))
pkt->pts = pkt->dts = AV_NOPTS_VALUE;
if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
int i;
uint8_t *sd = av_packet_get_side_data(pkt, AV_PKT_DATA_QUALITY_STATS,
NULL);
ost->quality = sd ? AV_RL32(sd) : -1;
ost->pict_type = sd ? sd[4] : AV_PICTURE_TYPE_NONE;
for (i = 0; i<FF_ARRAY_ELEMS(ost->error); i++) {
if (sd && i < sd[5])
ost->error[i] = AV_RL64(sd + 8 + 8*i);
else
ost->error[i] = -1;
}
if (ost->frame_rate.num && ost->is_cfr) {
if (pkt->duration > 0)
av_log(NULL, AV_LOG_WARNING, "Overriding packet duration by frame rate, this should not happen\n");
pkt->duration = av_rescale_q(1, av_inv_q(ost->frame_rate),
ost->mux_timebase);
}
}
// 将pkt 从ost->mux_timebase转成ost->st->time_base
// 即是从编码器转成 输出流的time_base
av_packet_rescale_ts(pkt, ost->mux_timebase, ost->st->time_base);
if (!(s->oformat->flags & AVFMT_NOTIMESTAMPS)) {
if (pkt->dts != AV_NOPTS_VALUE &&
pkt->pts != AV_NOPTS_VALUE &&
pkt->dts > pkt->pts) {
av_log(s, AV_LOG_WARNING, "Invalid DTS: %"PRId64" PTS: %"PRId64" in output stream %d:%d, replacing by guess\n",
pkt->dts, pkt->pts,
ost->file_index, ost->st->index);
pkt->pts =
pkt->dts = pkt->pts + pkt->dts + ost->last_mux_dts + 1
- FFMIN3(pkt->pts, pkt->dts, ost->last_mux_dts + 1)
- FFMAX3(pkt->pts, pkt->dts, ost->last_mux_dts + 1);
}
if ((st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO || st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO || st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) &&
pkt->dts != AV_NOPTS_VALUE &&
!(st->codecpar->codec_id == AV_CODEC_ID_VP9 && ost->stream_copy) &&
ost->last_mux_dts != AV_NOPTS_VALUE) {
int64_t max = ost->last_mux_dts + !(s->oformat->flags & AVFMT_TS_NONSTRICT);
if (pkt->dts < max) {
int loglevel = max - pkt->dts > 2 || st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO ? AV_LOG_WARNING : AV_LOG_DEBUG;
av_log(s, loglevel, "Non-monotonous DTS in output stream "
"%d:%d; previous: %"PRId64", current: %"PRId64"; ",
ost->file_index, ost->st->index, ost->last_mux_dts, pkt->dts);
if (exit_on_error) {
av_log(NULL, AV_LOG_FATAL, "aborting.\n");
exit_program(1);
}
av_log(s, loglevel, "changing to %"PRId64". This may result "
"in incorrect timestamps in the output file.\n",
max);
if (pkt->pts >= pkt->dts)
pkt->pts = FFMAX(pkt->pts, max);
pkt->dts = max;
}
}
}
ost->last_mux_dts = pkt->dts;
ost->data_size += pkt->size;
ost->packets_written++;
pkt->stream_index = ost->index;
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "muxer <- type:%s "
"pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s size:%d\n",
av_get_media_type_string(ost->enc_ctx->codec_type),
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, &ost->st->time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, &ost->st->time_base),
pkt->size
);
}
// 调用ffmpeg真正写入到文件或网络流
ret = av_interleaved_write_frame(s, pkt);
if (ret < 0) {
print_error("av_interleaved_write_frame()", ret);
main_return_code = 1;
close_all_output_streams(ost, MUXER_FINISHED | ENCODER_FINISHED, ENCODER_FINISHED);
}
av_packet_unref(pkt);
}本篇文章就分析到这里，欢迎关注，点赞，收藏，转发。
如果对文章感兴趣，也欢迎关注微信公众号“记录世界 from antonio”
本文暂时没有评论，来添加一个吧(●'◡'●)