Today learning ffmpeg/doc/examples/muxing. C

Program encoding 10 seconds of audio and video data, write parameter specified path, file suffix will determine the file packaging format, use audio and video encoder, call the command as follows:

➜ examples git:(master) qualify./muxing_g/TMP /mux.mp4 ➜ examples git:(master) qualify./muxing_g/TMP /mux.movCopy the code

Create AVFormatContext

    // Create AVFormatContext and guess the output format based on the file suffix
    avformat_alloc_output_context2(&oc, NULL.NULL, filename);
    if(! oc) {printf("Could not deduce output format from file extension: using MPEG.\n");
        // Output format cannot be inferred, use mPEG
        avformat_alloc_output_context2(&oc, NULL."mpeg", filename);
    }
Copy the code

Add video/audio stream to AVFormatContext

    if(fmt->video_codec ! = AV_CODEC_ID_NONE) { add_stream(&video_st, oc, &video_codec, fmt->video_codec); have_video =1;
        encode_video = 1;
    }
    if(fmt->audio_codec ! = AV_CODEC_ID_NONE) { add_stream(&audio_st, oc, &audio_codec, fmt->audio_codec); have_audio =1;
        encode_audio = 1;
    }
Copy the code

What does add_stream do

/* Add an output stream. */
static void add_stream(OutputStream *ost, AVFormatContext *oc,
                       const AVCodec **codec,
                       enum AVCodecID codec_id)
{
    AVCodecContext *c;
    int i;
    /* find the encoder */
    // find AVCodec for codec_id
    *codec = avcodec_find_encoder(codec_id);
    / / create the PKT
    ost->tmp_pkt = av_packet_alloc();
    / / create the AVStream
    ost->st = avformat_new_stream(oc, NULL);
    // Set the stream index
    ost->st->id = oc->nb_streams- 1;
    // Create the encoder context
    c = avcodec_alloc_context3(*codec);
    // Save the encoder context
    ost->enc = c;

    switch ((*codec)->type) {
    case AVMEDIA_TYPE_AUDIO:
         // Set the audio encoder context parameters (bit rate, sample rate, bit depth, channel layout, etc.).// Set the stream time base
        ost->st->time_base = (AVRational){ 1, c->sample_rate };
        break;

    case AVMEDIA_TYPE_VIDEO:
        // Set the encoder context parameters
        c->codec_id = codec_id;
        // Set the stream time base
        ost->st->time_base = (AVRational){ 1, STREAM_FRAME_RATE };
        c->time_base       = ost->st->time_base;
        // Encoder bit rate, frame rate, gop, resolution, Settings.break;

    default:
        break;
    }
    /* Some formats want stream headers to be separate. */
    // Handle the stream header
    if (oc->oformat->flags & AVFMT_GLOBALHEADER)
        c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
Copy the code

1. Find AVCodec based on codec_id

2. Call avformat_new_stream to create the stream

3. Set the stream index

4. Configure the encoder parameters according to the context in which the encoder is created by AVCodec

5. Set time_base for stream

6. Handle the flag bit for stream Headers

With the audio and video stream created, the next step is to prepare for writing

1. The stream is created above, and the encoder context is created. Open_video /open_audio continues to prepare for the write

2. When ready, open the file for writing

    /* Now that all the parameters are set, we can open the audio and * video codecs and allocate the necessary encode buffers. */
    if (have_video)
        open_video(oc, video_codec, &video_st, opt);

    if (have_audio)
        open_audio(oc, audio_codec, &audio_st, opt);

    // Print the oc information
    av_dump_format(oc, 0, filename, 1);

    /* open the output file, if needed */
    if(! (fmt->flags & AVFMT_NOFILE)) {// Open the file
        ret = avio_open(&oc->pb, filename, AVIO_FLAG_WRITE);
        if (ret < 0) {
            fprintf(stderr."Could not open '%s': %s\n", filename,
                    av_err2str(ret));
            return 1; }}Copy the code

See what Open_video does

static void open_video(AVFormatContext *oc, const AVCodec *codec,
                       OutputStream *ost, AVDictionary *opt_arg)
{
    int ret;
    AVCodecContext *c = ost->enc;
    AVDictionary *opt = NULL;
    // copy the contents of opt_arg to opt
    av_dict_copy(&opt, opt_arg, 0);
    /* open the codec */
    // Open the encoder
    ret = avcodec_open2(c, codec, &opt);
    / / release opt
    av_dict_free(&opt);
    if (ret < 0) {
        fprintf(stderr."Could not open video codec: %s\n", av_err2str(ret));
        exit(1);
    }

    /* allocate and init a re-usable frame */
    // Create a reusable AVFrame according to the format, width and height
    ost->frame = alloc_picture(c->pix_fmt, c->width, c->height);
    if(! ost->frame) {fprintf(stderr."Could not allocate video frame\n");
        exit(1);
    }

    /* If the output format is not YUV420P, then a temporary YUV420P * picture is needed too. It is then converted to the required * output format. */
    ost->tmp_frame = NULL;
    if(c->pix_fmt ! = AV_PIX_FMT_YUV420P) {// If pix_fmt is not yuv420p, create a yuv420p AVFrame and save it in oST ->tmp_frame
        ost->tmp_frame = alloc_picture(AV_PIX_FMT_YUV420P, c->width, c->height);
        if(! ost->tmp_frame) {fprintf(stderr."Could not allocate temporary picture\n");
            exit(1); }}/* copy the stream parameters to the muxer */
    // Copy the encoder's parameters to the stream's corresponding encoding parameters
    ret = avcodec_parameters_from_context(ost->st->codecpar, c);
    if (ret < 0) {
        fprintf(stderr."Could not copy the stream parameters\n");
        exit(1); }}Copy the code

1. Open encoder

2. Apply for resources and create an AVFrame to store pre-coding data

3. Call avCODEC_parameterS_FROm_context to copy the encoder’s encoding parameters into the STREAM’s COdecPAR

Write audio and video data to files

Write the file header

    /* Write the stream header, if any. */
    // Write the stream information to the file header
    ret = avformat_write_header(oc, &opt);
    if (ret < 0) {
        fprintf(stderr."Error occurred when opening output file: %s\n",
                av_err2str(ret));
        return 1;
    }
Copy the code

Alternate writing audio and video frames

    while (encode_video || encode_audio) {
        /* select the stream to encode */
        /* Alternately write encoded audio and video frames at the end of the video write or video_st.next_pts <= Audio_st.next_pts, write video or write audio */
        if(encode_video && (! encode_audio || av_compare_ts(video_st.next_pts, video_st.enc->time_base, audio_st.next_pts, audio_st.enc->time_base) <=0)) {
            // Write the encoded video frameencode_video = ! write_video_frame(oc, &video_st); }else {
            // Write the encoded audio frame
            encode_audio = !write_audio_frame(oc, &audio_st);
        }
    }
Copy the code

Write a trailer

    /* Write the trailer, if any. The trailer must be written before you * close the CodecContexts open when you wrote the header; otherwise * av_write_trailer() may try to use memory that was freed on * av_codec_close(). */
    av_write_trailer(oc);
Copy the code

Close encoder, close file, release resources

    /* Close each codec. */
    if (have_video)
        close_stream(oc, &video_st);
    if (have_audio)
        close_stream(oc, &audio_st);

    if(! (fmt->flags & AVFMT_NOFILE))/* Close the output file. */
        avio_closep(&oc->pb);

    /* free the stream */
    avformat_free_context(oc);
Copy the code

write_video_frame

static int write_video_frame(AVFormatContext *oc, OutputStream *ost)
{
    return write_frame(oc, ost->enc, ost->st, get_video_frame(ost), ost->tmp_pkt);
}
Copy the code

We simply call write_frame

static int write_frame(AVFormatContext *fmt_ctx, AVCodecContext *c, AVStream *st, AVFrame *frame, AVPacket *pkt)
{
    int ret;

    // send the frame to the encoder
    // Send the frame to the encoder to encode
    ret = avcodec_send_frame(c, frame);
    if (ret < 0) {
        fprintf(stderr."Error sending a frame to the encoder: %s\n",
                av_err2str(ret));
        exit(1);
    }

    while (ret >= 0) {
        // Read encoded PKT from encoder
        ret = avcodec_receive_packet(c, pkt);
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
            break;
        else if (ret < 0) {
            fprintf(stderr."Error encoding a frame: %s\n", av_err2str(ret));
            exit(1);
        }

        /* rescale output packet timestamp values from codec to stream timebase */
        // Adjust PKT's PTS, PKT's time base reference encoder's time base, convert it to reference stream's time base
        av_packet_rescale_ts(pkt, c->time_base, st->time_base);
        / / set PKT stream_index and stream corresponding consistent, audio and video respectively corresponding to different stream_index
        pkt->stream_index = st->index;

        /* Write the compressed frame to the media file. */
        log_packet(fmt_ctx, pkt);
        // Write PKT to the video file
        ret = av_interleaved_write_frame(fmt_ctx, pkt);
        /* pkt is now blank (av_interleaved_write_frame() takes ownership of * its contents and resets pkt), so that no unreferencing is necessary. * This would be different if one used av_write_frame(). */
        if (ret < 0) {
            fprintf(stderr."Error while writing output packet: %s\n", av_err2str(ret));
            exit(1); }}return ret == AVERROR_EOF ? 1 : 0;
}
Copy the code

1. Send AVFrame to encoder to encode

2. Read the PKT

3. Call av_packet_rescale_ts to adjust the PKT timestamp, and write PTS to the stream against time_base

4. Set PKT index

5. Call av_interleaved_write_frame to write PKT to a file

6. Returns the write result. When AVFrame is empty, the encoder is flushed and ret = AVERROR_EOF returns 1, ending the write

Each frame of frame data is filled with fake data from the get_video_frame method

static AVFrame *get_video_frame(OutputStream *ost)
{
    AVCodecContext *c = ost->enc;

    /* check if we want to generate more frames */
    if (av_compare_ts(ost->next_pts, c->time_base,
                      STREAM_DURATION, (AVRational){ 1.1 }) > 0)
        return NULL;

    /* when we pass a frame to the encoder, it may keep a reference to it * internally; make sure we do not overwrite it here */
    if (av_frame_make_writable(ost->frame) < 0)
        exit(1);

    if(c->pix_fmt ! = AV_PIX_FMT_YUV420P) {/* as we only generate a YUV420P picture, we must convert it * to the codec pixel format if needed */
        if(! ost->sws_ctx) { ost->sws_ctx = sws_getContext(c->width, c->height, AV_PIX_FMT_YUV420P, c->width, c->height, c->pix_fmt, SCALE_FLAGS,NULL.NULL.NULL);
            if(! ost->sws_ctx) {fprintf(stderr."Could not initialize the conversion context\n");
                exit(1);
            }
        }
        fill_yuv_image(ost->tmp_frame, ost->next_pts, c->width, c->height);
        sws_scale(ost->sws_ctx, (const uint8_t * const *) ost->tmp_frame->data,
                  ost->tmp_frame->linesize, 0, c->height, ost->frame->data,
                  ost->frame->linesize);
    } else {
        fill_yuv_image(ost->frame, ost->next_pts, c->width, c->height);
    }

    ost->frame->pts = ost->next_pts++;

    return ost->frame;
}
Copy the code

1. Call av_compare_ts to determine whether to continue generating a new frame. The initial rule is to write data for 10 seconds, and then stop writing after that

2. Av_frame_make_writable makes the current frame writable, but the frame referenced by the encoder is not writable. If this method is referenced, a new buF is created internally, making it writable

3. Fill in the data YUV, but also do the zoom processing, not to discuss

4. Update frame’s PTS

5. Returns the generated frame

write_audio_frame

/* * encode one audio frame and send it to the muxer * return 1 when encoding is finished, 0 otherwise */
static int write_audio_frame(AVFormatContext *oc, OutputStream *ost)
{
    AVCodecContext *c;
    AVFrame *frame;
    int ret;
    int dst_nb_samples;

    c = ost->enc;

    // Get the audio frame
    frame = get_audio_frame(ost);   

    if (frame) {
        /* convert samples from native format to destination codec format, using the resampler */
        /* compute destination number of samples */

        /* Calculate the number of samples that should be generated based on resampling */
        dst_nb_samples = av_rescale_rnd(swr_get_delay(ost->swr_ctx, c->sample_rate) + frame->nb_samples,
                                        c->sample_rate, c->sample_rate, AV_ROUND_UP);
        // Because the sampling rate is not changed, only the bit depth is changed, the number of samples remains unchanged
        av_assert0(dst_nb_samples == frame->nb_samples);

        /* when we pass a frame to the encoder, it may keep a reference to it * internally; * make sure we do not overwrite it here */
        // make ost->frame writable
        ret = av_frame_make_writable(ost->frame);
        if (ret < 0)
            exit(1);

        /* convert to destination format */
        / / re-sampling
        ret = swr_convert(ost->swr_ctx,
                          ost->frame->data, dst_nb_samples,
                          (const uint8_t **)frame->data, frame->nb_samples);
        if (ret < 0) {
            fprintf(stderr."Error while converting\n");
            exit(1);
        }

        frame = ost->frame;
        // Recalculate the audio PTS
        frame->pts = av_rescale_q(ost->samples_count, (AVRational){1, c->sample_rate}, c->time_base);
        / / samples_count calculation
        ost->samples_count += dst_nb_samples;
    }
    // Write the audio frame to the file
    return write_frame(oc, c, ost->st, frame, ost->tmp_pkt);
}
Copy the code

1. To prepare an AVFrame, call get_audio_frame to get an AVFrame and fill it with PCM fake data according to the encoding format

2. Call AV_rescale_rnd to calculate the number of resampling. If the sampling rate of the data source and the audio feed into the encoder is different, the sampling rate needs to be changed. The sample program only changes the sampling format, but does not change the sampling rate, and the number of samples remains unchanged

3. Resampling is done and the resampling data is saved in oST -> Frame

4. After resampling, the PTS of the frame needs to be updated

5. Call the write_frame encoding to write data to a file

Conclusion:

Learn how to encode and encapsulate audio and video data into files by using libavformat

1. Create AVFormatContext

2. Add the stream

3. Configure the codec, stream parameters

4. Encode frame, generate PKT, update PTS

5. Alternate write audio and video PKT

6. Close the codec and end writing the file