using System; using System.Runtime.InteropServices; using ADOFAIRenderer.Data; using FFmpeg.AutoGen; using MelonLoader; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; namespace ADOFAIRenderer.Util.FFmpeg { /// /// Class that encodes video by directly invoking (P/Invoke) FFmpeg's C native API. /// /// ── Lifecycle Flow ─────────────────────────────────────────────────── /// /// 1. Constructor: Stores encoding settings (resolution, FPS, codec, quality, etc.). /// /// 2. Start(outputPath): /// a) Create AVFormatContext for output container (mp4, etc.) /// b) Add video stream (AVStream) /// c) Try the candidate encoders of the codec in order and select the first one that opens /// (encoders of the active GPU vendor → other hardware encoders → software encoders; see GetEncoderCandidates) /// d) Write file header and initialize frame/scaler buffers /// /// 3. WriteFrame*(): Called for every frame /// a) Pre-converted NV12/YUV420P frame (GPU compute shader output): copied into the encoder frame /// RGBA frame: converted to the encoder pixel format with sws_scale (CPU) /// b) (In QSV hw_frames mode) Upload to GPU surface /// c) avcodec_send_frame → avcodec_receive_packet → Write to file /// /// 4. End(): Flush delayed frames remaining in the encoder and write trailer /// /// 5. Dispose(): Release all FFmpeg resources (context, frames, scaler, I/O) /// /// ── Threading ──────────────────────────────────────────────────────── /// Every public method takes _lock, so calls from different threads are serialized. /// In the renderer, the main Unity thread constructs and starts the encoder, and the render thread /// (RenderManager.RenderThread) sends the video frames and audio samples, calls End() and Dispose(). /// Dispose() may also be called from another thread to release an encoder whose render was stopped. /// After End() or Dispose(), the Write*/Send* methods return without doing anything. /// /// ── Timestamps ─────────────────────────────────────────────────────── /// Video PTS = number of frames submitted before the frame (codec time base 1 / FPS). /// Audio PTS = number of samples per channel submitted before the frame (codec time base 1 / sample rate). /// Both start at 0, so the two streams are aligned by the count of submitted frames and samples. /// /// ── Reference Documentation ───────────────────────────────────────── /// • FFmpeg Encoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html /// • Send/Receive Encoding API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// • Hardware Acceleration Overview: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html /// • HW Acceleration Wiki: https://trac.ffmpeg.org/wiki/HWAccelIntro /// internal sealed unsafe class FFmpegNativeEncoder : IDisposable { /// /// Struct containing configuration values to pass for audio encoding. /// public struct AudioSettings { /// /// Sample rate of input audio (Hz). e.g., 44100, 48000. If 0 or less, 48000 is used. /// The encoder runs at the same rate (no sample rate conversion), so the rate has to be one the audio codec supports. /// public int SampleRate; /// /// Number of input audio channels. e.g., 1 (mono), 2 (stereo). If 0 or less, 2 is used for the channel layout. /// The channel layout is the FFmpeg default layout for the count (av_channel_layout_default). /// public int Channels; /// /// Sample format of input audio (interleaved). Default is AV_SAMPLE_FMT_FLT (32-bit float). /// Unity's AudioClip.GetData() returns float[], so FLT is suitable. /// The value 0 (AV_SAMPLE_FMT_U8) is treated as "not set" and selects FLT. /// public AVSampleFormat InputSampleFormat; /// /// Audio codec ID to use. If default (0), AAC is used. /// e.g., AV_CODEC_ID_AAC, AV_CODEC_ID_FLAC, AV_CODEC_ID_OPUS, etc. /// public AVCodecID AudioCodecId; /// /// Audio bitrate (in bps). If default (0), 256000 bps is used. /// For AAC, 128000–320000 bps is common. /// public int Bitrate; } /// /// Struct containing configuration values to pass to the encoder. /// public struct EncoderSettings { /// /// Width resolution of output video (pixels). Also the width of every input frame (no scaling is performed). /// The 4:2:0 input size formulas (GetRequiredInputLength) assume an even value. /// public int Width; /// Height resolution of output video (pixels). Also the height of every input frame; even values assumed as for Width. public int Height; /// /// Frames per second (e.g., 60.0, 29.97, etc.). Converted to a rational number (RationalUtil.Double2MaxPrecisionInt) /// for the codec framerate and time base; every submitted frame advances the PTS by one frame interval. /// public double FPS; /// Video codec type to use (H264, H265, AV1, VP9, VVC). public Codec Codec; /// Bitrate control mode (CQP, CBR, VBR). public RateControlMode Mode; /// /// Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless) /// Clamped to the range of the selected encoder (see ApplyQualitySettings); a negative value selects 18. /// public int QualityValue; /// Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg. public int TargetBitrate; /// /// Maximum bitrate in VBR mode (in kbps). Determines peak quality limit. /// Used only when larger than TargetBitrate; otherwise the maximum is 1.5 × TargetBitrate. /// public int MaxBitrate; /// /// GOP (Group of Pictures) size (in seconds). Represents keyframe interval. /// Converted to frames as round(FPS × GopSize), at least 1. /// public double GopSize; /// Whether to force the use of software CPU encoder (hardware encoders are not tried). public bool ForceSoftware; /// /// Audio encoding settings. If null, encodes video only without audio stream. /// If configured, video and audio are muxed together. /// public AudioSettings? Audio; } // ────────────────────────────────────────────────────────────── // FFmpeg Core Struct Pointers // ────────────────────────────────────────────────────────────── /// /// Context managing format information for the output container (mp4, mkv, etc.). /// Used for writing file headers/trailers, recording packets, etc. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html /// private AVFormatContext* _pFormatContext = null; /// /// Context managing codec (encoder) state. /// All parameters required for encoding such as resolution, pixel format, framerate, and quality settings are configured here. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// private AVCodecContext* _pCodecContext = null; /// /// Video stream within the output file. /// A single container (format context) can contain multiple streams (video, audio, etc.); /// this encoder writes one video stream and, when audio is enabled, one audio stream (). /// Owned by and freed with it. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html /// private AVStream* _pVideoStream = null; // ────────────────────────────────────────────────────────────── // Frame Buffers and Colorspace Converter (Scaler) Pointers // ────────────────────────────────────────────────────────────── /// /// Frame structure pointing to raw RGBA frame data captured from Unity. /// Does not own pixel data; references external buffer via data[0] and linesize[0]. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html /// private AVFrame* _pSrcFrame = null; /// /// Frame converted to the pixel format required by the encoder (: NV12, YUV420P or YUV420P10LE). /// Actual pixel data memory is allocated via av_frame_get_buffer() (reference-counted buffers owned by the frame). /// Written by the CPU conversion (sws_scale) or by the copy of a pre-converted frame, then submitted to the encoder. /// private AVFrame* _pDstFrame = null; /// /// libswscale colorspace/resolution conversion context. /// Performs RGBA (Unity) → YUV420P/NV12 (encoder) conversion. /// sws_scale is used even when only converting colorspace without changing resolution. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// private SwsContext* _pSwsContext = null; /// Accumulated index of encoded frames. Used as the PTS (Presentation Time Stamp) value. private long _frameIndex; /// /// Whether the encoder accepts frames: set to true when Start() completes, /// and to false when End() runs (End() is not repeated) or Dispose() releases the resources. /// private bool _isInitialized; /// Whether Dispose() has already been called. Prevents duplicate cleanup. private bool _disposed; /// Lock object for synchronizing encoder state and resource access in multithreaded environments. private readonly object _lock = new(); /// /// Whether hw_frames_ctx mode (GPU frame pool) is active in the QSV encoder. /// If true, NV12 → QSV hardware surface upload is required every frame (SubmitDstFrame). /// If false, the frame in system memory () is passed directly to the encoder; /// this is the case for every non-QSV encoder and for QSV in system memory input mode. /// private bool _isQsvEncoder; /// /// Pixel format of the frames passed to the encoder (output of the color conversion). /// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane) /// - libvvenc: YUV420P10LE (10-bit, 3 planes) /// - Other encoders (software encoders, VideoToolbox): YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling) /// Set when the selected encoder opens; read by FrameReader (through SelectedPixelFormat) to choose the GPU conversion. /// /// Reference - Major pixel format relationships: /// RGBA (Unity source, 4-channel 8-bit) /// ↓ sws_scale conversion /// YUV420P (3 planes Y/U/V, 1/4 chroma resolution) or /// NV12 (1 Y plane + 1 UV interleaved plane, 1/4 chroma resolution) /// /// Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html /// private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P; /// /// Color range of the encoder input and output (codec context color_range of the selected encoder). /// Full range (JPEG) for every encoder except libvvenc, limited range (MPEG) for libvvenc. /// private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG; /// /// Pixel format of the finally selected and running encoder (the format of the frames the encoder receives). /// A pre-converted input passed with isColorConverted = true has to be in this format (NV12 or YUV420P). /// Valid after Start() returns; AV_PIX_FMT_YUV420P before. /// public AVPixelFormat SelectedPixelFormat => _dstPixFmt; /// /// Packet structure to temporarily hold encoded packet data. /// Cached as a member variable to prevent allocation/deallocation overhead for each frame. /// private AVPacket* _pPacket = null; /// /// GPU frame structure used in QSV hardware acceleration mode. /// Cached as a member variable to prevent allocation/deallocation overhead for each frame. /// Allocated only in GPU frame pool input mode; holds a pool surface only while one frame is uploaded and sent. /// private AVFrame* _pHwFrame = null; /// /// Whether the conversion path (GPU pre-converted or CPU RGBA) has been logged. /// The path is logged once, on the first frame after Start(). /// private bool _loggedConversionMode; // ────────────────────────────────────────────────────────────── // FFmpeg Struct Pointers for Audio Encoding // ────────────────────────────────────────────────────────────── /// /// Context managing audio codec (encoder) state. /// Audio encoding parameters such as sample rate, channel layout, and sample format are configured here. /// private AVCodecContext* _pAudioCodecContext = null; /// Audio stream within the output file. Owned by and freed with it. private AVStream* _pAudioStream = null; /// Packet structure to temporarily hold encoded audio packet data. private AVPacket* _pAudioPacket = null; /// /// Audio resampling context (SwrContext). /// Converts input PCM format (e.g., float interleaved) → encoder required format (e.g., float planar). /// Input and output use the same sample rate and channel layout, so only the sample format and layout in memory change. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html /// private SwrContext* _pSwrContext = null; /// /// Audio FIFO buffer. Collects input samples of arbitrary sizes and passes them in frame_size units required by the encoder. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html /// private AVAudioFifo* _pAudioFifo = null; /// /// Frame structure to pass to the audio encoder. Reused for every audio frame; /// holds frame_size samples per channel in the encoder sample format. /// private AVFrame* _pAudioFrame = null; /// Accumulated index of encoded audio samples (per channel). Used as the PTS value (time base 1 / sample rate). private long _audioSampleIndex; /// Whether audio encoding is enabled. private bool _audioEnabled; /// Bytes of one input sample of every channel (interleaved input format × channel count). private int _audioInputBytesPerSample; /// Encoding settings passed at construction time (immutable). private readonly EncoderSettings _settings; /// /// Stores the encoding settings. No FFmpeg resource is allocated and the output file is not created until Start(). /// /// Video (and optional audio) encoding settings. Copied (struct). public FFmpegNativeEncoder(EncoderSettings settings) { _settings = settings; } // ================================================================ // Utility Methods // ================================================================ /// /// Returns a list of candidate encoders to try for the selected codec in priority order. /// /// Order (GPUUtil.GetPrioritizedEncoders): /// 1. Hardware encoders of the active GPU vendor /// 2. The remaining encoders in the declaration order of VideoEncoder.All /// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has) /// With ForceSoftware, only the software encoders are returned. /// /// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based) /// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework) /// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK / oneVPL) /// - videotoolbox: Hardware encoder of Apple platforms (macOS) /// - lib*: CPU-based software encoders (available when the FFmpeg build includes the library, but slower) /// /// Iterates through this list in Start() to use the first workable encoder. /// private VideoEncoder[] GetEncoderCandidates() { return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray(); } /// /// Expands the candidate list into (encoder, input mode) attempts in order. /// A QSV encoder is attempted twice: system memory input mode (useFramePool = false) first, /// then GPU frame pool input mode (useFramePool = true). Other encoders are attempted once (useFramePool = false). /// private static System.Collections.Generic.IEnumerable<(VideoEncoder Encoder, bool UseFramePool)> ExpandQsvInputModes(VideoEncoder[] candidates) { foreach (var encoder in candidates) { yield return (encoder, false); if (encoder.Kind == EncoderKind.QSV) { yield return (encoder, true); } } } /// /// Converts an FFmpeg error code (negative integer) to a human-readable string. /// e.g., -1 → "Operation not permitted" /// /// Calls av_strerror() internally to write the message to a stack-allocated buffer (1024 bytes, NUL-terminated). /// Returns "Unknown error (Code: n)" when av_strerror has no description for the code. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html /// private string GetFFmpegErrorString(int errnum) { const ulong bufferSize = 1024; byte* buffer = stackalloc byte[(int)bufferSize]; if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0) { return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})"; } return $"Unknown error (Code: {errnum})"; } // ================================================================ // Encoder Initialization (Start) // ================================================================ /// /// Initializes the entire encoding pipeline. /// /// Execution steps: /// 1. Create container format context (AVFormatContext) for the output file /// 2. Add video stream (AVStream) /// 3. Iterate encoder candidates → configure codec context (AVCodecContext) → open encoder /// 4. Copy codec parameters to stream /// 5. Open output file and write header /// 6. Initialize frame buffers (AVFrame) and colorspace converter (SwsContext) /// /// On failure, calls Dispose() to clean up partially initialized resources before rethrowing the exception. /// The encoder object cannot be started again after a failure (it is disposed); a new instance is required. /// A file created by avio_open before the failure is left on disk (the caller deletes incomplete output). /// Does nothing when the encoder is already started or disposed. /// /// /// Output video file path (e.g., "C:/output/video.mp4"). The extension selects the container (mp4, mkv, webm, ...). /// /// /// No container matches the extension, no candidate encoder can be opened, the file cannot be opened, /// or the header cannot be written. Native library load failures (NotSupportedException) are wrapped with a hint /// about the FFmpeg DLLs in the UserLibs folder. /// public void Start(string outputPath) { lock (_lock) { if (_isInitialized || _disposed) return; try { // ── Step 1: Create output container format context ── // avformat_alloc_output_context2 automatically selects an appropriate // output format (mp4, mkv, etc.) based on the file extension (outputPath). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html fixed (AVFormatContext** ppFormatCtx = &_pFormatContext) { ffmpeg.avformat_alloc_output_context2(ppFormatCtx, null, null, outputPath); } if (_pFormatContext == null) throw new Exception("Failed to create output format context."); // ── Step 2: Add video stream ── // Creates a single video track (stream) inside the container (stream index 0). // The audio stream, when enabled, is added after the encoder is opened (Step 4.5). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html _pVideoStream = ffmpeg.avformat_new_stream(_pFormatContext, null); if (_pVideoStream == null) throw new Exception("Failed to create video stream."); // ── Step 3: Select and open encoder ── // Iterates through the priority list returned by GetEncoderCandidates() // to find the first available encoder. // Each attempt allocates a new codec context. An attempt that fails frees its codec context // (and its hardware device/frames references) before the next one, so after the loop // _pCodecContext is either the opened encoder or null. var candidates = GetEncoderCandidates(); var openedSuccessfully = false; foreach (var (encoder, useFramePool) in ExpandQsvInputModes(candidates)) { // avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc"). // Returns null if the encoder is not included in the build. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html var pCodec = ffmpeg.avcodec_find_encoder_by_name(encoder.Name); if (pCodec == null) { Melon.Logger.Warning($"Encoder '{encoder.Name}' not found in FFmpeg library. Testing next candidate codec."); continue; } // avcodec_alloc_context3: Allocates a codec context for the selected encoder. // All encoding parameters such as resolution, framerate, quality, etc., are set on this context. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html _pCodecContext = ffmpeg.avcodec_alloc_context3(pCodec); if (_pCodecContext == null) continue; // FF_COMPLIANCE_EXPERIMENTAL: allows avcodec_open2 to open encoders that libavcodec marks // as experimental (AV_CODEC_CAP_EXPERIMENTAL); at a stricter level avcodec_open2 returns AVERROR_EXPERIMENTAL. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/avcodec.c (avcodec_open2) _pCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL; // ── Basic video parameters configuration ── _pCodecContext->width = _settings.Width; _pCodecContext->height = _settings.Height; // ── Output color tags ── // Matrix: BT.709, the RGB→YUV matrix of both conversion paths // (GPU: RGB2YUV_BT709_Full in ColorConversion.compute, CPU: ConfigureSwsColorspace). // Players use this tag to convert the YUV frames back to RGB. // Range: full (0-255) for every encoder except libvvenc, limited (16-235) for libvvenc. // The FFmpeg libvvenc wrapper passes the matrix/primaries/transfer tags to VVenC but not the range, // so a VVC stream is read as limited range; its input is converted to limited range to match. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_color_format) _pCodecContext->color_range = encoder == VideoEncoder.LibVvenc ? AVColorRange.AVCOL_RANGE_MPEG : AVColorRange.AVCOL_RANGE_JPEG; _pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709; // thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count. // For libvvenc, the FFmpeg wrapper passes a positive thread_count to VVenC as its thread count // (m_numThreads); with 0 the wrapper leaves the VVenC default thread setting in place. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c _pCodecContext->thread_count = 0; // ── Pixel format configuration ── // Video encoders generally expect input in YUV420P or NV12 format. // // • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE) // and by VideoToolbox. // • NV12: Y plane + UV interleaved plane. Used by the hardware encoders NVENC, AMF and QSV. // selectedPixFmt is the format of the frames this class produces (_dstPixFmt once the encoder opens); // _pCodecContext->pix_fmt is the format the encoder is told to accept. // • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is // AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode: // system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly. // GPU frame pool input mode: pix_fmt stays AV_PIX_FMT_QSV, and SubmitDstFrame() uploads // each NV12 frame to a QSV surface with av_hwframe_transfer_data. // // Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html var selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P; if (encoder.Kind == EncoderKind.QSV) { selectedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12; _pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_QSV; } else if (encoder.Kind is EncoderKind.NVENC or EncoderKind.AMF) { selectedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12; _pCodecContext->pix_fmt = selectedPixFmt; } else if (encoder == VideoEncoder.LibVvenc) { // libvvenc: Only 10-bit YUV420P10LE supported (the wrapper's pix_fmts list is AV_PIX_FMT_YUV420P10 only). // Pre-converted GPU input is NV12/YUV420P only, so libvvenc receives its frames through the // CPU conversion (sws_scale) path; WritePreConvertedFrame rejects this format. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P10LE; _pCodecContext->pix_fmt = selectedPixFmt; } else { _pCodecContext->pix_fmt = selectedPixFmt; } // ── Framerate and timebase configuration ── // time_base: Base unit of time interval between frames. // e.g., 60fps → time_base = 1/60 sec (num=1, den=60) // Multiplying PTS (Presentation Time Stamp) value by time_base yields actual time in seconds. // PTS=150, time_base=1/60 → 150 * (1/60) = 2.5 seconds // // framerate: Framerate value giving hints to the encoder (reciprocal of time_base). // // RationalUtil: Converts FPS (double) into fractional form (numerator/denominator). // e.g., 60.00 -> 60/1 // // Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html var fpsRational = RationalUtil.Double2MaxPrecisionInt(_settings.FPS); _pCodecContext->time_base = new AVRational { num = fpsRational.Denominator, den = fpsRational.Numerator }; _pCodecContext->framerate = new AVRational { num = fpsRational.Numerator, den = fpsRational.Denominator }; // ── GOP (Group of Pictures) size configuration ── // GOP: Keyframe (I-frame) interval. e.g., gop_size=120, 60fps → keyframe every 2 seconds. // Keyframes contain full images and serve as seek points. // Too large reduces seek precision, too small increases file size. var calculatedGop = Math.Max(1, (int)Math.Round(_settings.FPS * _settings.GopSize)); if (encoder == VideoEncoder.LibVvenc) { // gop_size for libvvenc: calculatedGop rounded to the nearest multiple of 32 (an exact half, e.g. 48, rounds down), at least 32. // VVenC places intra frames at multiples of its GOP size (32 in its default GOP structure). // The FFmpeg 8.0 libvvenc wrapper reads gop_size only to select all-intra coding (gop_size == 1); // for any other value the intra period comes from the wrapper's "period" option // (seconds, default 1), so this value does not change the keyframe interval of the output. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (m_IntraPeriodSec) _pCodecContext->gop_size = Math.Max(32, ((calculatedGop + 15) / 32) * 32); } else { _pCodecContext->gop_size = calculatedGop; } // ── B-frame configuration ── // B-frames (bidirectional predictive frames) reference both earlier and later frames. // They raise the compression ratio and make the encoder output packets out of presentation order // (dts < pts for reordered frames). // // Configuration per encoder: // SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure). // The FFmpeg libsvtav1 wrapper does not read max_b_frames; the prediction structure is set // through svtav1-params, which the wrapper passes to svt_av1_enc_parse_parameter. // libvvenc: max_b_frames = -1 (encoder default GOP structure). // Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order). // // has_b_frames is set by the encoder itself and is not written here. // // Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md // (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2) // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libsvtav1.c (config_enc_params) if (encoder == VideoEncoder.LibSvtAv1) { ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0); } else if (encoder == VideoEncoder.LibVvenc) { // libvvenc uses the hierarchical GOP structure of VVenC (reordered frames). // max_b_frames = -1 leaves the B-frame count to the encoder. The FFmpeg 8.0 wrapper does not read // max_b_frames, so the GOP structure is the VVenC default in either case. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c _pCodecContext->max_b_frames = -1; } else { _pCodecContext->max_b_frames = 0; } // ── Global header flag ── // Some container formats (mp4, etc.) require codec configuration info (SPS/PPS, etc.) // to be recorded once in the file header rather than in each packet. // If format has AVFMT_GLOBALHEADER flag set, turn on AV_CODEC_FLAG_GLOBAL_HEADER // so the encoder stores configuration info in extradata. // Reference: https://ffmpeg.org/doxygen/trunk/avformat_8h.html if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0) { _pCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER; } // ── Apply quality/bitrate settings ── // Writes bit_rate / rc_* fields or the encoder's private quality option. // Private options are set on priv_data before avcodec_open2, which reads them. ApplyQualitySettings(encoder); // ── QSV Encoder Hardware Device Initialization ── // QSV (Quick Sync Video) uses fixed-function encoding hardware on Intel GPUs. // Using QSV in FFmpeg requires creating a hardware device context. // // 3 strategies are attempted sequentially below (child device: D3D11VA on Windows, VAAPI elsewhere): // Strategy 1) Create QSV device directly with child_device_type=d3d11va (or vaapi) option // → FFmpeg internally creates the child device and connects it to QSV // Strategy 2) Create the D3D11VA (or VAAPI) device first, then derive QSV device // → Effective when Strategy 1 fails on some drivers // Strategy 3) Create QSV device directly (legacy method) // → Fallback on older drivers/systems // // After obtaining device, the input mode is configured (see ExpandQsvInputModes): // Input mode 1, hw_device_ctx: Encoder receives NV12 frames from system memory (attempted first) // Input mode 2, hw_frames_ctx: Frames are uploaded to a QSV surface pool in GPU memory // (attempted when the encoder cannot be opened in input mode 1) // // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html if (encoder.Kind == EncoderKind.QSV) { // hwDeviceCtx: local reference to the QSV device, released below after the codec context // (or the frames context) has taken its own reference. AVBufferRef* hwDeviceCtx = null; var qsvReady = false; int err; var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); string childDeviceName = isWindows ? "d3d11va" : "vaapi"; AVHWDeviceType childDeviceType = isWindows ? AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA : AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI; // ── Strategy 1: child_device_type option ── // Passing "child_device_type" dictionary to av_hwdevice_ctx_create // lets FFmpeg internally create an appropriate lower device (Windows: D3D11, Linux: VAAPI) and connect to QSV session. { AVDictionary* opts = null; ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0); err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0); ffmpeg.av_dict_free(&opts); if (err >= 0) { qsvReady = true; Melon.Logger.Msg($"{encoder.Name}: Successfully created QSV device using child_device_type={childDeviceName}"); } else { Melon.Logger.Warning($"{encoder.Name}: child_device_type failed — {GetFFmpegErrorString(err)} (Code: {err})"); } } // ── Strategy 2: Lower device → QSV derive ── // Create lower device (D3D11VA / VAAPI) first, // then derive QSV device using av_hwdevice_ctx_create_derived. if (!qsvReady) { AVBufferRef* childDeviceCtx = null; err = ffmpeg.av_hwdevice_ctx_create(&childDeviceCtx, childDeviceType, null, null, 0); if (err >= 0) { err = ffmpeg.av_hwdevice_ctx_create_derived(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, childDeviceCtx, 0); if (err >= 0) { qsvReady = true; Melon.Logger.Msg($"{encoder.Name}: Successfully created device by deriving {childDeviceType} → QSV"); } // The derived QSV device holds its own reference to the child device, // so the local reference is released whether or not the derivation succeeded. ffmpeg.av_buffer_unref(&childDeviceCtx); } } // ── Strategy 3: Direct QSV creation (legacy) ── // No options: FFmpeg selects the child device type of the platform by itself. if (!qsvReady) { err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0); if (err >= 0) qsvReady = true; } if (!qsvReady) { Melon.Logger.Warning($"{encoder.Name}: All QSV device creation strategies failed. Testing next candidate codec."); { var pCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCtx); } _pCodecContext = null; continue; } if (!useFramePool) { // ── Input mode 1 (primary): hw_device_ctx + NV12 system memory ── // The encoder receives NV12 frames in system memory (RAM) and the QSV runtime uploads them to the GPU. // // pix_fmt is the software format NV12. // _isQsvEncoder = false: WriteFrame() passes the NV12 frame to the encoder without uploading it. _pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12; _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx); _isQsvEncoder = false; Melon.Logger.Msg($"{encoder.Name}: Using system memory input mode (hw_device_ctx)"); } else { // ── Input mode 2 (fallback): hw_frames_ctx GPU frame pool ── // Used when the encoder cannot be opened in input mode 1. // AVHWFramesContext pre-allocates a pool of QSV surfaces in GPU memory (pix_fmt AV_PIX_FMT_QSV), // and WriteFrame() uploads each NV12 frame to a surface of the pool (av_hwframe_transfer_data). // // Pool creation depends on the texture types the GPU/driver accepts // (e.g. Intel Arc A770 rejects NV12 textures with D3D11_BIND_RENDER_TARGET → E_INVALIDARG 0x80070057). // // initial_pool_size: Number of surfaces in the pool (QsvFramePoolSize). // The encoder keeps submitted surfaces until they are encoded, so the pool has to hold // the frames in the encoder pipeline plus the frame being uploaded. // Width/height are aligned to multiples of 32: texture alignment condition required by QSV hardware. // // Frame pool memory type (AVQSVFramesContext.frame_type, a combination of MFX_MEMTYPE_* flags): // Windows (D3D11): MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010) is tried first, then 0. // 0x0010 creates the D3D11 textures with D3D11_BIND_DECODER. // 0 lets FFmpeg select MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET (0x0020), // which creates the textures with D3D11_BIND_RENDER_TARGET. // Other platforms (VAAPI): 0. FFmpeg sets the memory type of a VAAPI child pool itself. // Both types are accepted by the QSV encoder as video memory input. // When every type fails, the next candidate encoder is tested. // // Reference: // AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html // av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html // AVQSVFramesContext.frame_type: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h // frame_type → D3D11 bind flags (qsv_init_child_ctx, qsv_get_d3d11va_bind_flags): https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.c // MFX_MEMTYPE_* values: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h bool framesCtxReady = false; int[] frameTypes = isWindows ? new[] { QsvMemTypeVideoMemoryDecoderTarget, 0 } : new[] { 0 }; foreach (var frameType in frameTypes) { AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx); if (hwFramesRef == null) break; var framesCtx = (AVHWFramesContext*)hwFramesRef->data; framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask) framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask) framesCtx->initial_pool_size = QsvFramePoolSize; SetQsvFrameType(framesCtx, frameType); err = ffmpeg.av_hwframe_ctx_init(hwFramesRef); if (err >= 0) { // Success: Connect hw_frames_ctx to codec context. // Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released. _pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef); _isQsvEncoder = true; framesCtxReady = true; Melon.Logger.Msg($"{encoder.Name}: Using GPU frame pool input mode (hw_frames_ctx, frame_type=0x{frameType:X4}, pool={QsvFramePoolSize})"); ffmpeg.av_buffer_unref(&hwFramesRef); _pHwFrame = ffmpeg.av_frame_alloc(); if (_pHwFrame == null) { // The codec context (and the frames context it holds) is released by Dispose in the catch block of Start. ffmpeg.av_buffer_unref(&hwDeviceCtx); throw new Exception("Failed to allocate QSV hardware frame structure."); } break; } Melon.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed with frame_type=0x{frameType:X4} — {GetFFmpegErrorString(err)} (Code: {err})"); ffmpeg.av_buffer_unref(&hwFramesRef); } if (!framesCtxReady) { Melon.Logger.Warning($"{encoder.Name}: GPU frame pool input mode is unavailable. Testing next candidate codec."); ffmpeg.av_buffer_unref(&hwDeviceCtx); { var pCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCtx); } _pCodecContext = null; continue; } } // Release the local reference of hwDeviceCtx. // The device stays alive through the references taken above: // system memory input mode: codec context hw_device_ctx (av_buffer_ref) // GPU frame pool input mode: the frames context (av_hwframe_ctx_alloc keeps a device reference), // held by codec context hw_frames_ctx // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html ffmpeg.av_buffer_unref(&hwDeviceCtx); } // ── Specify per-encoder speed preset ── // Presets determine the tradeoff between encoding speed ↔ compression efficiency. // Fast preset: Fast encoding, but larger file size // Slow preset: Slow encoding, but smaller file size at same quality // // Reference: https://ffmpeg.org/doxygen/trunk/group__opt__set__funcs.html if (encoder.Kind == EncoderKind.NVENC) { // NVENC preset range: p1 (fastest) ~ p7 (highest quality). p4 = medium (default) // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/nvenc_h264.c ("preset") ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "p4", 0); } else if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265) { // x264/x265 presets: ultrafast, superfast, veryfast, faster, fast, medium, slow, ... // CPU encoding easily becomes a bottleneck, so use fast presets ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "veryfast", 0); } else if (encoder == VideoEncoder.LibSvtAv1) { // SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality), library default 8. // Negative presets are for debugging; the FFmpeg option default -2 keeps the library default. // Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md (EncoderMode, --preset) ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "10", 0); } else if (encoder == VideoEncoder.LibAomAv1) { // libaom cpu-used: 0 (highest quality) ~ 8 (highest speed), FFmpeg option default 1 // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libaomenc.c ("cpu-used", "usage") ffmpeg.av_opt_set(_pCodecContext->priv_data, "cpu-used", "6", 0); // "realtime" usage (AOM_USAGE_REALTIME, value 1): Single pass, low latency encoding mode. // The other values are "good" (default) and "allintra". ffmpeg.av_opt_set(_pCodecContext->priv_data, "usage", "realtime", 0); } else if (encoder == VideoEncoder.LibVvenc) { // libvvenc preset: slower, slow, medium, fast, faster. Default is medium, change to faster to gain performance // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c ("preset") ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "faster", 0); } // Other encoders (AMF, QSV, VideoToolbox, libvpx-vp9) use their default presets / speed settings. // Setting unsupported options can cause errors, so skip. // The av_opt_set results above are not checked: an option the encoder does not accept leaves its default. // ── Open encoder ── // avcodec_open2: Initializes encoder based on all parameters configured above. // Returns >= 0 on success, enabling frame encoding via avcodec_send_frame/avcodec_receive_packet afterwards. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html Melon.Logger.Msg($"[Encoder Diagnostic] {encoder.Name}: {_pCodecContext->width}x{_pCodecContext->height}, pix_fmt={_pCodecContext->pix_fmt}, gop={_pCodecContext->gop_size}, max_b_frames={_pCodecContext->max_b_frames}, framerate={_pCodecContext->framerate.num}/{_pCodecContext->framerate.den}, thread_count={_pCodecContext->thread_count}, bit_rate={_pCodecContext->bit_rate}, rc_max_rate={_pCodecContext->rc_max_rate}, rc_buffer_size={_pCodecContext->rc_buffer_size}, mode={_settings.Mode}"); var openResult = ffmpeg.avcodec_open2(_pCodecContext, pCodec, null); if (openResult >= 0) { var isHardware = encoder.IsHardware; var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage"; Melon.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})"); _dstPixFmt = selectedPixFmt; _colorRange = _pCodecContext->color_range; openedSuccessfully = true; break; } // If open fails, free codec context and QSV frame state, then try next candidate. // avcodec_free_context also releases the hw_device_ctx / hw_frames_ctx references of the context, // which releases the QSV device and frame pool of this attempt. var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : ""; Melon.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate."); var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; _isQsvEncoder = false; if (_pHwFrame != null) { var pHwFrame = _pHwFrame; ffmpeg.av_frame_free(&pHwFrame); _pHwFrame = null; } } if (!openedSuccessfully || _pCodecContext == null) { throw new Exception("Failed to initialize any available video encoder (both hardware accelerated and CPU software)."); } // ── Step 4: Copy codec parameters to stream ── // Copy parameters set in codec context (resolution, codec ID, profile, etc.) // to stream's codecpar. Information to be written to container header. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext); if (paramResult < 0) throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})"); // Stream time base: unit of the timestamps stored in the container. // Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks. // When it is left unset, the muxer uses 1/90000, in which the interval of rates such as // 59.94 FPS (1501.5 ticks), 23.976 FPS (3753.75 ticks) or 165 FPS (545.45 ticks) is not an integer, // and the frame durations alternate between two values. // The muxer may still adjust the value in avformat_write_header (e.g. mp4 scales it to at least 10000); // packet timestamps are rescaled to the final value in SendFrameAndWritePackets. // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavformat/mux.c (init_muxer: default 1/90000) _pVideoStream->time_base = _pCodecContext->time_base; // ── Step 4.5 (Optional): Initialize audio stream ── // If EncoderSettings.Audio is set, add audio stream. // Must run before avformat_write_header: streams cannot be added after the header is written. if (_settings.Audio.HasValue) { InitializeAudioStream(_settings.Audio.Value); } // ── Step 5: Open output file I/O ── // Formats without AVFMT_NOFILE flag require actual file I/O. // Open file in write mode using avio_open and connect to pb (I/O context) of format context. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0) { var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE); if (avioResult < 0) throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})"); } // Write file header (container metadata; for mp4 the ftyp box, the moov index is written by av_write_trailer). // Packets can be written with av_interleaved_write_frame after this point. // The muxer may change the stream time bases here; packets are rescaled to the final values when written. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null); if (headerResult < 0) throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})"); // ── Step 6: Initialize frame buffers and colorspace converter ── // av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html _pSrcFrame = ffmpeg.av_frame_alloc(); _pDstFrame = ffmpeg.av_frame_alloc(); _pPacket = ffmpeg.av_packet_alloc(); if (_pSrcFrame == null || _pDstFrame == null || _pPacket == null) throw new Exception("Failed to allocate frame or packet structure."); AllocateDstFrameBuffer(); // sws_getContext: Initialize software scaler (colorspace converter). // Input: RGBA (Unity capture format, 4 bytes/pixel) // Output: YUV420P or NV12 (encoder required format) // // Colorspace is converted without resolution change here, but sws_scale // can also perform resizing simultaneously. // // SWS_POINT: Nearest neighbor sampling. The luma plane keeps the source resolution (1:1). // The chroma planes are subsampled to half width and height (4:2:0) with the same filter, // so each chroma sample takes one source pixel instead of averaging the 2x2 block. // This scaler is used only by the CPU conversion path (WriteFrame with RGBA input). // It is created for every encoder, also when all frames arrive pre-converted. // For libvvenc the output is YUV420P10LE (8-bit RGBA values scaled to 10 bits). // // Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html _pSwsContext = ffmpeg.sws_getContext( _settings.Width, _settings.Height, AVPixelFormat.AV_PIX_FMT_RGBA, _settings.Width, _settings.Height, _dstPixFmt, (int)SwsFlags.SWS_POINT, null, null, null); if (_pSwsContext == null) throw new Exception("Failed to initialize software scaler."); // BT.709 matrix and the encoder color range for the RGBA → YUV conversion (see ConfigureSwsColorspace). ConfigureSwsColorspace(); _isInitialized = true; _loggedConversionMode = false; } catch (NotSupportedException ex) { // FFmpeg.AutoGen throws NotSupportedException when a native function cannot be bound // (DLL missing, wrong version or corrupted). Dispose(); throw new Exception("Failed to load FFmpeg native library (DLL). Please check if FFmpeg DLL files (avcodec, avformat, etc.) in UserLibs folder are missing or corrupted, delete the DLL files in UserLibs, and restart the game.", ex); } catch (Exception) { // Clean up all partially allocated resources if failure occurs during initialization. Dispose(); throw; } } } // ================================================================ // Quality / Bitrate Settings // ================================================================ /// /// Sets quality parameters on the codec context according to the rate control mode (CBR, VBR, CQP) selected by the user. /// /// ── Bitrate Control Mode Description ── /// /// • CBR (Constant Bitrate): Maintains a constant bitrate. Suitable for streaming. /// → bit_rate = rc_max_rate = rc_min_rate (all identical), rc_buffer_size = 1 second of the bitrate /// → libx264/libx265 additionally receive HRD/CBR options. Other encoders select their rate control mode /// from these generic fields according to their own FFmpeg wrapper. /// /// • VBR (Variable Bitrate): Bitrate varies according to scene complexity. /// → bit_rate = target value, rc_max_rate = maximum allowed value, rc_buffer_size = 2 × rc_max_rate /// /// libvvenc receives bit_rate only in CBR/VBR (see the libvvenc notes in the method body). /// Bitrates in the settings are in kbps; the codec context fields are in bits per second (rc_buffer_size in bits). /// /// • CQP (Constant Quantization Parameter): Uses a fixed quantization value. /// → Lower value = higher quality/larger size, Higher value = lower quality/smaller size /// → Configuration method varies per encoder (see below) /// /// Reference - CQP options per encoder: /// NVENC: "qp" (single value applied to I/P/B frames) /// AMF: "qp_i" + "qp_p" (I-frame and P-frame configured separately) /// QSV: AVCodecContext.global_quality (Intel ICQ mode) /// Software: "crf" (Constant Rate Factor, smarter quality control) /// /// VideoToolbox encoders take the software branch ("crf") in CQP mode; when the encoder has no such option, /// TrySetOption logs that it was not applied and the encoder keeps its default rate control. /// /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// /// Encoder whose codec context (, allocated, not yet opened) is configured. /// The rate control mode is not CBR, VBR or CQP. private void ApplyQualitySettings(VideoEncoder encoder) { switch (_settings.Mode) { case RateControlMode.CBR: // kbps → bps conversion (FFmpeg bit_rate field is in bps) _pCodecContext->bit_rate = _settings.TargetBitrate * 1000L; // libvvenc: only bit_rate is set; VVenC's internal rate control handles the rest. // The FFmpeg 8.0 libvvenc wrapper reads bit_rate and, with VVenC 1.8.0 or later, rc_max_rate // (m_RCMaxBitrate); it does not read rc_min_rate or rc_buffer_size. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_rc_mode) if (encoder != VideoEncoder.LibVvenc) { // In CBR, set target/max/min bitrates all equal. // rc_buffer_size determines the bitrate buffering size (VBV/HRD buffer, in bits): // TargetBitrate × 1000 bits = one second of data at the target bitrate. _pCodecContext->rc_max_rate = _pCodecContext->rc_min_rate = _settings.TargetBitrate * 1000L; _pCodecContext->rc_buffer_size = _settings.TargetBitrate * 1000; // HRD (Hypothetical Reference Decoder) signaling for CBR output, used for streaming compatibility. // libx264: "nal-hrd=cbr" writes the NAL HRD information and pads filler data to keep the bitrate constant. // FFmpeg's option description: "Signal HRD information (requires vbv-bufsize; cbr not allowed in .mp4)". // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/libx264.c ("nal-hrd": none/vbr/cbr) // libx265: "x265-params" = "hrd=1" (--hrd: "Enable the signaling of HRD parameters to the decoder") // + "strict-cbr=1" (--strict-cbr: "Enables stricter conditions to control bitrate deviance // from the target bitrate in ABR mode"). // Reference: https://x265.readthedocs.io/en/master/cli.html if (encoder == VideoEncoder.Libx264) { TrySetOption("nal-hrd", "cbr"); } else if (encoder == VideoEncoder.Libx265) { TrySetOption("x265-params", "hrd=1:strict-cbr=1"); } // NVENC: without "rc", nvenc_setup_rate_control selects CONSTQP when "qp" is set and VBR otherwise; // bit_rate and rc_max_rate being equal does not select CBR. "rc=cbr" selects NV_ENC_PARAMS_RC_CBR. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/nvenc.c (nvenc_setup_rate_control), // https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/nvenc_h264.c ("rc" option, "cbr" constant) else if (encoder.Kind == EncoderKind.NVENC) { TrySetOption("rc", "cbr"); } } break; case RateControlMode.VBR: // kbps → bps conversion _pCodecContext->bit_rate = _settings.TargetBitrate * 1000L; // libvvenc: only bit_rate is set (see the CBR case for the fields its wrapper reads). if (encoder != VideoEncoder.LibVvenc) { // If MaxBitrate is larger than TargetBitrate, use as is; // otherwise set 1.5x TargetBitrate as max rate. var maxRate = _settings.MaxBitrate > _settings.TargetBitrate ? _settings.MaxBitrate * 1000L : _settings.TargetBitrate * 1500L; _pCodecContext->rc_max_rate = maxRate; // Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes // (in bits: two seconds of data at the maximum bitrate). _pCodecContext->rc_buffer_size = (int)(maxRate * 2); } break; case RateControlMode.CQP: // Valid quality range per encoder. The range of the private options is read from the // encoder's option definition (AVOption min/max); a declared minimum of -1 means "auto". // NVENC: "qp" H.264/HEVC 0~51, AV1 0~255 // (libavcodec/nvenc_h264.c, nvenc_hevc.c: "qp" -1~51 / libavcodec/nvenc_av1.c: "qp" -1~255) // The bounds of every encoder except QSV are read at run time (GetQualityRange); // the values listed here are those of the FFmpeg sources. // AMF: "qp_i" H.264/HEVC 0~51, AV1 0~255 // (libavcodec/amfenc_h264.c, amfenc_hevc.c: "qp_i" -1~51 / libavcodec/amfenc_av1.c: "qp_i" -1~255) // QSV: AVCodecContext.global_quality 1~51 // VVC: "qp" 0~63 (libavcodec/libvvenc.c: "qp" -1~63) // Software: "crf" x264/x265 0~51 (https://x265.readthedocs.io/en/master/cli.html: --crf 0..51.0), // VP9/AV1 0~63 (libavcodec/libvpxenc.c, libaomenc.c: "crf" -1~63 / libavcodec/libsvtav1.c: "crf" 0~63) // Reference: https://github.com/FFmpeg/FFmpeg/tree/master/libavcodec int qpMin, qpMax; if (encoder.Kind == EncoderKind.QSV) { // QSV runs in ICQ (Intelligent Constant Quality) mode when global_quality > 0 and rc_max_rate is 0; // the lower bound 1 keeps global_quality positive, so ICQ is selected (qsvenc.c select_rc_mode checks // this condition before the bitrate-based modes). The ICQ quality is clamped to 1~51 by FFmpeg: // libavcodec/qsvenc.c: q->param.mfx.ICQQuality = av_clip(avctx->global_quality, 1, 51); // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/qsvenc.c qpMin = 1; qpMax = 51; } else if (encoder.Kind == EncoderKind.NVENC) (qpMin, qpMax) = GetQualityRange("qp"); else if (encoder.Kind == EncoderKind.AMF) (qpMin, qpMax) = GetQualityRange("qp_i"); else if (encoder == VideoEncoder.LibVvenc) (qpMin, qpMax) = GetQualityRange("qp"); else (qpMin, qpMax) = GetQualityRange("crf"); // Software encoder (libx264, libx265, libvpx-vp9, etc.) // Default value 18 (used when QualityValue is negative): Quality close to "visually lossless" in H.264. // Provides similar level of quality in other codecs as well. // The value is clamped to [qpMin, qpMax] of the encoder; a clamped value is logged as a warning. const int defaultQuality = 18; var requestedQp = _settings.QualityValue; int qpValue; if (requestedQp < 0) { qpValue = Math.Clamp(defaultQuality, qpMin, qpMax); Melon.Logger.Warning($"{encoder.Name}: Invalid quality value {requestedQp}. Using {qpValue}."); } else { qpValue = Math.Clamp(requestedQp, qpMin, qpMax); if (qpValue != requestedQp) Melon.Logger.Warning($"{encoder.Name}: Quality value {requestedQp} is outside the supported range {qpMin}~{qpMax}. Using {qpValue}."); } if (encoder.Kind == EncoderKind.NVENC) { // NVENC: Applies same QP to I/P/B frames with single "qp" option TrySetOption("qp", qpValue.ToString()); } else if (encoder.Kind == EncoderKind.AMF) { // AMF: QP for I-frame (qp_i) and P-frame (qp_p) must be set separately. // Since B-frames are disabled, qp_b is not set. TrySetOption("qp_i", qpValue.ToString()); TrySetOption("qp_p", qpValue.ToString()); } else if (encoder.Kind == EncoderKind.QSV) { // QSV: ICQ (Intelligent Constant Quality) mode, configured with AVCodecContext.global_quality. _pCodecContext->global_quality = qpValue; } else if (encoder == VideoEncoder.LibVvenc) { // libvvenc: Specifies Constant QP mode with "qp" option TrySetOption("qp", qpValue.ToString()); } else { // Software encoders (libx264, libx265, libvpx-vp9, libsvtav1, libaom-av1) and VideoToolbox. // CRF (Constant Rate Factor): Smarter quality control than QP. // Automatically adjusts bitrate according to scene complexity to maintain consistent visual quality. // Range: 0 (lossless) ~ 51 (lowest quality) for x264/x265, 0~63 for VP9/AV1. Typically between 18 and 28. TrySetOption("crf", qpValue.ToString()); } break; default: throw new ArgumentOutOfRangeException(); } } /// /// Sets a private option (priv_data) of the encoder. /// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false. /// Must be called before avcodec_open2. The value string is parsed by av_opt_set according to the option type. /// Reference: https://ffmpeg.org/doxygen/trunk/group__opt__set__funcs.html /// /// Option name as declared by the encoder (e.g. "crf", "qp", "x265-params"). /// Option value as text. // ReSharper disable once UnusedMethodReturnValue.Local private bool TrySetOption(string name, string value) { var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0); if (ret >= 0) return true; Melon.Logger.Warning($"[Encoder] Option '{name}={value}' was not applied: {GetFFmpegErrorString(ret)} (Code: {ret})"); return false; } /// /// MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010): frames are in video memory and belong to video decoder render targets. /// Reference: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h /// private const int QsvMemTypeVideoMemoryDecoderTarget = 0x0010; /// /// Number of surfaces in the QSV GPU frame pool (AVHWFramesContext.initial_pool_size) of GPU frame pool input mode. /// Same value as the encoder frame pool of the FFmpeg hardware encoding example (initial_pool_size = 20). /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/doc/examples/vaapi_encode.c /// private const int QsvFramePoolSize = 20; /// /// Writes AVQSVFramesContext.frame_type of a QSV frames context (AVHWFramesContext.hwctx). /// FFmpeg.AutoGen has no AVQSVFramesContext type, so the field is written at its offset. /// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info /// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit). /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h /// Must be called before av_hwframe_ctx_init. Does nothing when hwctx is null. /// /// QSV frames context allocated by av_hwframe_ctx_alloc on a QSV device. /// Combination of MFX_MEMTYPE_* flags, or 0 to let FFmpeg choose. private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType) { if (framesCtx->hwctx == null) return; *(int*)((byte*)framesCtx->hwctx + IntPtr.Size + sizeof(int)) = frameType; } /// /// Largest declared maximum that is treated as an actual quality range. /// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used. /// The largest bounded quality range among the supported encoders is 0~255 (AV1 NVENC/AMF), while /// libx264/libx265 declare "crf" as -1~FLT_MAX (libavcodec/libx264.c, libx265.c). /// private const double MaxMeaningfulQualityLimit = 1000; /// /// Returns the valid range of a private quality option of the encoder from its option definition (AVOption min/max). /// e.g. 0~51 for H.264/HEVC, 0~63 for VP9/AV1/VVC, 0~255 for AV1 NVENC/AMF. /// The lower bound is at least 0 (a declared minimum of -1 means "auto"). /// When the option is missing or its declared maximum is unbounded (see ), returns 0~51, /// the QP scale of H.264/HEVC 8-bit (x264/x265 CRF 0~51). /// Reference: https://x265.readthedocs.io/en/master/cli.html (--crf 0..51.0) /// /// Name of the quality option in priv_data. private (int Min, int Max) GetQualityRange(string optionName) { const int fallbackMax = 51; var option = ffmpeg.av_opt_find(_pCodecContext->priv_data, optionName, null, 0, 0); if (option == null || !(option->max > 0 && option->max <= MaxMeaningfulQualityLimit)) return (0, fallbackMax); var min = Math.Max(0, (int)Math.Ceiling(option->min)); var max = (int)Math.Floor(option->max); return min <= max ? (min, max) : (0, fallbackMax); } // ================================================================ // Write Frame // ================================================================ /// /// Encodes a single frame receiving RGBA pixel data (byte* pointer) captured from Unity. /// /// Unity textures use Bottom-Left origin ((0,0) at bottom-left of screen), /// while video standards use Top-Left origin ((0,0) at top-left of screen). /// To resolve this, a "negative stride trick" is used: /// /// ── Negative stride (linesize) trick ── /// • Set data[0] to the start address of the last row (bottom-most line) of the buffer /// • Set linesize[0] to -stride (negative) /// → sws_scale reads rows in reverse, naturally performing vertical flip (vflip) /// → Highly efficient as no extra memory copy or separate flip operation is required /// /// stride = Width × 4 (RGBA has 4 bytes per pixel) /// /// The buffer is read synchronously by sws_scale inside this call and is not referenced after it returns. /// Does nothing when the encoder is not started, already ended or disposed. /// /// /// Starting address of Unity RGBA pixel data: Width × Height pixels, rows tightly packed (Width × 4 bytes), /// bottom row first. The length is not checked here; the public Send* methods validate it. /// public void WriteFrameFromBuffer(byte* buffer) { lock (_lock) { if (_disposed || !_isInitialized) return; // _pSrcFrame does not own memory: its data[0]/linesize[0] point into the caller's buffer for this call only. var stride = _settings.Width * 4; // RGBA = 4 bytes per pixel _pSrcFrame->data[0] = buffer + stride * (_settings.Height - 1); // Starting address of last row _pSrcFrame->linesize[0] = -stride; // Vertical flip with negative stride _pSrcFrame->width = _settings.Width; _pSrcFrame->height = _settings.Height; _pSrcFrame->format = (int)AVPixelFormat.AV_PIX_FMT_RGBA; WriteFrame(); } } /// /// Encodes a frame by obtaining a read-only pointer from Unity NativeArray<byte>. /// NativeArray is managed by Unity's native memory allocator, /// allowing processing of large amounts of data without GC (Garbage Collector) pressure. /// The array holds RGBA pixels in the layout of . /// /// RGBA frame data, at least Width × Height × 4 bytes. /// The array is smaller than one RGBA frame. public void WriteFrameFromNativeBuffer(NativeArray buffer) { lock (_lock) { if (_disposed || !_isInitialized) return; ValidateInputLength(buffer.Length, isColorConverted: false); WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr()); } } /// /// Number of bytes read from the input buffer of one frame. /// RGBA input (isColorConverted = false): Width × Height × 4 /// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV) /// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V) /// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it). /// The 4:2:0 formulas use integer halves of Width and Height (exact for even dimensions). /// /// True for pre-converted NV12/YUV420P input, false for RGBA input. private long GetRequiredInputLength(bool isColorConverted) { long w = _settings.Width, h = _settings.Height; if (!isColorConverted) return w * h * 4; return _dstPixFmt switch { AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2), AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2), _ => 0 }; } /// /// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it /// (GetRequiredInputLength), so that the copy does not read outside the buffer. /// private void ValidateInputLength(long length, bool isColorConverted) { var required = GetRequiredInputLength(isColorConverted); if (length < required) throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input."); } /// /// Encodes a frame from IntPtr (native pointer). /// Used by the renderer for frames read back into native memory (AsyncGPUReadback), /// and usable from external native plugins or custom capture sources. /// The memory is read synchronously within this call; the caller may reuse or free it after the call returns. /// Does nothing when the encoder is not started, already ended or disposed. /// /// Start of the frame data. /// Number of readable bytes at ; must be at least one frame (GetRequiredInputLength). /// /// True: the data is already in (NV12 or YUV420P, planes tightly packed, top row first). /// False: the data is RGBA, bottom row first (flipped while converting). /// /// is smaller than one frame. /// Pre-converted input while the encoder format is not NV12 or YUV420P. /// The encoder or the muxer reports an error. public void SendDataFromNativeBuffer(IntPtr pointer, int length, bool isColorConverted) { lock (_lock) { if (_disposed || !_isInitialized) return; if (!_loggedConversionMode) { _loggedConversionMode = true; Melon.Logger.Msg(isColorConverted ? "[Encoder] GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)" : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)"); } ValidateInputLength(length, isColorConverted); if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer()); else WriteFrameFromBuffer((byte*)pointer.ToPointer()); } } /// /// Encodes a frame from a managed byte[]. /// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer. /// The array is not referenced after the call returns (the renderer returns it to its ArrayPool). /// Data layout and exceptions as for . /// /// Frame data. May be larger than one frame (e.g. a pooled array). /// Number of valid bytes at the start of . /// True for pre-converted NV12/YUV420P data, false for RGBA. /// exceeds the array or is smaller than one frame. public void SendDataFromBuffer(byte[] buffer, int length, bool isColorConverted) { lock (_lock) { if (_disposed || !_isInitialized) return; if (length > buffer.Length) throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes)."); ValidateInputLength(length, isColorConverted); fixed (byte* pBuffer = buffer) { if (!_loggedConversionMode) { _loggedConversionMode = true; Melon.Logger.Msg(isColorConverted ? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)" : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)"); } if (isColorConverted) WritePreConvertedFrame(pBuffer); else WriteFrameFromBuffer(pBuffer); } } } /// /// Sets the RGB→YUV conversion of the CPU conversion path (_pSwsContext) to the BT.709 matrix /// and the color range of the encoder (_colorRange), so that it produces the same colors as the GPU path /// (RGB2YUV_BT709_Full in ColorConversion.compute) and as the tags of the output stream. /// A new libswscale context uses the BT.601 matrix (SWS_CS_DEFAULT = SWS_CS_ITU601) and the limited range. /// /// For RGB input, "table" holds the coefficients of the output YUV space and dstRange its range (1 = full, 0 = limited). /// The source (RGB) values, brightness, contrast and saturation keep the values of the context. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails, sws_getCoefficients) /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libswscale/swscale.h (SWS_CS_ITU709, SWS_CS_DEFAULT) /// private void ConfigureSwsColorspace() { int* currentInvTable; int* currentTable; int srcRange, dstRange, brightness, contrast, saturation; var ret = ffmpeg.sws_getColorspaceDetails(_pSwsContext, ¤tInvTable, &srcRange, ¤tTable, &dstRange, &brightness, &contrast, &saturation); if (ret < 0) { Melon.Logger.Warning($"[Encoder] Cannot read the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults."); return; } var coefficients = ffmpeg.sws_getCoefficients(ffmpeg.SWS_CS_ITU709); var invTable = new int_array4(); var table = new int_array4(); for (uint i = 0; i < 4; i++) { invTable[i] = currentInvTable[i]; table[i] = coefficients[i]; } var fullRange = _colorRange == AVColorRange.AVCOL_RANGE_JPEG; ret = ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, srcRange, table, fullRange ? 1 : 0, brightness, contrast, saturation); if (ret < 0) { Melon.Logger.Warning($"[Encoder] Cannot set the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults."); } } /// /// Sets the format, size, color range and matrix of the destination frame (_pDstFrame) and allocates a new pixel buffer for it. /// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers. /// Second argument 0 means default alignment. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer) /// private void AllocateDstFrameBuffer() { _pDstFrame->format = (int)_dstPixFmt; _pDstFrame->width = _settings.Width; _pDstFrame->height = _settings.Height; _pDstFrame->color_range = _colorRange; _pDstFrame->colorspace = AVColorSpace.AVCOL_SPC_BT709; var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0); if (ret < 0) throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})"); } /// /// Makes the pixel buffer of the destination frame (_pDstFrame) writable before the next frame is written into it. /// /// After avcodec_send_frame, the encoder may keep a reference to the buffer of the submitted frame /// until that frame is encoded (e.g. QSV in system memory input mode). The buffer is then shared and not writable /// (av_frame_is_writable returns 0). /// In that case the frame releases its reference (av_frame_unref) and a new buffer is allocated. /// The encoder keeps the previous buffer until it no longer needs it. The previous contents are not copied, /// because every plane is overwritten by the next frame. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_is_writable, av_frame_unref) /// private void PrepareDstFrameForWrite() { if (ffmpeg.av_frame_is_writable(_pDstFrame) != 0) return; ffmpeg.av_frame_unref(_pDstFrame); AllocateDstFrameBuffer(); } /// /// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data. /// /// Input layout (tightly packed, top row first, planes back to back): /// NV12: Y (Width × Height) | UV interleaved (Width × Height/2) /// YUV420P: Y (Width × Height) | U (Width/2 × Height/2) | V (Width/2 × Height/2) /// Each plane is copied into , whose rows can be padded (linesize ≥ row width). /// A plane is copied with one memory copy when the destination stride equals the row width, otherwise row by row. /// /// Start of the frame data; the length was validated by the caller. /// The encoder pixel format is neither NV12 nor YUV420P (e.g. YUV420P10LE). private void WritePreConvertedFrame(byte* buffer) { PrepareDstFrameForWrite(); var width = _settings.Width; var height = _settings.Height; switch (_dstPixFmt) { case AVPixelFormat.AV_PIX_FMT_NV12: { // Copy Y Plane var pSrcY = buffer; var pDstY = _pDstFrame->data[0]; var dstStrideY = _pDstFrame->linesize[0]; if (width == dstStrideY) { Buffer.MemoryCopy(pSrcY, pDstY, dstStrideY * height, width * height); } else { for (var y = 0; y < height; y++) { Buffer.MemoryCopy(pSrcY + y * width, pDstY + y * dstStrideY, width, width); } } // Copy UV Plane (positioned after Y plane size) // Each UV row holds Width/2 U/V pairs = Width bytes; there are Height/2 rows. var pSrcUV = buffer + width * height; var pDstUV = _pDstFrame->data[1]; var dstStrideUV = _pDstFrame->linesize[1]; var uvHeight = height / 2; if (width == dstStrideUV) { Buffer.MemoryCopy(pSrcUV, pDstUV, dstStrideUV * uvHeight, width * uvHeight); } else { for (var y = 0; y < uvHeight; y++) { Buffer.MemoryCopy(pSrcUV + y * width, pDstUV + y * dstStrideUV, width, width); } } break; } case AVPixelFormat.AV_PIX_FMT_YUV420P: { // Copy Y Plane var pSrcY = buffer; var pDstY = _pDstFrame->data[0]; var dstStrideY = _pDstFrame->linesize[0]; if (width == dstStrideY) { Buffer.MemoryCopy(pSrcY, pDstY, dstStrideY * height, width * height); } else { for (var y = 0; y < height; y++) { Buffer.MemoryCopy(pSrcY + y * width, pDstY + y * dstStrideY, width, width); } } // Copy U Plane (Width/2 × Height/2 bytes, positioned after the Y plane) var pSrcU = buffer + width * height; var pDstU = _pDstFrame->data[1]; var dstStrideU = _pDstFrame->linesize[1]; var chromaWidth = width / 2; var chromaHeight = height / 2; if (chromaWidth == dstStrideU) { Buffer.MemoryCopy(pSrcU, pDstU, dstStrideU * chromaHeight, chromaWidth * chromaHeight); } else { for (var y = 0; y < chromaHeight; y++) { Buffer.MemoryCopy(pSrcU + y * chromaWidth, pDstU + y * dstStrideU, chromaWidth, chromaWidth); } } // Copy V Plane (same size as U, positioned after the U plane) var pSrcV = pSrcU + chromaWidth * chromaHeight; var pDstV = _pDstFrame->data[2]; var dstStrideV = _pDstFrame->linesize[2]; if (chromaWidth == dstStrideV) { Buffer.MemoryCopy(pSrcV, pDstV, dstStrideV * chromaHeight, chromaWidth * chromaHeight); } else { for (var y = 0; y < chromaHeight; y++) { Buffer.MemoryCopy(pSrcV + y * chromaWidth, pDstV + y * dstStrideV, chromaWidth, chromaWidth); } } break; } default: throw new NotSupportedException($"This pixel format ({_dstPixFmt}) does not support direct copying of pre-converted buffers."); } SubmitDstFrame(); } /// /// Core logic for actual frame encoding. /// /// Processing flow: /// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable /// (Allocates a new buffer if encoder is still referencing previous frame's buffer) /// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion /// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface (SubmitDstFrame) /// 4. Pass to encoder after setting PTS (SubmitDstFrame) /// /// The source is , set up by WriteFrameFromBuffer. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale) /// private void WriteFrame() { PrepareDstFrameForWrite(); // ── Colorspace conversion: RGBA → YUV420P or NV12 ── // sws_scale parameters explanation: // _pSwsContext: Pre-initialized conversion context // _pSrcFrame->data: Input data plane array (RGBA uses data[0] only) // _pSrcFrame->linesize: Input stride per plane (bytes/row) // 0: First source row index to process // _settings.Height: Number of rows to process (= full height) // _pDstFrame->data: Output data plane array // _pDstFrame->linesize: Output stride per plane ffmpeg.sws_scale( _pSwsContext, _pSrcFrame->data, _pSrcFrame->linesize, 0, _settings.Height, _pDstFrame->data, _pDstFrame->linesize ); SubmitDstFrame(); } /// /// Passes the filled destination frame () to the encoder with the next video PTS /// and writes the packets the encoder returns. /// GPU frame pool input mode (QSV, _isQsvEncoder): the frame is uploaded to a pool surface (_pHwFrame), which is sent. /// Every other encoder / input mode: _pDstFrame itself is sent. /// _frameIndex is incremented when the frame's PTS is assigned, right before SendFrameAndWritePackets. /// /// Surface allocation, upload, encoding or packet writing fails. private void SubmitDstFrame() { if (_isQsvEncoder) { // ── QSV hw_frames_ctx mode: System memory → GPU surface upload ── // // av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool. // Fails (e.g. ENOMEM) when every surface of the pool is still held by the encoder (QsvFramePoolSize). // av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface. // // The frame is copied from system memory to the GPU surface on every call; // the encoder then reads the surface from GPU memory. // // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html ffmpeg.av_frame_unref(_pHwFrame); try { var ret = ffmpeg.av_hwframe_get_buffer(_pCodecContext->hw_frames_ctx, _pHwFrame, 0); if (ret < 0) throw new Exception($"Failed to allocate QSV hardware frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})"); ret = ffmpeg.av_hwframe_transfer_data(_pHwFrame, _pDstFrame, 0); if (ret < 0) throw new Exception($"Failed to upload QSV frame — {GetFFmpegErrorString(ret)} (Code: {ret})"); // PTS (Presentation Time Stamp): Point in time when this frame should be displayed. // Increments _frameIndex by 1, actual time calculated as PTS × time_base. _pHwFrame->pts = _frameIndex++; SendFrameAndWritePackets(_pHwFrame); } finally { // Release reference immediately after use so it can be returned to pool. // The encoder keeps its own reference to the surface until the frame is encoded. ffmpeg.av_frame_unref(_pHwFrame); } } else { // ── General Encoders (Software or hw_device_ctx mode) ── // Pass software frame directly to encoder. // In hw_device_ctx mode, encoder handles GPU upload internally. // avcodec_send_frame takes a new reference to (or a copy of) the frame data; _pDstFrame keeps its own // reference, and PrepareDstFrameForWrite allocates a new buffer if the old one is still shared. _pDstFrame->pts = _frameIndex++; SendFrameAndWritePackets(_pDstFrame); } } // ================================================================ // Encoder Send/Receive and Packet Writing // ================================================================ /// /// Encodes frames using FFmpeg's Send/Receive pattern and writes result packets to file. /// /// ── Send/Receive API Description ── /// Asynchronous encoding API introduced in FFmpeg 3.x: /// /// avcodec_send_frame(ctx, frame) /// → Pushes frame into encoder's internal queue. /// → Starts encoding once enough frames are gathered. /// /// avcodec_receive_packet(ctx, packet) /// → Pops encoded packet (compressed data). /// → EAGAIN: Output not ready yet (more frames needed) /// → EOF: Encoding complete (on flush) /// /// Since zero or more packets can result from a single send_frame call, /// receive_packet is repeated until EAGAIN/EOF is returned. /// /// ── Packet Timestamp Handling ── /// av_packet_rescale_ts: Converts PTS/DTS from codec's time_base to stream's time_base. /// Mandatory because codec and container may use different time_bases /// (Start sets the stream time base to the codec time base, but the muxer can change it in avformat_write_header). /// e.g., codec time_base=1/60, stream time_base=1/90000 → PTS 150 → PTS 225000 /// /// Errors: avcodec_send_frame errors (including AVERROR_EOF when called again after the flush) and packet /// receive/write errors are thrown as exceptions. avcodec_send_frame does not return EAGAIN here, /// because every call drains all available packets before returning. /// /// Reference: /// Send/Receive API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// Packet: https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html /// Interleaved Write: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html /// /// Frame to encode. Pass null to flush encoder (output delayed frames). private void SendFrameAndWritePackets(AVFrame* pFrame) { // Push frame to encoder internal queue var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame); if (response < 0) { throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})"); } // Pop all encoded packets and write to file. try { while (true) { response = ffmpeg.avcodec_receive_packet(_pCodecContext, _pPacket); // EAGAIN: Encoder needs more input frames → exit loop // EOF: Encoder flush complete (all delayed frames outputted) → exit loop if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break; if (response < 0) throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})"); // Packet duration: 1 tick of the codec time base (1 / framerate), the display time of one frame // at the constant frame rate of the output. Applied when the encoder leaves it unknown (0). // The encoders output 0 here because libavcodec clears the frame duration // unless AV_CODEC_FLAG_FRAME_DURATION is set. // The mp4/mov muxer takes the length of the last sample from the duration of the last packet; // with 0 the last frame has no display time and the edit list ends before it. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/encode.c (encode_send_frame_internal: frame duration) // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/movenc.c (ff_mov_write_packet: track_duration) if (_pPacket->duration <= 0) _pPacket->duration = 1; // Convert PTS/DTS from codec timebase → stream timebase ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base); _pPacket->stream_index = _pVideoStream->index; // av_interleaved_write_frame: Write packet to container. // "Interleaved" means packets from multiple streams (video+audio) are interleaved // and written in time order. This function is used even when only video is present. // It takes ownership of the packet data (the packet is left blank), and it buffers packets // until packets of the other streams with later timestamps arrive. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html (av_interleaved_write_frame) response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket); if (response < 0) throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})"); // Unref packet data reference for reuse in next receive_packet. ffmpeg.av_packet_unref(_pPacket); } } finally { // Safely unref packet reference even on abnormal exit (exception) ffmpeg.av_packet_unref(_pPacket); } } // ================================================================ // Encoding Termination (End) // ================================================================ /// /// Finalizes encoding. /// /// Most encoders queue frames in internal buffers (delayed frames) /// and perform encoding when enough frames have accumulated. In End(): /// /// 1. Send a null frame to switch encoder to "flush mode". /// → Encoder encodes and outputs all remaining frames in internal buffer. /// → Pops packets until receive_packet returns EOF. /// /// 2. av_write_trailer: Writes container trailer (end-of-file metadata). /// For MP4, the moov atom (index info) is completed here. /// ⚠️ If trailer is not written, the file becomes corrupted and unplayable! /// av_write_trailer also flushes the output buffer and returns the write error of the output (AVIOContext.error), if any. /// /// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete. /// End() runs once: after it returns or throws, further calls return immediately. /// End() does not release resources and does not close the file handle; Dispose() must still be called. /// Order: video flush → audio FIFO remainder and audio flush → trailer. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer) /// public void End() { lock (_lock) { if (_disposed || !_isInitialized) return; try { // 1. Flush delayed frames from video encoder (send null frame as "end of input" signal) SendFrameAndWritePackets(null); // 1.5. Flush remaining samples in audio FIFO + flush audio encoder if (_audioEnabled) { FlushAudioEncoder(); } // 2. Write container trailer (moov atom and file termination metadata) if (_pFormatContext != null) { var ret = ffmpeg.av_write_trailer(_pFormatContext); if (ret < 0) throw new Exception($"Failed to write the container trailer — {GetFFmpegErrorString(ret)} (Code: {ret})"); } } finally { _isInitialized = false; } } } // ================================================================ // Resource Cleanup (Dispose) // ================================================================ /// /// Releases every native resource. Safe to call more than once and after a failed Start(); also called by the finalizer. /// Cleanup order: /// /// 1. SwsContext (scaler) → independent, can be freed first /// 2. AVFrame (src/dst/hw) and AVPacket → the structures and their buffer references. /// Buffers are reference-counted: a buffer the encoder still references stays alive until the codec context is freed. /// 3. Video AVCodecContext (codec) → also releases its hw_frames_ctx/hw_device_ctx references /// (the QSV frame pool and device are destroyed when their last reference goes away) /// 4. Audio resources (DisposeAudioResources: resampler, FIFO, frame, packet, audio codec context) /// 5. AVFormatContext (format/IO) → closes the file handle (avio_closep), then frees the context and its streams /// /// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned. /// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it). /// /// Reference: /// av_frame_free: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html /// avcodec_free_context: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html /// avformat_free_context: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html /// avio_closep: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html /// public void Dispose() { lock (_lock) { if (_disposed) return; _disposed = true; _isInitialized = false; // Release Sws scaler if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } // Release source frame memory (frame referencing Unity RGBA data) if (_pSrcFrame != null) { var pFrame = _pSrcFrame; ffmpeg.av_frame_free(&pFrame); _pSrcFrame = null; } // Release destination frame memory (YUV420P/NV12 converted frame) if (_pDstFrame != null) { var pFrame = _pDstFrame; ffmpeg.av_frame_free(&pFrame); _pDstFrame = null; } // Release QSV hardware frame memory if (_pHwFrame != null) { var pFrame = _pHwFrame; ffmpeg.av_frame_free(&pFrame); _pHwFrame = null; } // Release packet memory if (_pPacket != null) { var pPacket = _pPacket; ffmpeg.av_packet_free(&pPacket); _pPacket = null; } // Release codec context // This function internally frees hw_frames_ctx and hw_device_ctx as well. if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; } // ── Release audio-related resources ── DisposeAudioResources(); // Close format context and I/O if (_pFormatContext != null) { // Must manually close avio when not AVFMT_NOFILE. // (Opened with avio_open, so close with avio_closep) if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0) { var pPb = _pFormatContext->pb; if (pPb != null) { ffmpeg.avio_closep(&pPb); _pFormatContext->pb = null; } } // Release format context (internal streams, etc. freed as well) ffmpeg.avformat_free_context(_pFormatContext); _pFormatContext = null; } GC.SuppressFinalize(this); } } /// /// Finalizer (destructor). Executed when collected by GC without Dispose() being called. /// Safety net to prevent native resource leaks. Runs on the GC finalizer thread; the output file is not finished. /// ⚠️ Finalizer execution timing is not guaranteed, so always manage resources /// using a using statement or explicit Dispose() call. /// ~FFmpegNativeEncoder() { Dispose(); } // ================================================================ // Audio Encoding // ================================================================ /// /// Initializes the audio stream. /// /// Execution steps: /// 1. Search audio encoder (codec) and allocate codec context /// 2. Set sample rate, channel layout, sample format, bitrate /// 3. Add audio stream (AVStream) and copy codec parameters /// 4. Initialize resampler (SwrContext) (input format → encoder format conversion) /// 5. Initialize FIFO buffer (arbitrary input size → collect in frame_size units) /// 6. Allocate audio frame and packet structures /// /// Called by Start() after the video encoder is opened and before the header is written. /// Defaults for unset values: codec AAC, 48000 Hz, 2 channels, 256000 bps, input format FLT. /// The encoder sample rate equals the input rate; a rate the codec does not support makes avcodec_open2 fail. /// Throws on any failure; Start() then disposes the encoder. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html /// /// Audio settings of . private void InitializeAudioStream(AudioSettings audioSettings) { // ── 1. Determine and search audio codec ── var audioCodecId = audioSettings.AudioCodecId != 0 ? audioSettings.AudioCodecId : AVCodecID.AV_CODEC_ID_AAC; var pAudioCodec = ffmpeg.avcodec_find_encoder(audioCodecId); if (pAudioCodec == null) throw new Exception($"Audio encoder not found: {audioCodecId}"); // ── 2. Allocate codec context and configure parameters ── _pAudioCodecContext = ffmpeg.avcodec_alloc_context3(pAudioCodec); if (_pAudioCodecContext == null) throw new Exception("Failed to allocate audio codec context."); var sampleRate = audioSettings.SampleRate > 0 ? audioSettings.SampleRate : 48000; var channels = audioSettings.Channels > 0 ? audioSettings.Channels : 2; var bitrate = audioSettings.Bitrate > 0 ? audioSettings.Bitrate : 256000; // Select the first sample format supported by the encoder. // Most encoders prefer AV_SAMPLE_FMT_FLTP (float planar). // AVCodec.sample_fmts is deprecated but still functional; avcodec_get_supported_config() is the replacement #pragma warning disable CS0618 var encoderSampleFmt = pAudioCodec->sample_fmts != null ? pAudioCodec->sample_fmts[0] : AVSampleFormat.AV_SAMPLE_FMT_FLTP; #pragma warning restore CS0618 _pAudioCodecContext->sample_rate = sampleRate; _pAudioCodecContext->sample_fmt = encoderSampleFmt; _pAudioCodecContext->bit_rate = bitrate; _pAudioCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL; // Configure channel layout (FFmpeg 7.x+: use AVChannelLayout) ffmpeg.av_channel_layout_default(&_pAudioCodecContext->ch_layout, channels); // time_base: Audio typically uses 1/sample_rate, so one PTS tick is one sample per channel. _pAudioCodecContext->time_base = new AVRational { num = 1, den = sampleRate }; // Global header flag (same rule as the video encoder): codec configuration (e.g. AAC AudioSpecificConfig) // goes into extradata for containers that store it in the header. if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0) { _pAudioCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER; } // Open encoder var ret = ffmpeg.avcodec_open2(_pAudioCodecContext, pAudioCodec, null); if (ret < 0) throw new Exception($"Failed to initialize audio encoder: {GetFFmpegErrorString(ret)} (Code: {ret})"); Melon.Logger.Msg($"Audio encoder successfully initialized: {Marshal.PtrToStringAnsi((IntPtr)pAudioCodec->name)} (sample_rate={sampleRate}, channels={channels}, fmt={encoderSampleFmt})"); // ── 3. Add audio stream ── _pAudioStream = ffmpeg.avformat_new_stream(_pFormatContext, null); if (_pAudioStream == null) throw new Exception("Failed to create audio stream."); if (ffmpeg.avcodec_parameters_from_context(_pAudioStream->codecpar, _pAudioCodecContext) < 0) throw new Exception("Failed to copy audio codec parameters to stream."); // Stream time base hint; the muxer may change it in avformat_write_header (packets are rescaled when written). _pAudioStream->time_base = _pAudioCodecContext->time_base; // ── 4. Initialize resampler (SwrContext) ── // Input: interleaved float (Unity AudioClip.GetData default) → Output: Encoder required format var inputSampleFmt = audioSettings.InputSampleFormat != 0 ? audioSettings.InputSampleFormat : AVSampleFormat.AV_SAMPLE_FMT_FLT; // Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes). // Uses audioSettings.Channels as given (not the defaulted channel count used for the layouts below). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample) _audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels; // Allocate SwrContext fixed (SwrContext** ppSwr = &_pSwrContext) { // Configure input channel layout AVChannelLayout inLayout; ffmpeg.av_channel_layout_default(&inLayout, channels); // Output: encoder layout/format/rate. Input: same layout and rate, input sample format. // Only the sample format (and interleaved → planar layout) is converted; no resampling takes place. // Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_alloc_set_opts2) ret = ffmpeg.swr_alloc_set_opts2( ppSwr, &_pAudioCodecContext->ch_layout, encoderSampleFmt, sampleRate, &inLayout, inputSampleFmt, sampleRate, 0, null); if (ret < 0 || _pSwrContext == null) throw new Exception($"Failed to configure audio resampler options: {GetFFmpegErrorString(ret)} (Code: {ret})"); } ret = ffmpeg.swr_init(_pSwrContext); if (ret < 0) throw new Exception($"Failed to initialize audio resampler: {GetFFmpegErrorString(ret)} (Code: {ret})"); // ── 5. Initialize FIFO buffer ── // Holds samples in the encoder format. Initial capacity 1 sample; av_audio_fifo_write grows it as needed. _pAudioFifo = ffmpeg.av_audio_fifo_alloc(encoderSampleFmt, channels, 1); if (_pAudioFifo == null) throw new Exception("Failed to allocate audio FIFO buffer."); // ── 6. Allocate audio frame and packet ── _pAudioFrame = ffmpeg.av_frame_alloc(); _pAudioPacket = ffmpeg.av_packet_alloc(); if (_pAudioFrame == null || _pAudioPacket == null) throw new Exception("Failed to allocate audio frame or packet structure."); // Determine frame size: Some codecs (e.g. AAC) have a fixed frame_size. // Use default 1024 if frame_size is 0 (e.g. PCM family). // frame_size is known after avcodec_open2 (e.g. AAC 1024, Opus 960 at 48 kHz). var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; _pAudioFrame->nb_samples = frameSize; _pAudioFrame->format = (int)encoderSampleFmt; ffmpeg.av_channel_layout_copy(&_pAudioFrame->ch_layout, &_pAudioCodecContext->ch_layout); _pAudioFrame->sample_rate = sampleRate; ret = ffmpeg.av_frame_get_buffer(_pAudioFrame, 0); if (ret < 0) throw new Exception($"Failed to allocate audio frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})"); _audioSampleIndex = 0; _audioEnabled = true; } /// /// Encodes raw PCM audio sample data into the audio stream. /// /// Can pass float[] array returned by Unity's AudioClip.GetData() or other PCM sources. /// Internally executes resampling → FIFO buffering → frame_size unit encoding pipeline. /// /// Thread safety: Safe to call from multiple threads as it uses _lock internally. /// Does nothing when audio is disabled, or the encoder is not started, already ended or disposed. /// Samples that do not fill a whole encoder frame stay in the FIFO until the next call or End(). /// /// /// PCM audio sample data in , interleaved (LRLRLR...). /// May be larger than needed (e.g. a pooled array); only the first numSamples × channels values are read. /// /// Number of samples per channel (= total float count / channel count). /// is negative or exceeds the array. /// Format conversion, FIFO, encoding or packet writing fails. public void WriteAudioSamples(float[] samples, int numSamples) { lock (_lock) { if (_disposed || !_isInitialized || !_audioEnabled) return; if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float)) throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats)."); fixed (float* pSamples = samples) { WriteAudioSamplesInternal((byte*)pSamples, numSamples); } } } /// /// Encodes raw PCM audio sample data from pointer into the audio stream. /// Same behavior as the float[] overload, without the length check: the caller guarantees that /// numSamples × channels interleaved samples in the input format are readable at . /// The memory is read within this call only. /// /// Pointer to PCM audio sample data (interleaved, input sample format). /// Number of samples per channel. public void WriteAudioSamples(IntPtr samples, int numSamples) { lock (_lock) { if (_disposed || !_isInitialized || !_audioEnabled) return; WriteAudioSamplesInternal((byte*)samples.ToPointer(), numSamples); } } /// /// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it. /// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer. /// Throws an exception when the buffer cannot be allocated. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable) /// private void MakeAudioFrameWritable() { var ret = ffmpeg.av_frame_make_writable(_pAudioFrame); if (ret < 0) throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})"); } /// /// Internal implementation of audio sample encoding. /// /// Processing flow: /// 1. Resample input PCM → encoder format via swr_convert /// (same sample rate: one output sample per input sample, only the format and planar layout change) /// 2. Add resampled data to FIFO buffer /// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder /// (each frame gets PTS = _audioSampleIndex, which then advances by frame_size) /// /// A temporary frame for the converted samples is allocated and freed on every call. /// When that frame structure cannot be allocated, the call returns without encoding the samples. /// /// Reference: /// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html /// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html /// /// Interleaved input samples (single plane). /// Number of samples per channel. private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples) { // ── 1. Resampling: Input format → Encoder format ── // Allocate frame for output buffer (temporary frame to hold resampling result) var pConvertedFrame = ffmpeg.av_frame_alloc(); if (pConvertedFrame == null) return; try { pConvertedFrame->nb_samples = numSamples; pConvertedFrame->format = (int)_pAudioCodecContext->sample_fmt; ffmpeg.av_channel_layout_copy(&pConvertedFrame->ch_layout, &_pAudioCodecContext->ch_layout); pConvertedFrame->sample_rate = _pAudioCodecContext->sample_rate; var ret = ffmpeg.av_frame_get_buffer(pConvertedFrame, 0); if (ret < 0) throw new Exception($"Failed to allocate resampling output frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})"); // swr_convert: Converts input data to format required by encoder. // Arguments: output planes and capacity (samples per channel), input planes and sample count. // &inputBuffer is a one-element plane array, because the input is interleaved. // Returns the number of samples per channel written to the output, or a negative error code. var convertedSamples = ffmpeg.swr_convert( _pSwrContext, (byte**)&pConvertedFrame->data, numSamples, &inputBuffer, numSamples); switch (convertedSamples) { case < 0: throw new Exception($"Audio resampling failed: {GetFFmpegErrorString(convertedSamples)} (Code: {convertedSamples})"); case 0: return; } // ── 2. Add to FIFO buffer ── var written = ffmpeg.av_audio_fifo_write(_pAudioFifo, (void**)&pConvertedFrame->data, convertedSamples); if (written < convertedSamples) throw new Exception("Failed to write to audio FIFO"); // ── 3. Pop from FIFO in frame_size units and encode ── var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize) { // The encoder may still reference the buffer of the previous frame; get a writable one first. MakeAudioFrameWritable(); _pAudioFrame->nb_samples = frameSize; var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize); if (read < frameSize) throw new Exception("Failed to read from audio FIFO"); _pAudioFrame->pts = _audioSampleIndex; _audioSampleIndex += frameSize; SendAudioFrameAndWritePackets(_pAudioFrame); } } finally { ffmpeg.av_frame_free(&pConvertedFrame); } } /// /// Sends audio frame to encoder and writes result packets to file. /// Uses same Send/Receive pattern as video SendFrameAndWritePackets. /// Packets are rescaled from the codec time base (1 / sample rate) to the audio stream time base. /// /// Audio frame to encode, or null to flush the audio encoder. /// Sending, receiving or writing fails. private void SendAudioFrameAndWritePackets(AVFrame* pFrame) { var response = ffmpeg.avcodec_send_frame(_pAudioCodecContext, pFrame); if (response < 0) { throw new Exception($"Error occurred while sending frame to audio encoder: {GetFFmpegErrorString(response)} (Code: {response})"); } try { while (true) { response = ffmpeg.avcodec_receive_packet(_pAudioCodecContext, _pAudioPacket); if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break; if (response < 0) { throw new Exception($"Error occurred while receiving packet from audio encoder: {GetFFmpegErrorString(response)} (Code: {response})"); } // Convert codec timebase → stream timebase ffmpeg.av_packet_rescale_ts(_pAudioPacket, _pAudioCodecContext->time_base, _pAudioStream->time_base); _pAudioPacket->stream_index = _pAudioStream->index; response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pAudioPacket); if (response < 0) { throw new Exception($"Error occurred while writing audio packet to file: {GetFFmpegErrorString(response)} (Code: {response})"); } ffmpeg.av_packet_unref(_pAudioPacket); } } finally { ffmpeg.av_packet_unref(_pAudioPacket); } } /// /// Flushes the audio encoder. /// Sends the samples remaining in the FIFO (fewer than frame_size) as one last short frame, /// then sends a null frame to output all delayed packets of the encoder. /// /// The short frame is handled by libavcodec (encode_send_frame_internal): /// encoders with AV_CODEC_CAP_SMALL_LAST_FRAME or AV_CODEC_CAP_VARIABLE_FRAME_SIZE receive it as is, /// other encoders receive it padded with silence up to frame_size (pad_last_frame). /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/encode.c /// private void FlushAudioEncoder() { if (_pAudioFifo == null || _pAudioCodecContext == null) return; // Encode remaining samples in FIFO (last chunk smaller than frame_size) var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo); if (remaining > 0) { MakeAudioFrameWritable(); _pAudioFrame->nb_samples = remaining; var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining); if (read > 0) { _pAudioFrame->pts = _audioSampleIndex; _audioSampleIndex += read; SendAudioFrameAndWritePackets(_pAudioFrame); } } // Flush encoder by sending null frame SendAudioFrameAndWritePackets(null); } /// /// Releases audio-related FFmpeg resources (resampler, FIFO, frame, packet, audio codec context) and disables audio. /// Called inside Dispose(). The audio stream itself belongs to the format context and is freed with it. /// private void DisposeAudioResources() { if (_pSwrContext != null) { fixed (SwrContext** ppSwr = &_pSwrContext) { ffmpeg.swr_free(ppSwr); } _pSwrContext = null; } if (_pAudioFifo != null) { ffmpeg.av_audio_fifo_free(_pAudioFifo); _pAudioFifo = null; } if (_pAudioFrame != null) { var pFrame = _pAudioFrame; ffmpeg.av_frame_free(&pFrame); _pAudioFrame = null; } if (_pAudioPacket != null) { var pPacket = _pAudioPacket; ffmpeg.av_packet_free(&pPacket); _pAudioPacket = null; } if (_pAudioCodecContext != null) { var pCtx = _pAudioCodecContext; ffmpeg.avcodec_free_context(&pCtx); _pAudioCodecContext = null; } _audioEnabled = false; } } }