ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Document FFmpeg Encoder And Decoder

Browse Source
This commit is contained in:
copecone committed 2026-10-15 16:17:12 +09:00
1 parent 131fabdbe1
commit 23054ad7c7
3 files changed
+636 -115
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Util/FFmpeg/FFmpegNativeEncoder.cs
+330 -86
View File
@@ -32,6 +32,18 @@ namespace ADOFAIRenderer.Util.FFmpeg
///
/// 5. Dispose(): Release all FFmpeg resources (context, frames, scaler, I/O)
///
/// ── Threading ────────────────────────────────────────────────────────
/// Every public method takes <c>_lock</c>, so calls from different threads are serialized.
/// In the renderer, the main thread constructs and starts the encoder, and the render thread
/// (RenderManager.RenderThread) sends the video frames and audio samples, calls End() and Dispose().
/// Dispose() may also be called from another thread to release an encoder whose render was stopped.
/// After End() or Dispose(), the Write*/Send* methods return without doing anything.
///
/// ── Timestamps ───────────────────────────────────────────────────────
/// Video PTS = number of frames submitted before the frame (codec time base 1 / FPS).
/// Audio PTS = number of samples per channel submitted before the frame (codec time base 1 / sample rate).
/// Both start at 0, so the two streams are aligned by the count of submitted frames and samples.
///
/// ── Reference Documentation ─────────────────────────────────────────
/// • FFmpeg Encoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
/// • Send/Receive Encoding API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
@@ -45,15 +57,22 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
public struct AudioSettings
{
/// <summary>Sample rate of input audio (Hz). e.g., 44100, 48000.</summary>
/// <summary>
/// Sample rate of input audio (Hz). e.g., 44100, 48000. If 0 or less, 48000 is used.
/// The encoder runs at the same rate (no sample rate conversion), so the rate has to be one the audio codec supports.
/// </summary>
public int SampleRate;
/// <summary>Number of input audio channels. e.g., 1 (mono), 2 (stereo).</summary>
/// <summary>
/// Number of input audio channels. e.g., 1 (mono), 2 (stereo). If 0 or less, 2 is used for the channel layout.
/// The channel layout is the FFmpeg default layout for the count (av_channel_layout_default).
/// </summary>
public int Channels;
/// <summary>
/// Sample format of input audio. Default is AV_SAMPLE_FMT_FLT (32-bit float).
/// Sample format of input audio (interleaved). Default is AV_SAMPLE_FMT_FLT (32-bit float).
/// Unity's AudioClip.GetData() returns float[], so FLT is suitable.
/// The value 0 (AV_SAMPLE_FMT_U8) is treated as "not set" and selects FLT.
/// </summary>
public AVSampleFormat InputSampleFormat;
@@ -75,13 +94,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
public struct EncoderSettings
{
/// <summary>Width resolution of output video (pixels).</summary>
/// <summary>
/// Width resolution of output video (pixels). Also the width of every input frame (no scaling is performed).
/// The 4:2:0 input size formulas (GetRequiredInputLength) assume an even value.
/// </summary>
public int Width;
/// <summary>Height resolution of output video (pixels).</summary>
/// <summary>Height resolution of output video (pixels). Also the height of every input frame; even values assumed as for Width.</summary>
public int Height;
/// <summary>Frames per second (e.g., 60.0, 29.97, etc.).</summary>
/// <summary>
/// Frames per second (e.g., 60.0, 29.97, etc.). Converted to a rational number (RationalUtil.Double2MaxPrecisionInt)
/// for the codec framerate and time base; every submitted frame advances the PTS by one frame interval.
/// </summary>
public double FPS;
/// <summary>Video codec type to use (H264, H265, AV1, VP9, VVC).</summary>
@@ -90,19 +115,28 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Bitrate control mode (CQP, CBR, VBR).</summary>
public RateControlMode Mode;
/// <summary>Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)</summary>
/// <summary>
/// Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)
/// Clamped to the range of the selected encoder (see ApplyQualitySettings); a negative value selects 18.
/// </summary>
public int QualityValue;
/// <summary>Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg.</summary>
public int TargetBitrate;
/// <summary>Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.</summary>
/// <summary>
/// Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.
/// Used only when larger than TargetBitrate; otherwise the maximum is 1.5 × TargetBitrate.
/// </summary>
public int MaxBitrate;
/// <summary>GOP (Group of Pictures) size (in seconds). Represents keyframe interval.</summary>
/// <summary>
/// GOP (Group of Pictures) size (in seconds). Represents keyframe interval.
/// Converted to frames as round(FPS × GopSize), at least 1.
/// </summary>
public double GopSize;
/// <summary>Whether to force the use of software CPU encoder.</summary>
/// <summary>Whether to force the use of software CPU encoder (hardware encoders are not tried).</summary>
public bool ForceSoftware;
/// <summary>
@@ -132,8 +166,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Video stream within the output file.
/// A single container (format context) can contain multiple streams (video, audio, etc.),
/// but here only one video stream is used.
/// A single container (format context) can contain multiple streams (video, audio, etc.);
/// this encoder writes one video stream and, when audio is enabled, one audio stream (<see cref="_pAudioStream"/>).
/// Owned by <see cref="_pFormatContext"/> and freed with it.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
/// </summary>
private AVStream* _pVideoStream = null;
@@ -150,8 +185,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
private AVFrame* _pSrcFrame = null;
/// <summary>
/// Frame converted to the pixel format required by the encoder (YUV420P or NV12).
/// Actual pixel data memory is allocated via av_frame_get_buffer().
/// Frame converted to the pixel format required by the encoder (<see cref="_dstPixFmt"/>: NV12, YUV420P or YUV420P10LE).
/// Actual pixel data memory is allocated via av_frame_get_buffer() (reference-counted buffers owned by the frame).
/// Written by the CPU conversion (sws_scale) or by the copy of a pre-converted frame, then submitted to the encoder.
/// </summary>
private AVFrame* _pDstFrame = null;
@@ -180,8 +216,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Whether hw_frames_ctx mode (GPU frame pool) is active in the QSV encoder.
/// If true, NV12 → QSV hardware surface upload is required every frame.
/// If false, passes NV12 frames in system memory directly to the encoder.
/// If true, NV12 → QSV hardware surface upload is required every frame (SubmitDstFrame).
/// If false, the frame in system memory (<see cref="_pDstFrame"/>) is passed directly to the encoder;
/// this is the case for every non-QSV encoder and for QSV in system memory input mode.
/// </summary>
private bool _isQsvEncoder;
@@ -189,7 +226,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Pixel format of the frames passed to the encoder (output of the color conversion).
/// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane)
/// - libvvenc: YUV420P10LE (10-bit, 3 planes)
/// - Other software encoders: YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling)
/// - Other encoders (software encoders, VideoToolbox): YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling)
/// Set when the selected encoder opens; read by FrameReader (through SelectedPixelFormat) to choose the GPU conversion.
///
/// Reference - Major pixel format relationships:
/// RGBA (Unity source, 4-channel 8-bit)
@@ -207,7 +245,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG;
/// <summary>Pixel format of the finally selected and running encoder.</summary>
/// <summary>
/// Pixel format of the finally selected and running encoder (the format of the frames the encoder receives).
/// A pre-converted input passed with isColorConverted = true has to be in this format (NV12 or YUV420P).
/// Valid after Start() returns; AV_PIX_FMT_YUV420P before.
/// </summary>
public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
/// <summary>
@@ -219,8 +261,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// GPU frame structure used in QSV hardware acceleration mode.
/// Cached as a member variable to prevent allocation/deallocation overhead for each frame.
/// Allocated only in GPU frame pool input mode; holds a pool surface only while one frame is uploaded and sent.
/// </summary>
private AVFrame* _pHwFrame = null;
/// <summary>
/// Whether the conversion path (GPU pre-converted or CPU RGBA) has been logged.
/// The path is logged once, on the first frame after Start().
/// </summary>
private bool _loggedConversionMode;
// ──────────────────────────────────────────────────────────────
@@ -233,7 +281,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVCodecContext* _pAudioCodecContext = null;
/// <summary>Audio stream within the output file.</summary>
/// <summary>Audio stream within the output file. Owned by <see cref="_pFormatContext"/> and freed with it.</summary>
private AVStream* _pAudioStream = null;
/// <summary>Packet structure to temporarily hold encoded audio packet data.</summary>
@@ -242,6 +290,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Audio resampling context (SwrContext).
/// Converts input PCM format (e.g., float interleaved) → encoder required format (e.g., float planar).
/// Input and output use the same sample rate and channel layout, so only the sample format and layout in memory change.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// </summary>
private SwrContext* _pSwrContext = null;
@@ -252,10 +301,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVAudioFifo* _pAudioFifo = null;
/// <summary>Frame structure to pass to the audio encoder.</summary>
/// <summary>
/// Frame structure to pass to the audio encoder. Reused for every audio frame;
/// holds frame_size samples per channel in the encoder sample format.
/// </summary>
private AVFrame* _pAudioFrame = null;
/// <summary>Accumulated index of encoded audio samples. Used as the PTS value.</summary>
/// <summary>Accumulated index of encoded audio samples (per channel). Used as the PTS value (time base 1 / sample rate).</summary>
private long _audioSampleIndex;
/// <summary>Whether audio encoding is enabled.</summary>
@@ -267,6 +319,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Encoding settings passed at construction time (immutable).</summary>
private readonly EncoderSettings _settings;
/// <summary>
/// Stores the encoding settings. No FFmpeg resource is allocated and the output file is not created until Start().
/// </summary>
/// <param name="settings">Video (and optional audio) encoding settings. Copied (struct).</param>
public FFmpegNativeEncoder(EncoderSettings settings)
{
_settings = settings;
@@ -285,10 +341,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has)
/// With ForceSoftware, only the software encoders are returned.
///
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
/// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK)
/// - lib*: CPU-based software encoders (always available, but slower)
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
/// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK / oneVPL)
/// - videotoolbox: Hardware encoder of Apple platforms (macOS)
/// - lib*: CPU-based software encoders (available when the FFmpeg build includes the library, but slower)
///
/// Iterates through this list in Start() to use the first workable encoder.
/// </summary>
@@ -318,7 +375,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Converts an FFmpeg error code (negative integer) to a human-readable string.
/// e.g., -1 → "Operation not permitted"
///
/// Calls av_strerror() internally to write the message to a stack-allocated buffer.
/// Calls av_strerror() internally to write the message to a stack-allocated buffer (1024 bytes, NUL-terminated).
/// Returns "Unknown error (Code: n)" when av_strerror has no description for the code.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
/// </summary>
private string GetFFmpegErrorString(int errnum)
@@ -348,8 +406,18 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 6. Initialize frame buffers (AVFrame) and colorspace converter (SwsContext)
///
/// On failure, calls Dispose() to clean up partially initialized resources before rethrowing the exception.
/// The encoder object cannot be started again after a failure (it is disposed); a new instance is required.
/// A file created by avio_open before the failure is left on disk (the caller deletes incomplete output).
/// Does nothing when the encoder is already started or disposed.
/// </summary>
/// <param name="outputPath">Output video file path (e.g., "C:\output\video.mp4")</param>
/// <param name="outputPath">
/// Output video file path (e.g., "C:/output/video.mp4"). The extension selects the container (mp4, mkv, webm, ...).
/// </param>
/// <exception cref="Exception">
/// No container matches the extension, no candidate encoder can be opened, the file cannot be opened,
/// or the header cannot be written. Native library load failures (NotSupportedException) are wrapped with a hint
/// about the FFmpeg DLLs in the UserLibs folder.
/// </exception>
public void Start(string outputPath)
{
lock (_lock)
@@ -370,8 +438,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pFormatContext == null) throw new Exception("Failed to create output format context.");
// ── Step 2: Add video stream ──
// Creates a single video track (stream) inside the container.
// Additional audio streams can be created here if needed.
// Creates a single video track (stream) inside the container (stream index 0).
// The audio stream, when enabled, is added after the encoder is opened (Step 4.5).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
_pVideoStream = ffmpeg.avformat_new_stream(_pFormatContext, null);
if (_pVideoStream == null) throw new Exception("Failed to create video stream.");
@@ -379,6 +447,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Step 3: Select and open encoder ──
// Iterates through the priority list returned by GetEncoderCandidates()
// to find the first available encoder.
// Each attempt allocates a new codec context. An attempt that fails frees its codec context
// (and its hardware device/frames references) before the next one, so after the loop
// _pCodecContext is either the opened encoder or null.
var candidates = GetEncoderCandidates();
var openedSuccessfully = false;
@@ -400,6 +471,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pCodecContext = ffmpeg.avcodec_alloc_context3(pCodec);
if (_pCodecContext == null) continue;
// FF_COMPLIANCE_EXPERIMENTAL: allows avcodec_open2 to open encoders that libavcodec marks
// as experimental (AV_CODEC_CAP_EXPERIMENTAL); at a stricter level avcodec_open2 returns AVERROR_EXPERIMENTAL.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/avcodec.c (avcodec_open2)
_pCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL;
// ── Basic video parameters configuration ──
@@ -419,15 +493,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
: AVColorRange.AVCOL_RANGE_JPEG;
_pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709;
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
// so setting 0 (auto) is required for parallel encoding utilizing all cores.
// For libvvenc, the FFmpeg wrapper passes a positive thread_count to VVenC as its thread count
// (m_numThreads); with 0 the wrapper leaves the VVenC default thread setting in place.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c
_pCodecContext->thread_count = 0;
// ── Pixel format configuration ──
// Video encoders generally expect input in YUV420P or NV12 format.
//
// • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE).
// • NV12: Y plane + UV interleaved plane. Used by the hardware encoders (NVENC, AMF, QSV).
// • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE)
// and by VideoToolbox.
// • NV12: Y plane + UV interleaved plane. Used by the hardware encoders NVENC, AMF and QSV.
// selectedPixFmt is the format of the frames this class produces (_dstPixFmt once the encoder opens);
// _pCodecContext->pix_fmt is the format the encoder is told to accept.
// • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is
// AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode:
// system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly.
@@ -448,7 +526,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc: Only 10-bit YUV420P10LE supported (yuv420p verified unsupported in this build)
// libvvenc: Only 10-bit YUV420P10LE supported (the wrapper's pix_fmts list is AV_PIX_FMT_YUV420P10 only).
// Pre-converted GPU input is NV12/YUV420P only, so libvvenc receives its frames through the
// CPU conversion (sws_scale) path; WritePreConvertedFrame rejects this format.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c
selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P10LE;
_pCodecContext->pix_fmt = selectedPixFmt;
}
@@ -483,7 +564,12 @@ namespace ADOFAIRenderer.Util.FFmpeg
var calculatedGop = Math.Max(1, (int)Math.Round(_settings.FPS * _settings.GopSize));
if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc (VVenC) requires IntraPeriod (gop_size) to be a multiple of its GOPSize (typically 32).
// gop_size for libvvenc: calculatedGop rounded to the nearest multiple of 32 (an exact half, e.g. 48, rounds down), at least 32.
// VVenC places intra frames at multiples of its GOP size (32 in its default GOP structure).
// The FFmpeg 8.0 libvvenc wrapper reads gop_size only to select all-intra coding (gop_size == 1);
// for any other value the intra period comes from the wrapper's "period" option
// (seconds, default 1), so this value does not change the keyframe interval of the output.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (m_IntraPeriodSec)
_pCodecContext->gop_size = Math.Max(32, ((calculatedGop + 15) / 32) * 32);
}
else
@@ -498,7 +584,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
//
// Configuration per encoder:
// SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure).
// Setting max_b_frames on AVCodecContext makes SVT-AV1 report a "bad parameter" error.
// The FFmpeg libsvtav1 wrapper does not read max_b_frames; the prediction structure is set
// through svtav1-params, which the wrapper passes to svt_av1_enc_parse_parameter.
// libvvenc: max_b_frames = -1 (encoder default GOP structure).
// Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order).
//
@@ -506,14 +593,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
//
// Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md
// (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2)
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libsvtav1.c (config_enc_params)
if (encoder == VideoEncoder.LibSvtAv1)
{
ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0);
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc relies on hierarchical GOP structures; forcing max_b_frames = 0 causes initialization failure.
// We explicitly set max_b_frames to -1 to allow libvvenc to use its default GOP reference distance.
// libvvenc uses the hierarchical GOP structure of VVenC (reordered frames).
// max_b_frames = -1 leaves the B-frame count to the encoder. The FFmpeg 8.0 wrapper does not read
// max_b_frames, so the GOP structure is the VVenC default in either case.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c
_pCodecContext->max_b_frames = -1;
}
else
@@ -533,16 +623,18 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
// ── Apply quality/bitrate settings ──
// Writes bit_rate / rc_* fields or the encoder's private quality option.
// Private options are set on priv_data before avcodec_open2, which reads them.
ApplyQualitySettings(encoder);
// ── QSV Encoder Hardware Device Initialization ──
// QSV (Quick Sync Video) uses fixed-function encoding hardware on Intel GPUs.
// Using QSV in FFmpeg requires creating a hardware device context.
//
// 3 strategies are attempted sequentially below:
// Strategy 1) Create QSV device directly with child_device_type=d3d11va option
// → FFmpeg internally creates D3D11 device as child and connects to QSV
// Strategy 2) Create D3D11VA device first, then derive QSV device
// 3 strategies are attempted sequentially below (child device: D3D11VA on Windows, VAAPI elsewhere):
// Strategy 1) Create QSV device directly with child_device_type=d3d11va (or vaapi) option
// → FFmpeg internally creates the child device and connects it to QSV
// Strategy 2) Create the D3D11VA (or VAAPI) device first, then derive QSV device
// → Effective when Strategy 1 fails on some drivers
// Strategy 3) Create QSV device directly (legacy method)
// → Fallback on older drivers/systems
@@ -555,6 +647,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (encoder.Kind == EncoderKind.QSV)
{
// hwDeviceCtx: local reference to the QSV device, released below after the codec context
// (or the frames context) has taken its own reference.
AVBufferRef* hwDeviceCtx = null;
var qsvReady = false;
int err;
@@ -597,11 +691,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
qsvReady = true;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Successfully created device by deriving {childDeviceType} → QSV");
}
// The derived QSV device holds its own reference to the child device,
// so the local reference is released whether or not the derivation succeeded.
ffmpeg.av_buffer_unref(&childDeviceCtx);
}
}
// ── Strategy 3: Direct QSV creation (legacy) ──
// No options: FFmpeg selects the child device type of the platform by itself.
if (!qsvReady)
{
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
@@ -734,6 +831,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (encoder.Kind == EncoderKind.NVENC)
{
// NVENC preset range: p1 (fastest) ~ p7 (highest quality). p4 = medium (default)
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/nvenc_h264.c ("preset")
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "p4", 0);
}
else if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265)
@@ -744,23 +842,29 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
else if (encoder == VideoEncoder.LibSvtAv1)
{
// SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality)
// SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality), library default 8.
// Negative presets are for debugging; the FFmpeg option default -2 keeps the library default.
// Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md (EncoderMode, --preset)
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "10", 0);
}
else if (encoder == VideoEncoder.LibAomAv1)
{
// libaom cpu-used: 0 (highest quality) ~ 8 (highest speed)
// libaom cpu-used: 0 (highest quality) ~ 8 (highest speed), FFmpeg option default 1
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libaomenc.c ("cpu-used", "usage")
ffmpeg.av_opt_set(_pCodecContext->priv_data, "cpu-used", "6", 0);
// "realtime" usage: Single pass, low latency encoding mode
// "realtime" usage (AOM_USAGE_REALTIME, value 1): Single pass, low latency encoding mode.
// The other values are "good" (default) and "allintra".
ffmpeg.av_opt_set(_pCodecContext->priv_data, "usage", "realtime", 0);
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc preset: slower, slow, medium, fast, faster. Default is medium, change to faster to gain performance
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c ("preset")
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "faster", 0);
}
// Other encoders (AMF, QSV, etc.) use default presets.
// Other encoders (AMF, QSV, VideoToolbox, libvpx-vp9) use their default presets / speed settings.
// Setting unsupported options can cause errors, so skip.
// The av_opt_set results above are not checked: an option the encoder does not accept leaves its default.
// ── Open encoder ──
// avcodec_open2: Initializes encoder based on all parameters configured above.
@@ -780,6 +884,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
// If open fails, free codec context and QSV frame state, then try next candidate.
// avcodec_free_context also releases the hw_device_ctx / hw_frames_ctx references of the context,
// which releases the QSV device and frame pool of this attempt.
var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
Melon<RenderMod>.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
var pCodecCtx = _pCodecContext;
@@ -819,6 +925,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Step 4.5 (Optional): Initialize audio stream ──
// If EncoderSettings.Audio is set, add audio stream.
// Must run before avformat_write_header: streams cannot be added after the header is written.
if (_settings.Audio.HasValue)
{
InitializeAudioStream(_settings.Audio.Value);
@@ -835,8 +942,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
}
// Write file header (ftyp, moov, container metadata).
// Write file header (container metadata; for mp4 the ftyp box, the moov index is written by av_write_trailer).
// Packets can be written with av_interleaved_write_frame after this point.
// The muxer may change the stream time bases here; packets are rescaled to the final values when written.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
if (headerResult < 0)
@@ -864,6 +972,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// The chroma planes are subsampled to half width and height (4:2:0) with the same filter,
// so each chroma sample takes one source pixel instead of averaging the 2x2 block.
// This scaler is used only by the CPU conversion path (WriteFrame with RGBA input).
// It is created for every encoder, also when all frames arrive pre-converted.
// For libvvenc the output is YUV420P10LE (8-bit RGBA values scaled to 10 bits).
//
// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
_pSwsContext = ffmpeg.sws_getContext(
@@ -874,6 +984,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSwsContext == null)
throw new Exception("Failed to initialize software scaler.");
// BT.709 matrix and the encoder color range for the RGBA → YUV conversion (see ConfigureSwsColorspace).
ConfigureSwsColorspace();
_isInitialized = true;
@@ -881,6 +992,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
catch (NotSupportedException ex)
{
// FFmpeg.AutoGen throws NotSupportedException when a native function cannot be bound
// (DLL missing, wrong version or corrupted).
Dispose();
throw new Exception("Failed to load FFmpeg native library (DLL). Please check if FFmpeg DLL files (avcodec, avformat, etc.) in UserLibs folder are missing or corrupted, delete the DLL files in UserLibs, and restart the game.", ex);
}
@@ -904,10 +1017,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// ── Bitrate Control Mode Description ──
///
/// • CBR (Constant Bitrate): Maintains a constant bitrate. Suitable for streaming.
/// → bit_rate = rc_max_rate = rc_min_rate (all identical)
/// → bit_rate = rc_max_rate = rc_min_rate (all identical), rc_buffer_size = 1 second of the bitrate
/// → libx264/libx265 additionally receive HRD/CBR options. Other encoders select their rate control mode
/// from these generic fields according to their own FFmpeg wrapper.
///
/// • VBR (Variable Bitrate): Bitrate varies according to scene complexity.
/// → bit_rate = target value, rc_max_rate = maximum allowed value
/// → bit_rate = target value, rc_max_rate = maximum allowed value, rc_buffer_size = 2 × rc_max_rate
///
/// libvvenc receives bit_rate only in CBR/VBR (see the libvvenc notes in the method body).
/// Bitrates in the settings are in kbps; the codec context fields are in bits per second (rc_buffer_size in bits).
///
/// • CQP (Constant Quantization Parameter): Uses a fixed quantization value.
/// → Lower value = higher quality/larger size, Higher value = lower quality/smaller size
@@ -919,8 +1037,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// QSV: AVCodecContext.global_quality (Intel ICQ mode)
/// Software: "crf" (Constant Rate Factor, smarter quality control)
///
/// VideoToolbox encoders take the software branch ("crf") in CQP mode; when the encoder has no such option,
/// TrySetOption logs that it was not applied and the encoder keeps its default rate control.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
/// </summary>
/// <param name="encoder">Encoder whose codec context (<see cref="_pCodecContext"/>, allocated, not yet opened) is configured.</param>
/// <exception cref="ArgumentOutOfRangeException">The rate control mode is not CBR, VBR or CQP.</exception>
private void ApplyQualitySettings(VideoEncoder encoder)
{
switch (_settings.Mode)
@@ -929,12 +1052,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
// kbps → bps conversion (FFmpeg bit_rate field is in bps)
_pCodecContext->bit_rate = _settings.TargetBitrate * 1000L;
// libvvenc does not support rc_min_rate/rc_max_rate/rc_buffer_size,
// so set bit_rate only and delegate the rest to internal rate control.
// libvvenc: only bit_rate is set; VVenC's internal rate control handles the rest.
// The FFmpeg 8.0 libvvenc wrapper reads bit_rate and, with VVenC 1.8.0 or later, rc_max_rate
// (m_RCMaxBitrate); it does not read rc_min_rate or rc_buffer_size.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_rc_mode)
if (encoder != VideoEncoder.LibVvenc)
{
// In CBR, set target/max/min bitrates all equal.
// rc_buffer_size determines the bitrate buffering size.
// rc_buffer_size determines the bitrate buffering size (VBV/HRD buffer, in bits):
// TargetBitrate × 1000 bits = one second of data at the target bitrate.
_pCodecContext->rc_max_rate = _pCodecContext->rc_min_rate = _settings.TargetBitrate * 1000L;
_pCodecContext->rc_buffer_size = _settings.TargetBitrate * 1000;
@@ -961,7 +1087,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// kbps → bps conversion
_pCodecContext->bit_rate = _settings.TargetBitrate * 1000L;
// Skip libvvenc as it does not support rc_max_rate/rc_buffer_size.
// libvvenc: only bit_rate is set (see the CBR case for the fields its wrapper reads).
if (encoder != VideoEncoder.LibVvenc)
{
// If MaxBitrate is larger than TargetBitrate, use as is;
@@ -970,7 +1096,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
? _settings.MaxBitrate * 1000L : _settings.TargetBitrate * 1500L;
_pCodecContext->rc_max_rate = maxRate;
// Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes.
// Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes
// (in bits: two seconds of data at the maximum bitrate).
_pCodecContext->rc_buffer_size = (int)(maxRate * 2);
}
break;
@@ -980,6 +1107,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// encoder's option definition (AVOption min/max); a declared minimum of -1 means "auto".
// NVENC: "qp" H.264/HEVC 0~51, AV1 0~255
// (libavcodec/nvenc_h264.c, nvenc_hevc.c: "qp" -1~51 / libavcodec/nvenc_av1.c: "qp" -1~255)
// The bounds of every encoder except QSV are read at run time (GetQualityRange);
// the values listed here are those of the FFmpeg sources.
// AMF: "qp_i" H.264/HEVC 0~51, AV1 0~255
// (libavcodec/amfenc_h264.c, amfenc_hevc.c: "qp_i" -1~51 / libavcodec/amfenc_av1.c: "qp_i" -1~255)
// QSV: AVCodecContext.global_quality 1~51
@@ -991,7 +1120,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (encoder.Kind == EncoderKind.QSV)
{
// QSV runs in ICQ (Intelligent Constant Quality) mode when global_quality > 0 and rc_max_rate is 0;
// global_quality 0 selects the encoder's default CQP. The ICQ quality is clamped to 1~51 by FFmpeg:
// the lower bound 1 keeps global_quality positive, so ICQ is selected (qsvenc.c select_rc_mode checks
// this condition before the bitrate-based modes). The ICQ quality is clamped to 1~51 by FFmpeg:
// libavcodec/qsvenc.c: q->param.mfx.ICQQuality = av_clip(avctx->global_quality, 1, 51);
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/qsvenc.c
qpMin = 1;
@@ -1002,8 +1132,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
else if (encoder == VideoEncoder.LibVvenc) (qpMin, qpMax) = GetQualityRange("qp");
else (qpMin, qpMax) = GetQualityRange("crf"); // Software encoder (libx264, libx265, libvpx-vp9, etc.)
// Default value 18: Quality close to "visually lossless" in H.264.
// Default value 18 (used when QualityValue is negative): Quality close to "visually lossless" in H.264.
// Provides similar level of quality in other codecs as well.
// The value is clamped to [qpMin, qpMax] of the encoder; a clamped value is logged as a warning.
const int defaultQuality = 18;
var requestedQp = _settings.QualityValue;
int qpValue;
@@ -1043,6 +1174,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
else
{
// Software encoders (libx264, libx265, libvpx-vp9, libsvtav1, libaom-av1) and VideoToolbox.
// CRF (Constant Rate Factor): Smarter quality control than QP.
// Automatically adjusts bitrate according to scene complexity to maintain consistent visual quality.
// Range: 0 (lossless) ~ 51 (lowest quality) for x264/x265, 0~63 for VP9/AV1. Typically between 18 and 28.
@@ -1057,7 +1189,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Sets a private option (priv_data) of the encoder.
/// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
/// Must be called before avcodec_open2. The value string is parsed by av_opt_set according to the option type.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__opt__set__funcs.html
/// </summary>
/// <param name="name">Option name as declared by the encoder (e.g. "crf", "qp", "x265-params").</param>
/// <param name="value">Option value as text.</param>
// ReSharper disable once UnusedMethodReturnValue.Local
private bool TrySetOption(string name, string value)
{
@@ -1087,7 +1223,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
/// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
/// Must be called before av_hwframe_ctx_init. Does nothing when hwctx is null.
/// </summary>
/// <param name="framesCtx">QSV frames context allocated by av_hwframe_ctx_alloc on a QSV device.</param>
/// <param name="frameType">Combination of MFX_MEMTYPE_* flags, or 0 to let FFmpeg choose.</param>
private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
{
if (framesCtx->hwctx == null) return;
@@ -1142,14 +1281,21 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// → Highly efficient as no extra memory copy or separate flip operation is required
///
/// stride = Width × 4 (RGBA has 4 bytes per pixel)
///
/// The buffer is read synchronously by sws_scale inside this call and is not referenced after it returns.
/// Does nothing when the encoder is not started, already ended or disposed.
/// </summary>
/// <param name="buffer">Starting address of Unity RGBA pixel data</param>
/// <param name="buffer">
/// Starting address of Unity RGBA pixel data: Width × Height pixels, rows tightly packed (Width × 4 bytes),
/// bottom row first. The length is not checked here; the public Send* methods validate it.
/// </param>
public void WriteFrameFromBuffer(byte* buffer)
{
lock (_lock)
{
if (_disposed || !_isInitialized) return;
// _pSrcFrame does not own memory: its data[0]/linesize[0] point into the caller's buffer for this call only.
var stride = _settings.Width * 4; // RGBA = 4 bytes per pixel
_pSrcFrame->data[0] = buffer + stride * (_settings.Height - 1); // Starting address of last row
_pSrcFrame->linesize[0] = -stride; // Vertical flip with negative stride
@@ -1165,7 +1311,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Encodes a frame by obtaining a read-only pointer from Unity NativeArray&lt;byte&gt;.
/// NativeArray is managed by Unity's native memory allocator,
/// allowing processing of large amounts of data without GC (Garbage Collector) pressure.
/// The array holds RGBA pixels in the layout of <see cref="WriteFrameFromBuffer"/>.
/// </summary>
/// <param name="buffer">RGBA frame data, at least Width × Height × 4 bytes.</param>
/// <exception cref="ArgumentException">The array is smaller than one RGBA frame.</exception>
public void WriteFrameFromNativeBuffer(NativeArray<byte> buffer)
{
lock (_lock)
@@ -1182,7 +1331,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV)
/// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
/// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
/// The 4:2:0 formulas use integer halves of Width and Height (exact for even dimensions).
/// </summary>
/// <param name="isColorConverted">True for pre-converted NV12/YUV420P input, false for RGBA input.</param>
private long GetRequiredInputLength(bool isColorConverted)
{
long w = _settings.Width, h = _settings.Height;
@@ -1209,8 +1360,20 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources.
/// Used by the renderer for frames read back into native memory (AsyncGPUReadback),
/// and usable from external native plugins or custom capture sources.
/// The memory is read synchronously within this call; the caller may reuse or free it after the call returns.
/// Does nothing when the encoder is not started, already ended or disposed.
/// </summary>
/// <param name="pointer">Start of the frame data.</param>
/// <param name="length">Number of readable bytes at <paramref name="pointer"/>; must be at least one frame (GetRequiredInputLength).</param>
/// <param name="isColorConverted">
/// True: the data is already in <see cref="SelectedPixelFormat"/> (NV12 or YUV420P, planes tightly packed, top row first).
/// False: the data is RGBA, bottom row first (flipped while converting).
/// </param>
/// <exception cref="ArgumentException"><paramref name="length"/> is smaller than one frame.</exception>
/// <exception cref="NotSupportedException">Pre-converted input while the encoder format is not NV12 or YUV420P.</exception>
/// <exception cref="Exception">The encoder or the muxer reports an error.</exception>
public void SendDataFromNativeBuffer(IntPtr pointer, int length, bool isColorConverted)
{
lock (_lock)
@@ -1234,7 +1397,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes a frame from a managed byte[].
/// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer.
/// The array is not referenced after the call returns (the renderer returns it to its ArrayPool).
/// Data layout and exceptions as for <see cref="SendDataFromNativeBuffer"/>.
/// </summary>
/// <param name="buffer">Frame data. May be larger than one frame (e.g. a pooled array).</param>
/// <param name="length">Number of valid bytes at the start of <paramref name="buffer"/>.</param>
/// <param name="isColorConverted">True for pre-converted NV12/YUV420P data, false for RGBA.</param>
/// <exception cref="ArgumentException"><paramref name="length"/> exceeds the array or is smaller than one frame.</exception>
public void SendDataFromBuffer(byte[] buffer, int length, bool isColorConverted)
{
lock (_lock)
@@ -1345,7 +1514,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data.
///
/// Input layout (tightly packed, top row first, planes back to back):
/// NV12: Y (Width × Height) | UV interleaved (Width × Height/2)
/// YUV420P: Y (Width × Height) | U (Width/2 × Height/2) | V (Width/2 × Height/2)
/// Each plane is copied into <see cref="_pDstFrame"/>, whose rows can be padded (linesize ≥ row width).
/// A plane is copied with one memory copy when the destination stride equals the row width, otherwise row by row.
/// </summary>
/// <param name="buffer">Start of the frame data; the length was validated by the caller.</param>
/// <exception cref="NotSupportedException">The encoder pixel format is neither NV12 nor YUV420P (e.g. YUV420P10LE).</exception>
private void WritePreConvertedFrame(byte* buffer)
{
PrepareDstFrameForWrite();
@@ -1375,6 +1552,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
// Copy UV Plane (positioned after Y plane size)
// Each UV row holds Width/2 U/V pairs = Width bytes; there are Height/2 rows.
var pSrcUV = buffer + width * height;
var pDstUV = _pDstFrame->data[1];
var dstStrideUV = _pDstFrame->linesize[1];
@@ -1413,7 +1591,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
// Copy U Plane
// Copy U Plane (Width/2 × Height/2 bytes, positioned after the Y plane)
var pSrcU = buffer + width * height;
var pDstU = _pDstFrame->data[1];
var dstStrideU = _pDstFrame->linesize[1];
@@ -1432,7 +1610,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
// Copy V Plane
// Copy V Plane (same size as U, positioned after the U plane)
var pSrcV = pSrcU + chromaWidth * chromaHeight;
var pDstV = _pDstFrame->data[2];
var dstStrideV = _pDstFrame->linesize[2];
@@ -1465,8 +1643,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable
/// (Allocates a new buffer if encoder is still referencing previous frame's buffer)
/// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface
/// 4. Pass to encoder after setting PTS
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface (SubmitDstFrame)
/// 4. Pass to encoder after setting PTS (SubmitDstFrame)
///
/// The source is <see cref="_pSrcFrame"/>, set up by WriteFrameFromBuffer.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale)
/// </summary>
@@ -1493,6 +1673,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
SubmitDstFrame();
}
/// <summary>
/// Passes the filled destination frame (<see cref="_pDstFrame"/>) to the encoder with the next video PTS
/// and writes the packets the encoder returns.
/// GPU frame pool input mode (QSV, _isQsvEncoder): the frame is uploaded to a pool surface (_pHwFrame), which is sent.
/// Every other encoder / input mode: _pDstFrame itself is sent.
/// _frameIndex is incremented when the frame's PTS is assigned, right before SendFrameAndWritePackets.
/// </summary>
/// <exception cref="Exception">Surface allocation, upload, encoding or packet writing fails.</exception>
private void SubmitDstFrame()
{
if (_isQsvEncoder)
@@ -1500,6 +1688,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── QSV hw_frames_ctx mode: System memory → GPU surface upload ──
//
// av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool.
// Fails (e.g. ENOMEM) when every surface of the pool is still held by the encoder (QsvFramePoolSize).
// av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface.
//
// The frame is copied from system memory to the GPU surface on every call;
@@ -1526,6 +1715,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
finally
{
// Release reference immediately after use so it can be returned to pool.
// The encoder keeps its own reference to the surface until the frame is encoded.
ffmpeg.av_frame_unref(_pHwFrame);
}
}
@@ -1534,6 +1724,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── General Encoders (Software or hw_device_ctx mode) ──
// Pass software frame directly to encoder.
// In hw_device_ctx mode, encoder handles GPU upload internally.
// avcodec_send_frame takes a new reference to (or a copy of) the frame data; _pDstFrame keeps its own
// reference, and PrepareDstFrameForWrite allocates a new buffer if the old one is still shared.
_pDstFrame->pts = _frameIndex++;
SendFrameAndWritePackets(_pDstFrame);
}
@@ -1563,9 +1755,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
///
/// ── Packet Timestamp Handling ──
/// av_packet_rescale_ts: Converts PTS/DTS from codec's time_base to stream's time_base.
/// Mandatory because codec and container may use different time_bases.
/// Mandatory because codec and container may use different time_bases
/// (Start sets the stream time base to the codec time base, but the muxer can change it in avformat_write_header).
/// e.g., codec time_base=1/60, stream time_base=1/90000 → PTS 150 → PTS 225000
///
/// Errors: avcodec_send_frame errors (including AVERROR_EOF when called again after the flush) and packet
/// receive/write errors are thrown as exceptions. avcodec_send_frame does not return EAGAIN here,
/// because every call drains all available packets before returning.
///
/// Reference:
/// Send/Receive API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
/// Packet: https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html
@@ -1611,6 +1808,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// av_interleaved_write_frame: Write packet to container.
// "Interleaved" means packets from multiple streams (video+audio) are interleaved
// and written in time order. This function is used even when only video is present.
// It takes ownership of the packet data (the packet is left blank), and it buffers packets
// until packets of the other streams with later timestamps arrive.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html (av_interleaved_write_frame)
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
if (response < 0)
throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
@@ -1647,6 +1847,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
///
/// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete.
/// End() runs once: after it returns or throws, further calls return immediately.
/// End() does not release resources and does not close the file handle; Dispose() must still be called.
/// Order: video flush → audio FIFO remainder and audio flush → trailer.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer)
@@ -1688,12 +1890,16 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ================================================================
/// <summary>
/// Actual resource cleanup logic. Cleanup order is important:
/// Releases every native resource. Safe to call more than once and after a failed Start(); also called by the finalizer.
/// Cleanup order:
///
/// 1. SwsContext (scaler) → independent, can be freed first
/// 2. AVFrame (src/dst) → freed before codec as codec may reference them
/// 3. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
/// 4. AVFormatContext (format/IO) → includes closing file handle
/// 2. AVFrame (src/dst/hw) and AVPacket → the structures and their buffer references.
/// Buffers are reference-counted: a buffer the encoder still references stays alive until the codec context is freed.
/// 3. Video AVCodecContext (codec) → also releases its hw_frames_ctx/hw_device_ctx references
/// (the QSV frame pool and device are destroyed when their last reference goes away)
/// 4. Audio resources (DisposeAudioResources: resampler, FIFO, frame, packet, audio codec context)
/// 5. AVFormatContext (format/IO) → closes the file handle (avio_closep), then frees the context and its streams
///
/// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned.
/// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it).
@@ -1788,7 +1994,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Finalizer (destructor). Executed when collected by GC without Dispose() being called.
/// Safety net to prevent native resource leaks.
/// Safety net to prevent native resource leaks. Runs on the GC finalizer thread; the output file is not finished.
/// ⚠️ Finalizer execution timing is not guaranteed, so always manage resources
/// using a using statement or explicit Dispose() call.
/// </summary>
@@ -1809,8 +2015,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 5. Initialize FIFO buffer (arbitrary input size → collect in frame_size units)
/// 6. Allocate audio frame and packet structures
///
/// Called by Start() after the video encoder is opened and before the header is written.
/// Defaults for unset values: codec AAC, 48000 Hz, 2 channels, 256000 bps, input format FLT.
/// The encoder sample rate equals the input rate; a rate the codec does not support makes avcodec_open2 fail.
/// Throws on any failure; Start() then disposes the encoder.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html
/// </summary>
/// <param name="audioSettings">Audio settings of <see cref="EncoderSettings.Audio"/>.</param>
private void InitializeAudioStream(AudioSettings audioSettings)
{
// ── 1. Determine and search audio codec ──
@@ -1846,10 +2058,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Configure channel layout (FFmpeg 7.x+: use AVChannelLayout)
ffmpeg.av_channel_layout_default(&_pAudioCodecContext->ch_layout, channels);
// time_base: Audio typically uses 1/sample_rate.
// time_base: Audio typically uses 1/sample_rate, so one PTS tick is one sample per channel.
_pAudioCodecContext->time_base = new AVRational { num = 1, den = sampleRate };
// Global header flag
// Global header flag (same rule as the video encoder): codec configuration (e.g. AAC AudioSpecificConfig)
// goes into extradata for containers that store it in the header.
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0)
{
_pAudioCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER;
@@ -1870,6 +2083,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (ffmpeg.avcodec_parameters_from_context(_pAudioStream->codecpar, _pAudioCodecContext) < 0)
throw new Exception("Failed to copy audio codec parameters to stream.");
// Stream time base hint; the muxer may change it in avformat_write_header (packets are rescaled when written).
_pAudioStream->time_base = _pAudioCodecContext->time_base;
// ── 4. Initialize resampler (SwrContext) ──
@@ -1879,6 +2093,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
: AVSampleFormat.AV_SAMPLE_FMT_FLT;
// Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
// Uses audioSettings.Channels as given (not the defaulted channel count used for the layouts below).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
_audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
@@ -1889,6 +2104,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
AVChannelLayout inLayout;
ffmpeg.av_channel_layout_default(&inLayout, channels);
// Output: encoder layout/format/rate. Input: same layout and rate, input sample format.
// Only the sample format (and interleaved → planar layout) is converted; no resampling takes place.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_alloc_set_opts2)
ret = ffmpeg.swr_alloc_set_opts2(
ppSwr,
&_pAudioCodecContext->ch_layout, encoderSampleFmt, sampleRate,
@@ -1904,6 +2122,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Failed to initialize audio resampler: {GetFFmpegErrorString(ret)} (Code: {ret})");
// ── 5. Initialize FIFO buffer ──
// Holds samples in the encoder format. Initial capacity 1 sample; av_audio_fifo_write grows it as needed.
_pAudioFifo = ffmpeg.av_audio_fifo_alloc(encoderSampleFmt, channels, 1);
if (_pAudioFifo == null)
throw new Exception("Failed to allocate audio FIFO buffer.");
@@ -1916,6 +2135,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Determine frame size: Some codecs (e.g. AAC) have a fixed frame_size.
// Use default 1024 if frame_size is 0 (e.g. PCM family).
// frame_size is known after avcodec_open2 (e.g. AAC 1024, Opus 960 at 48 kHz).
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
_pAudioFrame->nb_samples = frameSize;
_pAudioFrame->format = (int)encoderSampleFmt;
@@ -1937,9 +2157,16 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Internally executes resampling → FIFO buffering → frame_size unit encoding pipeline.
///
/// Thread safety: Safe to call from multiple threads as it uses _lock internally.
/// Does nothing when audio is disabled, or the encoder is not started, already ended or disposed.
/// Samples that do not fill a whole encoder frame stay in the FIFO until the next call or End().
/// </summary>
/// <param name="samples">PCM audio sample data. Must be in interleaved format (LRLRLR...).</param>
/// <param name="samples">
/// PCM audio sample data in <see cref="AudioSettings.InputSampleFormat"/>, interleaved (LRLRLR...).
/// May be larger than needed (e.g. a pooled array); only the first numSamples × channels values are read.
/// </param>
/// <param name="numSamples">Number of samples per channel (= total float count / channel count).</param>
/// <exception cref="ArgumentException"><paramref name="numSamples"/> is negative or exceeds the array.</exception>
/// <exception cref="Exception">Format conversion, FIFO, encoding or packet writing fails.</exception>
public void WriteAudioSamples(float[] samples, int numSamples)
{
lock (_lock)
@@ -1957,8 +2184,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes raw PCM audio sample data from pointer into the audio stream.
/// Same behavior as the float[] overload, without the length check: the caller guarantees that
/// numSamples × channels interleaved samples in the input format are readable at <paramref name="samples"/>.
/// The memory is read within this call only.
/// </summary>
/// <param name="samples">Pointer to PCM audio sample data.</param>
/// <param name="samples">Pointer to PCM audio sample data (interleaved, input sample format).</param>
/// <param name="numSamples">Number of samples per channel.</param>
public void WriteAudioSamples(IntPtr samples, int numSamples)
{
@@ -1969,18 +2199,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
/// <summary>
/// Internal implementation of audio sample encoding.
///
/// Processing flow:
/// 1. Resample input PCM → encoder format via swr_convert
/// 2. Add resampled data to FIFO buffer
/// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder
///
/// Reference:
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary>
/// <summary>
/// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
/// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
@@ -1994,6 +2212,25 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
}
/// <summary>
/// Internal implementation of audio sample encoding.
///
/// Processing flow:
/// 1. Resample input PCM → encoder format via swr_convert
/// (same sample rate: one output sample per input sample, only the format and planar layout change)
/// 2. Add resampled data to FIFO buffer
/// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder
/// (each frame gets PTS = _audioSampleIndex, which then advances by frame_size)
///
/// A temporary frame for the converted samples is allocated and freed on every call.
/// When that frame structure cannot be allocated, the call returns without encoding the samples.
///
/// Reference:
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary>
/// <param name="inputBuffer">Interleaved input samples (single plane).</param>
/// <param name="numSamples">Number of samples per channel.</param>
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
{
// ── 1. Resampling: Input format → Encoder format ──
@@ -2013,6 +2250,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Failed to allocate resampling output frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})");
// swr_convert: Converts input data to format required by encoder.
// Arguments: output planes and capacity (samples per channel), input planes and sample count.
// &inputBuffer is a one-element plane array, because the input is interleaved.
// Returns the number of samples per channel written to the output, or a negative error code.
var convertedSamples = ffmpeg.swr_convert(
_pSwrContext,
(byte**)&pConvertedFrame->data, numSamples,
@@ -2032,6 +2272,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
{
// The encoder may still reference the buffer of the previous frame; get a writable one first.
MakeAudioFrameWritable();
_pAudioFrame->nb_samples = frameSize;
@@ -2053,7 +2294,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Sends audio frame to encoder and writes result packets to file.
/// Uses same Send/Receive pattern as video SendFrameAndWritePackets.
/// Packets are rescaled from the codec time base (1 / sample rate) to the audio stream time base.
/// </summary>
/// <param name="pFrame">Audio frame to encode, or null to flush the audio encoder.</param>
/// <exception cref="Exception">Sending, receiving or writing fails.</exception>
private void SendAudioFrameAndWritePackets(AVFrame* pFrame)
{
var response = ffmpeg.avcodec_send_frame(_pAudioCodecContext, pFrame);
@@ -2128,8 +2372,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
/// <summary>
/// Releases audio-related FFmpeg resources.
/// Called inside Dispose().
/// Releases audio-related FFmpeg resources (resampler, FIFO, frame, packet, audio codec context) and disables audio.
/// Called inside Dispose(). The audio stream itself belongs to the format context and is freed with it.
/// </summary>
private void DisposeAudioResources()
{