ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Document FFmpeg Encoder And Decoder

Browse Source
This commit is contained in:
copecone committed 2026-10-15 16:17:12 +09:00
1 parent 19d0e258ac
commit 354c42fdd9
3 files changed
+636 -115
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Util/FFmpeg/FFmpegNativeDecoder.cs
+229 -28
View File
@@ -10,9 +10,19 @@ namespace ADOFAIRenderer.Util.FFmpeg;
/// <summary>
/// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke.
/// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and
/// Supports both GPU hardware-accelerated decoding (device types of <see cref="PreferredHwDeviceTypes"/>: D3D11VA, DXVA2,
/// CUDA, D3D12VA on Windows; VAAPI, CUDA or VideoToolbox on other platforms; others the decoder reports) and
/// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback.
///
/// Used by VideoBGPlayer to decode the background video of a level. Only the first video stream is decoded;
/// audio and other streams are discarded by the demuxer.
///
/// Threading: every public member takes <c>_lock</c>, so calls from different threads are serialized.
/// Width, Height, FPS, Duration and TotalFrames are set once in the constructor and read without the lock.
///
/// Timestamps returned by the TryRead* methods are the presentation times of the stream in seconds
/// (PTS × stream time base), so a stream with a non-zero start time returns timestamps starting at that time.
///
/// ── Architecture &amp; Lifecycle Flow ──────────────────────────────────────────────
///
/// 1. Construction &amp; Demuxing:
@@ -42,6 +52,7 @@ namespace ADOFAIRenderer.Util.FFmpeg;
/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
///
/// 4. Seeking &amp; Scrubbing (SeekToTime / SeekToFrame):
/// (Short forward distances skip a) and b) and only decode forward; see SeekToTime.)
/// a) av_seek_frame: Fast backward keyframe seek.
/// b) avcodec_flush_buffers: Clear internal decoder state.
/// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead;
@@ -72,7 +83,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// </summary>
private record struct HwConfigCandidate
{
/// <summary>Hardware device type to create with av_hwdevice_ctx_create (AVCodecHWConfig.device_type).</summary>
public AVHWDeviceType DeviceType;
/// <summary>Hardware surface pixel format the decoder outputs with this device (AVCodecHWConfig.pix_fmt, e.g. AV_PIX_FMT_D3D11).</summary>
public AVPixelFormat PixelFormat;
}
@@ -80,9 +94,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// Preferred hardware device types in priority order.
/// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config),
/// sorted by this list; configurations not in the list follow in reported order.
/// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Other platforms: VAAPI or VideoToolbox.
/// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Linux: CUDA (NVIDIA), then VAAPI. macOS: VideoToolbox.
/// A device type that cannot be created on the machine (no driver, wrong platform) fails creation and is skipped.
/// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by
/// avcodec_find_decoder report DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan in the bundled FFmpeg 8 build;
/// avcodec_find_decoder report hwaccel device types such as DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan (depending on the build);
/// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name.
/// </summary>
private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
@@ -116,6 +131,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.).
/// This class owns one reference; the codec context holds a second one (hw_device_ctx).
/// Both are released in Dispose (or when a hardware candidate fails). Null in software decoding.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html
/// </summary>
private AVBufferRef* _hwDeviceCtx = null;
@@ -137,6 +154,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Frame structure holding converted pixel data produced by <c>sws_scale</c> (e.g., RGBA or fallback YUV).
/// Owns no pixel memory: during one conversion its data[]/linesize[] point to the caller's output buffers,
/// and they are cleared before and after (ClearConvertedFramePointers).
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
/// </summary>
private AVFrame* _pConvertedFrame = null;
@@ -147,6 +166,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
/// </summary>
private AVFrame* _pCachedFrame = null;
/// <summary>
/// Whether <see cref="_pCachedFrame"/> holds a frame that the next <c>TryRead*</c> call returns
/// before any new frame is decoded. Set by DecodeToTarget, cleared when the frame is consumed or dropped.
/// </summary>
private bool _hasCachedFrame;
/// <summary>
@@ -172,7 +196,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// </summary>
private AVCodecContext_get_format? _getFormatDelegate;
/// <summary>Whether hardware-accelerated decoding is currently active.</summary>
/// <summary>
/// Whether hardware-accelerated decoding is currently active.
/// Set when a hardware candidate is configured; cleared when it fails, or by GetFormat when the stream
/// cannot be decoded on the hardware surface format (the decoder then outputs software frames).
/// </summary>
private bool _isHwDecoding;
/// <summary>Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV).</summary>
@@ -183,6 +211,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12).
/// Assumed NV12 when hardware decoding is configured, then updated from every transferred frame
/// (e.g. P010 for 10-bit streams). AV_PIX_FMT_NONE in software decoding.
/// </summary>
private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
@@ -190,11 +220,28 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
// Playback, Stream &amp; Scaler Synchronization Fields
// ──────────────────────────────────────────────────────────────
/// <summary>Index of the decoded video stream in the format context. -1 until a video stream is found.</summary>
private int _videoStreamIndex = -1;
/// <summary>
/// Index of the frame most recently returned (or skipped while seeking): round(timestamp × FPS) for frames with a timestamp,
/// the previous value + 1 for frames without one. 0 after opening.
/// </summary>
private long _currentFrameIndex;
/// <summary>Time base of the video stream (seconds per PTS unit). Used to convert PTS to seconds and seek targets to PTS.</summary>
private AVRational _timeBase;
/// <summary>Whether Open completed. The read and seek methods do nothing until then and after Dispose.</summary>
private bool _isOpened;
/// <summary>Whether Dispose has run. Prevents duplicate cleanup.</summary>
private bool _disposed;
/// <summary>
/// Whether the end of the input has been reached and the decoder was put into draining mode
/// (avcodec_send_packet with null). Reset by a keyframe seek (after avcodec_flush_buffers).
/// </summary>
private bool _isDraining;
/// <summary>Number of video packets skipped because the decoder rejected them as invalid data.</summary>
@@ -203,12 +250,21 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>Number of skipped packets for which a warning is logged.</summary>
private const int MaxLoggedSkippedPackets = 5;
/// <summary>Lock object serializing all public members (decoder state and native resources).</summary>
private readonly object _lock = new();
// Fields to detect SwsContext updates
// Parameters the current _pSwsContext was created with; a frame with different values recreates the context.
/// <summary>Source pixel format of the current scaler context.</summary>
private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
/// <summary>Destination pixel format of the current scaler context (RGBA, NV12 or YUV420P).</summary>
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
/// <summary>Source width of the current scaler context in pixels (the destination is always Width × Height).</summary>
private int _srcWidth;
/// <summary>Source height of the current scaler context in pixels.</summary>
private int _srcHeight;
/// <summary>YUV→RGB matrix id (<c>SWS_CS_*</c>) applied to the context. 0 when no matrix is set (non-RGBA output).</summary>
@@ -221,22 +277,31 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
// Public Properties
// ──────────────────────────────────────────────────────────────
/// <summary>Video width in pixels.</summary>
/// <summary>
/// Video width in pixels (codec context width after opening, or the stream parameters).
/// Every output of the TryRead* methods has this width; frames of another size are scaled to it.
/// </summary>
public int Width { get; private set; }
/// <summary>Video height in pixels.</summary>
/// <summary>Video height in pixels. Every output of the TryRead* methods has this height.</summary>
public int Height { get; private set; }
/// <summary>Frames per second.</summary>
/// <summary>
/// Frames per second: r_frame_rate of the stream, else avg_frame_rate, else 30.
/// For variable frame rate streams both values are approximations.
/// </summary>
public double FPS { get; private set; }
/// <summary>Total duration in seconds.</summary>
/// <summary>Total duration in seconds (stream duration, else container duration, else 0).</summary>
public double Duration { get; private set; }
/// <summary>Estimated total number of frames.</summary>
/// <summary>
/// Estimated total number of frames: the frame count of the stream header (nb_frames) when present,
/// otherwise round(duration × FPS); 0 when the duration is unknown.
/// </summary>
public long TotalFrames { get; private set; }
/// <summary>Index of the currently read frame.</summary>
/// <summary>Index of the currently read frame (the frame most recently returned or skipped; see _currentFrameIndex).</summary>
public long CurrentFrameIndex
{
get
@@ -248,7 +313,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
/// <summary>Current playback time in seconds.</summary>
/// <summary>Current playback time in seconds: CurrentFrameIndex / FPS (0 when FPS is not positive).</summary>
public double CurrentTime
{
get
@@ -266,6 +331,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// in system RAM (typically <see cref="AVPixelFormat.AV_PIX_FMT_NV12"/>) rather than the GPU surface format
/// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines.
/// In software mode, returns the codec context's pixel format (e.g., YUV420P).
/// Before the first hardware frame is transferred, NV12 is returned as the expected download format;
/// the actual format (e.g. P010 for 10-bit video) is known after the first transfer.
/// The TryRead* methods convert any decoded format to their output format, so this value only selects the fastest path.
/// </summary>
public AVPixelFormat PixelFormat
{
@@ -287,6 +355,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]).
/// Taken from the codec context, else from the stream parameters; AVCOL_RANGE_UNSPECIFIED when neither carries a tag.
/// This property reads the color_range tags only; it does not inspect the pixel format (YUVJ* formats are full range).
/// </summary>
public AVColorRange ColorRange
{
@@ -394,10 +464,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Initializes a new instance of <see cref="FFmpegNativeDecoder"/> and opens the specified video file.
/// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back
/// to multi-threaded software decoding if unavailable.
/// Automatically attempts hardware acceleration (device types reported by the decoder, in the order of
/// <see cref="PreferredHwDeviceTypes"/>), falling back to multi-threaded software decoding if unavailable.
/// </summary>
/// <param name="videoPath">Absolute or relative file path to the video file.</param>
/// <exception cref="Exception">
/// The file cannot be opened, has no video stream, or its codec has no decoder or cannot be opened.
/// All native resources allocated so far are released before the exception propagates.
/// </exception>
public FFmpegNativeDecoder(string videoPath)
{
Open(videoPath);
@@ -425,10 +499,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
// 2. Retrieve stream information
// Reads (and buffers) the start of the file to fill in codec parameters, frame rates and durations.
if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0)
throw new Exception("Failed to retrieve stream info.");
// 3. Find primary video stream
// 3. Find primary video stream (the first stream whose codec type is video)
_videoStreamIndex = -1;
for (int i = 0; i < _pFormatContext->nb_streams; i++)
{
@@ -456,6 +531,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_timeBase = stream->time_base;
// 4. Calculate FPS
// r_frame_rate: lowest frame rate that represents all timestamps accurately (guessed by libavformat).
// avg_frame_rate: average frame rate. Both are approximations for variable frame rate streams.
// Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html
if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0)
{
FPS = ffmpeg.av_q2d(stream->r_frame_rate);
@@ -470,6 +548,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
// 5. Find decoder by codec ID
// avcodec_find_decoder returns the default decoder registered for the ID (e.g. libdav1d for AV1
// when the build includes it, which reports no hardware configuration).
var codecId = stream->codecpar->codec_id;
var decoder = ffmpeg.avcodec_find_decoder(codecId);
if (decoder == null)
@@ -480,6 +560,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (_pCodecContext == null)
throw new Exception("Failed to allocate codec context.");
// Copies the stream parameters (size, pixel format, extradata, color tags, ...) into the codec context.
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
throw new Exception("Failed to copy codec parameters.");
@@ -492,6 +573,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
// Multi-threading settings for improved software decoding performance (auto thread count)
// thread_type = FF_THREAD_FRAME selects frame threading only (several frames decoded in parallel,
// adds one frame of delay per thread); slice threading is not enabled.
// TryInitializeHwDecoder leaves a fresh codec context when every hardware candidate fails.
// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
_pCodecContext->thread_count = 0;
_pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME;
@@ -505,6 +589,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height;
// 8. Calculate total frames and duration
// nb_frames is the frame count stored in the container (0 when unknown);
// otherwise the count is estimated from the stream or container duration and FPS.
if (stream->nb_frames > 0)
{
TotalFrames = stream->nb_frames;
@@ -531,6 +617,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
: _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0;
// 9. Allocate frame and packet buffers
// Structures only; pixel buffers are attached by the decoder, av_hwframe_transfer_data or av_frame_move_ref.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
_pDecodedFrame = ffmpeg.av_frame_alloc();
_pHwTransferFrame = ffmpeg.av_frame_alloc();
@@ -552,8 +639,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Searches for supported hardware device configurations via <c>avcodec_get_hw_config</c>,
/// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA),
/// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA, VAAPI, ...),
/// registers the <c>get_format</c> callback, and attempts to open the hardware decoder.
///
/// Candidates are tried in the order of <see cref="PreferredHwDeviceTypes"/>, then the remaining reported order.
/// When a candidate fails to open, its codec context and device are freed and a fresh codec context
/// (with the stream parameters) is allocated for the next candidate or for the software fallback.
/// On success, <see cref="_hwDeviceCtx"/>, <see cref="_hwPixFmt"/> and <see cref="_hwDeviceType"/> describe the device
/// and the codec context is open.
/// </summary>
/// <param name="decoder">Target decoder.</param>
/// <param name="stream">Source video stream.</param>
@@ -562,6 +655,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
{
// ReSharper disable once InconsistentNaming
// Collect all supported hardware configs from decoder
// AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX (0x01): the decoder supports the hardware format through hw_device_ctx,
// which has to be set on the codec context before avcodec_open2. Only these configurations are used here.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/codec.h
const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01;
var candidates = new List<HwConfigCandidate>();
for (int i = 0; ; i++)
@@ -616,6 +712,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV)
{
// QSV: first with an explicit child device (D3D11VA on Windows, VAAPI elsewhere),
// then without options (FFmpeg selects the child device type).
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
string childDeviceName = isWindows ? "d3d11va" : "vaapi";
AVDictionary* opts = null;
@@ -631,6 +729,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
else
{
// Default device of the type (device = null: the first / default adapter chosen by FFmpeg).
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0);
}
@@ -646,6 +745,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
// Assign hardware context and format selection callback
// _hwDeviceCtx takes over the reference created above; the codec context receives its own reference.
_hwDeviceCtx = hwDeviceCtx;
_hwPixFmt = candidate.PixelFormat;
_hwDeviceType = candidate.DeviceType;
@@ -663,6 +763,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
continue;
}
// The delegate is kept in a field so that it is not collected while native code can still call it
// (FFmpeg.AutoGen marshals it to a function pointer). With thread_count = 1 it is called on the calling
// thread inside avcodec_send_packet / avcodec_receive_frame, which run under _lock.
_getFormatDelegate = GetFormat;
_pCodecContext->get_format = _getFormatDelegate;
@@ -681,7 +784,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
return true;
}
// Failed to open codec with this hardware configuration, clean up and try next
// Failed to open codec with this hardware configuration, clean up and try next.
// Freeing the codec context releases its hw_device_ctx reference; _hwDeviceCtx holds the last one.
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate...");
if (_pCodecContext != null)
@@ -717,8 +821,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// Callback invoked by libavcodec during decoding to negotiate the output pixel format.
/// Confirms the hardware surface format (<see cref="_hwPixFmt"/>) if offered by the decoder,
/// or falls back to software pixel formats via <c>avcodec_default_get_format</c>.
/// Called at the start of decoding and again when stream parameters change (e.g. resolution).
/// The fallback clears <see cref="_isHwDecoding"/>; the decoder then outputs frames in system memory
/// with the thread count set for hardware decoding (1).
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
/// </summary>
/// <param name="ctx">Codec context negotiating the format.</param>
/// <param name="pixFmts">Formats the decoder can output for the stream, terminated by AV_PIX_FMT_NONE (hardware formats first).</param>
/// <returns>The selected pixel format.</returns>
private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts)
{
for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++)
@@ -744,9 +854,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// then sends it to the decoder via <c>avcodec_send_packet</c>.
/// A packet that the decoder rejects as invalid data (<c>AVERROR_INVALIDDATA</c>, e.g. a damaged packet)
/// is skipped and the next packet is read. Any other decoder error is thrown.
/// Packets of other streams are discarded (they are normally not returned, see AVDISCARD_ALL in Open).
/// Called only after avcodec_receive_frame returned EAGAIN, so avcodec_send_packet accepts the packet.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
/// </summary>
/// <returns>True if a video packet was sent; false on EOF.</returns>
/// <returns>True if a video packet was sent; false on EOF or on a read error (both end the stream).</returns>
private bool ReadAndSendPacket()
{
while (true)
@@ -757,6 +869,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (_pPacket->stream_index == _videoStreamIndex)
{
// avcodec_send_packet takes its own reference to (or copy of) the packet data,
// so the packet is unreferenced right away for the next av_read_frame.
ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket);
ffmpeg.av_packet_unref(_pPacket);
@@ -785,9 +899,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV),
/// it is automatically downloaded to system RAM via <c>av_hwframe_transfer_data</c> into <c>_pHwTransferFrame</c>,
/// preserving presentation timestamps via <c>av_frame_copy_props</c>.
///
/// Decode loop: avcodec_receive_frame → on EAGAIN read and send the next packet → retry.
/// When the input ends, a null packet switches the decoder to draining mode; the remaining delayed frames
/// are returned, then avcodec_receive_frame returns AVERROR_EOF.
/// The returned frame (<see cref="_pDecodedFrame"/> or <see cref="_pHwTransferFrame"/>) holds its data until
/// the caller unreferences it (av_frame_unref) or the next frame is received.
/// </summary>
/// <param name="outFrame">The software frame containing pixel data in system RAM.</param>
/// <returns>True if a frame was successfully decoded and available; false on EOF or draining.</returns>
/// <exception cref="Exception">Decoding, packet sending or the GPU-to-host transfer fails.</exception>
private bool TryReceiveFrame(out AVFrame* outFrame)
{
outFrame = null;
@@ -800,6 +921,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
// Check if the received frame is on a hardware surface
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
{
// av_hwframe_transfer_data with an empty destination frame allocates a system memory buffer
// in the first format the device supports for download (NV12 for 8-bit 4:2:0 on the common APIs).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html (av_hwframe_transfer_data)
ffmpeg.av_frame_unref(_pHwTransferFrame);
int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0);
if (transferRet < 0)
@@ -809,6 +933,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
// Copy presentation timestamp and metadata properties
// (pts, best_effort_timestamp, color tags; av_hwframe_transfer_data copies pixel data only).
// The hardware surface is then released back to the decoder's pool.
ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame);
ffmpeg.av_frame_unref(_pDecodedFrame);
@@ -822,6 +948,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
return true;
}
// EAGAIN: the decoder needs more input. Feed the next video packet, or start draining at the end of the input.
// The return value of the flush call (null packet) is not checked; a failure shows up as an error or EOF
// from the next avcodec_receive_frame.
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
{
if (_isDraining) return false;
@@ -832,6 +961,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
ffmpeg.avcodec_send_packet(_pCodecContext, null);
}
}
// EOF: every delayed frame has been returned after draining.
else if (ret == ffmpeg.AVERROR_EOF)
{
return false;
@@ -886,8 +1016,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats, dimensions,
/// or (for RGBA output) the colorspace / range of the source frame change.
/// For RGBA output the YUV→RGB matrix and the source range are taken from <paramref name="colorFrame"/>.
/// For NV12/YUV420P output no matrix is set; libswscale converts between the formats with its default range handling.
/// The destination size is always <see cref="Width"/> × <see cref="Height"/>; SWS_BILINEAR filters scaling and chroma resampling.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
/// </summary>
/// <exception cref="Exception">sws_getContext cannot create a context for the conversion.</exception>
/// <param name="srcFormat">Pixel format of the source frame.</param>
/// <param name="srcWidth">Width of the source frame in pixels.</param>
/// <param name="srcHeight">Height of the source frame in pixels.</param>
@@ -933,6 +1066,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
}
// A new context uses the BT.601 matrix and the range implied by the source pixel format (full for YUVJ*,
// limited otherwise); set the frame's matrix and range for YUV → RGBA.
if (swsColorspace != 0)
{
ApplySwsColorspace(swsColorspace, fullRange);
@@ -952,13 +1087,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
// Returns a negative value when the context cannot change its colorspace details (e.g. YUV to YUV conversion).
// A negative return value means the details cannot be read; the context then keeps its default matrix and range.
if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation) < 0)
{
return;
}
// inv_table: coefficients of the source YUV space (used for YUV → RGB); table: coefficients of the destination.
// A failure of sws_setColorspaceDetails leaves the defaults of the context and is not reported.
var coefficients = ffmpeg.sws_getCoefficients(swsColorspace);
var invTable = new int_array4();
var table = new int_array4();
@@ -975,6 +1112,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state.
/// Used by seeking to inspect candidate frames without falsely incrementing frame indices.
/// Uses best_effort_timestamp (libavcodec's estimate from PTS/DTS), else pts, converted with the stream time base.
/// A frame without either timestamp gets <see cref="_currentFrameIndex"/> / FPS.
/// </summary>
private double GetFrameTimestamp(AVFrame* frame)
{
@@ -994,6 +1133,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter.
/// With a timestamp, <see cref="_currentFrameIndex"/> = round(timestamp × FPS); without one, it is incremented by 1.
/// </summary>
private double GetTimestampAndUpdateIndex(AVFrame* frame)
{
@@ -1016,6 +1156,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Resets all data plane pointers and strides in <see cref="_pConvertedFrame"/> to zero.
/// Prevents stale or dangling pointers from previous conversions being retained across frames.
/// The frame never owns buffers (buf[] stays empty), so av_frame_free in Dispose does not touch caller memory.
/// 8 = AV_NUM_DATA_POINTERS, the size of AVFrame.data / linesize.
/// </summary>
private void ClearConvertedFramePointers()
{
@@ -1034,12 +1176,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies it in 32-bit RGBA format using libswscale.
/// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P).
/// The YUV→RGB matrix and range follow the frame's tags (see InitOrUpdateSwsContext). Frames of another size
/// are scaled to Width × Height. A frame cached by a seek is returned first.
/// </summary>
/// <param name="outBuffer">Pointer to destination memory storing RGBA data.</param>
/// <param name="outBuffer">Pointer to destination memory storing RGBA data (at least outStride × Height bytes).</param>
/// <param name="outStride">Byte stride per row of destination memory (Width * 4).</param>
/// <param name="vflip">Whether to vertically flip the output to match Unity texture coordinates.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
/// <returns>True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer.</returns>
/// <exception cref="Exception">Decoding, transfer or scaler creation fails.</exception>
public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
{
timestamp = 0;
@@ -1063,6 +1208,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
timestamp = GetTimestampAndUpdateIndex(frame);
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame);
// Destination: the caller's buffer. vflip uses a negative stride starting at the last row,
// so sws_scale writes the rows bottom-up (Unity textures have their origin at the bottom-left).
ClearConvertedFramePointers();
if (vflip)
{
@@ -1082,6 +1229,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_pConvertedFrame->data, _pConvertedFrame->linesize
);
// Release the decoded (or cached) frame's buffers; the output is now in the caller's buffer.
ClearConvertedFramePointers();
ffmpeg.av_frame_unref(frame);
return true;
@@ -1090,6 +1238,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer.
/// Same as the byte* overload.
/// </summary>
public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp)
{
@@ -1098,7 +1247,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies it in RGBA format to a Unity NativeArray.
/// The rows are tightly packed (stride Width × 4).
/// </summary>
/// <exception cref="ArgumentException">The array is not created or is smaller than Width × Height × 4 bytes.</exception>
public bool TryReadFrame(NativeArray<byte> outBuffer, bool vflip, out double timestamp)
{
if (!outBuffer.IsCreated)
@@ -1119,16 +1270,19 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format.
/// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed,
/// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing.
/// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed:
/// the planes are copied row by row into the caller's buffers (e.g. texture data for Compute Shader processing).
/// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale.
/// The Y/UV values keep the range of the source (see <see cref="ColorRange"/>); no YUV→RGB conversion happens here.
/// A frame cached by a seek is returned first.
/// </summary>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (yStride × Height bytes; Width bytes per row are written).</param>
/// <param name="yStride">Row stride in bytes for Y plane.</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes).</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (uvStride × ceil(Height / 2) bytes; ceil(Width / 2) × 2 bytes per row are written).</param>
/// <param name="uvStride">Row stride in bytes for UV plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
/// <returns>True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer.</returns>
/// <exception cref="Exception">Decoding, transfer or scaler creation fails.</exception>
// ReSharper disable once InconsistentNaming
public bool TryReadNV12Frame(
byte* yBuffer, int yStride,
@@ -1155,6 +1309,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
timestamp = GetTimestampAndUpdateIndex(frame);
// Conversion path: any other format (e.g. YUV420P from software decoding, P010 from 10-bit hardware decoding)
// or a frame size different from Width × Height. sws_scale writes directly into the caller's planes.
if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height)
{
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12);
@@ -1175,12 +1331,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
else
{
// Direct copy path: NV12 frame of the output size.
// The UV plane has ceil(Height / 2) rows of ceil(Width / 2) U/V byte pairs.
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = ((w + 1) / 2) * 2;
// Copy Y plane (row by row to safely skip linesize GPU padding)
// Buffer.MemoryCopy(source, destination, destinationSizeInBytes, bytesToCopy): the destination size
// argument is the stride, so a stride smaller than the row width throws instead of overrunning.
for (int y = 0; y < h; y++)
{
Buffer.MemoryCopy(
@@ -1208,6 +1368,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers.
/// Same as the byte* overload.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadNV12Frame(
@@ -1224,7 +1385,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays.
/// The planes are tightly packed: Y stride Width, UV stride ceil(Width / 2) × 2 with ceil(Height / 2) rows.
/// </summary>
/// <exception cref="ArgumentException">An array is not created or is smaller than its plane.</exception>
// ReSharper disable once InconsistentNaming
public bool TryReadNV12Frame(
NativeArray<byte> yBuffer,
@@ -1261,7 +1424,18 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// Reads the next video frame and copies its planar Y, U, and V channel data separately.
/// For native YUV420P sources, color conversion is bypassed and memory is copied directly.
/// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale.
/// YUVJ420P (full range YUV420P) is copied as well; the values keep the source range (see <see cref="ColorRange"/>).
/// A frame cached by a seek is returned first.
/// </summary>
/// <param name="yBuffer">Destination of the Y plane (yStride × Height bytes; Width bytes per row are written).</param>
/// <param name="yStride">Row stride in bytes of the Y plane.</param>
/// <param name="uBuffer">Destination of the U plane (uStride × ceil(Height / 2) bytes; ceil(Width / 2) bytes per row).</param>
/// <param name="uStride">Row stride in bytes of the U plane.</param>
/// <param name="vBuffer">Destination of the V plane (same size as U).</param>
/// <param name="vStride">Row stride in bytes of the V plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer.</returns>
/// <exception cref="Exception">Decoding, transfer or scaler creation fails.</exception>
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
byte* yBuffer, int yStride,
@@ -1289,6 +1463,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
timestamp = GetTimestampAndUpdateIndex(frame);
// Conversion path: any format other than YUV420P/YUVJ420P, or a frame size different from Width × Height.
if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height)
{
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P);
@@ -1311,6 +1486,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
else
{
// Direct copy path: each plane row by row (source rows can be padded, linesize ≥ row width).
int w = Width;
int h = Height;
int uvW = (w + 1) / 2;
@@ -1354,6 +1530,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers.
/// Same as the byte* overload.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
@@ -1372,7 +1549,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays.
/// The planes are tightly packed: Y stride Width, U/V stride ceil(Width / 2) with ceil(Height / 2) rows.
/// </summary>
/// <exception cref="ArgumentException">An array is not created or is smaller than its plane.</exception>
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
NativeArray<byte> yBuffer,
@@ -1419,8 +1598,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// for the next <c>TryRead*</c> call.
/// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred.
/// When the end of the stream is reached before the target, no frame is cached.
/// Every discarded frame updates <see cref="_currentFrameIndex"/>; the cached target frame updates it when it is read.
/// </summary>
/// <param name="timeSeconds">Target time in seconds.</param>
/// <exception cref="Exception">Decoding or the GPU-to-host transfer of the target frame fails.</exception>
private void DecodeToTarget(double timeSeconds)
{
while (true)
@@ -1451,6 +1632,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
else
{
// System memory frame: move the buffer references into the cache without copying
// (_pDecodedFrame is left blank for the next receive).
ffmpeg.av_frame_unref(_pCachedFrame);
ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame);
}
@@ -1465,6 +1648,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
continue;
}
// Same input feeding and draining as TryReceiveFrame.
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
{
if (_isDraining) return;
@@ -1496,8 +1680,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read).
/// A backward keyframe seek is followed by a forward decode up to the target.
/// The target frame is transferred and preserved in <see cref="_pCachedFrame"/>; when it is already there, nothing is decoded.
/// Negative times are treated as 0. The time is in the timeline of the stream timestamps (a stream that starts at
/// a non-zero time has no frame before its start; the first frame is returned for earlier targets).
/// When the target is past the last frame, no frame is cached and the next TryRead* call returns false.
/// </summary>
/// <param name="timeSeconds">Target time in seconds to seek to.</param>
/// <exception cref="Exception">Seeking (including the fallback seek to the start) or decoding fails.</exception>
public void SeekToTime(double timeSeconds)
{
lock (_lock)
@@ -1546,6 +1734,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_hasCachedFrame = false;
}
// Keyframe seek. With a stream index, the timestamp is in the stream time base.
// AVSEEK_FLAG_BACKWARD: seek to the nearest keyframe at or before the target, so that decoding forward
// from there reaches the target frame.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_seek_frame)
long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase));
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
@@ -1559,6 +1751,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
// Flush decoder internal buffers
// Drops the frames buffered from before the seek and ends draining mode, so new packets are accepted again.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
ffmpeg.avcodec_flush_buffers(_pCodecContext);
_isDraining = false;
@@ -1569,9 +1762,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
/// <summary>
/// Seeks to a specific frame index.
/// Seeks to a specific frame index: <see cref="SeekToTime"/> with frameIndex / FPS seconds.
/// </summary>
/// <param name="frameIndex">Target frame index.</param>
/// <param name="frameIndex">Target frame index (0-based, in units of 1 / FPS).</param>
public void SeekToFrame(long frameIndex)
{
SeekToTime(frameIndex / FPS);
@@ -1582,7 +1775,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Translates an FFmpeg integer error code to a readable string using <c>av_strerror</c>.
/// Translates an FFmpeg integer error code to a readable string using <c>av_strerror</c>
/// (1024-byte stack buffer; "Unknown error (Code: n)" when there is no description).
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
/// </summary>
private string GetFFmpegErrorString(int errnum)
@@ -1598,6 +1792,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures.
/// Safe to call more than once; also called by the constructor on failure and by the finalizer.
/// Order: scaler → frames → packet → codec context (releases its hw_device_ctx reference)
/// → this object's device reference → input file and format context.
/// After Dispose, the read and seek methods return without doing anything.
/// </summary>
public void Dispose()
{
@@ -1681,6 +1879,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
/// <summary>
/// Finalizer. Releases the native resources when the decoder was not disposed (runs on the GC finalizer thread).
/// </summary>
~FFmpegNativeDecoder()
{
Dispose();
Util/FFmpeg/FFmpegNativeEncoder.cs
+330 -86
View File
@@ -32,6 +32,18 @@ namespace ADOFAIRenderer.Util.FFmpeg
///
/// 5. Dispose(): Release all FFmpeg resources (context, frames, scaler, I/O)
///
/// ── Threading ────────────────────────────────────────────────────────
/// Every public method takes <c>_lock</c>, so calls from different threads are serialized.
/// In the renderer, the main thread constructs and starts the encoder, and the render thread
/// (RenderManager.RenderThread) sends the video frames and audio samples, calls End() and Dispose().
/// Dispose() may also be called from another thread to release an encoder whose render was stopped.
/// After End() or Dispose(), the Write*/Send* methods return without doing anything.
///
/// ── Timestamps ───────────────────────────────────────────────────────
/// Video PTS = number of frames submitted before the frame (codec time base 1 / FPS).
/// Audio PTS = number of samples per channel submitted before the frame (codec time base 1 / sample rate).
/// Both start at 0, so the two streams are aligned by the count of submitted frames and samples.
///
/// ── Reference Documentation ─────────────────────────────────────────
/// • FFmpeg Encoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
/// • Send/Receive Encoding API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
@@ -45,15 +57,22 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
public struct AudioSettings
{
/// <summary>Sample rate of input audio (Hz). e.g., 44100, 48000.</summary>
/// <summary>
/// Sample rate of input audio (Hz). e.g., 44100, 48000. If 0 or less, 48000 is used.
/// The encoder runs at the same rate (no sample rate conversion), so the rate has to be one the audio codec supports.
/// </summary>
public int SampleRate;
/// <summary>Number of input audio channels. e.g., 1 (mono), 2 (stereo).</summary>
/// <summary>
/// Number of input audio channels. e.g., 1 (mono), 2 (stereo). If 0 or less, 2 is used for the channel layout.
/// The channel layout is the FFmpeg default layout for the count (av_channel_layout_default).
/// </summary>
public int Channels;
/// <summary>
/// Sample format of input audio. Default is AV_SAMPLE_FMT_FLT (32-bit float).
/// Sample format of input audio (interleaved). Default is AV_SAMPLE_FMT_FLT (32-bit float).
/// Unity's AudioClip.GetData() returns float[], so FLT is suitable.
/// The value 0 (AV_SAMPLE_FMT_U8) is treated as "not set" and selects FLT.
/// </summary>
public AVSampleFormat InputSampleFormat;
@@ -75,13 +94,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
public struct EncoderSettings
{
/// <summary>Width resolution of output video (pixels).</summary>
/// <summary>
/// Width resolution of output video (pixels). Also the width of every input frame (no scaling is performed).
/// The 4:2:0 input size formulas (GetRequiredInputLength) assume an even value.
/// </summary>
public int Width;
/// <summary>Height resolution of output video (pixels).</summary>
/// <summary>Height resolution of output video (pixels). Also the height of every input frame; even values assumed as for Width.</summary>
public int Height;
/// <summary>Frames per second (e.g., 60.0, 29.97, etc.).</summary>
/// <summary>
/// Frames per second (e.g., 60.0, 29.97, etc.). Converted to a rational number (RationalUtil.Double2MaxPrecisionInt)
/// for the codec framerate and time base; every submitted frame advances the PTS by one frame interval.
/// </summary>
public double FPS;
/// <summary>Video codec type to use (H264, H265, AV1, VP9, VVC).</summary>
@@ -90,19 +115,28 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Bitrate control mode (CQP, CBR, VBR).</summary>
public RateControlMode Mode;
/// <summary>Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)</summary>
/// <summary>
/// Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)
/// Clamped to the range of the selected encoder (see ApplyQualitySettings); a negative value selects 18.
/// </summary>
public int QualityValue;
/// <summary>Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg.</summary>
public int TargetBitrate;
/// <summary>Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.</summary>
/// <summary>
/// Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.
/// Used only when larger than TargetBitrate; otherwise the maximum is 1.5 × TargetBitrate.
/// </summary>
public int MaxBitrate;
/// <summary>GOP (Group of Pictures) size (in seconds). Represents keyframe interval.</summary>
/// <summary>
/// GOP (Group of Pictures) size (in seconds). Represents keyframe interval.
/// Converted to frames as round(FPS × GopSize), at least 1.
/// </summary>
public double GopSize;
/// <summary>Whether to force the use of software CPU encoder.</summary>
/// <summary>Whether to force the use of software CPU encoder (hardware encoders are not tried).</summary>
public bool ForceSoftware;
/// <summary>
@@ -132,8 +166,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Video stream within the output file.
/// A single container (format context) can contain multiple streams (video, audio, etc.),
/// but here only one video stream is used.
/// A single container (format context) can contain multiple streams (video, audio, etc.);
/// this encoder writes one video stream and, when audio is enabled, one audio stream (<see cref="_pAudioStream"/>).
/// Owned by <see cref="_pFormatContext"/> and freed with it.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
/// </summary>
private AVStream* _pVideoStream = null;
@@ -150,8 +185,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
private AVFrame* _pSrcFrame = null;
/// <summary>
/// Frame converted to the pixel format required by the encoder (YUV420P or NV12).
/// Actual pixel data memory is allocated via av_frame_get_buffer().
/// Frame converted to the pixel format required by the encoder (<see cref="_dstPixFmt"/>: NV12, YUV420P or YUV420P10LE).
/// Actual pixel data memory is allocated via av_frame_get_buffer() (reference-counted buffers owned by the frame).
/// Written by the CPU conversion (sws_scale) or by the copy of a pre-converted frame, then submitted to the encoder.
/// </summary>
private AVFrame* _pDstFrame = null;
@@ -180,8 +216,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Whether hw_frames_ctx mode (GPU frame pool) is active in the QSV encoder.
/// If true, NV12 → QSV hardware surface upload is required every frame.
/// If false, passes NV12 frames in system memory directly to the encoder.
/// If true, NV12 → QSV hardware surface upload is required every frame (SubmitDstFrame).
/// If false, the frame in system memory (<see cref="_pDstFrame"/>) is passed directly to the encoder;
/// this is the case for every non-QSV encoder and for QSV in system memory input mode.
/// </summary>
private bool _isQsvEncoder;
@@ -189,7 +226,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Pixel format of the frames passed to the encoder (output of the color conversion).
/// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane)
/// - libvvenc: YUV420P10LE (10-bit, 3 planes)
/// - Other software encoders: YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling)
/// - Other encoders (software encoders, VideoToolbox): YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling)
/// Set when the selected encoder opens; read by FrameReader (through SelectedPixelFormat) to choose the GPU conversion.
///
/// Reference - Major pixel format relationships:
/// RGBA (Unity source, 4-channel 8-bit)
@@ -207,7 +245,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG;
/// <summary>Pixel format of the finally selected and running encoder.</summary>
/// <summary>
/// Pixel format of the finally selected and running encoder (the format of the frames the encoder receives).
/// A pre-converted input passed with isColorConverted = true has to be in this format (NV12 or YUV420P).
/// Valid after Start() returns; AV_PIX_FMT_YUV420P before.
/// </summary>
public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
/// <summary>
@@ -219,8 +261,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// GPU frame structure used in QSV hardware acceleration mode.
/// Cached as a member variable to prevent allocation/deallocation overhead for each frame.
/// Allocated only in GPU frame pool input mode; holds a pool surface only while one frame is uploaded and sent.
/// </summary>
private AVFrame* _pHwFrame = null;
/// <summary>
/// Whether the conversion path (GPU pre-converted or CPU RGBA) has been logged.
/// The path is logged once, on the first frame after Start().
/// </summary>
private bool _loggedConversionMode;
// ──────────────────────────────────────────────────────────────
@@ -233,7 +281,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVCodecContext* _pAudioCodecContext = null;
/// <summary>Audio stream within the output file.</summary>
/// <summary>Audio stream within the output file. Owned by <see cref="_pFormatContext"/> and freed with it.</summary>
private AVStream* _pAudioStream = null;
/// <summary>Packet structure to temporarily hold encoded audio packet data.</summary>
@@ -242,6 +290,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Audio resampling context (SwrContext).
/// Converts input PCM format (e.g., float interleaved) → encoder required format (e.g., float planar).
/// Input and output use the same sample rate and channel layout, so only the sample format and layout in memory change.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// </summary>
private SwrContext* _pSwrContext = null;
@@ -252,10 +301,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVAudioFifo* _pAudioFifo = null;
/// <summary>Frame structure to pass to the audio encoder.</summary>
/// <summary>
/// Frame structure to pass to the audio encoder. Reused for every audio frame;
/// holds frame_size samples per channel in the encoder sample format.
/// </summary>
private AVFrame* _pAudioFrame = null;
/// <summary>Accumulated index of encoded audio samples. Used as the PTS value.</summary>
/// <summary>Accumulated index of encoded audio samples (per channel). Used as the PTS value (time base 1 / sample rate).</summary>
private long _audioSampleIndex;
/// <summary>Whether audio encoding is enabled.</summary>
@@ -267,6 +319,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Encoding settings passed at construction time (immutable).</summary>
private readonly EncoderSettings _settings;
/// <summary>
/// Stores the encoding settings. No FFmpeg resource is allocated and the output file is not created until Start().
/// </summary>
/// <param name="settings">Video (and optional audio) encoding settings. Copied (struct).</param>
public FFmpegNativeEncoder(EncoderSettings settings)
{
_settings = settings;
@@ -285,10 +341,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has)
/// With ForceSoftware, only the software encoders are returned.
///
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
/// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK)
/// - lib*: CPU-based software encoders (always available, but slower)
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
/// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK / oneVPL)
/// - videotoolbox: Hardware encoder of Apple platforms (macOS)
/// - lib*: CPU-based software encoders (available when the FFmpeg build includes the library, but slower)
///
/// Iterates through this list in Start() to use the first workable encoder.
/// </summary>
@@ -318,7 +375,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Converts an FFmpeg error code (negative integer) to a human-readable string.
/// e.g., -1 → "Operation not permitted"
///
/// Calls av_strerror() internally to write the message to a stack-allocated buffer.
/// Calls av_strerror() internally to write the message to a stack-allocated buffer (1024 bytes, NUL-terminated).
/// Returns "Unknown error (Code: n)" when av_strerror has no description for the code.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
/// </summary>
private string GetFFmpegErrorString(int errnum)
@@ -348,8 +406,18 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 6. Initialize frame buffers (AVFrame) and colorspace converter (SwsContext)
///
/// On failure, calls Dispose() to clean up partially initialized resources before rethrowing the exception.
/// The encoder object cannot be started again after a failure (it is disposed); a new instance is required.
/// A file created by avio_open before the failure is left on disk (the caller deletes incomplete output).
/// Does nothing when the encoder is already started or disposed.
/// </summary>
/// <param name="outputPath">Output video file path (e.g., "C:\output\video.mp4")</param>
/// <param name="outputPath">
/// Output video file path (e.g., "C:/output/video.mp4"). The extension selects the container (mp4, mkv, webm, ...).
/// </param>
/// <exception cref="Exception">
/// No container matches the extension, no candidate encoder can be opened, the file cannot be opened,
/// or the header cannot be written. Native library load failures (NotSupportedException) are wrapped with a hint
/// about the FFmpeg DLLs in the UserLibs folder.
/// </exception>
public void Start(string outputPath)
{
lock (_lock)
@@ -370,8 +438,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pFormatContext == null) throw new Exception("Failed to create output format context.");
// ── Step 2: Add video stream ──
// Creates a single video track (stream) inside the container.
// Additional audio streams can be created here if needed.
// Creates a single video track (stream) inside the container (stream index 0).
// The audio stream, when enabled, is added after the encoder is opened (Step 4.5).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
_pVideoStream = ffmpeg.avformat_new_stream(_pFormatContext, null);
if (_pVideoStream == null) throw new Exception("Failed to create video stream.");
@@ -379,6 +447,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Step 3: Select and open encoder ──
// Iterates through the priority list returned by GetEncoderCandidates()
// to find the first available encoder.
// Each attempt allocates a new codec context. An attempt that fails frees its codec context
// (and its hardware device/frames references) before the next one, so after the loop
// _pCodecContext is either the opened encoder or null.
var candidates = GetEncoderCandidates();
var openedSuccessfully = false;
@@ -400,6 +471,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pCodecContext = ffmpeg.avcodec_alloc_context3(pCodec);
if (_pCodecContext == null) continue;
// FF_COMPLIANCE_EXPERIMENTAL: allows avcodec_open2 to open encoders that libavcodec marks
// as experimental (AV_CODEC_CAP_EXPERIMENTAL); at a stricter level avcodec_open2 returns AVERROR_EXPERIMENTAL.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/avcodec.c (avcodec_open2)
_pCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL;
// ── Basic video parameters configuration ──
@@ -419,15 +493,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
: AVColorRange.AVCOL_RANGE_JPEG;
_pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709;
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
// so setting 0 (auto) is required for parallel encoding utilizing all cores.
// For libvvenc, the FFmpeg wrapper passes a positive thread_count to VVenC as its thread count
// (m_numThreads); with 0 the wrapper leaves the VVenC default thread setting in place.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c
_pCodecContext->thread_count = 0;
// ── Pixel format configuration ──
// Video encoders generally expect input in YUV420P or NV12 format.
//
// • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE).
// • NV12: Y plane + UV interleaved plane. Used by the hardware encoders (NVENC, AMF, QSV).
// • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE)
// and by VideoToolbox.
// • NV12: Y plane + UV interleaved plane. Used by the hardware encoders NVENC, AMF and QSV.
// selectedPixFmt is the format of the frames this class produces (_dstPixFmt once the encoder opens);
// _pCodecContext->pix_fmt is the format the encoder is told to accept.
// • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is
// AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode:
// system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly.
@@ -448,7 +526,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc: Only 10-bit YUV420P10LE supported (yuv420p verified unsupported in this build)
// libvvenc: Only 10-bit YUV420P10LE supported (the wrapper's pix_fmts list is AV_PIX_FMT_YUV420P10 only).
// Pre-converted GPU input is NV12/YUV420P only, so libvvenc receives its frames through the
// CPU conversion (sws_scale) path; WritePreConvertedFrame rejects this format.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c
selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P10LE;
_pCodecContext->pix_fmt = selectedPixFmt;
}
@@ -483,7 +564,12 @@ namespace ADOFAIRenderer.Util.FFmpeg
var calculatedGop = Math.Max(1, (int)Math.Round(_settings.FPS * _settings.GopSize));
if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc (VVenC) requires IntraPeriod (gop_size) to be a multiple of its GOPSize (typically 32).
// gop_size for libvvenc: calculatedGop rounded to the nearest multiple of 32 (an exact half, e.g. 48, rounds down), at least 32.
// VVenC places intra frames at multiples of its GOP size (32 in its default GOP structure).
// The FFmpeg 8.0 libvvenc wrapper reads gop_size only to select all-intra coding (gop_size == 1);
// for any other value the intra period comes from the wrapper's "period" option
// (seconds, default 1), so this value does not change the keyframe interval of the output.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (m_IntraPeriodSec)
_pCodecContext->gop_size = Math.Max(32, ((calculatedGop + 15) / 32) * 32);
}
else
@@ -498,7 +584,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
//
// Configuration per encoder:
// SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure).
// Setting max_b_frames on AVCodecContext makes SVT-AV1 report a "bad parameter" error.
// The FFmpeg libsvtav1 wrapper does not read max_b_frames; the prediction structure is set
// through svtav1-params, which the wrapper passes to svt_av1_enc_parse_parameter.
// libvvenc: max_b_frames = -1 (encoder default GOP structure).
// Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order).
//
@@ -506,14 +593,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
//
// Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md
// (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2)
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libsvtav1.c (config_enc_params)
if (encoder == VideoEncoder.LibSvtAv1)
{
ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0);
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc relies on hierarchical GOP structures; forcing max_b_frames = 0 causes initialization failure.
// We explicitly set max_b_frames to -1 to allow libvvenc to use its default GOP reference distance.
// libvvenc uses the hierarchical GOP structure of VVenC (reordered frames).
// max_b_frames = -1 leaves the B-frame count to the encoder. The FFmpeg 8.0 wrapper does not read
// max_b_frames, so the GOP structure is the VVenC default in either case.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c
_pCodecContext->max_b_frames = -1;
}
else
@@ -533,16 +623,18 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
// ── Apply quality/bitrate settings ──
// Writes bit_rate / rc_* fields or the encoder's private quality option.
// Private options are set on priv_data before avcodec_open2, which reads them.
ApplyQualitySettings(encoder);
// ── QSV Encoder Hardware Device Initialization ──
// QSV (Quick Sync Video) uses fixed-function encoding hardware on Intel GPUs.
// Using QSV in FFmpeg requires creating a hardware device context.
//
// 3 strategies are attempted sequentially below:
// Strategy 1) Create QSV device directly with child_device_type=d3d11va option
// → FFmpeg internally creates D3D11 device as child and connects to QSV
// Strategy 2) Create D3D11VA device first, then derive QSV device
// 3 strategies are attempted sequentially below (child device: D3D11VA on Windows, VAAPI elsewhere):
// Strategy 1) Create QSV device directly with child_device_type=d3d11va (or vaapi) option
// → FFmpeg internally creates the child device and connects it to QSV
// Strategy 2) Create the D3D11VA (or VAAPI) device first, then derive QSV device
// → Effective when Strategy 1 fails on some drivers
// Strategy 3) Create QSV device directly (legacy method)
// → Fallback on older drivers/systems
@@ -555,6 +647,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (encoder.Kind == EncoderKind.QSV)
{
// hwDeviceCtx: local reference to the QSV device, released below after the codec context
// (or the frames context) has taken its own reference.
AVBufferRef* hwDeviceCtx = null;
var qsvReady = false;
int err;
@@ -597,11 +691,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
qsvReady = true;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Successfully created device by deriving {childDeviceType} → QSV");
}
// The derived QSV device holds its own reference to the child device,
// so the local reference is released whether or not the derivation succeeded.
ffmpeg.av_buffer_unref(&childDeviceCtx);
}
}
// ── Strategy 3: Direct QSV creation (legacy) ──
// No options: FFmpeg selects the child device type of the platform by itself.
if (!qsvReady)
{
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
@@ -734,6 +831,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (encoder.Kind == EncoderKind.NVENC)
{
// NVENC preset range: p1 (fastest) ~ p7 (highest quality). p4 = medium (default)
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/nvenc_h264.c ("preset")
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "p4", 0);
}
else if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265)
@@ -744,23 +842,29 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
else if (encoder == VideoEncoder.LibSvtAv1)
{
// SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality)
// SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality), library default 8.
// Negative presets are for debugging; the FFmpeg option default -2 keeps the library default.
// Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md (EncoderMode, --preset)
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "10", 0);
}
else if (encoder == VideoEncoder.LibAomAv1)
{
// libaom cpu-used: 0 (highest quality) ~ 8 (highest speed)
// libaom cpu-used: 0 (highest quality) ~ 8 (highest speed), FFmpeg option default 1
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libaomenc.c ("cpu-used", "usage")
ffmpeg.av_opt_set(_pCodecContext->priv_data, "cpu-used", "6", 0);
// "realtime" usage: Single pass, low latency encoding mode
// "realtime" usage (AOM_USAGE_REALTIME, value 1): Single pass, low latency encoding mode.
// The other values are "good" (default) and "allintra".
ffmpeg.av_opt_set(_pCodecContext->priv_data, "usage", "realtime", 0);
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc preset: slower, slow, medium, fast, faster. Default is medium, change to faster to gain performance
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c ("preset")
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "faster", 0);
}
// Other encoders (AMF, QSV, etc.) use default presets.
// Other encoders (AMF, QSV, VideoToolbox, libvpx-vp9) use their default presets / speed settings.
// Setting unsupported options can cause errors, so skip.
// The av_opt_set results above are not checked: an option the encoder does not accept leaves its default.
// ── Open encoder ──
// avcodec_open2: Initializes encoder based on all parameters configured above.
@@ -780,6 +884,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
// If open fails, free codec context and QSV frame state, then try next candidate.
// avcodec_free_context also releases the hw_device_ctx / hw_frames_ctx references of the context,
// which releases the QSV device and frame pool of this attempt.
var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
Melon<RenderMod>.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
var pCodecCtx = _pCodecContext;
@@ -819,6 +925,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Step 4.5 (Optional): Initialize audio stream ──
// If EncoderSettings.Audio is set, add audio stream.
// Must run before avformat_write_header: streams cannot be added after the header is written.
if (_settings.Audio.HasValue)
{
InitializeAudioStream(_settings.Audio.Value);
@@ -835,8 +942,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
}
// Write file header (ftyp, moov, container metadata).
// Write file header (container metadata; for mp4 the ftyp box, the moov index is written by av_write_trailer).
// Packets can be written with av_interleaved_write_frame after this point.
// The muxer may change the stream time bases here; packets are rescaled to the final values when written.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
if (headerResult < 0)
@@ -864,6 +972,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// The chroma planes are subsampled to half width and height (4:2:0) with the same filter,
// so each chroma sample takes one source pixel instead of averaging the 2x2 block.
// This scaler is used only by the CPU conversion path (WriteFrame with RGBA input).
// It is created for every encoder, also when all frames arrive pre-converted.
// For libvvenc the output is YUV420P10LE (8-bit RGBA values scaled to 10 bits).
//
// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
_pSwsContext = ffmpeg.sws_getContext(
@@ -874,6 +984,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSwsContext == null)
throw new Exception("Failed to initialize software scaler.");
// BT.709 matrix and the encoder color range for the RGBA → YUV conversion (see ConfigureSwsColorspace).
ConfigureSwsColorspace();
_isInitialized = true;
@@ -881,6 +992,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
catch (NotSupportedException ex)
{
// FFmpeg.AutoGen throws NotSupportedException when a native function cannot be bound
// (DLL missing, wrong version or corrupted).
Dispose();
throw new Exception("Failed to load FFmpeg native library (DLL). Please check if FFmpeg DLL files (avcodec, avformat, etc.) in UserLibs folder are missing or corrupted, delete the DLL files in UserLibs, and restart the game.", ex);
}
@@ -904,10 +1017,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// ── Bitrate Control Mode Description ──
///
/// • CBR (Constant Bitrate): Maintains a constant bitrate. Suitable for streaming.
/// → bit_rate = rc_max_rate = rc_min_rate (all identical)
/// → bit_rate = rc_max_rate = rc_min_rate (all identical), rc_buffer_size = 1 second of the bitrate
/// → libx264/libx265 additionally receive HRD/CBR options. Other encoders select their rate control mode
/// from these generic fields according to their own FFmpeg wrapper.
///
/// • VBR (Variable Bitrate): Bitrate varies according to scene complexity.
/// → bit_rate = target value, rc_max_rate = maximum allowed value
/// → bit_rate = target value, rc_max_rate = maximum allowed value, rc_buffer_size = 2 × rc_max_rate
///
/// libvvenc receives bit_rate only in CBR/VBR (see the libvvenc notes in the method body).
/// Bitrates in the settings are in kbps; the codec context fields are in bits per second (rc_buffer_size in bits).
///
/// • CQP (Constant Quantization Parameter): Uses a fixed quantization value.
/// → Lower value = higher quality/larger size, Higher value = lower quality/smaller size
@@ -919,8 +1037,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// QSV: AVCodecContext.global_quality (Intel ICQ mode)
/// Software: "crf" (Constant Rate Factor, smarter quality control)
///
/// VideoToolbox encoders take the software branch ("crf") in CQP mode; when the encoder has no such option,
/// TrySetOption logs that it was not applied and the encoder keeps its default rate control.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
/// </summary>
/// <param name="encoder">Encoder whose codec context (<see cref="_pCodecContext"/>, allocated, not yet opened) is configured.</param>
/// <exception cref="ArgumentOutOfRangeException">The rate control mode is not CBR, VBR or CQP.</exception>
private void ApplyQualitySettings(VideoEncoder encoder)
{
switch (_settings.Mode)
@@ -929,12 +1052,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
// kbps → bps conversion (FFmpeg bit_rate field is in bps)
_pCodecContext->bit_rate = _settings.TargetBitrate * 1000L;
// libvvenc does not support rc_min_rate/rc_max_rate/rc_buffer_size,
// so set bit_rate only and delegate the rest to internal rate control.
// libvvenc: only bit_rate is set; VVenC's internal rate control handles the rest.
// The FFmpeg 8.0 libvvenc wrapper reads bit_rate and, with VVenC 1.8.0 or later, rc_max_rate
// (m_RCMaxBitrate); it does not read rc_min_rate or rc_buffer_size.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_rc_mode)
if (encoder != VideoEncoder.LibVvenc)
{
// In CBR, set target/max/min bitrates all equal.
// rc_buffer_size determines the bitrate buffering size.
// rc_buffer_size determines the bitrate buffering size (VBV/HRD buffer, in bits):
// TargetBitrate × 1000 bits = one second of data at the target bitrate.
_pCodecContext->rc_max_rate = _pCodecContext->rc_min_rate = _settings.TargetBitrate * 1000L;
_pCodecContext->rc_buffer_size = _settings.TargetBitrate * 1000;
@@ -961,7 +1087,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// kbps → bps conversion
_pCodecContext->bit_rate = _settings.TargetBitrate * 1000L;
// Skip libvvenc as it does not support rc_max_rate/rc_buffer_size.
// libvvenc: only bit_rate is set (see the CBR case for the fields its wrapper reads).
if (encoder != VideoEncoder.LibVvenc)
{
// If MaxBitrate is larger than TargetBitrate, use as is;
@@ -970,7 +1096,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
? _settings.MaxBitrate * 1000L : _settings.TargetBitrate * 1500L;
_pCodecContext->rc_max_rate = maxRate;
// Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes.
// Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes
// (in bits: two seconds of data at the maximum bitrate).
_pCodecContext->rc_buffer_size = (int)(maxRate * 2);
}
break;
@@ -980,6 +1107,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// encoder's option definition (AVOption min/max); a declared minimum of -1 means "auto".
// NVENC: "qp" H.264/HEVC 0~51, AV1 0~255
// (libavcodec/nvenc_h264.c, nvenc_hevc.c: "qp" -1~51 / libavcodec/nvenc_av1.c: "qp" -1~255)
// The bounds of every encoder except QSV are read at run time (GetQualityRange);
// the values listed here are those of the FFmpeg sources.
// AMF: "qp_i" H.264/HEVC 0~51, AV1 0~255
// (libavcodec/amfenc_h264.c, amfenc_hevc.c: "qp_i" -1~51 / libavcodec/amfenc_av1.c: "qp_i" -1~255)
// QSV: AVCodecContext.global_quality 1~51
@@ -991,7 +1120,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (encoder.Kind == EncoderKind.QSV)
{
// QSV runs in ICQ (Intelligent Constant Quality) mode when global_quality > 0 and rc_max_rate is 0;
// global_quality 0 selects the encoder's default CQP. The ICQ quality is clamped to 1~51 by FFmpeg:
// the lower bound 1 keeps global_quality positive, so ICQ is selected (qsvenc.c select_rc_mode checks
// this condition before the bitrate-based modes). The ICQ quality is clamped to 1~51 by FFmpeg:
// libavcodec/qsvenc.c: q->param.mfx.ICQQuality = av_clip(avctx->global_quality, 1, 51);
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/qsvenc.c
qpMin = 1;
@@ -1002,8 +1132,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
else if (encoder == VideoEncoder.LibVvenc) (qpMin, qpMax) = GetQualityRange("qp");
else (qpMin, qpMax) = GetQualityRange("crf"); // Software encoder (libx264, libx265, libvpx-vp9, etc.)
// Default value 18: Quality close to "visually lossless" in H.264.
// Default value 18 (used when QualityValue is negative): Quality close to "visually lossless" in H.264.
// Provides similar level of quality in other codecs as well.
// The value is clamped to [qpMin, qpMax] of the encoder; a clamped value is logged as a warning.
const int defaultQuality = 18;
var requestedQp = _settings.QualityValue;
int qpValue;
@@ -1043,6 +1174,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
else
{
// Software encoders (libx264, libx265, libvpx-vp9, libsvtav1, libaom-av1) and VideoToolbox.
// CRF (Constant Rate Factor): Smarter quality control than QP.
// Automatically adjusts bitrate according to scene complexity to maintain consistent visual quality.
// Range: 0 (lossless) ~ 51 (lowest quality) for x264/x265, 0~63 for VP9/AV1. Typically between 18 and 28.
@@ -1057,7 +1189,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Sets a private option (priv_data) of the encoder.
/// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
/// Must be called before avcodec_open2. The value string is parsed by av_opt_set according to the option type.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__opt__set__funcs.html
/// </summary>
/// <param name="name">Option name as declared by the encoder (e.g. "crf", "qp", "x265-params").</param>
/// <param name="value">Option value as text.</param>
// ReSharper disable once UnusedMethodReturnValue.Local
private bool TrySetOption(string name, string value)
{
@@ -1087,7 +1223,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
/// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
/// Must be called before av_hwframe_ctx_init. Does nothing when hwctx is null.
/// </summary>
/// <param name="framesCtx">QSV frames context allocated by av_hwframe_ctx_alloc on a QSV device.</param>
/// <param name="frameType">Combination of MFX_MEMTYPE_* flags, or 0 to let FFmpeg choose.</param>
private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
{
if (framesCtx->hwctx == null) return;
@@ -1142,14 +1281,21 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// → Highly efficient as no extra memory copy or separate flip operation is required
///
/// stride = Width × 4 (RGBA has 4 bytes per pixel)
///
/// The buffer is read synchronously by sws_scale inside this call and is not referenced after it returns.
/// Does nothing when the encoder is not started, already ended or disposed.
/// </summary>
/// <param name="buffer">Starting address of Unity RGBA pixel data</param>
/// <param name="buffer">
/// Starting address of Unity RGBA pixel data: Width × Height pixels, rows tightly packed (Width × 4 bytes),
/// bottom row first. The length is not checked here; the public Send* methods validate it.
/// </param>
public void WriteFrameFromBuffer(byte* buffer)
{
lock (_lock)
{
if (_disposed || !_isInitialized) return;
// _pSrcFrame does not own memory: its data[0]/linesize[0] point into the caller's buffer for this call only.
var stride = _settings.Width * 4; // RGBA = 4 bytes per pixel
_pSrcFrame->data[0] = buffer + stride * (_settings.Height - 1); // Starting address of last row
_pSrcFrame->linesize[0] = -stride; // Vertical flip with negative stride
@@ -1165,7 +1311,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Encodes a frame by obtaining a read-only pointer from Unity NativeArray&lt;byte&gt;.
/// NativeArray is managed by Unity's native memory allocator,
/// allowing processing of large amounts of data without GC (Garbage Collector) pressure.
/// The array holds RGBA pixels in the layout of <see cref="WriteFrameFromBuffer"/>.
/// </summary>
/// <param name="buffer">RGBA frame data, at least Width × Height × 4 bytes.</param>
/// <exception cref="ArgumentException">The array is smaller than one RGBA frame.</exception>
public void WriteFrameFromNativeBuffer(NativeArray<byte> buffer)
{
lock (_lock)
@@ -1182,7 +1331,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV)
/// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
/// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
/// The 4:2:0 formulas use integer halves of Width and Height (exact for even dimensions).
/// </summary>
/// <param name="isColorConverted">True for pre-converted NV12/YUV420P input, false for RGBA input.</param>
private long GetRequiredInputLength(bool isColorConverted)
{
long w = _settings.Width, h = _settings.Height;
@@ -1209,8 +1360,20 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources.
/// Used by the renderer for frames read back into native memory (AsyncGPUReadback),
/// and usable from external native plugins or custom capture sources.
/// The memory is read synchronously within this call; the caller may reuse or free it after the call returns.
/// Does nothing when the encoder is not started, already ended or disposed.
/// </summary>
/// <param name="pointer">Start of the frame data.</param>
/// <param name="length">Number of readable bytes at <paramref name="pointer"/>; must be at least one frame (GetRequiredInputLength).</param>
/// <param name="isColorConverted">
/// True: the data is already in <see cref="SelectedPixelFormat"/> (NV12 or YUV420P, planes tightly packed, top row first).
/// False: the data is RGBA, bottom row first (flipped while converting).
/// </param>
/// <exception cref="ArgumentException"><paramref name="length"/> is smaller than one frame.</exception>
/// <exception cref="NotSupportedException">Pre-converted input while the encoder format is not NV12 or YUV420P.</exception>
/// <exception cref="Exception">The encoder or the muxer reports an error.</exception>
public void SendDataFromNativeBuffer(IntPtr pointer, int length, bool isColorConverted)
{
lock (_lock)
@@ -1234,7 +1397,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes a frame from a managed byte[].
/// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer.
/// The array is not referenced after the call returns (the renderer returns it to its ArrayPool).
/// Data layout and exceptions as for <see cref="SendDataFromNativeBuffer"/>.
/// </summary>
/// <param name="buffer">Frame data. May be larger than one frame (e.g. a pooled array).</param>
/// <param name="length">Number of valid bytes at the start of <paramref name="buffer"/>.</param>
/// <param name="isColorConverted">True for pre-converted NV12/YUV420P data, false for RGBA.</param>
/// <exception cref="ArgumentException"><paramref name="length"/> exceeds the array or is smaller than one frame.</exception>
public void SendDataFromBuffer(byte[] buffer, int length, bool isColorConverted)
{
lock (_lock)
@@ -1345,7 +1514,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data.
///
/// Input layout (tightly packed, top row first, planes back to back):
/// NV12: Y (Width × Height) | UV interleaved (Width × Height/2)
/// YUV420P: Y (Width × Height) | U (Width/2 × Height/2) | V (Width/2 × Height/2)
/// Each plane is copied into <see cref="_pDstFrame"/>, whose rows can be padded (linesize ≥ row width).
/// A plane is copied with one memory copy when the destination stride equals the row width, otherwise row by row.
/// </summary>
/// <param name="buffer">Start of the frame data; the length was validated by the caller.</param>
/// <exception cref="NotSupportedException">The encoder pixel format is neither NV12 nor YUV420P (e.g. YUV420P10LE).</exception>
private void WritePreConvertedFrame(byte* buffer)
{
PrepareDstFrameForWrite();
@@ -1375,6 +1552,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
// Copy UV Plane (positioned after Y plane size)
// Each UV row holds Width/2 U/V pairs = Width bytes; there are Height/2 rows.
var pSrcUV = buffer + width * height;
var pDstUV = _pDstFrame->data[1];
var dstStrideUV = _pDstFrame->linesize[1];
@@ -1413,7 +1591,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
// Copy U Plane
// Copy U Plane (Width/2 × Height/2 bytes, positioned after the Y plane)
var pSrcU = buffer + width * height;
var pDstU = _pDstFrame->data[1];
var dstStrideU = _pDstFrame->linesize[1];
@@ -1432,7 +1610,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
// Copy V Plane
// Copy V Plane (same size as U, positioned after the U plane)
var pSrcV = pSrcU + chromaWidth * chromaHeight;
var pDstV = _pDstFrame->data[2];
var dstStrideV = _pDstFrame->linesize[2];
@@ -1465,8 +1643,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable
/// (Allocates a new buffer if encoder is still referencing previous frame's buffer)
/// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface
/// 4. Pass to encoder after setting PTS
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface (SubmitDstFrame)
/// 4. Pass to encoder after setting PTS (SubmitDstFrame)
///
/// The source is <see cref="_pSrcFrame"/>, set up by WriteFrameFromBuffer.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale)
/// </summary>
@@ -1493,6 +1673,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
SubmitDstFrame();
}
/// <summary>
/// Passes the filled destination frame (<see cref="_pDstFrame"/>) to the encoder with the next video PTS
/// and writes the packets the encoder returns.
/// GPU frame pool input mode (QSV, _isQsvEncoder): the frame is uploaded to a pool surface (_pHwFrame), which is sent.
/// Every other encoder / input mode: _pDstFrame itself is sent.
/// _frameIndex is incremented when the frame's PTS is assigned, right before SendFrameAndWritePackets.
/// </summary>
/// <exception cref="Exception">Surface allocation, upload, encoding or packet writing fails.</exception>
private void SubmitDstFrame()
{
if (_isQsvEncoder)
@@ -1500,6 +1688,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── QSV hw_frames_ctx mode: System memory → GPU surface upload ──
//
// av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool.
// Fails (e.g. ENOMEM) when every surface of the pool is still held by the encoder (QsvFramePoolSize).
// av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface.
//
// The frame is copied from system memory to the GPU surface on every call;
@@ -1526,6 +1715,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
finally
{
// Release reference immediately after use so it can be returned to pool.
// The encoder keeps its own reference to the surface until the frame is encoded.
ffmpeg.av_frame_unref(_pHwFrame);
}
}
@@ -1534,6 +1724,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── General Encoders (Software or hw_device_ctx mode) ──
// Pass software frame directly to encoder.
// In hw_device_ctx mode, encoder handles GPU upload internally.
// avcodec_send_frame takes a new reference to (or a copy of) the frame data; _pDstFrame keeps its own
// reference, and PrepareDstFrameForWrite allocates a new buffer if the old one is still shared.
_pDstFrame->pts = _frameIndex++;
SendFrameAndWritePackets(_pDstFrame);
}
@@ -1563,9 +1755,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
///
/// ── Packet Timestamp Handling ──
/// av_packet_rescale_ts: Converts PTS/DTS from codec's time_base to stream's time_base.
/// Mandatory because codec and container may use different time_bases.
/// Mandatory because codec and container may use different time_bases
/// (Start sets the stream time base to the codec time base, but the muxer can change it in avformat_write_header).
/// e.g., codec time_base=1/60, stream time_base=1/90000 → PTS 150 → PTS 225000
///
/// Errors: avcodec_send_frame errors (including AVERROR_EOF when called again after the flush) and packet
/// receive/write errors are thrown as exceptions. avcodec_send_frame does not return EAGAIN here,
/// because every call drains all available packets before returning.
///
/// Reference:
/// Send/Receive API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
/// Packet: https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html
@@ -1611,6 +1808,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// av_interleaved_write_frame: Write packet to container.
// "Interleaved" means packets from multiple streams (video+audio) are interleaved
// and written in time order. This function is used even when only video is present.
// It takes ownership of the packet data (the packet is left blank), and it buffers packets
// until packets of the other streams with later timestamps arrive.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html (av_interleaved_write_frame)
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
if (response < 0)
throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
@@ -1647,6 +1847,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
///
/// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete.
/// End() runs once: after it returns or throws, further calls return immediately.
/// End() does not release resources and does not close the file handle; Dispose() must still be called.
/// Order: video flush → audio FIFO remainder and audio flush → trailer.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer)
@@ -1688,12 +1890,16 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ================================================================
/// <summary>
/// Actual resource cleanup logic. Cleanup order is important:
/// Releases every native resource. Safe to call more than once and after a failed Start(); also called by the finalizer.
/// Cleanup order:
///
/// 1. SwsContext (scaler) → independent, can be freed first
/// 2. AVFrame (src/dst) → freed before codec as codec may reference them
/// 3. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
/// 4. AVFormatContext (format/IO) → includes closing file handle
/// 2. AVFrame (src/dst/hw) and AVPacket → the structures and their buffer references.
/// Buffers are reference-counted: a buffer the encoder still references stays alive until the codec context is freed.
/// 3. Video AVCodecContext (codec) → also releases its hw_frames_ctx/hw_device_ctx references
/// (the QSV frame pool and device are destroyed when their last reference goes away)
/// 4. Audio resources (DisposeAudioResources: resampler, FIFO, frame, packet, audio codec context)
/// 5. AVFormatContext (format/IO) → closes the file handle (avio_closep), then frees the context and its streams
///
/// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned.
/// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it).
@@ -1788,7 +1994,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Finalizer (destructor). Executed when collected by GC without Dispose() being called.
/// Safety net to prevent native resource leaks.
/// Safety net to prevent native resource leaks. Runs on the GC finalizer thread; the output file is not finished.
/// ⚠️ Finalizer execution timing is not guaranteed, so always manage resources
/// using a using statement or explicit Dispose() call.
/// </summary>
@@ -1809,8 +2015,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 5. Initialize FIFO buffer (arbitrary input size → collect in frame_size units)
/// 6. Allocate audio frame and packet structures
///
/// Called by Start() after the video encoder is opened and before the header is written.
/// Defaults for unset values: codec AAC, 48000 Hz, 2 channels, 256000 bps, input format FLT.
/// The encoder sample rate equals the input rate; a rate the codec does not support makes avcodec_open2 fail.
/// Throws on any failure; Start() then disposes the encoder.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html
/// </summary>
/// <param name="audioSettings">Audio settings of <see cref="EncoderSettings.Audio"/>.</param>
private void InitializeAudioStream(AudioSettings audioSettings)
{
// ── 1. Determine and search audio codec ──
@@ -1846,10 +2058,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Configure channel layout (FFmpeg 7.x+: use AVChannelLayout)
ffmpeg.av_channel_layout_default(&_pAudioCodecContext->ch_layout, channels);
// time_base: Audio typically uses 1/sample_rate.
// time_base: Audio typically uses 1/sample_rate, so one PTS tick is one sample per channel.
_pAudioCodecContext->time_base = new AVRational { num = 1, den = sampleRate };
// Global header flag
// Global header flag (same rule as the video encoder): codec configuration (e.g. AAC AudioSpecificConfig)
// goes into extradata for containers that store it in the header.
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0)
{
_pAudioCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER;
@@ -1870,6 +2083,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (ffmpeg.avcodec_parameters_from_context(_pAudioStream->codecpar, _pAudioCodecContext) < 0)
throw new Exception("Failed to copy audio codec parameters to stream.");
// Stream time base hint; the muxer may change it in avformat_write_header (packets are rescaled when written).
_pAudioStream->time_base = _pAudioCodecContext->time_base;
// ── 4. Initialize resampler (SwrContext) ──
@@ -1879,6 +2093,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
: AVSampleFormat.AV_SAMPLE_FMT_FLT;
// Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
// Uses audioSettings.Channels as given (not the defaulted channel count used for the layouts below).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
_audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
@@ -1889,6 +2104,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
AVChannelLayout inLayout;
ffmpeg.av_channel_layout_default(&inLayout, channels);
// Output: encoder layout/format/rate. Input: same layout and rate, input sample format.
// Only the sample format (and interleaved → planar layout) is converted; no resampling takes place.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_alloc_set_opts2)
ret = ffmpeg.swr_alloc_set_opts2(
ppSwr,
&_pAudioCodecContext->ch_layout, encoderSampleFmt, sampleRate,
@@ -1904,6 +2122,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Failed to initialize audio resampler: {GetFFmpegErrorString(ret)} (Code: {ret})");
// ── 5. Initialize FIFO buffer ──
// Holds samples in the encoder format. Initial capacity 1 sample; av_audio_fifo_write grows it as needed.
_pAudioFifo = ffmpeg.av_audio_fifo_alloc(encoderSampleFmt, channels, 1);
if (_pAudioFifo == null)
throw new Exception("Failed to allocate audio FIFO buffer.");
@@ -1916,6 +2135,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Determine frame size: Some codecs (e.g. AAC) have a fixed frame_size.
// Use default 1024 if frame_size is 0 (e.g. PCM family).
// frame_size is known after avcodec_open2 (e.g. AAC 1024, Opus 960 at 48 kHz).
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
_pAudioFrame->nb_samples = frameSize;
_pAudioFrame->format = (int)encoderSampleFmt;
@@ -1937,9 +2157,16 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Internally executes resampling → FIFO buffering → frame_size unit encoding pipeline.
///
/// Thread safety: Safe to call from multiple threads as it uses _lock internally.
/// Does nothing when audio is disabled, or the encoder is not started, already ended or disposed.
/// Samples that do not fill a whole encoder frame stay in the FIFO until the next call or End().
/// </summary>
/// <param name="samples">PCM audio sample data. Must be in interleaved format (LRLRLR...).</param>
/// <param name="samples">
/// PCM audio sample data in <see cref="AudioSettings.InputSampleFormat"/>, interleaved (LRLRLR...).
/// May be larger than needed (e.g. a pooled array); only the first numSamples × channels values are read.
/// </param>
/// <param name="numSamples">Number of samples per channel (= total float count / channel count).</param>
/// <exception cref="ArgumentException"><paramref name="numSamples"/> is negative or exceeds the array.</exception>
/// <exception cref="Exception">Format conversion, FIFO, encoding or packet writing fails.</exception>
public void WriteAudioSamples(float[] samples, int numSamples)
{
lock (_lock)
@@ -1957,8 +2184,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Encodes raw PCM audio sample data from pointer into the audio stream.
/// Same behavior as the float[] overload, without the length check: the caller guarantees that
/// numSamples × channels interleaved samples in the input format are readable at <paramref name="samples"/>.
/// The memory is read within this call only.
/// </summary>
/// <param name="samples">Pointer to PCM audio sample data.</param>
/// <param name="samples">Pointer to PCM audio sample data (interleaved, input sample format).</param>
/// <param name="numSamples">Number of samples per channel.</param>
public void WriteAudioSamples(IntPtr samples, int numSamples)
{
@@ -1969,18 +2199,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
/// <summary>
/// Internal implementation of audio sample encoding.
///
/// Processing flow:
/// 1. Resample input PCM → encoder format via swr_convert
/// 2. Add resampled data to FIFO buffer
/// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder
///
/// Reference:
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary>
/// <summary>
/// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
/// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
@@ -1994,6 +2212,25 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
}
/// <summary>
/// Internal implementation of audio sample encoding.
///
/// Processing flow:
/// 1. Resample input PCM → encoder format via swr_convert
/// (same sample rate: one output sample per input sample, only the format and planar layout change)
/// 2. Add resampled data to FIFO buffer
/// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder
/// (each frame gets PTS = _audioSampleIndex, which then advances by frame_size)
///
/// A temporary frame for the converted samples is allocated and freed on every call.
/// When that frame structure cannot be allocated, the call returns without encoding the samples.
///
/// Reference:
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary>
/// <param name="inputBuffer">Interleaved input samples (single plane).</param>
/// <param name="numSamples">Number of samples per channel.</param>
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
{
// ── 1. Resampling: Input format → Encoder format ──
@@ -2013,6 +2250,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new Exception($"Failed to allocate resampling output frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})");
// swr_convert: Converts input data to format required by encoder.
// Arguments: output planes and capacity (samples per channel), input planes and sample count.
// &inputBuffer is a one-element plane array, because the input is interleaved.
// Returns the number of samples per channel written to the output, or a negative error code.
var convertedSamples = ffmpeg.swr_convert(
_pSwrContext,
(byte**)&pConvertedFrame->data, numSamples,
@@ -2032,6 +2272,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
{
// The encoder may still reference the buffer of the previous frame; get a writable one first.
MakeAudioFrameWritable();
_pAudioFrame->nb_samples = frameSize;
@@ -2053,7 +2294,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Sends audio frame to encoder and writes result packets to file.
/// Uses same Send/Receive pattern as video SendFrameAndWritePackets.
/// Packets are rescaled from the codec time base (1 / sample rate) to the audio stream time base.
/// </summary>
/// <param name="pFrame">Audio frame to encode, or null to flush the audio encoder.</param>
/// <exception cref="Exception">Sending, receiving or writing fails.</exception>
private void SendAudioFrameAndWritePackets(AVFrame* pFrame)
{
var response = ffmpeg.avcodec_send_frame(_pAudioCodecContext, pFrame);
@@ -2128,8 +2372,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
/// <summary>
/// Releases audio-related FFmpeg resources.
/// Called inside Dispose().
/// Releases audio-related FFmpeg resources (resampler, FIFO, frame, packet, audio codec context) and disables audio.
/// Called inside Dispose(). The audio stream itself belongs to the format context and is freed with it.
/// </summary>
private void DisposeAudioResources()
{
Util/FFmpeg/VideoEncoder.cs
+77 -1
View File
@@ -6,52 +6,107 @@ using ADOFAIRenderer.Data;
namespace ADOFAIRenderer.Util.FFmpeg;
/// <summary>
/// Family of a video encoder. Selects the encoder-specific configuration in <see cref="FFmpegNativeEncoder"/>
/// (input pixel format, hardware device setup, quality option names, speed preset).
/// </summary>
[SuppressMessage("ReSharper", "InconsistentNaming")]
internal enum EncoderKind
{
/// <summary>CPU encoder library linked into FFmpeg (libx264, libx265, libvpx-vp9, libsvtav1, libaom-av1, libvvenc).</summary>
Software,
/// <summary>NVIDIA NVENC hardware encoder (*_nvenc).</summary>
NVENC,
/// <summary>AMD Advanced Media Framework hardware encoder (*_amf).</summary>
AMF,
/// <summary>Intel Quick Sync Video hardware encoder (*_qsv), driven through a QSV hardware device context.</summary>
QSV,
/// <summary>Apple VideoToolbox hardware encoder (*_videotoolbox, macOS).</summary>
VideoToolbox
}
/// <summary>
/// Known FFmpeg video encoder: FFmpeg encoder name, output codec, GPU vendor and encoder family.
///
/// Instances exist only as the static fields below (the constructor is private); compare them with == or
/// <see cref="Equals(VideoEncoder)"/>, which compare <see cref="Name"/> case-insensitively.
/// <see cref="All"/> lists every encoder, and its order is the fallback order used by
/// GPUUtil.GetPrioritizedEncoders after the encoders of the active GPU vendor.
///
/// An entry does not mean that the encoder exists in the loaded FFmpeg build or works on the current machine:
/// FFmpegNativeEncoder.Start looks each one up with avcodec_find_encoder_by_name and opens it, and skips the
/// encoders that are missing or fail to open.
/// Reference: https://ffmpeg.org/ffmpeg-codecs.html (Video Encoders)
/// </summary>
[SuppressMessage("ReSharper", "InconsistentNaming")]
internal sealed class VideoEncoder : IEquatable<VideoEncoder>
{
// ReSharper disable MemberCanBePrivate.Global
// Each field: FFmpeg encoder name, output codec, GPU vendor whose hardware runs it
// (Unknown for CPU encoders), encoder family.
// ── H.264 ──
/// <summary>NVIDIA NVENC hardware encoder for H.264 (FFmpeg encoder "h264_nvenc").</summary>
public static readonly VideoEncoder H264_NVENC = new("h264_nvenc", Codec.H264, GPUVendor.NVIDIA, EncoderKind.NVENC);
/// <summary>Intel Quick Sync Video hardware encoder for H.264 (FFmpeg encoder "h264_qsv").</summary>
public static readonly VideoEncoder H264_QSV = new("h264_qsv", Codec.H264, GPUVendor.Intel, EncoderKind.QSV);
/// <summary>AMD AMF hardware encoder for H.264 (FFmpeg encoder "h264_amf").</summary>
public static readonly VideoEncoder H264_AMF = new("h264_amf", Codec.H264, GPUVendor.AMD, EncoderKind.AMF);
/// <summary>Apple VideoToolbox hardware encoder for H.264 (FFmpeg encoder "h264_videotoolbox").</summary>
public static readonly VideoEncoder H264_VideoToolbox = new("h264_videotoolbox", Codec.H264, GPUVendor.Apple, EncoderKind.VideoToolbox);
/// <summary>CPU software encoder for H.264 (FFmpeg encoder "libx264").</summary>
public static readonly VideoEncoder Libx264 = new("libx264", Codec.H264, GPUVendor.Unknown, EncoderKind.Software);
// ── H.265 (HEVC) ──
/// <summary>NVIDIA NVENC hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_nvenc").</summary>
public static readonly VideoEncoder HEVC_NVENC = new("hevc_nvenc", Codec.H265, GPUVendor.NVIDIA, EncoderKind.NVENC);
/// <summary>Intel Quick Sync Video hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_qsv").</summary>
public static readonly VideoEncoder HEVC_QSV = new("hevc_qsv", Codec.H265, GPUVendor.Intel, EncoderKind.QSV);
/// <summary>AMD AMF hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_amf").</summary>
public static readonly VideoEncoder HEVC_AMF = new("hevc_amf", Codec.H265, GPUVendor.AMD, EncoderKind.AMF);
/// <summary>Apple VideoToolbox hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_videotoolbox").</summary>
public static readonly VideoEncoder HEVC_VideoToolbox = new("hevc_videotoolbox", Codec.H265, GPUVendor.Apple, EncoderKind.VideoToolbox);
/// <summary>CPU software encoder for H.265 (HEVC) (FFmpeg encoder "libx265").</summary>
public static readonly VideoEncoder Libx265 = new("libx265", Codec.H265, GPUVendor.Unknown, EncoderKind.Software);
// ── VP9 ──
// libavcodec has no "vp9_nvenc" encoder (NVENC does not encode VP9), so avcodec_find_encoder_by_name returns null
// for this entry and FFmpegNativeEncoder.Start skips it with a "not found" warning.
/// <summary>NVIDIA NVENC entry for VP9 (FFmpeg encoder name "vp9_nvenc"); never found in libavcodec, see above.</summary>
public static readonly VideoEncoder VP9_NVENC = new("vp9_nvenc", Codec.VP9, GPUVendor.NVIDIA, EncoderKind.NVENC);
/// <summary>Intel Quick Sync Video hardware encoder for VP9 (FFmpeg encoder "vp9_qsv").</summary>
public static readonly VideoEncoder VP9_QSV = new("vp9_qsv", Codec.VP9, GPUVendor.Intel, EncoderKind.QSV);
/// <summary>CPU software encoder for VP9 (FFmpeg encoder "libvpx-vp9").</summary>
public static readonly VideoEncoder LibvpxVP9 = new("libvpx-vp9", Codec.VP9, GPUVendor.Unknown, EncoderKind.Software);
// ── AV1 ──
/// <summary>NVIDIA NVENC hardware encoder for AV1 (FFmpeg encoder "av1_nvenc").</summary>
public static readonly VideoEncoder AV1_NVENC = new("av1_nvenc", Codec.AV1, GPUVendor.NVIDIA, EncoderKind.NVENC);
/// <summary>Intel Quick Sync Video hardware encoder for AV1 (FFmpeg encoder "av1_qsv").</summary>
public static readonly VideoEncoder AV1_QSV = new("av1_qsv", Codec.AV1, GPUVendor.Intel, EncoderKind.QSV);
/// <summary>AMD AMF hardware encoder for AV1 (FFmpeg encoder "av1_amf").</summary>
public static readonly VideoEncoder AV1_AMF = new("av1_amf", Codec.AV1, GPUVendor.AMD, EncoderKind.AMF);
/// <summary>CPU software encoder for AV1 (FFmpeg encoder "libsvtav1").</summary>
public static readonly VideoEncoder LibSvtAv1 = new("libsvtav1", Codec.AV1, GPUVendor.Unknown, EncoderKind.Software);
/// <summary>CPU software encoder for AV1 (FFmpeg encoder "libaom-av1").</summary>
public static readonly VideoEncoder LibAomAv1 = new("libaom-av1", Codec.AV1, GPUVendor.Unknown, EncoderKind.Software);
// ── VVC ──
/// <summary>CPU software encoder for H.266 (VVC) (FFmpeg encoder "libvvenc").</summary>
public static readonly VideoEncoder LibVvenc = new("libvvenc", Codec.VVC, GPUVendor.Unknown, EncoderKind.Software);
// ReSharper restore MemberCanBePrivate.Global
/// <summary>
/// Every known encoder, grouped by codec. Within a codec the order is NVENC → QSV → AMF → VideoToolbox → software
/// (for the families the codec has); GPUUtil.GetPrioritizedEncoders keeps this order for the encoders that do not
/// belong to the active GPU vendor.
/// </summary>
public static readonly IReadOnlyList<VideoEncoder> All = new[]
{
H264_NVENC, H264_QSV, H264_AMF, H264_VideoToolbox, Libx264,
@@ -61,15 +116,26 @@ internal sealed class VideoEncoder : IEquatable<VideoEncoder>
LibVvenc
};
/// <summary>Lookup table of <see cref="All"/> by <see cref="Name"/> (case-insensitive), used by <see cref="FromName"/>.</summary>
private static readonly Dictionary<string, VideoEncoder> ByName =
All.ToDictionary(e => e.Name, StringComparer.OrdinalIgnoreCase);
/// <summary>FFmpeg encoder name passed to avcodec_find_encoder_by_name (e.g. "h264_nvenc", "libx264").</summary>
public string Name { get; }
/// <summary>Video codec of the bitstream the encoder produces.</summary>
public Codec Codec { get; }
/// <summary>GPU vendor whose hardware runs the encoder. <see cref="GPUVendor.Unknown"/> for CPU (software) encoders.</summary>
public GPUVendor Vendor { get; }
/// <summary>Encoder family (software or hardware API).</summary>
public EncoderKind Kind { get; }
/// <summary>True for every family except <see cref="EncoderKind.Software"/>.</summary>
public bool IsHardware => Kind != EncoderKind.Software;
/// <summary>Creates an entry. Used only by the static fields of this class.</summary>
private VideoEncoder(string name, Codec codec, GPUVendor vendor, EncoderKind kind)
{
Name = name;
@@ -78,36 +144,46 @@ internal sealed class VideoEncoder : IEquatable<VideoEncoder>
Kind = kind;
}
/// <summary>
/// Returns the known encoder with the given FFmpeg name (case-insensitive),
/// or null when <paramref name="name"/> is null, empty or not in <see cref="All"/>.
/// </summary>
public static VideoEncoder? FromName(string name)
{
return string.IsNullOrEmpty(name) ? null : ByName.GetValueOrDefault(name);
}
/// <summary>True when <paramref name="other"/> is the same entry or has the same <see cref="Name"/> (case-insensitive).</summary>
public bool Equals(VideoEncoder other)
{
if (ReferenceEquals(null, other)) return false;
return ReferenceEquals(this, other) || string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase);
}
/// <summary>True when <paramref name="obj"/> is a <see cref="VideoEncoder"/> equal to this one (see <see cref="Equals(VideoEncoder)"/>).</summary>
public override bool Equals(object? obj)
{
return ReferenceEquals(this, obj) || obj is VideoEncoder other && Equals(other);
}
/// <summary>Case-insensitive hash of <see cref="Name"/>, consistent with <see cref="Equals(VideoEncoder)"/>.</summary>
public override int GetHashCode()
{
return StringComparer.OrdinalIgnoreCase.GetHashCode(Name);
}
/// <summary>Value equality through <see cref="object.Equals(object, object)"/>; two null operands are equal.</summary>
public static bool operator ==(VideoEncoder left, VideoEncoder right)
{
return Equals(left, right);
}
/// <summary>Negation of the <c>==</c> operator.</summary>
public static bool operator !=(VideoEncoder left, VideoEncoder right)
{
return !Equals(left, right);
}
/// <summary>Returns the FFmpeg encoder name (<see cref="Name"/>).</summary>
public override string ToString() => Name;
}