ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Cleanup Comments

Browse Source
This commit is contained in:
copecone committed 2026-10-12 19:48:15 +09:00
1 parent f1e926f758
commit 4e556a4632
3 files changed
+83 -45
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Util/FFmpeg/FFmpegNativeDecoder.cs
+16 -9
View File
@@ -23,7 +23,7 @@ namespace ADOFAIRenderer.Util.FFmpeg;
/// 2. Codec & Hardware Device Discovery:
/// a) avcodec_find_decoder: Locate codec implementation by codec ID.
/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder.
/// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox).
/// c) HW device creation for the reported configurations, in the order of PreferredHwDeviceTypes.
/// d) Attach hw_device_ctx and register custom get_format callback.
/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding
/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME).
@@ -35,8 +35,10 @@ namespace ADOFAIRenderer.Util.FFmpeg;
/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data
/// into system RAM as NV12 (av_frame_copy_props preserves timestamps).
/// d) (Output Dispatch):
/// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback.
/// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback.
/// - TryReadNV12Frame: NV12 frames are copied row by row (Buffer.MemoryCopy) into the output buffers,
/// other formats are converted with sws_scale.
/// - TryReadYUV420PFrame: YUV420P/YUVJ420P planes are copied row by row into the output buffers,
/// other formats are converted with sws_scale.
/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
///
/// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame):
@@ -76,9 +78,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Preferred hardware device types in priority order.
/// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration,
/// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA.
/// Non-Windows systems fall back to VAAPI or VideoToolbox.
/// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config),
/// sorted by this list; configurations not in the list follow in reported order.
/// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Other platforms: VAAPI or VideoToolbox.
/// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by
/// avcodec_find_decoder report DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan in the bundled FFmpeg 8 build;
/// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name.
/// </summary>
private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
{
@@ -661,9 +666,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_getFormatDelegate = GetFormat;
_pCodecContext->get_format = _getFormatDelegate;
// Hardware decoding requires single-threaded decoding (thread_count = 1).
// Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function
// GPU decoders and causes surface pool allocation exhaustion or crashes in get_format.
// Hardware decoding runs with one decoder thread (thread_count = 1).
// The GPU performs the decoding; the decoder thread only parses packets and submits them.
// With frame threading, libavcodec adds one hardware surface per thread to the frame pool
// (avcodec_get_hw_frames_parameters). The software fallback in GetFormat keeps this thread count.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/decode.c
_pCodecContext->thread_count = 1;
// Attempt opening codec context with hardware configuration
Util/FFmpeg/FFmpegNativeEncoder.cs
+58 -33
View File
@@ -18,12 +18,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 2. Start(outputPath):
/// a) Create AVFormatContext for output container (mp4, etc.)
/// b) Add video stream (AVStream)
/// c) Try hardware encoders (NVENC → AMF → QSV) → software encoders (libx264, etc.)
/// in order to select an available encoder
/// c) Try the candidate encoders of the codec in order and select the first one that opens
/// (encoders of the active GPU vendor → other hardware encoders → software encoders; see GetEncoderCandidates)
/// d) Write file header and initialize frame/scaler buffers
///
/// 3. WriteFrame*(): Called for every frame
/// a) Convert Unity RGBA original frame to NV12/YUV420P using sws_scale
/// a) Pre-converted NV12/YUV420P frame (GPU compute shader output): copied into the encoder frame
/// RGBA frame: converted to the encoder pixel format with sws_scale (CPU)
/// b) (In QSV hw_frames mode) Upload to GPU surface
/// c) avcodec_send_frame → avcodec_receive_packet → Write to file
///
@@ -90,13 +91,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
public RateControlMode Mode;
/// <summary>Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)</summary>
public int QualityValue; // CRF
public int QualityValue;
/// <summary>Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg.</summary>
public int TargetBitrate; // VBR
public int TargetBitrate;
/// <summary>Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.</summary>
public int MaxBitrate; // VBR
public int MaxBitrate;
/// <summary>GOP (Group of Pictures) size (in seconds). Represents keyframe interval.</summary>
public double GopSize;
@@ -165,7 +166,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Accumulated index of encoded frames. Used as the PTS (Presentation Time Stamp) value.</summary>
private long _frameIndex;
/// <summary>Whether Start() completed successfully and encoding is ready.</summary>
/// <summary>
/// Whether the encoder accepts frames: set to true when Start() completes,
/// and to false when End() runs (End() is not repeated) or Dispose() releases the resources.
/// </summary>
private bool _isInitialized;
/// <summary>Whether Dispose() has already been called. Prevents duplicate cleanup.</summary>
@@ -182,9 +186,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
private bool _isQsvEncoder;
/// <summary>
/// Target pixel format for the encoder.
/// - General encoders (NVENC, AMF, Software): YUV420P (most common 4:2:0 chroma subsampling)
/// - QSV encoder: NV12 (Y plane + UV interleaved plane, preferred format for Intel hardware)
/// Pixel format of the frames passed to the encoder (output of the color conversion).
/// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane)
/// - libvvenc: YUV420P10LE (10-bit, 3 planes)
/// - Other software encoders: YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling)
///
/// Reference - Major pixel format relationships:
/// RGBA (Unity source, 4-channel 8-bit)
@@ -285,7 +290,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Returns a list of candidate encoders to try for the selected codec in priority order.
///
/// Priority: Hardware encoders (NVENC → AMF → QSV) → Software encoders (libx264, etc.)
/// Order (GPUUtil.GetPrioritizedEncoders):
/// 1. Hardware encoders of the active GPU vendor
/// 2. The remaining encoders in the declaration order of VideoEncoder.All
/// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has)
/// With ForceSoftware, only the software encoders are returned.
///
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
@@ -416,11 +425,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Pixel format configuration ──
// Video encoders generally expect input in YUV420P or NV12 format.
//
// • YUV420P: 3 independent planes Y/U/V. Used by most software encoders.
// • NV12: Y plane + UV interleaved plane. Preferred by hardware encoders (NVENC, AMF, QSV).
// • QSV Special Note: pix_fmt of codec context is AV_PIX_FMT_QSV (hardware surface identifier),
// while actual sws_scale output is set to NV12 (software format).
// FFmpeg internally handles mapping between NV12 ↔ QSV surfaces.
// • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE).
// • NV12: Y plane + UV interleaved plane. Used by the hardware encoders (NVENC, AMF, QSV).
// • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is
// AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode:
// system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly.
// GPU frame pool input mode: pix_fmt stays AV_PIX_FMT_QSV, and SubmitDstFrame() uploads
// each NV12 frame to a QSV surface with av_hwframe_transfer_data.
//
// Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html
var selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
@@ -479,17 +490,21 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pCodecContext->gop_size = calculatedGop;
}
// ── Disable B-frames ──
// B-frames (Bidirectional predictive frames): Increase compression ratio by referencing before/after frames,
// but increase encoding latency and may cause compatibility issues on some encoders.
// In real-time recording scenarios, disable them to minimize latency.
// ── B-frame configuration ──
// B-frames (bidirectional predictive frames) reference both earlier and later frames.
// They raise the compression ratio and make the encoder output packets out of presentation order
// (dts < pts for reordered frames).
//
// Reference: has_b_frames is a read-only field internally set by the encoder,
// so do not set it directly.
// Configuration per encoder:
// SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure).
// Setting max_b_frames on AVCodecContext makes SVT-AV1 report a "bad parameter" error.
// libvvenc: max_b_frames = -1 (encoder default GOP structure).
// Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order).
//
// SVT-AV1 Special Note: Setting max_b_frames directly on AVCodecContext
// causes a "bad parameter" error, so control via svtav1-params.
// pred-struct=1: Low Delay (IPPP) structure disabling B-frames.
// has_b_frames is set by the encoder itself and is not written here.
//
// Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md
// (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2)
if (encoder == VideoEncoder.LibSvtAv1)
{
ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0);
@@ -696,8 +711,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
// Transfer ownership of hwDeviceCtx to codec context,
// so release local reference (reference count management).
// Release the local reference of hwDeviceCtx.
// The device stays alive through the references taken above:
// system memory input mode: codec context hw_device_ctx (av_buffer_ref)
// GPU frame pool input mode: the frames context (av_hwframe_ctx_alloc keeps a device reference),
// held by codec context hw_frames_ctx
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html
ffmpeg.av_buffer_unref(&hwDeviceCtx);
}
@@ -833,8 +851,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Colorspace is converted without resolution change here, but sws_scale
// can also perform resizing simultaneously.
//
// SWS_POINT: Nearest neighbor interpolation. Since only colorspace is converted without resolution change,
// interpolation algorithm does not affect output quality, and SWS_POINT is fastest.
// SWS_POINT: Nearest neighbor sampling. The luma plane keeps the source resolution (1:1).
// The chroma planes are subsampled to half width and height (4:2:0) with the same filter,
// so each chroma sample takes one source pixel instead of averaging the 2x2 block.
// This scaler is used only by the CPU conversion path (WriteFrame with RGBA input).
//
// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
_pSwsContext = ffmpeg.sws_getContext(
@@ -1389,8 +1409,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool.
// av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface.
//
// Through this process, the encoder can read frames directly from GPU memory,
// minimizing PCIe bandwidth usage.
// The frame is copied from system memory to the GPU surface on every call;
// the encoder then reads the surface from GPU memory.
//
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
@@ -1953,8 +1973,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Flushes the audio encoder.
/// Pads and encodes remaining samples (less than frame_size) in FIFO,
/// then sends null frame to output all delayed frames in encoder.
/// Sends the samples remaining in the FIFO (fewer than frame_size) as one last short frame,
/// then sends a null frame to output all delayed packets of the encoder.
///
/// The short frame is handled by libavcodec (encode_send_frame_internal):
/// encoders with AV_CODEC_CAP_SMALL_LAST_FRAME or AV_CODEC_CAP_VARIABLE_FRAME_SIZE receive it as is,
/// other encoders receive it padded with silence up to frame_size (pad_last_frame).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/encode.c
/// </summary>
private void FlushAudioEncoder()
{