From 4e556a46324b74eed301ced850d2ec78ce7091b85f20f1eb232907d17f3a7b10 Mon Sep 17 00:00:00 2001 From: copecone Date: Mon, 12 Oct 2026 19:48:15 +0900 Subject: [PATCH] Cleanup Comments --- Util/FFmpeg/FFmpegNativeDecoder.cs | 25 +++++--- Util/FFmpeg/FFmpegNativeEncoder.cs | 91 +++++++++++++++++++----------- Util/OS/OSNotification.cs | 12 +++- 3 files changed, 83 insertions(+), 45 deletions(-) diff --git a/Util/FFmpeg/FFmpegNativeDecoder.cs b/Util/FFmpeg/FFmpegNativeDecoder.cs index d85038c..e9adc68 100644 --- a/Util/FFmpeg/FFmpegNativeDecoder.cs +++ b/Util/FFmpeg/FFmpegNativeDecoder.cs @@ -23,7 +23,7 @@ namespace ADOFAIRenderer.Util.FFmpeg; /// 2. Codec & Hardware Device Discovery: /// a) avcodec_find_decoder: Locate codec implementation by codec ID. /// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder. -/// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox). +/// c) HW device creation for the reported configurations, in the order of PreferredHwDeviceTypes. /// d) Attach hw_device_ctx and register custom get_format callback. /// e) If HW initialization fails or is unsupported, fall back to CPU software decoding /// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME). @@ -35,8 +35,10 @@ namespace ADOFAIRenderer.Util.FFmpeg; /// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data /// into system RAM as NV12 (av_frame_copy_props preserves timestamps). /// d) (Output Dispatch): -/// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback. -/// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback. +/// - TryReadNV12Frame: NV12 frames are copied row by row (Buffer.MemoryCopy) into the output buffers, +/// other formats are converted with sws_scale. +/// - TryReadYUV420PFrame: YUV420P/YUVJ420P planes are copied row by row into the output buffers, +/// other formats are converted with sws_scale. /// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip). /// /// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame): @@ -76,9 +78,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Preferred hardware device types in priority order. - /// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration, - /// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA. - /// Non-Windows systems fall back to VAAPI or VideoToolbox. + /// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config), + /// sorted by this list; configurations not in the list follow in reported order. + /// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Other platforms: VAAPI or VideoToolbox. + /// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by + /// avcodec_find_decoder report DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan in the bundled FFmpeg 8 build; + /// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name. /// private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[] { @@ -661,9 +666,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _getFormatDelegate = GetFormat; _pCodecContext->get_format = _getFormatDelegate; - // Hardware decoding requires single-threaded decoding (thread_count = 1). - // Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function - // GPU decoders and causes surface pool allocation exhaustion or crashes in get_format. + // Hardware decoding runs with one decoder thread (thread_count = 1). + // The GPU performs the decoding; the decoder thread only parses packets and submits them. + // With frame threading, libavcodec adds one hardware surface per thread to the frame pool + // (avcodec_get_hw_frames_parameters). The software fallback in GetFormat keeps this thread count. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/decode.c _pCodecContext->thread_count = 1; // Attempt opening codec context with hardware configuration diff --git a/Util/FFmpeg/FFmpegNativeEncoder.cs b/Util/FFmpeg/FFmpegNativeEncoder.cs index debd386..5d3a773 100644 --- a/Util/FFmpeg/FFmpegNativeEncoder.cs +++ b/Util/FFmpeg/FFmpegNativeEncoder.cs @@ -18,12 +18,13 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 2. Start(outputPath): /// a) Create AVFormatContext for output container (mp4, etc.) /// b) Add video stream (AVStream) - /// c) Try hardware encoders (NVENC → AMF → QSV) → software encoders (libx264, etc.) - /// in order to select an available encoder + /// c) Try the candidate encoders of the codec in order and select the first one that opens + /// (encoders of the active GPU vendor → other hardware encoders → software encoders; see GetEncoderCandidates) /// d) Write file header and initialize frame/scaler buffers /// /// 3. WriteFrame*(): Called for every frame - /// a) Convert Unity RGBA original frame to NV12/YUV420P using sws_scale + /// a) Pre-converted NV12/YUV420P frame (GPU compute shader output): copied into the encoder frame + /// RGBA frame: converted to the encoder pixel format with sws_scale (CPU) /// b) (In QSV hw_frames mode) Upload to GPU surface /// c) avcodec_send_frame → avcodec_receive_packet → Write to file /// @@ -90,13 +91,13 @@ namespace ADOFAIRenderer.Util.FFmpeg public RateControlMode Mode; /// Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless) - public int QualityValue; // CRF + public int QualityValue; /// Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg. - public int TargetBitrate; // VBR + public int TargetBitrate; /// Maximum bitrate in VBR mode (in kbps). Determines peak quality limit. - public int MaxBitrate; // VBR + public int MaxBitrate; /// GOP (Group of Pictures) size (in seconds). Represents keyframe interval. public double GopSize; @@ -165,7 +166,10 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Accumulated index of encoded frames. Used as the PTS (Presentation Time Stamp) value. private long _frameIndex; - /// Whether Start() completed successfully and encoding is ready. + /// + /// Whether the encoder accepts frames: set to true when Start() completes, + /// and to false when End() runs (End() is not repeated) or Dispose() releases the resources. + /// private bool _isInitialized; /// Whether Dispose() has already been called. Prevents duplicate cleanup. @@ -182,9 +186,10 @@ namespace ADOFAIRenderer.Util.FFmpeg private bool _isQsvEncoder; /// - /// Target pixel format for the encoder. - /// - General encoders (NVENC, AMF, Software): YUV420P (most common 4:2:0 chroma subsampling) - /// - QSV encoder: NV12 (Y plane + UV interleaved plane, preferred format for Intel hardware) + /// Pixel format of the frames passed to the encoder (output of the color conversion). + /// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane) + /// - libvvenc: YUV420P10LE (10-bit, 3 planes) + /// - Other software encoders: YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling) /// /// Reference - Major pixel format relationships: /// RGBA (Unity source, 4-channel 8-bit) @@ -285,7 +290,11 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Returns a list of candidate encoders to try for the selected codec in priority order. /// - /// Priority: Hardware encoders (NVENC → AMF → QSV) → Software encoders (libx264, etc.) + /// Order (GPUUtil.GetPrioritizedEncoders): + /// 1. Hardware encoders of the active GPU vendor + /// 2. The remaining encoders in the declaration order of VideoEncoder.All + /// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has) + /// With ForceSoftware, only the software encoders are returned. /// /// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based) /// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework) @@ -416,11 +425,13 @@ namespace ADOFAIRenderer.Util.FFmpeg // ── Pixel format configuration ── // Video encoders generally expect input in YUV420P or NV12 format. // - // • YUV420P: 3 independent planes Y/U/V. Used by most software encoders. - // • NV12: Y plane + UV interleaved plane. Preferred by hardware encoders (NVENC, AMF, QSV). - // • QSV Special Note: pix_fmt of codec context is AV_PIX_FMT_QSV (hardware surface identifier), - // while actual sws_scale output is set to NV12 (software format). - // FFmpeg internally handles mapping between NV12 ↔ QSV surfaces. + // • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE). + // • NV12: Y plane + UV interleaved plane. Used by the hardware encoders (NVENC, AMF, QSV). + // • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is + // AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode: + // system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly. + // GPU frame pool input mode: pix_fmt stays AV_PIX_FMT_QSV, and SubmitDstFrame() uploads + // each NV12 frame to a QSV surface with av_hwframe_transfer_data. // // Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html var selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P; @@ -479,17 +490,21 @@ namespace ADOFAIRenderer.Util.FFmpeg _pCodecContext->gop_size = calculatedGop; } - // ── Disable B-frames ── - // B-frames (Bidirectional predictive frames): Increase compression ratio by referencing before/after frames, - // but increase encoding latency and may cause compatibility issues on some encoders. - // In real-time recording scenarios, disable them to minimize latency. + // ── B-frame configuration ── + // B-frames (bidirectional predictive frames) reference both earlier and later frames. + // They raise the compression ratio and make the encoder output packets out of presentation order + // (dts < pts for reordered frames). // - // Reference: has_b_frames is a read-only field internally set by the encoder, - // so do not set it directly. + // Configuration per encoder: + // SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure). + // Setting max_b_frames on AVCodecContext makes SVT-AV1 report a "bad parameter" error. + // libvvenc: max_b_frames = -1 (encoder default GOP structure). + // Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order). // - // SVT-AV1 Special Note: Setting max_b_frames directly on AVCodecContext - // causes a "bad parameter" error, so control via svtav1-params. - // pred-struct=1: Low Delay (IPPP) structure disabling B-frames. + // has_b_frames is set by the encoder itself and is not written here. + // + // Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md + // (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2) if (encoder == VideoEncoder.LibSvtAv1) { ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0); @@ -696,8 +711,11 @@ namespace ADOFAIRenderer.Util.FFmpeg } } - // Transfer ownership of hwDeviceCtx to codec context, - // so release local reference (reference count management). + // Release the local reference of hwDeviceCtx. + // The device stays alive through the references taken above: + // system memory input mode: codec context hw_device_ctx (av_buffer_ref) + // GPU frame pool input mode: the frames context (av_hwframe_ctx_alloc keeps a device reference), + // held by codec context hw_frames_ctx // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html ffmpeg.av_buffer_unref(&hwDeviceCtx); } @@ -833,8 +851,10 @@ namespace ADOFAIRenderer.Util.FFmpeg // Colorspace is converted without resolution change here, but sws_scale // can also perform resizing simultaneously. // - // SWS_POINT: Nearest neighbor interpolation. Since only colorspace is converted without resolution change, - // interpolation algorithm does not affect output quality, and SWS_POINT is fastest. + // SWS_POINT: Nearest neighbor sampling. The luma plane keeps the source resolution (1:1). + // The chroma planes are subsampled to half width and height (4:2:0) with the same filter, + // so each chroma sample takes one source pixel instead of averaging the 2x2 block. + // This scaler is used only by the CPU conversion path (WriteFrame with RGBA input). // // Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html _pSwsContext = ffmpeg.sws_getContext( @@ -1389,8 +1409,8 @@ namespace ADOFAIRenderer.Util.FFmpeg // av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool. // av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface. // - // Through this process, the encoder can read frames directly from GPU memory, - // minimizing PCIe bandwidth usage. + // The frame is copied from system memory to the GPU surface on every call; + // the encoder then reads the surface from GPU memory. // // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html @@ -1953,8 +1973,13 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Flushes the audio encoder. - /// Pads and encodes remaining samples (less than frame_size) in FIFO, - /// then sends null frame to output all delayed frames in encoder. + /// Sends the samples remaining in the FIFO (fewer than frame_size) as one last short frame, + /// then sends a null frame to output all delayed packets of the encoder. + /// + /// The short frame is handled by libavcodec (encode_send_frame_internal): + /// encoders with AV_CODEC_CAP_SMALL_LAST_FRAME or AV_CODEC_CAP_VARIABLE_FRAME_SIZE receive it as is, + /// other encoders receive it padded with silence up to frame_size (pad_last_frame). + /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/encode.c /// private void FlushAudioEncoder() { diff --git a/Util/OS/OSNotification.cs b/Util/OS/OSNotification.cs index 7632e9a..c755050 100644 --- a/Util/OS/OSNotification.cs +++ b/Util/OS/OSNotification.cs @@ -9,8 +9,12 @@ namespace ADOFAIRenderer.Util.OS; /// /// Sends native desktop notifications on Windows, macOS, and Linux. /// -/// - Windows: Uses Win32 Shell_NotifyIcon (system tray balloon) via P/Invoke. -/// No PowerShell, no UWP, no WinRT dependencies. +/// - Windows: Toast notification shown by a hidden powershell.exe process +/// (WinRT ToastNotificationManager, script passed with -EncodedCommand). +/// With a folder path, the script also registers the adofai-renderer:// URL protocol under +/// HKCU\Software\Classes\adofai-renderer (open command: explorer.exe "<folder>") +/// so the toast button opens the folder; the key is overwritten on each notification and stays registered. +/// When PowerShell fails, a Win32 Shell_NotifyIcon balloon is shown via P/Invoke. /// /// /// - macOS: Uses osascript with AppleScript's display notification. @@ -88,7 +92,9 @@ internal static class OSNotification /// /// Attempts to show a Windows Toast notification via PowerShell. - /// Returns true if the process launched successfully, false otherwise. + /// With a folder path, registers the adofai-renderer:// protocol in HKCU first (see class summary). + /// Waits up to 5 seconds for the process. Returns true when it exits with code 0, + /// false when it exits with another code, does not exit within 5 seconds, or fails to start. /// private static bool TrySendToast(string title, string message, string? folderPath) {