Prefer QSV System Memory Input With GPU Frame Pool Fallback

Util/FFmpeg/FFmpegNativeEncoder.cs

@@ -63,7 +63,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             public AVCodecID AudioCodecId;
 
             /// <summary>
-            /// Audio bitrate (in bps). If default (0), 128000 bps is used.
+            /// Audio bitrate (in bps). If default (0), 256000 bps is used.
             /// For AAC, 128000–320000 bps is common.
             /// </summary>
             public int Bitrate;
@@ -299,6 +299,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
             return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray();
         }
 
+        /// <summary>
+        /// Expands the candidate list into (encoder, input mode) attempts in order.
+        /// A QSV encoder is attempted twice: system memory input mode (useFramePool = false) first,
+        /// then GPU frame pool input mode (useFramePool = true). Other encoders are attempted once (useFramePool = false).
+        /// </summary>
+        private static System.Collections.Generic.IEnumerable<(VideoEncoder Encoder, bool UseFramePool)> ExpandQsvInputModes(VideoEncoder[] candidates)
+        {
+            foreach (var encoder in candidates)
+            {
+                yield return (encoder, false);
+                if (encoder.Kind == EncoderKind.QSV)
+                {
+                    yield return (encoder, true);
+                }
+            }
+        }
+
         /// <summary>
         /// Converts an FFmpeg error code (negative integer) to a human-readable string.
         /// e.g., -1 → "Operation not permitted"
@@ -367,7 +384,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                     var candidates = GetEncoderCandidates();
                     var openedSuccessfully = false;
 
-                    foreach (var encoder in candidates)
+                    foreach (var (encoder, useFramePool) in ExpandQsvInputModes(candidates))
                     {
                         // avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc").
                         // Returns null if the encoder is not included in the build.
@@ -514,9 +531,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
                         //   Strategy 3) Create QSV device directly (legacy method)
                         //               → Fallback on older drivers/systems
                         //
-                        // After obtaining device, attempt using GPU frame pool:
+                        // After obtaining device, the input mode is configured (see ExpandQsvInputModes):
-                        //   hw_frames_ctx: Pre-allocate frame pool in GPU VRAM (zero-copy, highest performance)
+                        //   Input mode 1, hw_device_ctx: Encoder receives NV12 frames from system memory (attempted first)
-                        //   hw_device_ctx: Encoder reads frames directly from system memory (better compatibility)
+                        //   Input mode 2, hw_frames_ctx: Frames are uploaded to a QSV surface pool in GPU memory
+                        //                                (attempted when the encoder cannot be opened in input mode 1)
                         //
                         // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
                         if (encoder.Kind == EncoderKind.QSV)
@@ -585,64 +603,97 @@ namespace ADOFAIRenderer.Util.FFmpeg
                                 continue;
                             }
 
-                            // ── Attempt hw_frames_ctx: Pre-create GPU frame pool ──
+                            if (!useFramePool)
-                            // AVHWFramesContext pre-allocates frame pool in GPU VRAM,
-                            // eliminating memory allocation/deallocation overhead for each frame.
-                            //
-                            // Pros: Zero-copy capability, highest performance
-                            // Cons: May fail depending on GPU/driver resolution/format constraints
-                            //       (e.g. specific texture size constraints on Intel Arc → E_INVALIDARG)
-                            //
-                            // initial_pool_size: Number of frames to pre-allocate. Matches encoder pipeline depth.
-                            // Align resolution to multiples of 32: Texture alignment condition required by QSV hardware.
-                            //
-                            // Reference:
-                            //   AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
-                            //   av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
-                            bool framesCtxReady = false;
-                            AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
-                            if (hwFramesRef != null)
-                            {
-                                var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
-                                framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV;       // Hardware surface format
-                                framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12;    // Software-side format
-                                framesCtx->width = (_settings.Width + 31) & ~31;          // 32-multiple alignment (bitmask)
-                                framesCtx->height = (_settings.Height + 31) & ~31;        // 32-multiple alignment (bitmask)
-                                framesCtx->initial_pool_size = 4;                         // Frame pool size
-
-                                err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
-                                if (err >= 0)
-                                {
-                                    // Success: Connect hw_frames_ctx to codec context.
-                                    // Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
-                                    _pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
-                                    _isQsvEncoder = true;
-                                    framesCtxReady = true;
-                                    Melon<RenderMod>.Logger.Msg($"{encoder.Name}: hw_frames_ctx initialization succeeded (GPU frame pool mode)");
-
-                                    _pHwFrame = ffmpeg.av_frame_alloc();
-                                    if (_pHwFrame == null)
-                                        throw new Exception("Failed to allocate QSV hardware frame structure.");
-                                }
-                                else
-                                {
-                                    Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed — {GetFFmpegErrorString(err)} (Code: {err}). Falling back to hw_device_ctx mode.");
-                                }
-                                ffmpeg.av_buffer_unref(&hwFramesRef);
-                            }
-
-                            // ── When hw_frames_ctx fails: Fallback to hw_device_ctx + NV12 system memory ──
-                            // In hw_device_ctx mode, the encoder receives NV12 frames from system memory (RAM)
-                            // directly and uploads to GPU internally.
-                            // Slightly slower than hw_frames_ctx, but compatibility is much better.
-                            //
-                            // Change pix_fmt to NV12: Use software format directly instead of hardware surface (QSV).
-                            // _isQsvEncoder = false: Skip manual upload in WriteFrame().
-                            if (!framesCtxReady)
                             {
+                                // ── Input mode 1 (primary): hw_device_ctx + NV12 system memory ──
+                                // The encoder receives NV12 frames in system memory (RAM) and the QSV runtime uploads them to the GPU.
+                                //
+                                // pix_fmt is the software format NV12.
+                                // _isQsvEncoder = false: WriteFrame() passes the NV12 frame to the encoder without uploading it.
                                 _pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
                                 _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
-                                _isQsvEncoder = false; // System memory mode, no manual frame upload required
+                                _isQsvEncoder = false;
+                                Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using system memory input mode (hw_device_ctx)");
+                            }
+                            else
+                            {
+                                // ── Input mode 2 (fallback): hw_frames_ctx GPU frame pool ──
+                                // Used when the encoder cannot be opened in input mode 1.
+                                // AVHWFramesContext pre-allocates a pool of QSV surfaces in GPU memory (pix_fmt AV_PIX_FMT_QSV),
+                                // and WriteFrame() uploads each NV12 frame to a surface of the pool (av_hwframe_transfer_data).
+                                //
+                                // Pool creation depends on the texture types the GPU/driver accepts
+                                // (e.g. Intel Arc A770 rejects NV12 textures with D3D11_BIND_RENDER_TARGET → E_INVALIDARG 0x80070057).
+                                //
+                                // initial_pool_size: Number of surfaces in the pool (QsvFramePoolSize).
+                                //   The encoder keeps submitted surfaces until they are encoded, so the pool has to hold
+                                //   the frames in the encoder pipeline plus the frame being uploaded.
+                                // Width/height are aligned to multiples of 32: texture alignment condition required by QSV hardware.
+                                //
+                                // Frame pool memory type (AVQSVFramesContext.frame_type, a combination of MFX_MEMTYPE_* flags):
+                                //   Windows (D3D11): MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010) is tried first, then 0.
+                                //     0x0010 creates the D3D11 textures with D3D11_BIND_DECODER.
+                                //     0 lets FFmpeg select MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET (0x0020),
+                                //     which creates the textures with D3D11_BIND_RENDER_TARGET.
+                                //   Other platforms (VAAPI): 0. FFmpeg sets the memory type of a VAAPI child pool itself.
+                                // Both types are accepted by the QSV encoder as video memory input.
+                                // When every type fails, the next candidate encoder is tested.
+                                //
+                                // Reference:
+                                //   AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
+                                //   av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
+                                //   AVQSVFramesContext.frame_type: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
+                                //   frame_type → D3D11 bind flags (qsv_init_child_ctx, qsv_get_d3d11va_bind_flags): https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.c
+                                //   MFX_MEMTYPE_* values: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
+                                bool framesCtxReady = false;
+                                int[] frameTypes = isWindows
+                                    ? new[] { QsvMemTypeVideoMemoryDecoderTarget, 0 }
+                                    : new[] { 0 };
+                                foreach (var frameType in frameTypes)
+                                {
+                                    AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
+                                    if (hwFramesRef == null) break;
+
+                                    var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
+                                    framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV;       // Hardware surface format
+                                    framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12;    // Software-side format
+                                    framesCtx->width = (_settings.Width + 31) & ~31;          // 32-multiple alignment (bitmask)
+                                    framesCtx->height = (_settings.Height + 31) & ~31;        // 32-multiple alignment (bitmask)
+                                    framesCtx->initial_pool_size = QsvFramePoolSize;
+                                    SetQsvFrameType(framesCtx, frameType);
+
+                                    err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
+                                    if (err >= 0)
+                                    {
+                                        // Success: Connect hw_frames_ctx to codec context.
+                                        // Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
+                                        _pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
+                                        _isQsvEncoder = true;
+                                        framesCtxReady = true;
+                                        Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using GPU frame pool input mode (hw_frames_ctx, frame_type=0x{frameType:X4}, pool={QsvFramePoolSize})");
+                                        ffmpeg.av_buffer_unref(&hwFramesRef);
+
+                                        _pHwFrame = ffmpeg.av_frame_alloc();
+                                        if (_pHwFrame == null)
+                                            throw new Exception("Failed to allocate QSV hardware frame structure.");
+                                        break;
+                                    }
+
+                                    Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed with frame_type=0x{frameType:X4} — {GetFFmpegErrorString(err)} (Code: {err})");
+                                    ffmpeg.av_buffer_unref(&hwFramesRef);
+                                }
+
+                                if (!framesCtxReady)
+                                {
+                                    Melon<RenderMod>.Logger.Warning($"{encoder.Name}: GPU frame pool input mode is unavailable. Testing next candidate codec.");
+                                    ffmpeg.av_buffer_unref(&hwDeviceCtx);
+                                    {
+                                        var pCtx = _pCodecContext;
+                                        ffmpeg.avcodec_free_context(&pCtx);
+                                    }
+                                    _pCodecContext = null;
+                                    continue;
+                                }
                             }
 
                             // Transfer ownership of hwDeviceCtx to codec context,
@@ -704,11 +755,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
                             break;
                         }
 
-                        // If open fails, free codec context and try next encoder.
+                        // If open fails, free codec context and QSV frame state, then try next candidate.
-                        Melon<RenderMod>.Logger.Warning($"Failed to initialize hardware encoder {encoder.Name}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate codec.");
+                        var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
+                        Melon<RenderMod>.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
                         var pCodecCtx = _pCodecContext;
                         ffmpeg.avcodec_free_context(&pCodecCtx);
                         _pCodecContext = null;
+                        _isQsvEncoder = false;
+                        if (_pHwFrame != null)
+                        {
+                            var pHwFrame = _pHwFrame;
+                            ffmpeg.av_frame_free(&pHwFrame);
+                            _pHwFrame = null;
+                        }
                     }
 
                     if (!openedSuccessfully || _pCodecContext == null)
@@ -977,6 +1036,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// Sets a private option (priv_data) of the encoder.
         /// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
         /// </summary>
+        // ReSharper disable once UnusedMethodReturnValue.Local
         private bool TrySetOption(string name, string value)
         {
             var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0);
@@ -986,6 +1046,32 @@ namespace ADOFAIRenderer.Util.FFmpeg
             return false;
         }
 
+        /// <summary>
+        /// MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010): frames are in video memory and belong to video decoder render targets.
+        /// Reference: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
+        /// </summary>
+        private const int QsvMemTypeVideoMemoryDecoderTarget = 0x0010;
+
+        /// <summary>
+        /// Number of surfaces in the QSV GPU frame pool (AVHWFramesContext.initial_pool_size) of GPU frame pool input mode.
+        /// Same value as the encoder frame pool of the FFmpeg hardware encoding example (initial_pool_size = 20).
+        /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/doc/examples/vaapi_encode.c
+        /// </summary>
+        private const int QsvFramePoolSize = 20;
+
+        /// <summary>
+        /// Writes AVQSVFramesContext.frame_type of a QSV frames context (AVHWFramesContext.hwctx).
+        /// FFmpeg.AutoGen has no AVQSVFramesContext type, so the field is written at its offset.
+        /// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
+        /// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
+        /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
+        /// </summary>
+        private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
+        {
+            if (framesCtx->hwctx == null) return;
+            *(int*)((byte*)framesCtx->hwctx + IntPtr.Size + sizeof(int)) = frameType;
+        }
+
         /// <summary>
         /// Largest declared maximum that is treated as an actual quality range.
         /// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used.