diff --git a/Util/FFmpeg/FFmpegNativeEncoder.cs b/Util/FFmpeg/FFmpegNativeEncoder.cs
index da8a20e..b4cb7cc 100644
--- a/Util/FFmpeg/FFmpegNativeEncoder.cs
+++ b/Util/FFmpeg/FFmpegNativeEncoder.cs
@@ -63,7 +63,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
public AVCodecID AudioCodecId;
///
- /// Audio bitrate (in bps). If default (0), 128000 bps is used.
+ /// Audio bitrate (in bps). If default (0), 256000 bps is used.
/// For AAC, 128000–320000 bps is common.
///
public int Bitrate;
@@ -299,6 +299,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray();
}
+ ///
+ /// Expands the candidate list into (encoder, input mode) attempts in order.
+ /// A QSV encoder is attempted twice: system memory input mode (useFramePool = false) first,
+ /// then GPU frame pool input mode (useFramePool = true). Other encoders are attempted once (useFramePool = false).
+ ///
+ private static System.Collections.Generic.IEnumerable<(VideoEncoder Encoder, bool UseFramePool)> ExpandQsvInputModes(VideoEncoder[] candidates)
+ {
+ foreach (var encoder in candidates)
+ {
+ yield return (encoder, false);
+ if (encoder.Kind == EncoderKind.QSV)
+ {
+ yield return (encoder, true);
+ }
+ }
+ }
+
///
/// Converts an FFmpeg error code (negative integer) to a human-readable string.
/// e.g., -1 → "Operation not permitted"
@@ -367,7 +384,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var candidates = GetEncoderCandidates();
var openedSuccessfully = false;
- foreach (var encoder in candidates)
+ foreach (var (encoder, useFramePool) in ExpandQsvInputModes(candidates))
{
// avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc").
// Returns null if the encoder is not included in the build.
@@ -514,9 +531,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Strategy 3) Create QSV device directly (legacy method)
// → Fallback on older drivers/systems
//
- // After obtaining device, attempt using GPU frame pool:
- // hw_frames_ctx: Pre-allocate frame pool in GPU VRAM (zero-copy, highest performance)
- // hw_device_ctx: Encoder reads frames directly from system memory (better compatibility)
+ // After obtaining device, the input mode is configured (see ExpandQsvInputModes):
+ // Input mode 1, hw_device_ctx: Encoder receives NV12 frames from system memory (attempted first)
+ // Input mode 2, hw_frames_ctx: Frames are uploaded to a QSV surface pool in GPU memory
+ // (attempted when the encoder cannot be opened in input mode 1)
//
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (encoder.Kind == EncoderKind.QSV)
@@ -585,64 +603,97 @@ namespace ADOFAIRenderer.Util.FFmpeg
continue;
}
- // ── Attempt hw_frames_ctx: Pre-create GPU frame pool ──
- // AVHWFramesContext pre-allocates frame pool in GPU VRAM,
- // eliminating memory allocation/deallocation overhead for each frame.
- //
- // Pros: Zero-copy capability, highest performance
- // Cons: May fail depending on GPU/driver resolution/format constraints
- // (e.g. specific texture size constraints on Intel Arc → E_INVALIDARG)
- //
- // initial_pool_size: Number of frames to pre-allocate. Matches encoder pipeline depth.
- // Align resolution to multiples of 32: Texture alignment condition required by QSV hardware.
- //
- // Reference:
- // AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
- // av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
- bool framesCtxReady = false;
- AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
- if (hwFramesRef != null)
- {
- var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
- framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
- framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
- framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
- framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
- framesCtx->initial_pool_size = 4; // Frame pool size
-
- err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
- if (err >= 0)
- {
- // Success: Connect hw_frames_ctx to codec context.
- // Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
- _pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
- _isQsvEncoder = true;
- framesCtxReady = true;
- Melon.Logger.Msg($"{encoder.Name}: hw_frames_ctx initialization succeeded (GPU frame pool mode)");
-
- _pHwFrame = ffmpeg.av_frame_alloc();
- if (_pHwFrame == null)
- throw new Exception("Failed to allocate QSV hardware frame structure.");
- }
- else
- {
- Melon.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed — {GetFFmpegErrorString(err)} (Code: {err}). Falling back to hw_device_ctx mode.");
- }
- ffmpeg.av_buffer_unref(&hwFramesRef);
- }
-
- // ── When hw_frames_ctx fails: Fallback to hw_device_ctx + NV12 system memory ──
- // In hw_device_ctx mode, the encoder receives NV12 frames from system memory (RAM)
- // directly and uploads to GPU internally.
- // Slightly slower than hw_frames_ctx, but compatibility is much better.
- //
- // Change pix_fmt to NV12: Use software format directly instead of hardware surface (QSV).
- // _isQsvEncoder = false: Skip manual upload in WriteFrame().
- if (!framesCtxReady)
+ if (!useFramePool)
{
+ // ── Input mode 1 (primary): hw_device_ctx + NV12 system memory ──
+ // The encoder receives NV12 frames in system memory (RAM) and the QSV runtime uploads them to the GPU.
+ //
+ // pix_fmt is the software format NV12.
+ // _isQsvEncoder = false: WriteFrame() passes the NV12 frame to the encoder without uploading it.
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
- _isQsvEncoder = false; // System memory mode, no manual frame upload required
+ _isQsvEncoder = false;
+ Melon.Logger.Msg($"{encoder.Name}: Using system memory input mode (hw_device_ctx)");
+ }
+ else
+ {
+ // ── Input mode 2 (fallback): hw_frames_ctx GPU frame pool ──
+ // Used when the encoder cannot be opened in input mode 1.
+ // AVHWFramesContext pre-allocates a pool of QSV surfaces in GPU memory (pix_fmt AV_PIX_FMT_QSV),
+ // and WriteFrame() uploads each NV12 frame to a surface of the pool (av_hwframe_transfer_data).
+ //
+ // Pool creation depends on the texture types the GPU/driver accepts
+ // (e.g. Intel Arc A770 rejects NV12 textures with D3D11_BIND_RENDER_TARGET → E_INVALIDARG 0x80070057).
+ //
+ // initial_pool_size: Number of surfaces in the pool (QsvFramePoolSize).
+ // The encoder keeps submitted surfaces until they are encoded, so the pool has to hold
+ // the frames in the encoder pipeline plus the frame being uploaded.
+ // Width/height are aligned to multiples of 32: texture alignment condition required by QSV hardware.
+ //
+ // Frame pool memory type (AVQSVFramesContext.frame_type, a combination of MFX_MEMTYPE_* flags):
+ // Windows (D3D11): MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010) is tried first, then 0.
+ // 0x0010 creates the D3D11 textures with D3D11_BIND_DECODER.
+ // 0 lets FFmpeg select MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET (0x0020),
+ // which creates the textures with D3D11_BIND_RENDER_TARGET.
+ // Other platforms (VAAPI): 0. FFmpeg sets the memory type of a VAAPI child pool itself.
+ // Both types are accepted by the QSV encoder as video memory input.
+ // When every type fails, the next candidate encoder is tested.
+ //
+ // Reference:
+ // AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
+ // av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
+ // AVQSVFramesContext.frame_type: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
+ // frame_type → D3D11 bind flags (qsv_init_child_ctx, qsv_get_d3d11va_bind_flags): https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.c
+ // MFX_MEMTYPE_* values: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
+ bool framesCtxReady = false;
+ int[] frameTypes = isWindows
+ ? new[] { QsvMemTypeVideoMemoryDecoderTarget, 0 }
+ : new[] { 0 };
+ foreach (var frameType in frameTypes)
+ {
+ AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
+ if (hwFramesRef == null) break;
+
+ var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
+ framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
+ framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
+ framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
+ framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
+ framesCtx->initial_pool_size = QsvFramePoolSize;
+ SetQsvFrameType(framesCtx, frameType);
+
+ err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
+ if (err >= 0)
+ {
+ // Success: Connect hw_frames_ctx to codec context.
+ // Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
+ _pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
+ _isQsvEncoder = true;
+ framesCtxReady = true;
+ Melon.Logger.Msg($"{encoder.Name}: Using GPU frame pool input mode (hw_frames_ctx, frame_type=0x{frameType:X4}, pool={QsvFramePoolSize})");
+ ffmpeg.av_buffer_unref(&hwFramesRef);
+
+ _pHwFrame = ffmpeg.av_frame_alloc();
+ if (_pHwFrame == null)
+ throw new Exception("Failed to allocate QSV hardware frame structure.");
+ break;
+ }
+
+ Melon.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed with frame_type=0x{frameType:X4} — {GetFFmpegErrorString(err)} (Code: {err})");
+ ffmpeg.av_buffer_unref(&hwFramesRef);
+ }
+
+ if (!framesCtxReady)
+ {
+ Melon.Logger.Warning($"{encoder.Name}: GPU frame pool input mode is unavailable. Testing next candidate codec.");
+ ffmpeg.av_buffer_unref(&hwDeviceCtx);
+ {
+ var pCtx = _pCodecContext;
+ ffmpeg.avcodec_free_context(&pCtx);
+ }
+ _pCodecContext = null;
+ continue;
+ }
}
// Transfer ownership of hwDeviceCtx to codec context,
@@ -704,11 +755,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
break;
}
- // If open fails, free codec context and try next encoder.
- Melon.Logger.Warning($"Failed to initialize hardware encoder {encoder.Name}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate codec.");
+ // If open fails, free codec context and QSV frame state, then try next candidate.
+ var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
+ Melon.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
var pCodecCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCodecCtx);
_pCodecContext = null;
+ _isQsvEncoder = false;
+ if (_pHwFrame != null)
+ {
+ var pHwFrame = _pHwFrame;
+ ffmpeg.av_frame_free(&pHwFrame);
+ _pHwFrame = null;
+ }
}
if (!openedSuccessfully || _pCodecContext == null)
@@ -977,6 +1036,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Sets a private option (priv_data) of the encoder.
/// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
///
+ // ReSharper disable once UnusedMethodReturnValue.Local
private bool TrySetOption(string name, string value)
{
var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0);
@@ -986,6 +1046,32 @@ namespace ADOFAIRenderer.Util.FFmpeg
return false;
}
+ ///
+ /// MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010): frames are in video memory and belong to video decoder render targets.
+ /// Reference: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
+ ///
+ private const int QsvMemTypeVideoMemoryDecoderTarget = 0x0010;
+
+ ///
+ /// Number of surfaces in the QSV GPU frame pool (AVHWFramesContext.initial_pool_size) of GPU frame pool input mode.
+ /// Same value as the encoder frame pool of the FFmpeg hardware encoding example (initial_pool_size = 20).
+ /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/doc/examples/vaapi_encode.c
+ ///
+ private const int QsvFramePoolSize = 20;
+
+ ///
+ /// Writes AVQSVFramesContext.frame_type of a QSV frames context (AVHWFramesContext.hwctx).
+ /// FFmpeg.AutoGen has no AVQSVFramesContext type, so the field is written at its offset.
+ /// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
+ /// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
+ /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
+ ///
+ private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
+ {
+ if (framesCtx->hwctx == null) return;
+ *(int*)((byte*)framesCtx->hwctx + IntPtr.Size + sizeof(int)) = frameType;
+ }
+
///
/// Largest declared maximum that is treated as an actual quality range.
/// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used.