8 changed files with 425 additions and 125 deletions

No files matched your search

Binary file not shown.
Binary file not shown.
Binary file not shown.
+41 -12
View File
@@ -226,6 +226,11 @@ internal class RenderManager : MonoBehaviour
private GameObject? _debugMonitorObj; private GameObject? _debugMonitorObj;
private GameObject? _escapeHandlerObj; private GameObject? _escapeHandlerObj;
internal string? CurrentOutPath; internal string? CurrentOutPath;
// Output file of the current render that is not finished yet.
// Set just before the encoder opens the file, cleared by the render thread after the encoder finished the file (End).
// Only this file is deleted when a render is cancelled, fails or the game exits (DeleteIncompleteOutputFile).
private string? _incompleteOutputPath;
private bool _renderSetup; private bool _renderSetup;
private void Awake() private void Awake()
@@ -286,6 +291,32 @@ internal class RenderManager : MonoBehaviour
{ {
RenderThread.Join(200); RenderThread.Join(200);
} }
// The encoders are disposed above, which closes the output file. An unfinished file of this render is deleted.
DeleteIncompleteOutputFile();
}
/// <summary>
/// Deletes the output file of the current render when the render did not finish it (_incompleteOutputPath).
/// Files of earlier, finished renders are never deleted.
/// Returns true when a file was deleted.
/// </summary>
private bool DeleteIncompleteOutputFile()
{
var path = Interlocked.Exchange(ref _incompleteOutputPath, null);
if (string.IsNullOrEmpty(path) || !File.Exists(path)) return false;
try
{
File.Delete(path);
Melon<RenderMod>.Logger.Msg($"[File Cleanup] Deleted incomplete video: {path}");
return true;
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[File Cleanup] Failed to delete incomplete video: {ex.Message}");
return false;
}
} }
private void Update() private void Update()
@@ -634,6 +665,7 @@ internal class RenderManager : MonoBehaviour
{ {
_allVideoFFmpegInstances.Add(videoFFmpeg); _allVideoFFmpegInstances.Add(videoFFmpeg);
} }
Volatile.Write(ref _incompleteOutputPath, CurrentOutPath);
videoFFmpeg.Start(CurrentOutPath); videoFFmpeg.Start(CurrentOutPath);
} }
catch (Exception) catch (Exception)
@@ -735,7 +767,13 @@ internal class RenderManager : MonoBehaviour
FrameAvailableEvent.WaitOne(50); FrameAvailableEvent.WaitOne(50);
} }
if (!IsRenderAborted) videoFFmpeg.End(); if (!IsRenderAborted)
{
// End throws when the file cannot be finished (e.g. the trailer cannot be written),
// which is handled as a render failure by the catch block below.
videoFFmpeg.End();
Volatile.Write(ref _incompleteOutputPath, null);
}
videoFFmpeg.Dispose(); videoFFmpeg.Dispose();
lock (_allVideoFFmpegInstances) lock (_allVideoFFmpegInstances)
@@ -862,18 +900,9 @@ internal class RenderManager : MonoBehaviour
scrController.instance.noFailInfiniteMargin = false; scrController.instance.noFailInfiniteMargin = false;
} }
// 6. Immediately delete cancelled incomplete temporary video file // 6. Immediately delete the incomplete video file of this render
if (!string.IsNullOrEmpty(CurrentOutPath) && File.Exists(CurrentOutPath)) if (DeleteIncompleteOutputFile())
{ {
try
{
File.Delete(CurrentOutPath);
Melon<RenderMod>.Logger.Msg($"[File Cleanup] Deleted incomplete video: {CurrentOutPath}");
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[File Cleanup] Failed to delete temporary video: {ex.Message}");
}
CurrentOutPath = null; CurrentOutPath = null;
} }
+1 -1
View File
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine; using UnityEngine;
using Object = UnityEngine.Object; using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.7-INDEV", "ADOFAI Editor Forum")] [assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")] [assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer; namespace ADOFAIRenderer;
+158 -4
View File
@@ -206,6 +206,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
private int _srcWidth; private int _srcWidth;
private int _srcHeight; private int _srcHeight;
/// <summary>YUV→RGB matrix id (<c>SWS_CS_*</c>) applied to the context. 0 when no matrix is set (non-RGBA output).</summary>
private int _srcSwsColorspace;
/// <summary>Whether the source range applied to the context is the full range.</summary>
private bool _srcFullRange;
// ────────────────────────────────────────────────────────────── // ──────────────────────────────────────────────────────────────
// Public Properties // Public Properties
// ────────────────────────────────────────────────────────────── // ──────────────────────────────────────────────────────────────
@@ -296,6 +302,59 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
} }
} }
/// <summary>
/// Color matrix id used to convert the YUV frames of this stream to RGB, selected from the colorspace tag of the stream.
/// The ids are shared with the <c>ColorMatrix</c> uniform of ColorConversion.compute:
/// 0 = BT.709 (also used when the stream has no usable tag), 1 = BT.601, 2 = BT.2020 (non-constant luminance).
/// </summary>
public int ColorMatrix
{
get
{
lock (_lock)
{
if (_pCodecContext != null && _pCodecContext->colorspace != AVColorSpace.AVCOL_SPC_UNSPECIFIED)
{
return GetColorMatrixId(_pCodecContext->colorspace);
}
if (_pFormatContext != null && _videoStreamIndex >= 0)
{
return GetColorMatrixId(_pFormatContext->streams[_videoStreamIndex]->codecpar->color_space);
}
return ColorMatrixBT709;
}
}
}
/// <summary>Color matrix id of BT.709.</summary>
// ReSharper disable InconsistentNaming
public const int ColorMatrixBT709 = 0;
/// <summary>Color matrix id of BT.601 (SMPTE 170M / BT.470 BG).</summary>
public const int ColorMatrixBT601 = 1;
/// <summary>Color matrix id of BT.2020 (non-constant luminance).</summary>
public const int ColorMatrixBT2020 = 2;
// ReSharper restore InconsistentNaming
/// <summary>
/// Converts a colorspace tag to a color matrix id (see <see cref="ColorMatrix"/>).
/// Tags that have no matrix of their own (unspecified, RGB, other standards) use BT.709.
/// </summary>
private static int GetColorMatrixId(AVColorSpace colorspace)
{
switch (colorspace)
{
case AVColorSpace.AVCOL_SPC_SMPTE170M:
case AVColorSpace.AVCOL_SPC_BT470BG:
return ColorMatrixBT601;
case AVColorSpace.AVCOL_SPC_BT2020_NCL:
return ColorMatrixBT2020;
default:
return ColorMatrixBT709;
}
}
/// <summary> /// <summary>
/// Gets a value indicating whether hardware-accelerated decoding is currently active. /// Gets a value indicating whether hardware-accelerated decoding is currently active.
/// </summary> /// </summary>
@@ -778,12 +837,67 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
} }
/// <summary> /// <summary>
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats or dimensions change. /// Selects the YUV→RGB conversion matrix (libswscale <c>SWS_CS_*</c>) from the colorspace tag of a frame.
/// The matrix set is the same as that of the compute shader path (ColorConversion.compute, <c>ColorMatrix</c> uniform):
/// BT.709 (also for frames without a usable tag), BT.601 and BT.2020, so both paths give the same colors.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_getCoefficients)
/// </summary>
private static int GetSwsColorspace(AVColorSpace colorspace)
{
switch (GetColorMatrixId(colorspace))
{
case ColorMatrixBT601:
return ffmpeg.SWS_CS_ITU601;
case ColorMatrixBT2020:
return ffmpeg.SWS_CS_BT2020;
default:
return ffmpeg.SWS_CS_ITU709;
}
}
/// <summary>
/// Returns true when the Y/U/V samples of the frame use the full range (0-255) instead of the limited range (16-235 / 16-240).
/// The range is given by the frame's color_range tag or by a deprecated full range pixel format (YUVJ*).
/// </summary>
private static bool IsFullRange(AVFrame* frame)
{
if (frame->color_range == AVColorRange.AVCOL_RANGE_JPEG) return true;
switch ((AVPixelFormat)frame->format)
{
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
case AVPixelFormat.AV_PIX_FMT_YUVJ422P:
case AVPixelFormat.AV_PIX_FMT_YUVJ444P:
case AVPixelFormat.AV_PIX_FMT_YUVJ440P:
return true;
default:
return false;
}
}
/// <summary>
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats, dimensions,
/// or (for RGBA output) the colorspace / range of the source frame change.
/// For RGBA output the YUV→RGB matrix and the source range are taken from <paramref name="colorFrame"/>.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
/// </summary> /// </summary>
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat) /// <param name="srcFormat">Pixel format of the source frame.</param>
/// <param name="srcWidth">Width of the source frame in pixels.</param>
/// <param name="srcHeight">Height of the source frame in pixels.</param>
/// <param name="dstFormat">Pixel format of the output.</param>
/// <param name="colorFrame">Source frame whose colorspace / range tags are applied. Required for RGBA output.</param>
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat, AVFrame* colorFrame = null)
{ {
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat) var swsColorspace = 0;
var fullRange = false;
if (colorFrame != null && dstFormat == AVPixelFormat.AV_PIX_FMT_RGBA)
{
swsColorspace = GetSwsColorspace(colorFrame->colorspace);
fullRange = IsFullRange(colorFrame);
}
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat
&& _srcSwsColorspace == swsColorspace && _srcFullRange == fullRange)
{ {
return; return;
} }
@@ -798,6 +912,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_srcWidth = srcWidth; _srcWidth = srcWidth;
_srcHeight = srcHeight; _srcHeight = srcHeight;
_dstPixFmt = dstFormat; _dstPixFmt = dstFormat;
_srcSwsColorspace = swsColorspace;
_srcFullRange = fullRange;
_pSwsContext = ffmpeg.sws_getContext( _pSwsContext = ffmpeg.sws_getContext(
srcWidth, srcHeight, srcFormat, srcWidth, srcHeight, srcFormat,
@@ -809,6 +925,44 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
{ {
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height})."); throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
} }
if (swsColorspace != 0)
{
ApplySwsColorspace(swsColorspace, fullRange);
}
}
/// <summary>
/// Sets the YUV→RGB matrix and the source range of <see cref="_pSwsContext"/>.
/// The destination range, brightness, contrast and saturation keep the values of the context.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails)
/// </summary>
/// <param name="swsColorspace">Matrix id (<c>SWS_CS_*</c>) of the source.</param>
/// <param name="fullRange">True when the source uses the full range.</param>
private void ApplySwsColorspace(int swsColorspace, bool fullRange)
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
// Returns a negative value when the context cannot change its colorspace details (e.g. YUV to YUV conversion).
if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation) < 0)
{
return;
}
var coefficients = ffmpeg.sws_getCoefficients(swsColorspace);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = coefficients[i];
table[i] = currentTable[i];
}
ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, fullRange ? 1 : 0, table, dstRange,
brightness, contrast, saturation);
} }
/// <summary> /// <summary>
@@ -900,7 +1054,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (frame == null) return false; if (frame == null) return false;
timestamp = GetTimestampAndUpdateIndex(frame); timestamp = GetTimestampAndUpdateIndex(frame);
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA); InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame);
ClearConvertedFramePointers(); ClearConvertedFramePointers();
if (vflip) if (vflip)
+214 -108
View File
@@ -63,7 +63,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
public AVCodecID AudioCodecId; public AVCodecID AudioCodecId;
/// <summary> /// <summary>
/// Audio bitrate (in bps). If default (0), 128000 bps is used. /// Audio bitrate (in bps). If default (0), 256000 bps is used.
/// For AAC, 128000–320000 bps is common. /// For AAC, 128000–320000 bps is common.
/// </summary> /// </summary>
public int Bitrate; public int Bitrate;
@@ -299,6 +299,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray(); return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray();
} }
/// <summary>
/// Expands the candidate list into (encoder, input mode) attempts in order.
/// A QSV encoder is attempted twice: system memory input mode (useFramePool = false) first,
/// then GPU frame pool input mode (useFramePool = true). Other encoders are attempted once (useFramePool = false).
/// </summary>
private static System.Collections.Generic.IEnumerable<(VideoEncoder Encoder, bool UseFramePool)> ExpandQsvInputModes(VideoEncoder[] candidates)
{
foreach (var encoder in candidates)
{
yield return (encoder, false);
if (encoder.Kind == EncoderKind.QSV)
{
yield return (encoder, true);
}
}
}
/// <summary> /// <summary>
/// Converts an FFmpeg error code (negative integer) to a human-readable string. /// Converts an FFmpeg error code (negative integer) to a human-readable string.
/// e.g., -1 → "Operation not permitted" /// e.g., -1 → "Operation not permitted"
@@ -367,7 +384,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var candidates = GetEncoderCandidates(); var candidates = GetEncoderCandidates();
var openedSuccessfully = false; var openedSuccessfully = false;
foreach (var encoder in candidates) foreach (var (encoder, useFramePool) in ExpandQsvInputModes(candidates))
{ {
// avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc"). // avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc").
// Returns null if the encoder is not included in the build. // Returns null if the encoder is not included in the build.
@@ -514,9 +531,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Strategy 3) Create QSV device directly (legacy method) // Strategy 3) Create QSV device directly (legacy method)
// → Fallback on older drivers/systems // → Fallback on older drivers/systems
// //
// After obtaining device, attempt using GPU frame pool: // After obtaining device, the input mode is configured (see ExpandQsvInputModes):
// hw_frames_ctx: Pre-allocate frame pool in GPU VRAM (zero-copy, highest performance) // Input mode 1, hw_device_ctx: Encoder receives NV12 frames from system memory (attempted first)
// hw_device_ctx: Encoder reads frames directly from system memory (better compatibility) // Input mode 2, hw_frames_ctx: Frames are uploaded to a QSV surface pool in GPU memory
// (attempted when the encoder cannot be opened in input mode 1)
// //
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (encoder.Kind == EncoderKind.QSV) if (encoder.Kind == EncoderKind.QSV)
@@ -585,64 +603,97 @@ namespace ADOFAIRenderer.Util.FFmpeg
continue; continue;
} }
// ── Attempt hw_frames_ctx: Pre-create GPU frame pool ── if (!useFramePool)
// AVHWFramesContext pre-allocates frame pool in GPU VRAM,
// eliminating memory allocation/deallocation overhead for each frame.
//
// Pros: Zero-copy capability, highest performance
// Cons: May fail depending on GPU/driver resolution/format constraints
// (e.g. specific texture size constraints on Intel Arc → E_INVALIDARG)
//
// initial_pool_size: Number of frames to pre-allocate. Matches encoder pipeline depth.
// Align resolution to multiples of 32: Texture alignment condition required by QSV hardware.
//
// Reference:
// AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
// av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
bool framesCtxReady = false;
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
if (hwFramesRef != null)
{
var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
framesCtx->initial_pool_size = 4; // Frame pool size
err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
if (err >= 0)
{
// Success: Connect hw_frames_ctx to codec context.
// Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
_pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
_isQsvEncoder = true;
framesCtxReady = true;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: hw_frames_ctx initialization succeeded (GPU frame pool mode)");
_pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null)
throw new Exception("Failed to allocate QSV hardware frame structure.");
}
else
{
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed — {GetFFmpegErrorString(err)} (Code: {err}). Falling back to hw_device_ctx mode.");
}
ffmpeg.av_buffer_unref(&hwFramesRef);
}
// ── When hw_frames_ctx fails: Fallback to hw_device_ctx + NV12 system memory ──
// In hw_device_ctx mode, the encoder receives NV12 frames from system memory (RAM)
// directly and uploads to GPU internally.
// Slightly slower than hw_frames_ctx, but compatibility is much better.
//
// Change pix_fmt to NV12: Use software format directly instead of hardware surface (QSV).
// _isQsvEncoder = false: Skip manual upload in WriteFrame().
if (!framesCtxReady)
{ {
// ── Input mode 1 (primary): hw_device_ctx + NV12 system memory ──
// The encoder receives NV12 frames in system memory (RAM) and the QSV runtime uploads them to the GPU.
//
// pix_fmt is the software format NV12.
// _isQsvEncoder = false: WriteFrame() passes the NV12 frame to the encoder without uploading it.
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12; _pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx); _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
_isQsvEncoder = false; // System memory mode, no manual frame upload required _isQsvEncoder = false;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using system memory input mode (hw_device_ctx)");
}
else
{
// ── Input mode 2 (fallback): hw_frames_ctx GPU frame pool ──
// Used when the encoder cannot be opened in input mode 1.
// AVHWFramesContext pre-allocates a pool of QSV surfaces in GPU memory (pix_fmt AV_PIX_FMT_QSV),
// and WriteFrame() uploads each NV12 frame to a surface of the pool (av_hwframe_transfer_data).
//
// Pool creation depends on the texture types the GPU/driver accepts
// (e.g. Intel Arc A770 rejects NV12 textures with D3D11_BIND_RENDER_TARGET → E_INVALIDARG 0x80070057).
//
// initial_pool_size: Number of surfaces in the pool (QsvFramePoolSize).
// The encoder keeps submitted surfaces until they are encoded, so the pool has to hold
// the frames in the encoder pipeline plus the frame being uploaded.
// Width/height are aligned to multiples of 32: texture alignment condition required by QSV hardware.
//
// Frame pool memory type (AVQSVFramesContext.frame_type, a combination of MFX_MEMTYPE_* flags):
// Windows (D3D11): MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010) is tried first, then 0.
// 0x0010 creates the D3D11 textures with D3D11_BIND_DECODER.
// 0 lets FFmpeg select MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET (0x0020),
// which creates the textures with D3D11_BIND_RENDER_TARGET.
// Other platforms (VAAPI): 0. FFmpeg sets the memory type of a VAAPI child pool itself.
// Both types are accepted by the QSV encoder as video memory input.
// When every type fails, the next candidate encoder is tested.
//
// Reference:
// AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
// av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
// AVQSVFramesContext.frame_type: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
// frame_type → D3D11 bind flags (qsv_init_child_ctx, qsv_get_d3d11va_bind_flags): https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.c
// MFX_MEMTYPE_* values: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
bool framesCtxReady = false;
int[] frameTypes = isWindows
? new[] { QsvMemTypeVideoMemoryDecoderTarget, 0 }
: new[] { 0 };
foreach (var frameType in frameTypes)
{
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
if (hwFramesRef == null) break;
var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
framesCtx->initial_pool_size = QsvFramePoolSize;
SetQsvFrameType(framesCtx, frameType);
err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
if (err >= 0)
{
// Success: Connect hw_frames_ctx to codec context.
// Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
_pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
_isQsvEncoder = true;
framesCtxReady = true;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using GPU frame pool input mode (hw_frames_ctx, frame_type=0x{frameType:X4}, pool={QsvFramePoolSize})");
ffmpeg.av_buffer_unref(&hwFramesRef);
_pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null)
throw new Exception("Failed to allocate QSV hardware frame structure.");
break;
}
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed with frame_type=0x{frameType:X4} — {GetFFmpegErrorString(err)} (Code: {err})");
ffmpeg.av_buffer_unref(&hwFramesRef);
}
if (!framesCtxReady)
{
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: GPU frame pool input mode is unavailable. Testing next candidate codec.");
ffmpeg.av_buffer_unref(&hwDeviceCtx);
{
var pCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCtx);
}
_pCodecContext = null;
continue;
}
} }
// Transfer ownership of hwDeviceCtx to codec context, // Transfer ownership of hwDeviceCtx to codec context,
@@ -704,11 +755,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
break; break;
} }
// If open fails, free codec context and try next encoder. // If open fails, free codec context and QSV frame state, then try next candidate.
Melon<RenderMod>.Logger.Warning($"Failed to initialize hardware encoder {encoder.Name}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate codec."); var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
Melon<RenderMod>.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
var pCodecCtx = _pCodecContext; var pCodecCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCodecCtx); ffmpeg.avcodec_free_context(&pCodecCtx);
_pCodecContext = null; _pCodecContext = null;
_isQsvEncoder = false;
if (_pHwFrame != null)
{
var pHwFrame = _pHwFrame;
ffmpeg.av_frame_free(&pHwFrame);
_pHwFrame = null;
}
} }
if (!openedSuccessfully || _pCodecContext == null) if (!openedSuccessfully || _pCodecContext == null)
@@ -765,16 +824,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSrcFrame == null || _pDstFrame == null || _pPacket == null) if (_pSrcFrame == null || _pDstFrame == null || _pPacket == null)
throw new Exception("Failed to allocate frame or packet structure."); throw new Exception("Failed to allocate frame or packet structure.");
// Set format and size of destination frame, then allocate actual pixel buffer. AllocateDstFrameBuffer();
// av_frame_get_buffer: Allocates memory matching format, width, height and
// sets data[]/linesize[] pointers. Second argument 0 means default alignment.
_pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
if (ffmpeg.av_frame_get_buffer(_pDstFrame, 0) < 0)
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer.");
// sws_getContext: Initialize software scaler (colorspace converter). // sws_getContext: Initialize software scaler (colorspace converter).
// Input: RGBA (Unity capture format, 4 bytes/pixel) // Input: RGBA (Unity capture format, 4 bytes/pixel)
@@ -977,6 +1027,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Sets a private option (priv_data) of the encoder. /// Sets a private option (priv_data) of the encoder.
/// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false. /// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
/// </summary> /// </summary>
// ReSharper disable once UnusedMethodReturnValue.Local
private bool TrySetOption(string name, string value) private bool TrySetOption(string name, string value)
{ {
var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0); var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0);
@@ -986,6 +1037,32 @@ namespace ADOFAIRenderer.Util.FFmpeg
return false; return false;
} }
/// <summary>
/// MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010): frames are in video memory and belong to video decoder render targets.
/// Reference: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
/// </summary>
private const int QsvMemTypeVideoMemoryDecoderTarget = 0x0010;
/// <summary>
/// Number of surfaces in the QSV GPU frame pool (AVHWFramesContext.initial_pool_size) of GPU frame pool input mode.
/// Same value as the encoder frame pool of the FFmpeg hardware encoding example (initial_pool_size = 20).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/doc/examples/vaapi_encode.c
/// </summary>
private const int QsvFramePoolSize = 20;
/// <summary>
/// Writes AVQSVFramesContext.frame_type of a QSV frames context (AVHWFramesContext.hwctx).
/// FFmpeg.AutoGen has no AVQSVFramesContext type, so the field is written at its offset.
/// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
/// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
/// </summary>
private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
{
if (framesCtx->hwctx == null) return;
*(int*)((byte*)framesCtx->hwctx + IntPtr.Size + sizeof(int)) = frameType;
}
/// <summary> /// <summary>
/// Largest declared maximum that is treated as an actual quality range. /// Largest declared maximum that is treated as an actual quality range.
/// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used. /// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used.
@@ -1115,12 +1192,50 @@ namespace ADOFAIRenderer.Util.FFmpeg
} }
} }
/// <summary>
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
/// Second argument 0 means default alignment.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
/// </summary>
private void AllocateDstFrameBuffer()
{
_pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
if (ret < 0)
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
}
/// <summary>
/// Makes the pixel buffer of the destination frame (_pDstFrame) writable before the next frame is written into it.
///
/// After avcodec_send_frame, the encoder may keep a reference to the buffer of the submitted frame
/// until that frame is encoded (e.g. QSV in system memory input mode). The buffer is then shared and not writable
/// (av_frame_is_writable returns 0).
/// In that case the frame releases its reference (av_frame_unref) and a new buffer is allocated.
/// The encoder keeps the previous buffer until it no longer needs it. The previous contents are not copied,
/// because every plane is overwritten by the next frame.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_is_writable, av_frame_unref)
/// </summary>
private void PrepareDstFrameForWrite()
{
if (ffmpeg.av_frame_is_writable(_pDstFrame) != 0) return;
ffmpeg.av_frame_unref(_pDstFrame);
AllocateDstFrameBuffer();
}
/// <summary> /// <summary>
/// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data. /// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data.
/// </summary> /// </summary>
private void WritePreConvertedFrame(byte* buffer) private void WritePreConvertedFrame(byte* buffer)
{ {
ffmpeg.av_frame_make_writable(_pDstFrame); PrepareDstFrameForWrite();
var width = _settings.Width; var width = _settings.Width;
var height = _settings.Height; var height = _settings.Height;
@@ -1234,20 +1349,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Core logic for actual frame encoding. /// Core logic for actual frame encoding.
/// ///
/// Processing flow: /// Processing flow:
/// 1. av_frame_make_writable: Check if destination frame buffer is writable /// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable
/// (Allocates a new buffer if encoder is still referencing previous frame's buffer) /// (Allocates a new buffer if encoder is still referencing previous frame's buffer)
/// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion /// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface /// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface
/// 4. Pass to encoder after setting PTS /// 4. Pass to encoder after setting PTS
/// ///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale) /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale)
/// </summary> /// </summary>
private void WriteFrame() private void WriteFrame()
{ {
// Ensure writable state since encoder might still be referencing previous frame's buffer. PrepareDstFrameForWrite();
// Internally allocates a new buffer if referenced.
ffmpeg.av_frame_make_writable(_pDstFrame);
// ── Colorspace conversion: RGBA → YUV420P or NV12 ── // ── Colorspace conversion: RGBA → YUV420P or NV12 ──
// sws_scale parameters explanation: // sws_scale parameters explanation:
@@ -1408,8 +1520,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 2. av_write_trailer: Writes container trailer (end-of-file metadata). /// 2. av_write_trailer: Writes container trailer (end-of-file metadata).
/// For MP4, the moov atom (index info) is completed here. /// For MP4, the moov atom (index info) is completed here.
/// ⚠️ If trailer is not written, the file becomes corrupted and unplayable! /// ⚠️ If trailer is not written, the file becomes corrupted and unplayable!
/// av_write_trailer also flushes the output buffer and returns the write error of the output (AVIOContext.error), if any.
///
/// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete.
/// End() runs once: after it returns or throws, further calls return immediately.
/// ///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer)
/// </summary> /// </summary>
public void End() public void End()
{ {
@@ -1417,37 +1534,29 @@ namespace ADOFAIRenderer.Util.FFmpeg
{ {
if (_disposed || !_isInitialized) return; if (_disposed || !_isInitialized) return;
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
try try
{ {
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
SendFrameAndWritePackets(null); SendFrameAndWritePackets(null);
}
catch (Exception ex)
{
// Log error occurring during flush process
Melon<RenderMod>.Logger.Error($"Error during video encoder flush: {ex.Message}");
}
// 1.5. Flush remaining samples in audio FIFO + flush audio encoder // 1.5. Flush remaining samples in audio FIFO + flush audio encoder
if (_audioEnabled) if (_audioEnabled)
{
try
{ {
FlushAudioEncoder(); FlushAudioEncoder();
} }
catch (Exception ex)
// 2. Write container trailer (moov atom and file termination metadata)
if (_pFormatContext != null)
{ {
Melon<RenderMod>.Logger.Error($"Error during audio encoder flush: {ex.Message}"); var ret = ffmpeg.av_write_trailer(_pFormatContext);
if (ret < 0)
throw new Exception($"Failed to write the container trailer — {GetFFmpegErrorString(ret)} (Code: {ret})");
} }
} }
finally
// 2. Write container trailer (moov atom and file termination metadata)
if (_pFormatContext != null)
{ {
ffmpeg.av_write_trailer(_pFormatContext); _isInitialized = false;
} }
_isInitialized = false;
} }
} }
@@ -1458,11 +1567,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary> /// <summary>
/// Actual resource cleanup logic. Cleanup order is important: /// Actual resource cleanup logic. Cleanup order is important:
/// ///
/// 1. Call End() (if not called yet) → flush delayed frames + write trailer /// 1. SwsContext (scaler) → independent, can be freed first
/// 2. SwsContext (scaler) → independent, can be freed first /// 2. AVFrame (src/dst) → freed before codec as codec may reference them
/// 3. AVFrame (src/dst) → freed before codec as codec may reference them /// 3. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
/// 4. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together /// 4. AVFormatContext (format/IO) → includes closing file handle
/// 5. AVFormatContext (format/IO) → includes closing file handle ///
/// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned.
/// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it).
/// ///
/// Reference: /// Reference:
/// av_frame_free: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html /// av_frame_free: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
@@ -1476,12 +1587,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
{ {
if (_disposed) return; if (_disposed) return;
_disposed = true; _disposed = true;
_isInitialized = false;
// Trigger End() safely in case Dispose is called without calling End() first
if (_isInitialized)
{
try { End(); } catch { /* Ignore */ }
}
// Release Sws scaler // Release Sws scaler
if (_pSwsContext != null) if (_pSwsContext != null)
+11
View File
@@ -20,6 +20,7 @@ internal class VideoBGPlayer : MonoBehaviour
private static readonly int UTex = Shader.PropertyToID("_UTex"); private static readonly int UTex = Shader.PropertyToID("_UTex");
private static readonly int VTex = Shader.PropertyToID("_VTex"); private static readonly int VTex = Shader.PropertyToID("_VTex");
private static readonly int ColorRange = Shader.PropertyToID("ColorRange"); private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
private static readonly int ColorMatrix = Shader.PropertyToID("ColorMatrix");
[SuppressMessage("ReSharper", "InconsistentNaming")] [SuppressMessage("ReSharper", "InconsistentNaming")]
private enum GPUMode private enum GPUMode
@@ -189,6 +190,14 @@ internal class VideoBGPlayer : MonoBehaviour
var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0; var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
var targetVideoTime = songTime + vidOffset; var targetVideoTime = songTime + vidOffset;
// When looping is enabled, the playback position is the elapsed time wrapped into [0, Duration).
// When it is disabled, the position keeps increasing and the last frame stays on screen after the end.
var videoDuration = _decoder.Duration;
if (_isLooping && targetVideoTime >= 0 && videoDuration > 0)
{
targetVideoTime %= videoDuration;
}
if (targetVideoTime < 0) if (targetVideoTime < 0)
{ {
if (_targetTexture) if (_targetTexture)
@@ -307,6 +316,7 @@ internal class VideoBGPlayer : MonoBehaviour
_computeShader.SetInt(Width, w); _computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h); _computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0); _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
_computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
int threadGroupsX = (w + 7) / 8; int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8; int threadGroupsY = (h + 7) / 8;
@@ -341,6 +351,7 @@ internal class VideoBGPlayer : MonoBehaviour
_computeShader.SetInt(Width, w); _computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h); _computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0); _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
_computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
int threadGroupsX = (w + 7) / 8; int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8; int threadGroupsY = (h + 7) / 8;