SHA256
Compare commits
6
Commits
| Author | SHA256 | Date | |
|---|---|---|---|
|
|
f153bf70aa
|
||
|
|
93539307b1
|
||
|
|
c0e902fc1a
|
||
|
|
dd2585ee9f
|
||
|
|
1a672cbf59
|
||
|
|
6d5880fe19
|
No files matched your search
Binary file not shown.
Binary file not shown.
Binary file not shown.
+41
-12
@@ -226,6 +226,11 @@ internal class RenderManager : MonoBehaviour
|
||||
private GameObject? _debugMonitorObj;
|
||||
private GameObject? _escapeHandlerObj;
|
||||
internal string? CurrentOutPath;
|
||||
|
||||
// Output file of the current render that is not finished yet.
|
||||
// Set just before the encoder opens the file, cleared by the render thread after the encoder finished the file (End).
|
||||
// Only this file is deleted when a render is cancelled, fails or the game exits (DeleteIncompleteOutputFile).
|
||||
private string? _incompleteOutputPath;
|
||||
private bool _renderSetup;
|
||||
|
||||
private void Awake()
|
||||
@@ -286,6 +291,32 @@ internal class RenderManager : MonoBehaviour
|
||||
{
|
||||
RenderThread.Join(200);
|
||||
}
|
||||
|
||||
// The encoders are disposed above, which closes the output file. An unfinished file of this render is deleted.
|
||||
DeleteIncompleteOutputFile();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the output file of the current render when the render did not finish it (_incompleteOutputPath).
|
||||
/// Files of earlier, finished renders are never deleted.
|
||||
/// Returns true when a file was deleted.
|
||||
/// </summary>
|
||||
private bool DeleteIncompleteOutputFile()
|
||||
{
|
||||
var path = Interlocked.Exchange(ref _incompleteOutputPath, null);
|
||||
if (string.IsNullOrEmpty(path) || !File.Exists(path)) return false;
|
||||
|
||||
try
|
||||
{
|
||||
File.Delete(path);
|
||||
Melon<RenderMod>.Logger.Msg($"[File Cleanup] Deleted incomplete video: {path}");
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Melon<RenderMod>.Logger.Error($"[File Cleanup] Failed to delete incomplete video: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void Update()
|
||||
@@ -634,6 +665,7 @@ internal class RenderManager : MonoBehaviour
|
||||
{
|
||||
_allVideoFFmpegInstances.Add(videoFFmpeg);
|
||||
}
|
||||
Volatile.Write(ref _incompleteOutputPath, CurrentOutPath);
|
||||
videoFFmpeg.Start(CurrentOutPath);
|
||||
}
|
||||
catch (Exception)
|
||||
@@ -735,7 +767,13 @@ internal class RenderManager : MonoBehaviour
|
||||
FrameAvailableEvent.WaitOne(50);
|
||||
}
|
||||
|
||||
if (!IsRenderAborted) videoFFmpeg.End();
|
||||
if (!IsRenderAborted)
|
||||
{
|
||||
// End throws when the file cannot be finished (e.g. the trailer cannot be written),
|
||||
// which is handled as a render failure by the catch block below.
|
||||
videoFFmpeg.End();
|
||||
Volatile.Write(ref _incompleteOutputPath, null);
|
||||
}
|
||||
videoFFmpeg.Dispose();
|
||||
|
||||
lock (_allVideoFFmpegInstances)
|
||||
@@ -862,18 +900,9 @@ internal class RenderManager : MonoBehaviour
|
||||
scrController.instance.noFailInfiniteMargin = false;
|
||||
}
|
||||
|
||||
// 6. Immediately delete cancelled incomplete temporary video file
|
||||
if (!string.IsNullOrEmpty(CurrentOutPath) && File.Exists(CurrentOutPath))
|
||||
// 6. Immediately delete the incomplete video file of this render
|
||||
if (DeleteIncompleteOutputFile())
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(CurrentOutPath);
|
||||
Melon<RenderMod>.Logger.Msg($"[File Cleanup] Deleted incomplete video: {CurrentOutPath}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Melon<RenderMod>.Logger.Error($"[File Cleanup] Failed to delete temporary video: {ex.Message}");
|
||||
}
|
||||
CurrentOutPath = null;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
|
||||
using UnityEngine;
|
||||
using Object = UnityEngine.Object;
|
||||
|
||||
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.7-INDEV", "ADOFAI Editor Forum")]
|
||||
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")]
|
||||
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
|
||||
|
||||
namespace ADOFAIRenderer;
|
||||
|
||||
@@ -206,6 +206,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||||
private int _srcWidth;
|
||||
private int _srcHeight;
|
||||
|
||||
/// <summary>YUV→RGB matrix id (<c>SWS_CS_*</c>) applied to the context. 0 when no matrix is set (non-RGBA output).</summary>
|
||||
private int _srcSwsColorspace;
|
||||
|
||||
/// <summary>Whether the source range applied to the context is the full range.</summary>
|
||||
private bool _srcFullRange;
|
||||
|
||||
// ──────────────────────────────────────────────────────────────
|
||||
// Public Properties
|
||||
// ──────────────────────────────────────────────────────────────
|
||||
@@ -296,6 +302,59 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Color matrix id used to convert the YUV frames of this stream to RGB, selected from the colorspace tag of the stream.
|
||||
/// The ids are shared with the <c>ColorMatrix</c> uniform of ColorConversion.compute:
|
||||
/// 0 = BT.709 (also used when the stream has no usable tag), 1 = BT.601, 2 = BT.2020 (non-constant luminance).
|
||||
/// </summary>
|
||||
public int ColorMatrix
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_pCodecContext != null && _pCodecContext->colorspace != AVColorSpace.AVCOL_SPC_UNSPECIFIED)
|
||||
{
|
||||
return GetColorMatrixId(_pCodecContext->colorspace);
|
||||
}
|
||||
if (_pFormatContext != null && _videoStreamIndex >= 0)
|
||||
{
|
||||
return GetColorMatrixId(_pFormatContext->streams[_videoStreamIndex]->codecpar->color_space);
|
||||
}
|
||||
return ColorMatrixBT709;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Color matrix id of BT.709.</summary>
|
||||
// ReSharper disable InconsistentNaming
|
||||
public const int ColorMatrixBT709 = 0;
|
||||
|
||||
/// <summary>Color matrix id of BT.601 (SMPTE 170M / BT.470 BG).</summary>
|
||||
public const int ColorMatrixBT601 = 1;
|
||||
|
||||
/// <summary>Color matrix id of BT.2020 (non-constant luminance).</summary>
|
||||
public const int ColorMatrixBT2020 = 2;
|
||||
// ReSharper restore InconsistentNaming
|
||||
|
||||
/// <summary>
|
||||
/// Converts a colorspace tag to a color matrix id (see <see cref="ColorMatrix"/>).
|
||||
/// Tags that have no matrix of their own (unspecified, RGB, other standards) use BT.709.
|
||||
/// </summary>
|
||||
private static int GetColorMatrixId(AVColorSpace colorspace)
|
||||
{
|
||||
switch (colorspace)
|
||||
{
|
||||
case AVColorSpace.AVCOL_SPC_SMPTE170M:
|
||||
case AVColorSpace.AVCOL_SPC_BT470BG:
|
||||
return ColorMatrixBT601;
|
||||
case AVColorSpace.AVCOL_SPC_BT2020_NCL:
|
||||
return ColorMatrixBT2020;
|
||||
default:
|
||||
return ColorMatrixBT709;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether hardware-accelerated decoding is currently active.
|
||||
/// </summary>
|
||||
@@ -778,12 +837,67 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats or dimensions change.
|
||||
/// Selects the YUV→RGB conversion matrix (libswscale <c>SWS_CS_*</c>) from the colorspace tag of a frame.
|
||||
/// The matrix set is the same as that of the compute shader path (ColorConversion.compute, <c>ColorMatrix</c> uniform):
|
||||
/// BT.709 (also for frames without a usable tag), BT.601 and BT.2020, so both paths give the same colors.
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_getCoefficients)
|
||||
/// </summary>
|
||||
private static int GetSwsColorspace(AVColorSpace colorspace)
|
||||
{
|
||||
switch (GetColorMatrixId(colorspace))
|
||||
{
|
||||
case ColorMatrixBT601:
|
||||
return ffmpeg.SWS_CS_ITU601;
|
||||
case ColorMatrixBT2020:
|
||||
return ffmpeg.SWS_CS_BT2020;
|
||||
default:
|
||||
return ffmpeg.SWS_CS_ITU709;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true when the Y/U/V samples of the frame use the full range (0-255) instead of the limited range (16-235 / 16-240).
|
||||
/// The range is given by the frame's color_range tag or by a deprecated full range pixel format (YUVJ*).
|
||||
/// </summary>
|
||||
private static bool IsFullRange(AVFrame* frame)
|
||||
{
|
||||
if (frame->color_range == AVColorRange.AVCOL_RANGE_JPEG) return true;
|
||||
|
||||
switch ((AVPixelFormat)frame->format)
|
||||
{
|
||||
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
|
||||
case AVPixelFormat.AV_PIX_FMT_YUVJ422P:
|
||||
case AVPixelFormat.AV_PIX_FMT_YUVJ444P:
|
||||
case AVPixelFormat.AV_PIX_FMT_YUVJ440P:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats, dimensions,
|
||||
/// or (for RGBA output) the colorspace / range of the source frame change.
|
||||
/// For RGBA output the YUV→RGB matrix and the source range are taken from <paramref name="colorFrame"/>.
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
||||
/// </summary>
|
||||
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat)
|
||||
/// <param name="srcFormat">Pixel format of the source frame.</param>
|
||||
/// <param name="srcWidth">Width of the source frame in pixels.</param>
|
||||
/// <param name="srcHeight">Height of the source frame in pixels.</param>
|
||||
/// <param name="dstFormat">Pixel format of the output.</param>
|
||||
/// <param name="colorFrame">Source frame whose colorspace / range tags are applied. Required for RGBA output.</param>
|
||||
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat, AVFrame* colorFrame = null)
|
||||
{
|
||||
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat)
|
||||
var swsColorspace = 0;
|
||||
var fullRange = false;
|
||||
if (colorFrame != null && dstFormat == AVPixelFormat.AV_PIX_FMT_RGBA)
|
||||
{
|
||||
swsColorspace = GetSwsColorspace(colorFrame->colorspace);
|
||||
fullRange = IsFullRange(colorFrame);
|
||||
}
|
||||
|
||||
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat
|
||||
&& _srcSwsColorspace == swsColorspace && _srcFullRange == fullRange)
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -798,6 +912,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||||
_srcWidth = srcWidth;
|
||||
_srcHeight = srcHeight;
|
||||
_dstPixFmt = dstFormat;
|
||||
_srcSwsColorspace = swsColorspace;
|
||||
_srcFullRange = fullRange;
|
||||
|
||||
_pSwsContext = ffmpeg.sws_getContext(
|
||||
srcWidth, srcHeight, srcFormat,
|
||||
@@ -809,6 +925,44 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||||
{
|
||||
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
|
||||
}
|
||||
|
||||
if (swsColorspace != 0)
|
||||
{
|
||||
ApplySwsColorspace(swsColorspace, fullRange);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the YUV→RGB matrix and the source range of <see cref="_pSwsContext"/>.
|
||||
/// The destination range, brightness, contrast and saturation keep the values of the context.
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails)
|
||||
/// </summary>
|
||||
/// <param name="swsColorspace">Matrix id (<c>SWS_CS_*</c>) of the source.</param>
|
||||
/// <param name="fullRange">True when the source uses the full range.</param>
|
||||
private void ApplySwsColorspace(int swsColorspace, bool fullRange)
|
||||
{
|
||||
int* currentInvTable;
|
||||
int* currentTable;
|
||||
int srcRange, dstRange, brightness, contrast, saturation;
|
||||
|
||||
// Returns a negative value when the context cannot change its colorspace details (e.g. YUV to YUV conversion).
|
||||
if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, ¤tInvTable, &srcRange, ¤tTable, &dstRange,
|
||||
&brightness, &contrast, &saturation) < 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var coefficients = ffmpeg.sws_getCoefficients(swsColorspace);
|
||||
var invTable = new int_array4();
|
||||
var table = new int_array4();
|
||||
for (uint i = 0; i < 4; i++)
|
||||
{
|
||||
invTable[i] = coefficients[i];
|
||||
table[i] = currentTable[i];
|
||||
}
|
||||
|
||||
ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, fullRange ? 1 : 0, table, dstRange,
|
||||
brightness, contrast, saturation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -900,7 +1054,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||||
if (frame == null) return false;
|
||||
|
||||
timestamp = GetTimestampAndUpdateIndex(frame);
|
||||
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA);
|
||||
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame);
|
||||
|
||||
ClearConvertedFramePointers();
|
||||
if (vflip)
|
||||
|
||||
@@ -63,7 +63,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
public AVCodecID AudioCodecId;
|
||||
|
||||
/// <summary>
|
||||
/// Audio bitrate (in bps). If default (0), 128000 bps is used.
|
||||
/// Audio bitrate (in bps). If default (0), 256000 bps is used.
|
||||
/// For AAC, 128000–320000 bps is common.
|
||||
/// </summary>
|
||||
public int Bitrate;
|
||||
@@ -299,6 +299,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands the candidate list into (encoder, input mode) attempts in order.
|
||||
/// A QSV encoder is attempted twice: system memory input mode (useFramePool = false) first,
|
||||
/// then GPU frame pool input mode (useFramePool = true). Other encoders are attempted once (useFramePool = false).
|
||||
/// </summary>
|
||||
private static System.Collections.Generic.IEnumerable<(VideoEncoder Encoder, bool UseFramePool)> ExpandQsvInputModes(VideoEncoder[] candidates)
|
||||
{
|
||||
foreach (var encoder in candidates)
|
||||
{
|
||||
yield return (encoder, false);
|
||||
if (encoder.Kind == EncoderKind.QSV)
|
||||
{
|
||||
yield return (encoder, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts an FFmpeg error code (negative integer) to a human-readable string.
|
||||
/// e.g., -1 → "Operation not permitted"
|
||||
@@ -367,7 +384,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
var candidates = GetEncoderCandidates();
|
||||
var openedSuccessfully = false;
|
||||
|
||||
foreach (var encoder in candidates)
|
||||
foreach (var (encoder, useFramePool) in ExpandQsvInputModes(candidates))
|
||||
{
|
||||
// avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc").
|
||||
// Returns null if the encoder is not included in the build.
|
||||
@@ -514,9 +531,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
// Strategy 3) Create QSV device directly (legacy method)
|
||||
// → Fallback on older drivers/systems
|
||||
//
|
||||
// After obtaining device, attempt using GPU frame pool:
|
||||
// hw_frames_ctx: Pre-allocate frame pool in GPU VRAM (zero-copy, highest performance)
|
||||
// hw_device_ctx: Encoder reads frames directly from system memory (better compatibility)
|
||||
// After obtaining device, the input mode is configured (see ExpandQsvInputModes):
|
||||
// Input mode 1, hw_device_ctx: Encoder receives NV12 frames from system memory (attempted first)
|
||||
// Input mode 2, hw_frames_ctx: Frames are uploaded to a QSV surface pool in GPU memory
|
||||
// (attempted when the encoder cannot be opened in input mode 1)
|
||||
//
|
||||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||||
if (encoder.Kind == EncoderKind.QSV)
|
||||
@@ -585,30 +603,64 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Attempt hw_frames_ctx: Pre-create GPU frame pool ──
|
||||
// AVHWFramesContext pre-allocates frame pool in GPU VRAM,
|
||||
// eliminating memory allocation/deallocation overhead for each frame.
|
||||
if (!useFramePool)
|
||||
{
|
||||
// ── Input mode 1 (primary): hw_device_ctx + NV12 system memory ──
|
||||
// The encoder receives NV12 frames in system memory (RAM) and the QSV runtime uploads them to the GPU.
|
||||
//
|
||||
// Pros: Zero-copy capability, highest performance
|
||||
// Cons: May fail depending on GPU/driver resolution/format constraints
|
||||
// (e.g. specific texture size constraints on Intel Arc → E_INVALIDARG)
|
||||
// pix_fmt is the software format NV12.
|
||||
// _isQsvEncoder = false: WriteFrame() passes the NV12 frame to the encoder without uploading it.
|
||||
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
||||
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
|
||||
_isQsvEncoder = false;
|
||||
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using system memory input mode (hw_device_ctx)");
|
||||
}
|
||||
else
|
||||
{
|
||||
// ── Input mode 2 (fallback): hw_frames_ctx GPU frame pool ──
|
||||
// Used when the encoder cannot be opened in input mode 1.
|
||||
// AVHWFramesContext pre-allocates a pool of QSV surfaces in GPU memory (pix_fmt AV_PIX_FMT_QSV),
|
||||
// and WriteFrame() uploads each NV12 frame to a surface of the pool (av_hwframe_transfer_data).
|
||||
//
|
||||
// initial_pool_size: Number of frames to pre-allocate. Matches encoder pipeline depth.
|
||||
// Align resolution to multiples of 32: Texture alignment condition required by QSV hardware.
|
||||
// Pool creation depends on the texture types the GPU/driver accepts
|
||||
// (e.g. Intel Arc A770 rejects NV12 textures with D3D11_BIND_RENDER_TARGET → E_INVALIDARG 0x80070057).
|
||||
//
|
||||
// initial_pool_size: Number of surfaces in the pool (QsvFramePoolSize).
|
||||
// The encoder keeps submitted surfaces until they are encoded, so the pool has to hold
|
||||
// the frames in the encoder pipeline plus the frame being uploaded.
|
||||
// Width/height are aligned to multiples of 32: texture alignment condition required by QSV hardware.
|
||||
//
|
||||
// Frame pool memory type (AVQSVFramesContext.frame_type, a combination of MFX_MEMTYPE_* flags):
|
||||
// Windows (D3D11): MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010) is tried first, then 0.
|
||||
// 0x0010 creates the D3D11 textures with D3D11_BIND_DECODER.
|
||||
// 0 lets FFmpeg select MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET (0x0020),
|
||||
// which creates the textures with D3D11_BIND_RENDER_TARGET.
|
||||
// Other platforms (VAAPI): 0. FFmpeg sets the memory type of a VAAPI child pool itself.
|
||||
// Both types are accepted by the QSV encoder as video memory input.
|
||||
// When every type fails, the next candidate encoder is tested.
|
||||
//
|
||||
// Reference:
|
||||
// AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
|
||||
// av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||||
// AVQSVFramesContext.frame_type: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
|
||||
// frame_type → D3D11 bind flags (qsv_init_child_ctx, qsv_get_d3d11va_bind_flags): https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.c
|
||||
// MFX_MEMTYPE_* values: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
|
||||
bool framesCtxReady = false;
|
||||
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
|
||||
if (hwFramesRef != null)
|
||||
int[] frameTypes = isWindows
|
||||
? new[] { QsvMemTypeVideoMemoryDecoderTarget, 0 }
|
||||
: new[] { 0 };
|
||||
foreach (var frameType in frameTypes)
|
||||
{
|
||||
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
|
||||
if (hwFramesRef == null) break;
|
||||
|
||||
var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
|
||||
framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
|
||||
framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
|
||||
framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
|
||||
framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
|
||||
framesCtx->initial_pool_size = 4; // Frame pool size
|
||||
framesCtx->initial_pool_size = QsvFramePoolSize;
|
||||
SetQsvFrameType(framesCtx, frameType);
|
||||
|
||||
err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
|
||||
if (err >= 0)
|
||||
@@ -618,31 +670,30 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
_pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
|
||||
_isQsvEncoder = true;
|
||||
framesCtxReady = true;
|
||||
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: hw_frames_ctx initialization succeeded (GPU frame pool mode)");
|
||||
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using GPU frame pool input mode (hw_frames_ctx, frame_type=0x{frameType:X4}, pool={QsvFramePoolSize})");
|
||||
ffmpeg.av_buffer_unref(&hwFramesRef);
|
||||
|
||||
_pHwFrame = ffmpeg.av_frame_alloc();
|
||||
if (_pHwFrame == null)
|
||||
throw new Exception("Failed to allocate QSV hardware frame structure.");
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed — {GetFFmpegErrorString(err)} (Code: {err}). Falling back to hw_device_ctx mode.");
|
||||
}
|
||||
|
||||
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed with frame_type=0x{frameType:X4} — {GetFFmpegErrorString(err)} (Code: {err})");
|
||||
ffmpeg.av_buffer_unref(&hwFramesRef);
|
||||
}
|
||||
|
||||
// ── When hw_frames_ctx fails: Fallback to hw_device_ctx + NV12 system memory ──
|
||||
// In hw_device_ctx mode, the encoder receives NV12 frames from system memory (RAM)
|
||||
// directly and uploads to GPU internally.
|
||||
// Slightly slower than hw_frames_ctx, but compatibility is much better.
|
||||
//
|
||||
// Change pix_fmt to NV12: Use software format directly instead of hardware surface (QSV).
|
||||
// _isQsvEncoder = false: Skip manual upload in WriteFrame().
|
||||
if (!framesCtxReady)
|
||||
{
|
||||
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
||||
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
|
||||
_isQsvEncoder = false; // System memory mode, no manual frame upload required
|
||||
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: GPU frame pool input mode is unavailable. Testing next candidate codec.");
|
||||
ffmpeg.av_buffer_unref(&hwDeviceCtx);
|
||||
{
|
||||
var pCtx = _pCodecContext;
|
||||
ffmpeg.avcodec_free_context(&pCtx);
|
||||
}
|
||||
_pCodecContext = null;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Transfer ownership of hwDeviceCtx to codec context,
|
||||
@@ -704,11 +755,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
break;
|
||||
}
|
||||
|
||||
// If open fails, free codec context and try next encoder.
|
||||
Melon<RenderMod>.Logger.Warning($"Failed to initialize hardware encoder {encoder.Name}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate codec.");
|
||||
// If open fails, free codec context and QSV frame state, then try next candidate.
|
||||
var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
|
||||
Melon<RenderMod>.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
|
||||
var pCodecCtx = _pCodecContext;
|
||||
ffmpeg.avcodec_free_context(&pCodecCtx);
|
||||
_pCodecContext = null;
|
||||
_isQsvEncoder = false;
|
||||
if (_pHwFrame != null)
|
||||
{
|
||||
var pHwFrame = _pHwFrame;
|
||||
ffmpeg.av_frame_free(&pHwFrame);
|
||||
_pHwFrame = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!openedSuccessfully || _pCodecContext == null)
|
||||
@@ -765,16 +824,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
if (_pSrcFrame == null || _pDstFrame == null || _pPacket == null)
|
||||
throw new Exception("Failed to allocate frame or packet structure.");
|
||||
|
||||
// Set format and size of destination frame, then allocate actual pixel buffer.
|
||||
// av_frame_get_buffer: Allocates memory matching format, width, height and
|
||||
// sets data[]/linesize[] pointers. Second argument 0 means default alignment.
|
||||
_pDstFrame->format = (int)_dstPixFmt;
|
||||
_pDstFrame->width = _settings.Width;
|
||||
_pDstFrame->height = _settings.Height;
|
||||
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
|
||||
|
||||
if (ffmpeg.av_frame_get_buffer(_pDstFrame, 0) < 0)
|
||||
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer.");
|
||||
AllocateDstFrameBuffer();
|
||||
|
||||
// sws_getContext: Initialize software scaler (colorspace converter).
|
||||
// Input: RGBA (Unity capture format, 4 bytes/pixel)
|
||||
@@ -977,6 +1027,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
/// Sets a private option (priv_data) of the encoder.
|
||||
/// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
|
||||
/// </summary>
|
||||
// ReSharper disable once UnusedMethodReturnValue.Local
|
||||
private bool TrySetOption(string name, string value)
|
||||
{
|
||||
var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0);
|
||||
@@ -986,6 +1037,32 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010): frames are in video memory and belong to video decoder render targets.
|
||||
/// Reference: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
|
||||
/// </summary>
|
||||
private const int QsvMemTypeVideoMemoryDecoderTarget = 0x0010;
|
||||
|
||||
/// <summary>
|
||||
/// Number of surfaces in the QSV GPU frame pool (AVHWFramesContext.initial_pool_size) of GPU frame pool input mode.
|
||||
/// Same value as the encoder frame pool of the FFmpeg hardware encoding example (initial_pool_size = 20).
|
||||
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/doc/examples/vaapi_encode.c
|
||||
/// </summary>
|
||||
private const int QsvFramePoolSize = 20;
|
||||
|
||||
/// <summary>
|
||||
/// Writes AVQSVFramesContext.frame_type of a QSV frames context (AVHWFramesContext.hwctx).
|
||||
/// FFmpeg.AutoGen has no AVQSVFramesContext type, so the field is written at its offset.
|
||||
/// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
|
||||
/// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
|
||||
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
|
||||
/// </summary>
|
||||
private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
|
||||
{
|
||||
if (framesCtx->hwctx == null) return;
|
||||
*(int*)((byte*)framesCtx->hwctx + IntPtr.Size + sizeof(int)) = frameType;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Largest declared maximum that is treated as an actual quality range.
|
||||
/// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used.
|
||||
@@ -1115,12 +1192,50 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
|
||||
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
|
||||
/// Second argument 0 means default alignment.
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
|
||||
/// </summary>
|
||||
private void AllocateDstFrameBuffer()
|
||||
{
|
||||
_pDstFrame->format = (int)_dstPixFmt;
|
||||
_pDstFrame->width = _settings.Width;
|
||||
_pDstFrame->height = _settings.Height;
|
||||
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
|
||||
|
||||
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
|
||||
if (ret < 0)
|
||||
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes the pixel buffer of the destination frame (_pDstFrame) writable before the next frame is written into it.
|
||||
///
|
||||
/// After avcodec_send_frame, the encoder may keep a reference to the buffer of the submitted frame
|
||||
/// until that frame is encoded (e.g. QSV in system memory input mode). The buffer is then shared and not writable
|
||||
/// (av_frame_is_writable returns 0).
|
||||
/// In that case the frame releases its reference (av_frame_unref) and a new buffer is allocated.
|
||||
/// The encoder keeps the previous buffer until it no longer needs it. The previous contents are not copied,
|
||||
/// because every plane is overwritten by the next frame.
|
||||
///
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_is_writable, av_frame_unref)
|
||||
/// </summary>
|
||||
private void PrepareDstFrameForWrite()
|
||||
{
|
||||
if (ffmpeg.av_frame_is_writable(_pDstFrame) != 0) return;
|
||||
|
||||
ffmpeg.av_frame_unref(_pDstFrame);
|
||||
AllocateDstFrameBuffer();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data.
|
||||
/// </summary>
|
||||
private void WritePreConvertedFrame(byte* buffer)
|
||||
{
|
||||
ffmpeg.av_frame_make_writable(_pDstFrame);
|
||||
PrepareDstFrameForWrite();
|
||||
|
||||
var width = _settings.Width;
|
||||
var height = _settings.Height;
|
||||
@@ -1234,20 +1349,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
/// Core logic for actual frame encoding.
|
||||
///
|
||||
/// Processing flow:
|
||||
/// 1. av_frame_make_writable: Check if destination frame buffer is writable
|
||||
/// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable
|
||||
/// (Allocates a new buffer if encoder is still referencing previous frame's buffer)
|
||||
/// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion
|
||||
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface
|
||||
/// 4. Pass to encoder after setting PTS
|
||||
///
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale)
|
||||
/// </summary>
|
||||
private void WriteFrame()
|
||||
{
|
||||
// Ensure writable state since encoder might still be referencing previous frame's buffer.
|
||||
// Internally allocates a new buffer if referenced.
|
||||
ffmpeg.av_frame_make_writable(_pDstFrame);
|
||||
PrepareDstFrameForWrite();
|
||||
|
||||
// ── Colorspace conversion: RGBA → YUV420P or NV12 ──
|
||||
// sws_scale parameters explanation:
|
||||
@@ -1408,8 +1520,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
/// 2. av_write_trailer: Writes container trailer (end-of-file metadata).
|
||||
/// For MP4, the moov atom (index info) is completed here.
|
||||
/// ⚠️ If trailer is not written, the file becomes corrupted and unplayable!
|
||||
/// av_write_trailer also flushes the output buffer and returns the write error of the output (AVIOContext.error), if any.
|
||||
///
|
||||
/// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete.
|
||||
/// End() runs once: after it returns or throws, further calls return immediately.
|
||||
///
|
||||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
|
||||
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer)
|
||||
/// </summary>
|
||||
public void End()
|
||||
{
|
||||
@@ -1417,39 +1534,31 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
{
|
||||
if (_disposed || !_isInitialized) return;
|
||||
|
||||
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
|
||||
try
|
||||
{
|
||||
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
|
||||
SendFrameAndWritePackets(null);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Log error occurring during flush process
|
||||
Melon<RenderMod>.Logger.Error($"Error during video encoder flush: {ex.Message}");
|
||||
}
|
||||
|
||||
// 1.5. Flush remaining samples in audio FIFO + flush audio encoder
|
||||
if (_audioEnabled)
|
||||
{
|
||||
try
|
||||
{
|
||||
FlushAudioEncoder();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Melon<RenderMod>.Logger.Error($"Error during audio encoder flush: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Write container trailer (moov atom and file termination metadata)
|
||||
if (_pFormatContext != null)
|
||||
{
|
||||
ffmpeg.av_write_trailer(_pFormatContext);
|
||||
var ret = ffmpeg.av_write_trailer(_pFormatContext);
|
||||
if (ret < 0)
|
||||
throw new Exception($"Failed to write the container trailer — {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||||
}
|
||||
|
||||
}
|
||||
finally
|
||||
{
|
||||
_isInitialized = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Resource Cleanup (Dispose)
|
||||
@@ -1458,11 +1567,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
/// <summary>
|
||||
/// Actual resource cleanup logic. Cleanup order is important:
|
||||
///
|
||||
/// 1. Call End() (if not called yet) → flush delayed frames + write trailer
|
||||
/// 2. SwsContext (scaler) → independent, can be freed first
|
||||
/// 3. AVFrame (src/dst) → freed before codec as codec may reference them
|
||||
/// 4. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
|
||||
/// 5. AVFormatContext (format/IO) → includes closing file handle
|
||||
/// 1. SwsContext (scaler) → independent, can be freed first
|
||||
/// 2. AVFrame (src/dst) → freed before codec as codec may reference them
|
||||
/// 3. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
|
||||
/// 4. AVFormatContext (format/IO) → includes closing file handle
|
||||
///
|
||||
/// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned.
|
||||
/// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it).
|
||||
///
|
||||
/// Reference:
|
||||
/// av_frame_free: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
|
||||
@@ -1476,12 +1587,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
|
||||
// Trigger End() safely in case Dispose is called without calling End() first
|
||||
if (_isInitialized)
|
||||
{
|
||||
try { End(); } catch { /* Ignore */ }
|
||||
}
|
||||
_isInitialized = false;
|
||||
|
||||
// Release Sws scaler
|
||||
if (_pSwsContext != null)
|
||||
|
||||
@@ -20,6 +20,7 @@ internal class VideoBGPlayer : MonoBehaviour
|
||||
private static readonly int UTex = Shader.PropertyToID("_UTex");
|
||||
private static readonly int VTex = Shader.PropertyToID("_VTex");
|
||||
private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
|
||||
private static readonly int ColorMatrix = Shader.PropertyToID("ColorMatrix");
|
||||
|
||||
[SuppressMessage("ReSharper", "InconsistentNaming")]
|
||||
private enum GPUMode
|
||||
@@ -189,6 +190,14 @@ internal class VideoBGPlayer : MonoBehaviour
|
||||
var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
|
||||
var targetVideoTime = songTime + vidOffset;
|
||||
|
||||
// When looping is enabled, the playback position is the elapsed time wrapped into [0, Duration).
|
||||
// When it is disabled, the position keeps increasing and the last frame stays on screen after the end.
|
||||
var videoDuration = _decoder.Duration;
|
||||
if (_isLooping && targetVideoTime >= 0 && videoDuration > 0)
|
||||
{
|
||||
targetVideoTime %= videoDuration;
|
||||
}
|
||||
|
||||
if (targetVideoTime < 0)
|
||||
{
|
||||
if (_targetTexture)
|
||||
@@ -307,6 +316,7 @@ internal class VideoBGPlayer : MonoBehaviour
|
||||
_computeShader.SetInt(Width, w);
|
||||
_computeShader.SetInt(Height, h);
|
||||
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
|
||||
_computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
|
||||
|
||||
int threadGroupsX = (w + 7) / 8;
|
||||
int threadGroupsY = (h + 7) / 8;
|
||||
@@ -341,6 +351,7 @@ internal class VideoBGPlayer : MonoBehaviour
|
||||
_computeShader.SetInt(Width, w);
|
||||
_computeShader.SetInt(Height, h);
|
||||
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
|
||||
_computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
|
||||
|
||||
int threadGroupsX = (w + 7) / 8;
|
||||
int threadGroupsY = (h + 7) / 8;
|
||||
|
||||
Reference in new issue
Block a user