11 Commits
9 changed files with 646 additions and 220 deletions

No files matched your search

+1
View File
@@ -48,3 +48,4 @@ desktop.ini
# Developer Utility Files
/REVIEW_CHECKLIST.md
/REVIEW_FFMPEG.md
Binary file not shown.
Binary file not shown.
Binary file not shown.
+41 -12
View File
@@ -226,6 +226,11 @@ internal class RenderManager : MonoBehaviour
private GameObject? _debugMonitorObj;
private GameObject? _escapeHandlerObj;
internal string? CurrentOutPath;
// Output file of the current render that is not finished yet.
// Set just before the encoder opens the file, cleared by the render thread after the encoder finished the file (End).
// Only this file is deleted when a render is cancelled, fails or the game exits (DeleteIncompleteOutputFile).
private string? _incompleteOutputPath;
private bool _renderSetup;
private void Awake()
@@ -286,6 +291,32 @@ internal class RenderManager : MonoBehaviour
{
RenderThread.Join(200);
}
// The encoders are disposed above, which closes the output file. An unfinished file of this render is deleted.
DeleteIncompleteOutputFile();
}
/// <summary>
/// Deletes the output file of the current render when the render did not finish it (_incompleteOutputPath).
/// Files of earlier, finished renders are never deleted.
/// Returns true when a file was deleted.
/// </summary>
private bool DeleteIncompleteOutputFile()
{
var path = Interlocked.Exchange(ref _incompleteOutputPath, null);
if (string.IsNullOrEmpty(path) || !File.Exists(path)) return false;
try
{
File.Delete(path);
Melon<RenderMod>.Logger.Msg($"[File Cleanup] Deleted incomplete video: {path}");
return true;
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[File Cleanup] Failed to delete incomplete video: {ex.Message}");
return false;
}
}
private void Update()
@@ -634,6 +665,7 @@ internal class RenderManager : MonoBehaviour
{
_allVideoFFmpegInstances.Add(videoFFmpeg);
}
Volatile.Write(ref _incompleteOutputPath, CurrentOutPath);
videoFFmpeg.Start(CurrentOutPath);
}
catch (Exception)
@@ -735,7 +767,13 @@ internal class RenderManager : MonoBehaviour
FrameAvailableEvent.WaitOne(50);
}
if (!IsRenderAborted) videoFFmpeg.End();
if (!IsRenderAborted)
{
// End throws when the file cannot be finished (e.g. the trailer cannot be written),
// which is handled as a render failure by the catch block below.
videoFFmpeg.End();
Volatile.Write(ref _incompleteOutputPath, null);
}
videoFFmpeg.Dispose();
lock (_allVideoFFmpegInstances)
@@ -862,18 +900,9 @@ internal class RenderManager : MonoBehaviour
scrController.instance.noFailInfiniteMargin = false;
}
// 6. Immediately delete cancelled incomplete temporary video file
if (!string.IsNullOrEmpty(CurrentOutPath) && File.Exists(CurrentOutPath))
// 6. Immediately delete the incomplete video file of this render
if (DeleteIncompleteOutputFile())
{
try
{
File.Delete(CurrentOutPath);
Melon<RenderMod>.Logger.Msg($"[File Cleanup] Deleted incomplete video: {CurrentOutPath}");
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[File Cleanup] Failed to delete temporary video: {ex.Message}");
}
CurrentOutPath = null;
}
+1 -1
View File
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine;
using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.6-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer;
+292 -115
View File
@@ -191,6 +191,13 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
private bool _isOpened;
private bool _disposed;
private bool _isDraining;
/// <summary>Number of video packets skipped because the decoder rejected them as invalid data.</summary>
private int _skippedPacketCount;
/// <summary>Number of skipped packets for which a warning is logged.</summary>
private const int MaxLoggedSkippedPackets = 5;
private readonly object _lock = new();
// Fields to detect SwsContext updates
@@ -199,6 +206,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
private int _srcWidth;
private int _srcHeight;
/// <summary>YUV→RGB matrix id (<c>SWS_CS_*</c>) applied to the context. 0 when no matrix is set (non-RGBA output).</summary>
private int _srcSwsColorspace;
/// <summary>Whether the source range applied to the context is the full range.</summary>
private bool _srcFullRange;
// ──────────────────────────────────────────────────────────────
// Public Properties
// ──────────────────────────────────────────────────────────────
@@ -289,6 +302,59 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
/// <summary>
/// Color matrix id used to convert the YUV frames of this stream to RGB, selected from the colorspace tag of the stream.
/// The ids are shared with the <c>ColorMatrix</c> uniform of ColorConversion.compute:
/// 0 = BT.709 (also used when the stream has no usable tag), 1 = BT.601, 2 = BT.2020 (non-constant luminance).
/// </summary>
public int ColorMatrix
{
get
{
lock (_lock)
{
if (_pCodecContext != null && _pCodecContext->colorspace != AVColorSpace.AVCOL_SPC_UNSPECIFIED)
{
return GetColorMatrixId(_pCodecContext->colorspace);
}
if (_pFormatContext != null && _videoStreamIndex >= 0)
{
return GetColorMatrixId(_pFormatContext->streams[_videoStreamIndex]->codecpar->color_space);
}
return ColorMatrixBT709;
}
}
}
/// <summary>Color matrix id of BT.709.</summary>
// ReSharper disable InconsistentNaming
public const int ColorMatrixBT709 = 0;
/// <summary>Color matrix id of BT.601 (SMPTE 170M / BT.470 BG).</summary>
public const int ColorMatrixBT601 = 1;
/// <summary>Color matrix id of BT.2020 (non-constant luminance).</summary>
public const int ColorMatrixBT2020 = 2;
// ReSharper restore InconsistentNaming
/// <summary>
/// Converts a colorspace tag to a color matrix id (see <see cref="ColorMatrix"/>).
/// Tags that have no matrix of their own (unspecified, RGB, other standards) use BT.709.
/// </summary>
private static int GetColorMatrixId(AVColorSpace colorspace)
{
switch (colorspace)
{
case AVColorSpace.AVCOL_SPC_SMPTE170M:
case AVColorSpace.AVCOL_SPC_BT470BG:
return ColorMatrixBT601;
case AVColorSpace.AVCOL_SPC_BT2020_NCL:
return ColorMatrixBT2020;
default:
return ColorMatrixBT709;
}
}
/// <summary>
/// Gets a value indicating whether hardware-accelerated decoding is currently active.
/// </summary>
@@ -371,6 +437,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (_videoStreamIndex == -1)
throw new Exception("Failed to find video stream.");
// Only the video stream is used. The demuxer does not need to return the packets of the other streams
// (audio, subtitles, ...) once they are marked as discarded.
// Reference: AVStream.discard - "Selects which packets can be discarded at will and do not need to be demuxed."
// https://ffmpeg.org/doxygen/trunk/structAVStream.html
for (int i = 0; i < _pFormatContext->nb_streams; i++)
{
if (i != _videoStreamIndex)
_pFormatContext->streams[i]->discard = AVDiscard.AVDISCARD_ALL;
}
var stream = _pFormatContext->streams[_videoStreamIndex];
_timeBase = stream->time_base;
@@ -441,9 +517,13 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
TotalFrames = 0;
}
Duration = _pFormatContext->duration > 0
? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE
: (stream->duration > 0 ? stream->duration * ffmpeg.av_q2d(stream->time_base) : 0);
// Duration of the video stream (AVStream.duration, in the stream time base).
// When the stream does not carry a duration, the container duration (AVFormatContext.duration,
// in AV_TIME_BASE units, the longest of all streams) is used.
// Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html, https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
Duration = stream->duration > 0
? stream->duration * ffmpeg.av_q2d(stream->time_base)
: _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0;
// 9. Allocate frame and packet buffers
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
@@ -655,6 +735,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary>
/// Reads packets from the container until a packet belonging to the video stream is found,
/// then sends it to the decoder via <c>avcodec_send_packet</c>.
/// A packet that the decoder rejects as invalid data (<c>AVERROR_INVALIDDATA</c>, e.g. a damaged packet)
/// is skipped and the next packet is read. Any other decoder error is thrown.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
/// </summary>
/// <returns>True if a video packet was sent; false on EOF.</returns>
@@ -670,6 +752,18 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
{
ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket);
ffmpeg.av_packet_unref(_pPacket);
if (ret == ffmpeg.AVERROR_INVALIDDATA)
{
// The warning is logged for the first few packets only (a damaged file can have many).
_skippedPacketCount++;
if (_skippedPacketCount <= MaxLoggedSkippedPackets)
{
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Skipped an invalid video packet ({_skippedPacketCount}): {GetFFmpegErrorString(ret)}");
}
continue;
}
if (ret < 0)
{
throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}");
@@ -743,12 +837,67 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
/// <summary>
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats or dimensions change.
/// Selects the YUV→RGB conversion matrix (libswscale <c>SWS_CS_*</c>) from the colorspace tag of a frame.
/// The matrix set is the same as that of the compute shader path (ColorConversion.compute, <c>ColorMatrix</c> uniform):
/// BT.709 (also for frames without a usable tag), BT.601 and BT.2020, so both paths give the same colors.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_getCoefficients)
/// </summary>
private static int GetSwsColorspace(AVColorSpace colorspace)
{
switch (GetColorMatrixId(colorspace))
{
case ColorMatrixBT601:
return ffmpeg.SWS_CS_ITU601;
case ColorMatrixBT2020:
return ffmpeg.SWS_CS_BT2020;
default:
return ffmpeg.SWS_CS_ITU709;
}
}
/// <summary>
/// Returns true when the Y/U/V samples of the frame use the full range (0-255) instead of the limited range (16-235 / 16-240).
/// The range is given by the frame's color_range tag or by a deprecated full range pixel format (YUVJ*).
/// </summary>
private static bool IsFullRange(AVFrame* frame)
{
if (frame->color_range == AVColorRange.AVCOL_RANGE_JPEG) return true;
switch ((AVPixelFormat)frame->format)
{
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
case AVPixelFormat.AV_PIX_FMT_YUVJ422P:
case AVPixelFormat.AV_PIX_FMT_YUVJ444P:
case AVPixelFormat.AV_PIX_FMT_YUVJ440P:
return true;
default:
return false;
}
}
/// <summary>
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats, dimensions,
/// or (for RGBA output) the colorspace / range of the source frame change.
/// For RGBA output the YUV→RGB matrix and the source range are taken from <paramref name="colorFrame"/>.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
/// </summary>
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat)
/// <param name="srcFormat">Pixel format of the source frame.</param>
/// <param name="srcWidth">Width of the source frame in pixels.</param>
/// <param name="srcHeight">Height of the source frame in pixels.</param>
/// <param name="dstFormat">Pixel format of the output.</param>
/// <param name="colorFrame">Source frame whose colorspace / range tags are applied. Required for RGBA output.</param>
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat, AVFrame* colorFrame = null)
{
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat)
var swsColorspace = 0;
var fullRange = false;
if (colorFrame != null && dstFormat == AVPixelFormat.AV_PIX_FMT_RGBA)
{
swsColorspace = GetSwsColorspace(colorFrame->colorspace);
fullRange = IsFullRange(colorFrame);
}
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat
&& _srcSwsColorspace == swsColorspace && _srcFullRange == fullRange)
{
return;
}
@@ -763,6 +912,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_srcWidth = srcWidth;
_srcHeight = srcHeight;
_dstPixFmt = dstFormat;
_srcSwsColorspace = swsColorspace;
_srcFullRange = fullRange;
_pSwsContext = ffmpeg.sws_getContext(
srcWidth, srcHeight, srcFormat,
@@ -774,6 +925,44 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
{
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
}
if (swsColorspace != 0)
{
ApplySwsColorspace(swsColorspace, fullRange);
}
}
/// <summary>
/// Sets the YUV→RGB matrix and the source range of <see cref="_pSwsContext"/>.
/// The destination range, brightness, contrast and saturation keep the values of the context.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails)
/// </summary>
/// <param name="swsColorspace">Matrix id (<c>SWS_CS_*</c>) of the source.</param>
/// <param name="fullRange">True when the source uses the full range.</param>
private void ApplySwsColorspace(int swsColorspace, bool fullRange)
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
// Returns a negative value when the context cannot change its colorspace details (e.g. YUV to YUV conversion).
if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation) < 0)
{
return;
}
var coefficients = ffmpeg.sws_getCoefficients(swsColorspace);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = coefficients[i];
table[i] = currentTable[i];
}
ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, fullRange ? 1 : 0, table, dstRange,
brightness, contrast, saturation);
}
/// <summary>
@@ -865,7 +1054,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (frame == null) return false;
timestamp = GetTimestampAndUpdateIndex(frame);
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA);
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame);
ClearConvertedFramePointers();
if (vflip)
@@ -1212,107 +1401,24 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Decodes and discards a specified number of frames immediately without GPU-to-CPU transfer overhead.
/// Maximum distance in frames over which <see cref="SeekToTime"/> moves forward by decoding.
/// Decoding forward needs no keyframe seek and no decoder flush, which is cheaper than seeking for short distances.
/// </summary>
/// <param name="count">Number of frames to discard.</param>
private void SkipFrames(int count)
{
lock (_lock)
{
if (!_isOpened || _disposed) return;
if (count <= 0) return;
int skipped = 0;
if (_hasCachedFrame)
{
ffmpeg.av_frame_unref(_pCachedFrame);
_hasCachedFrame = false;
skipped++;
}
while (skipped < count)
{
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
if (ret == 0)
{
GetTimestampAndUpdateIndex(_pDecodedFrame);
ffmpeg.av_frame_unref(_pDecodedFrame);
skipped++;
continue;
}
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
{
if (_isDraining) break;
if (!ReadAndSendPacket())
{
_isDraining = true;
ffmpeg.avcodec_send_packet(_pCodecContext, null);
}
}
else if (ret == ffmpeg.AVERROR_EOF)
{
break;
}
else
{
throw new Exception($"Error skipping frames: {GetFFmpegErrorString(ret)}");
}
}
}
}
private const int MaxForwardDecodeFrames = 60;
/// <summary>
/// Seeks to a specific playback time in seconds using backward keyframe seek followed by forward decode.
/// Intermediate frames are discarded without GPU-to-host memory transfer to maximize seek speed;
/// only the target frame is transferred and preserved in <see cref="_pCachedFrame"/>.
/// Decodes forward from the current decoder position until the first frame whose presentation time is at least
/// <paramref name="timeSeconds"/> minus half a frame, and keeps that frame in <see cref="_pCachedFrame"/>
/// for the next <c>TryRead*</c> call.
/// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred.
/// When the end of the stream is reached before the target, no frame is cached.
/// </summary>
/// <param name="timeSeconds">Target time in seconds to seek to.</param>
public void SeekToTime(double timeSeconds)
/// <param name="timeSeconds">Target time in seconds.</param>
private void DecodeToTarget(double timeSeconds)
{
lock (_lock)
{
if (!_isOpened || _disposed) return;
if (timeSeconds < 0) timeSeconds = 0;
double targetFrameIndex = timeSeconds * FPS;
// If already at target index or very close future, skip frames instead of seek to avoid I-frame seek latency
bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60;
if (useSkip)
{
SkipFrames((int)(targetFrameIndex - _currentFrameIndex));
return;
}
if (_hasCachedFrame)
{
ffmpeg.av_frame_unref(_pCachedFrame);
_hasCachedFrame = false;
}
long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase));
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
// Fallback to beginning of file on seek failure
ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}");
}
}
// Flush decoder internal buffers
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
ffmpeg.avcodec_flush_buffers(_pCodecContext);
_isDraining = false;
// Decode frames until reaching the target presentation timestamp
while (true)
{
ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
if (ret == 0)
{
double currentPts = GetFrameTimestamp(_pDecodedFrame);
@@ -1343,28 +1449,18 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
_hasCachedFrame = true;
break;
return;
}
// Discard non-matching intermediate frame without transferring to CPU
long ptsValue = _pDecodedFrame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE
? _pDecodedFrame->best_effort_timestamp
: _pDecodedFrame->pts;
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
{
_currentFrameIndex++;
}
else
{
_currentFrameIndex = (long)Math.Round(currentPts * FPS);
}
GetTimestampAndUpdateIndex(_pDecodedFrame);
ffmpeg.av_frame_unref(_pDecodedFrame);
continue;
}
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
{
if (_isDraining) break;
if (_isDraining) return;
if (!ReadAndSendPacket())
{
@@ -1374,7 +1470,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
else if (ret == ffmpeg.AVERROR_EOF)
{
break;
return;
}
else
{
@@ -1382,6 +1478,87 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
}
/// <summary>
/// Moves the playback position to a time in seconds.
/// The next <c>TryRead*</c> call returns the first frame whose presentation time is at least
/// <paramref name="timeSeconds"/> minus half a frame (the frame nearest to the target time).
/// Both paths below return that same frame:
/// - Forward decode: the target is ahead of the current position by less than <see cref="MaxForwardDecodeFrames"/> frames.
/// Frames are decoded and discarded up to the target.
/// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read).
/// A backward keyframe seek is followed by a forward decode up to the target.
/// The target frame is transferred and preserved in <see cref="_pCachedFrame"/>; when it is already there, nothing is decoded.
/// </summary>
/// <param name="timeSeconds">Target time in seconds to seek to.</param>
public void SeekToTime(double timeSeconds)
{
lock (_lock)
{
if (!_isOpened || _disposed) return;
if (timeSeconds < 0) timeSeconds = 0;
// Earliest presentation time that satisfies the target (target minus half a frame).
double threshold = timeSeconds - (0.5 / FPS);
// Index of the first frame that satisfies the target. The epsilon keeps a frame that lies exactly on the
// margin from being excluded by floating point rounding.
double targetFrameIndex = Math.Ceiling(timeSeconds * FPS - 0.5 - 1e-6);
if (_hasCachedFrame)
{
// A frame is already positioned for the next read (e.g. by a previous SeekToTime call).
double cachedPts = GetFrameTimestamp(_pCachedFrame);
if (cachedPts >= threshold)
{
// The cached frame is the first frame that satisfies the target.
// When the target lies before it, the frames in between are no longer available and a keyframe seek is required.
if (cachedPts - threshold < 1.0 / FPS) return;
}
else if (targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
{
// The cached frame lies before the target: drop it and continue decoding forward.
GetTimestampAndUpdateIndex(_pCachedFrame);
ffmpeg.av_frame_unref(_pCachedFrame);
_hasCachedFrame = false;
DecodeToTarget(timeSeconds);
return;
}
}
else if (targetFrameIndex > _currentFrameIndex && targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
{
// The target frame has not been read yet and is close: decode forward instead of seeking.
DecodeToTarget(timeSeconds);
return;
}
if (_hasCachedFrame)
{
ffmpeg.av_frame_unref(_pCachedFrame);
_hasCachedFrame = false;
}
long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase));
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
// Fallback to beginning of file on seek failure
ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}");
}
}
// Flush decoder internal buffers
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
ffmpeg.avcodec_flush_buffers(_pCodecContext);
_isDraining = false;
// Decode frames until reaching the target presentation timestamp
DecodeToTarget(timeSeconds);
}
}
/// <summary>
+300 -92
View File
@@ -63,7 +63,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
public AVCodecID AudioCodecId;
/// <summary>
/// Audio bitrate (in bps). If default (0), 128000 bps is used.
/// Audio bitrate (in bps). If default (0), 256000 bps is used.
/// For AAC, 128000–320000 bps is common.
/// </summary>
public int Bitrate;
@@ -83,7 +83,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Frames per second (e.g., 60.0, 29.97, etc.).</summary>
public double FPS;
/// <summary>Video codec type to use (H264, H265, AV1, VP9).</summary>
/// <summary>Video codec type to use (H264, H265, AV1, VP9, VVC).</summary>
public Codec Codec;
/// <summary>Bitrate control mode (CQP, CBR, VBR).</summary>
@@ -299,6 +299,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray();
}
/// <summary>
/// Expands the candidate list into (encoder, input mode) attempts in order.
/// A QSV encoder is attempted twice: system memory input mode (useFramePool = false) first,
/// then GPU frame pool input mode (useFramePool = true). Other encoders are attempted once (useFramePool = false).
/// </summary>
private static System.Collections.Generic.IEnumerable<(VideoEncoder Encoder, bool UseFramePool)> ExpandQsvInputModes(VideoEncoder[] candidates)
{
foreach (var encoder in candidates)
{
yield return (encoder, false);
if (encoder.Kind == EncoderKind.QSV)
{
yield return (encoder, true);
}
}
}
/// <summary>
/// Converts an FFmpeg error code (negative integer) to a human-readable string.
/// e.g., -1 → "Operation not permitted"
@@ -367,7 +384,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var candidates = GetEncoderCandidates();
var openedSuccessfully = false;
foreach (var encoder in candidates)
foreach (var (encoder, useFramePool) in ExpandQsvInputModes(candidates))
{
// avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc").
// Returns null if the encoder is not included in the build.
@@ -514,9 +531,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Strategy 3) Create QSV device directly (legacy method)
// → Fallback on older drivers/systems
//
// After obtaining device, attempt using GPU frame pool:
// hw_frames_ctx: Pre-allocate frame pool in GPU VRAM (zero-copy, highest performance)
// hw_device_ctx: Encoder reads frames directly from system memory (better compatibility)
// After obtaining device, the input mode is configured (see ExpandQsvInputModes):
// Input mode 1, hw_device_ctx: Encoder receives NV12 frames from system memory (attempted first)
// Input mode 2, hw_frames_ctx: Frames are uploaded to a QSV surface pool in GPU memory
// (attempted when the encoder cannot be opened in input mode 1)
//
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (encoder.Kind == EncoderKind.QSV)
@@ -585,30 +603,64 @@ namespace ADOFAIRenderer.Util.FFmpeg
continue;
}
// ── Attempt hw_frames_ctx: Pre-create GPU frame pool ──
// AVHWFramesContext pre-allocates frame pool in GPU VRAM,
// eliminating memory allocation/deallocation overhead for each frame.
if (!useFramePool)
{
// ── Input mode 1 (primary): hw_device_ctx + NV12 system memory ──
// The encoder receives NV12 frames in system memory (RAM) and the QSV runtime uploads them to the GPU.
//
// Pros: Zero-copy capability, highest performance
// Cons: May fail depending on GPU/driver resolution/format constraints
// (e.g. specific texture size constraints on Intel Arc → E_INVALIDARG)
// pix_fmt is the software format NV12.
// _isQsvEncoder = false: WriteFrame() passes the NV12 frame to the encoder without uploading it.
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
_isQsvEncoder = false;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using system memory input mode (hw_device_ctx)");
}
else
{
// ── Input mode 2 (fallback): hw_frames_ctx GPU frame pool ──
// Used when the encoder cannot be opened in input mode 1.
// AVHWFramesContext pre-allocates a pool of QSV surfaces in GPU memory (pix_fmt AV_PIX_FMT_QSV),
// and WriteFrame() uploads each NV12 frame to a surface of the pool (av_hwframe_transfer_data).
//
// initial_pool_size: Number of frames to pre-allocate. Matches encoder pipeline depth.
// Align resolution to multiples of 32: Texture alignment condition required by QSV hardware.
// Pool creation depends on the texture types the GPU/driver accepts
// (e.g. Intel Arc A770 rejects NV12 textures with D3D11_BIND_RENDER_TARGET → E_INVALIDARG 0x80070057).
//
// initial_pool_size: Number of surfaces in the pool (QsvFramePoolSize).
// The encoder keeps submitted surfaces until they are encoded, so the pool has to hold
// the frames in the encoder pipeline plus the frame being uploaded.
// Width/height are aligned to multiples of 32: texture alignment condition required by QSV hardware.
//
// Frame pool memory type (AVQSVFramesContext.frame_type, a combination of MFX_MEMTYPE_* flags):
// Windows (D3D11): MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010) is tried first, then 0.
// 0x0010 creates the D3D11 textures with D3D11_BIND_DECODER.
// 0 lets FFmpeg select MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET (0x0020),
// which creates the textures with D3D11_BIND_RENDER_TARGET.
// Other platforms (VAAPI): 0. FFmpeg sets the memory type of a VAAPI child pool itself.
// Both types are accepted by the QSV encoder as video memory input.
// When every type fails, the next candidate encoder is tested.
//
// Reference:
// AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
// av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
// AVQSVFramesContext.frame_type: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
// frame_type → D3D11 bind flags (qsv_init_child_ctx, qsv_get_d3d11va_bind_flags): https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.c
// MFX_MEMTYPE_* values: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
bool framesCtxReady = false;
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
if (hwFramesRef != null)
int[] frameTypes = isWindows
? new[] { QsvMemTypeVideoMemoryDecoderTarget, 0 }
: new[] { 0 };
foreach (var frameType in frameTypes)
{
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
if (hwFramesRef == null) break;
var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
framesCtx->initial_pool_size = 4; // Frame pool size
framesCtx->initial_pool_size = QsvFramePoolSize;
SetQsvFrameType(framesCtx, frameType);
err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
if (err >= 0)
@@ -618,31 +670,30 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
_isQsvEncoder = true;
framesCtxReady = true;
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: hw_frames_ctx initialization succeeded (GPU frame pool mode)");
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Using GPU frame pool input mode (hw_frames_ctx, frame_type=0x{frameType:X4}, pool={QsvFramePoolSize})");
ffmpeg.av_buffer_unref(&hwFramesRef);
_pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null)
throw new Exception("Failed to allocate QSV hardware frame structure.");
break;
}
else
{
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed — {GetFFmpegErrorString(err)} (Code: {err}). Falling back to hw_device_ctx mode.");
}
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed with frame_type=0x{frameType:X4} — {GetFFmpegErrorString(err)} (Code: {err})");
ffmpeg.av_buffer_unref(&hwFramesRef);
}
// ── When hw_frames_ctx fails: Fallback to hw_device_ctx + NV12 system memory ──
// In hw_device_ctx mode, the encoder receives NV12 frames from system memory (RAM)
// directly and uploads to GPU internally.
// Slightly slower than hw_frames_ctx, but compatibility is much better.
//
// Change pix_fmt to NV12: Use software format directly instead of hardware surface (QSV).
// _isQsvEncoder = false: Skip manual upload in WriteFrame().
if (!framesCtxReady)
{
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
_isQsvEncoder = false; // System memory mode, no manual frame upload required
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: GPU frame pool input mode is unavailable. Testing next candidate codec.");
ffmpeg.av_buffer_unref(&hwDeviceCtx);
{
var pCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCtx);
}
_pCodecContext = null;
continue;
}
}
// Transfer ownership of hwDeviceCtx to codec context,
@@ -704,11 +755,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
break;
}
// If open fails, free codec context and try next encoder.
Melon<RenderMod>.Logger.Warning($"Failed to initialize hardware encoder {encoder.Name}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate codec.");
// If open fails, free codec context and QSV frame state, then try next candidate.
var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : "";
Melon<RenderMod>.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate.");
var pCodecCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCodecCtx);
_pCodecContext = null;
_isQsvEncoder = false;
if (_pHwFrame != null)
{
var pHwFrame = _pHwFrame;
ffmpeg.av_frame_free(&pHwFrame);
_pHwFrame = null;
}
}
if (!openedSuccessfully || _pCodecContext == null)
@@ -723,6 +782,16 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0)
throw new Exception("Failed to copy codec parameters to stream.");
// Stream time base: unit of the timestamps stored in the container.
// Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks.
// When it is left unset, the muxer uses 1/90000, in which the interval of rates such as
// 59.94 fps (1501.5 ticks), 23.976 fps (3753.75 ticks) or 165 fps (545.45 ticks) is not an integer,
// and the frame durations alternate between two values.
// The muxer may still adjust the value in avformat_write_header (e.g. mp4 scales it to at least 10000);
// packet timestamps are rescaled to the final value in SendFrameAndWritePackets.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavformat/mux.c (init_muxer: default 1/90000)
_pVideoStream->time_base = _pCodecContext->time_base;
// ── Step 4.5 (Optional): Initialize audio stream ──
// If EncoderSettings.Audio is set, add audio stream.
if (_settings.Audio.HasValue)
@@ -755,16 +824,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSrcFrame == null || _pDstFrame == null || _pPacket == null)
throw new Exception("Failed to allocate frame or packet structure.");
// Set format and size of destination frame, then allocate actual pixel buffer.
// av_frame_get_buffer: Allocates memory matching format, width, height and
// sets data[]/linesize[] pointers. Second argument 0 means default alignment.
_pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
if (ffmpeg.av_frame_get_buffer(_pDstFrame, 0) < 0)
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer.");
AllocateDstFrameBuffer();
// sws_getContext: Initialize software scaler (colorspace converter).
// Input: RGBA (Unity capture format, 4 bytes/pixel)
@@ -825,7 +885,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Reference - CQP options per encoder:
/// NVENC: "qp" (single value applied to I/P/B frames)
/// AMF: "qp_i" + "qp_p" (I-frame and P-frame configured separately)
/// QSV: "global_quality" (Intel unique option)
/// QSV: AVCodecContext.global_quality (Intel ICQ mode)
/// Software: "crf" (Constant Rate Factor, smarter quality control)
///
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
@@ -847,11 +907,21 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pCodecContext->rc_max_rate = _pCodecContext->rc_min_rate = _settings.TargetBitrate * 1000L;
_pCodecContext->rc_buffer_size = _settings.TargetBitrate * 1000;
// nal-hrd=cbr: x264/x265 dedicated option. Generates CBR bitstream matching
// NAL HRD (Hypothetical Reference Decoder) specification. May be required for streaming compatibility.
if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265)
// HRD (Hypothetical Reference Decoder) signaling for CBR output, used for streaming compatibility.
// libx264: "nal-hrd=cbr" writes the NAL HRD information and pads filler data to keep the bitrate constant.
// FFmpeg's option description: "Signal HRD information (requires vbv-bufsize; cbr not allowed in .mp4)".
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/libx264.c ("nal-hrd": none/vbr/cbr)
// libx265: "x265-params" = "hrd=1" (--hrd: "Enable the signaling of HRD parameters to the decoder")
// + "strict-cbr=1" (--strict-cbr: "Enables stricter conditions to control bitrate deviance
// from the target bitrate in ABR mode").
// Reference: https://x265.readthedocs.io/en/master/cli.html
if (encoder == VideoEncoder.Libx264)
{
ffmpeg.av_opt_set(_pCodecContext->priv_data, "nal-hrd", "cbr", 0);
TrySetOption("nal-hrd", "cbr");
}
else if (encoder == VideoEncoder.Libx265)
{
TrySetOption("x265-params", "hrd=1:strict-cbr=1");
}
}
break;
@@ -875,38 +945,77 @@ namespace ADOFAIRenderer.Util.FFmpeg
break;
case RateControlMode.CQP:
// Valid quality range per encoder. The range of the private options is read from the
// encoder's option definition (AVOption min/max); a declared minimum of -1 means "auto".
// NVENC: "qp" H.264/HEVC 0~51, AV1 0~255
// (libavcodec/nvenc_h264.c, nvenc_hevc.c: "qp" -1~51 / libavcodec/nvenc_av1.c: "qp" -1~255)
// AMF: "qp_i" H.264/HEVC 0~51, AV1 0~255
// (libavcodec/amfenc_h264.c, amfenc_hevc.c: "qp_i" -1~51 / libavcodec/amfenc_av1.c: "qp_i" -1~255)
// QSV: AVCodecContext.global_quality 1~51
// VVC: "qp" 0~63 (libavcodec/libvvenc.c: "qp" -1~63)
// Software: "crf" x264/x265 0~51 (https://x265.readthedocs.io/en/master/cli.html: --crf 0..51.0),
// VP9/AV1 0~63 (libavcodec/libvpxenc.c, libaomenc.c: "crf" -1~63 / libavcodec/libsvtav1.c: "crf" 0~63)
// Reference: https://github.com/FFmpeg/FFmpeg/tree/master/libavcodec
int qpMin, qpMax;
if (encoder.Kind == EncoderKind.QSV)
{
// QSV runs in ICQ (Intelligent Constant Quality) mode when global_quality > 0 and rc_max_rate is 0;
// global_quality 0 selects the encoder's default CQP. The ICQ quality is clamped to 1~51 by FFmpeg:
// libavcodec/qsvenc.c: q->param.mfx.ICQQuality = av_clip(avctx->global_quality, 1, 51);
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/qsvenc.c
qpMin = 1;
qpMax = 51;
}
else if (encoder.Kind == EncoderKind.NVENC) (qpMin, qpMax) = GetQualityRange("qp");
else if (encoder.Kind == EncoderKind.AMF) (qpMin, qpMax) = GetQualityRange("qp_i");
else if (encoder == VideoEncoder.LibVvenc) (qpMin, qpMax) = GetQualityRange("qp");
else (qpMin, qpMax) = GetQualityRange("crf"); // Software encoder (libx264, libx265, libvpx-vp9, etc.)
// Default value 18: Quality close to "visually lossless" in H.264.
// Provides similar level of quality in other codecs as well.
var qpValue = _settings.QualityValue is >= 0 and <= 51 ? _settings.QualityValue : 18;
const int defaultQuality = 18;
var requestedQp = _settings.QualityValue;
int qpValue;
if (requestedQp < 0)
{
qpValue = Math.Clamp(defaultQuality, qpMin, qpMax);
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: Invalid quality value {requestedQp}. Using {qpValue}.");
}
else
{
qpValue = Math.Clamp(requestedQp, qpMin, qpMax);
if (qpValue != requestedQp)
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: Quality value {requestedQp} is outside the supported range {qpMin}~{qpMax}. Using {qpValue}.");
}
if (encoder.Kind == EncoderKind.NVENC)
{
// NVENC: Applies same QP to I/P/B frames with single "qp" option
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp", qpValue.ToString(), 0);
TrySetOption("qp", qpValue.ToString());
}
else if (encoder.Kind == EncoderKind.AMF)
{
// AMF: QP for I-frame (qp_i) and P-frame (qp_p) must be set separately.
// Since B-frames are disabled, qp_b is not set.
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp_i", qpValue.ToString(), 0);
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp_p", qpValue.ToString(), 0);
TrySetOption("qp_i", qpValue.ToString());
TrySetOption("qp_p", qpValue.ToString());
}
else if (encoder.Kind == EncoderKind.QSV)
{
// QSV: Uses ICQ (Intelligent Constant Quality) mode with "global_quality" option
ffmpeg.av_opt_set(_pCodecContext->priv_data, "global_quality", qpValue.ToString(), 0);
// QSV: ICQ (Intelligent Constant Quality) mode, configured with AVCodecContext.global_quality.
_pCodecContext->global_quality = qpValue;
}
else if (encoder == VideoEncoder.LibVvenc)
{
// libvvenc: Specifies Constant QP mode with "qp" option
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp", qpValue.ToString(), 0);
TrySetOption("qp", qpValue.ToString());
}
else // Software encoder (libx264, libx265, libvpx-vp9, etc.)
else
{
// CRF (Constant Rate Factor): Smarter quality control than QP.
// Automatically adjusts bitrate according to scene complexity to maintain consistent visual quality.
// Range: 0 (lossless) ~ 51 (lowest quality). Typically between 18 and 28.
ffmpeg.av_opt_set(_pCodecContext->priv_data, "crf", qpValue.ToString(), 0);
// Range: 0 (lossless) ~ 51 (lowest quality) for x264/x265, 0~63 for VP9/AV1. Typically between 18 and 28.
TrySetOption("crf", qpValue.ToString());
}
break;
default:
@@ -914,6 +1023,76 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
/// <summary>
/// Sets a private option (priv_data) of the encoder.
/// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false.
/// </summary>
// ReSharper disable once UnusedMethodReturnValue.Local
private bool TrySetOption(string name, string value)
{
var ret = ffmpeg.av_opt_set(_pCodecContext->priv_data, name, value, 0);
if (ret >= 0) return true;
Melon<RenderMod>.Logger.Warning($"[Encoder] Option '{name}={value}' was not applied: {GetFFmpegErrorString(ret)} (Code: {ret})");
return false;
}
/// <summary>
/// MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET (0x0010): frames are in video memory and belong to video decoder render targets.
/// Reference: https://github.com/intel/libvpl/blob/main/api/vpl/mfxstructures.h
/// </summary>
private const int QsvMemTypeVideoMemoryDecoderTarget = 0x0010;
/// <summary>
/// Number of surfaces in the QSV GPU frame pool (AVHWFramesContext.initial_pool_size) of GPU frame pool input mode.
/// Same value as the encoder frame pool of the FFmpeg hardware encoding example (initial_pool_size = 20).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/doc/examples/vaapi_encode.c
/// </summary>
private const int QsvFramePoolSize = 20;
/// <summary>
/// Writes AVQSVFramesContext.frame_type of a QSV frames context (AVHWFramesContext.hwctx).
/// FFmpeg.AutoGen has no AVQSVFramesContext type, so the field is written at its offset.
/// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info
/// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h
/// </summary>
private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType)
{
if (framesCtx->hwctx == null) return;
*(int*)((byte*)framesCtx->hwctx + IntPtr.Size + sizeof(int)) = frameType;
}
/// <summary>
/// Largest declared maximum that is treated as an actual quality range.
/// A declared maximum above this value is treated as unbounded, and the fallback range 0~51 is used.
/// The largest bounded quality range among the supported encoders is 0~255 (AV1 NVENC/AMF), while
/// libx264/libx265 declare "crf" as -1~FLT_MAX (libavcodec/libx264.c, libx265.c).
/// </summary>
private const double MaxMeaningfulQualityLimit = 1000;
/// <summary>
/// Returns the valid range of a private quality option of the encoder from its option definition (AVOption min/max).
/// e.g. 0~51 for H.264/HEVC, 0~63 for VP9/AV1/VVC, 0~255 for AV1 NVENC/AMF.
/// The lower bound is at least 0 (a declared minimum of -1 means "auto").
/// When the option is missing or its declared maximum is unbounded (see <see cref="MaxMeaningfulQualityLimit"/>), returns 0~51,
/// the QP scale of H.264/HEVC 8-bit (x264/x265 CRF 0~51).
/// Reference: https://x265.readthedocs.io/en/master/cli.html (--crf 0..51.0)
/// </summary>
/// <param name="optionName">Name of the quality option in priv_data.</param>
private (int Min, int Max) GetQualityRange(string optionName)
{
const int fallbackMax = 51;
var option = ffmpeg.av_opt_find(_pCodecContext->priv_data, optionName, null, 0, 0);
if (option == null || !(option->max > 0 && option->max <= MaxMeaningfulQualityLimit))
return (0, fallbackMax);
var min = Math.Max(0, (int)Math.Ceiling(option->min));
var max = (int)Math.Floor(option->max);
return min <= max ? (min, max) : (0, fallbackMax);
}
// ================================================================
// Write Frame
// ================================================================
@@ -1013,12 +1192,50 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
}
/// <summary>
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
/// Second argument 0 means default alignment.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
/// </summary>
private void AllocateDstFrameBuffer()
{
_pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
if (ret < 0)
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
}
/// <summary>
/// Makes the pixel buffer of the destination frame (_pDstFrame) writable before the next frame is written into it.
///
/// After avcodec_send_frame, the encoder may keep a reference to the buffer of the submitted frame
/// until that frame is encoded (e.g. QSV in system memory input mode). The buffer is then shared and not writable
/// (av_frame_is_writable returns 0).
/// In that case the frame releases its reference (av_frame_unref) and a new buffer is allocated.
/// The encoder keeps the previous buffer until it no longer needs it. The previous contents are not copied,
/// because every plane is overwritten by the next frame.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_is_writable, av_frame_unref)
/// </summary>
private void PrepareDstFrameForWrite()
{
if (ffmpeg.av_frame_is_writable(_pDstFrame) != 0) return;
ffmpeg.av_frame_unref(_pDstFrame);
AllocateDstFrameBuffer();
}
/// <summary>
/// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data.
/// </summary>
private void WritePreConvertedFrame(byte* buffer)
{
ffmpeg.av_frame_make_writable(_pDstFrame);
PrepareDstFrameForWrite();
var width = _settings.Width;
var height = _settings.Height;
@@ -1132,20 +1349,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Core logic for actual frame encoding.
///
/// Processing flow:
/// 1. av_frame_make_writable: Check if destination frame buffer is writable
/// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable
/// (Allocates a new buffer if encoder is still referencing previous frame's buffer)
/// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface
/// 4. Pass to encoder after setting PTS
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale)
/// </summary>
private void WriteFrame()
{
// Ensure writable state since encoder might still be referencing previous frame's buffer.
// Internally allocates a new buffer if referenced.
ffmpeg.av_frame_make_writable(_pDstFrame);
PrepareDstFrameForWrite();
// ── Colorspace conversion: RGBA → YUV420P or NV12 ──
// sws_scale parameters explanation:
@@ -1306,8 +1520,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 2. av_write_trailer: Writes container trailer (end-of-file metadata).
/// For MP4, the moov atom (index info) is completed here.
/// ⚠️ If trailer is not written, the file becomes corrupted and unplayable!
/// av_write_trailer also flushes the output buffer and returns the write error of the output (AVIOContext.error), if any.
///
/// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete.
/// End() runs once: after it returns or throws, further calls return immediately.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer)
/// </summary>
public void End()
{
@@ -1315,39 +1534,31 @@ namespace ADOFAIRenderer.Util.FFmpeg
{
if (_disposed || !_isInitialized) return;
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
try
{
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
SendFrameAndWritePackets(null);
}
catch (Exception ex)
{
// Log error occurring during flush process
Melon<RenderMod>.Logger.Error($"Error during video encoder flush: {ex.Message}");
}
// 1.5. Flush remaining samples in audio FIFO + flush audio encoder
if (_audioEnabled)
{
try
{
FlushAudioEncoder();
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"Error during audio encoder flush: {ex.Message}");
}
}
// 2. Write container trailer (moov atom and file termination metadata)
if (_pFormatContext != null)
{
ffmpeg.av_write_trailer(_pFormatContext);
var ret = ffmpeg.av_write_trailer(_pFormatContext);
if (ret < 0)
throw new Exception($"Failed to write the container trailer — {GetFFmpegErrorString(ret)} (Code: {ret})");
}
}
finally
{
_isInitialized = false;
}
}
}
// ================================================================
// Resource Cleanup (Dispose)
@@ -1356,11 +1567,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>
/// Actual resource cleanup logic. Cleanup order is important:
///
/// 1. Call End() (if not called yet) → flush delayed frames + write trailer
/// 2. SwsContext (scaler) → independent, can be freed first
/// 3. AVFrame (src/dst) → freed before codec as codec may reference them
/// 4. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
/// 5. AVFormatContext (format/IO) → includes closing file handle
/// 1. SwsContext (scaler) → independent, can be freed first
/// 2. AVFrame (src/dst) → freed before codec as codec may reference them
/// 3. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
/// 4. AVFormatContext (format/IO) → includes closing file handle
///
/// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned.
/// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it).
///
/// Reference:
/// av_frame_free: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
@@ -1374,12 +1587,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
{
if (_disposed) return;
_disposed = true;
// Trigger End() safely in case Dispose is called without calling End() first
if (_isInitialized)
{
try { End(); } catch { /* Ignore */ }
}
_isInitialized = false;
// Release Sws scaler
if (_pSwsContext != null)
+11
View File
@@ -20,6 +20,7 @@ internal class VideoBGPlayer : MonoBehaviour
private static readonly int UTex = Shader.PropertyToID("_UTex");
private static readonly int VTex = Shader.PropertyToID("_VTex");
private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
private static readonly int ColorMatrix = Shader.PropertyToID("ColorMatrix");
[SuppressMessage("ReSharper", "InconsistentNaming")]
private enum GPUMode
@@ -189,6 +190,14 @@ internal class VideoBGPlayer : MonoBehaviour
var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
var targetVideoTime = songTime + vidOffset;
// When looping is enabled, the playback position is the elapsed time wrapped into [0, Duration).
// When it is disabled, the position keeps increasing and the last frame stays on screen after the end.
var videoDuration = _decoder.Duration;
if (_isLooping && targetVideoTime >= 0 && videoDuration > 0)
{
targetVideoTime %= videoDuration;
}
if (targetVideoTime < 0)
{
if (_targetTexture)
@@ -307,6 +316,7 @@ internal class VideoBGPlayer : MonoBehaviour
_computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
_computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8;
@@ -341,6 +351,7 @@ internal class VideoBGPlayer : MonoBehaviour
_computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
_computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8;