9 Commits
9 changed files with 347 additions and 76 deletions

No files matched your search

+3 -1
View File
@@ -520,7 +520,9 @@ public class RenderSession
}
RenderManager.Instance.levelPath = levelPath;
RenderManager.Instance.Configuration = renderConfig;
// The render uses its own copy of the configuration: values adjusted during the render
// (e.g. the audio sample rate in SetupPlay) stay in this copy and do not change the caller's object.
RenderManager.Instance.Configuration = renderConfig.Clone();
RenderManager.Instance.stage = RenderStage.Load;
+87 -11
View File
@@ -5,7 +5,6 @@ using System.IO;
using System.Threading;
using System.Buffers;
using System.Diagnostics;
using System.Linq;
using ADOFAIRenderer.API;
using ADOFAIRenderer.Core.Data;
using ADOFAIRenderer.Data;
@@ -81,7 +80,27 @@ internal class RenderManager : MonoBehaviour
public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene;
public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render;
/// <summary>
/// Maximum number of frames in ImageQueue. The main thread waits (throttles) while the queue holds more frames.
/// </summary>
public const int MaxQueuedFrames = 64;
/// <summary>
/// Upper bound of the readback buffer bytes of the frames in ImageQueue (4 GiB).
/// The frame limit becomes min(MaxQueuedFrames, MaxQueuedBytes / bytes per frame) (see ComputeQueueFrameLimit),
/// so it only lowers the frame count at resolutions where MaxQueuedFrames frames exceed this size.
/// </summary>
public const long MaxQueuedBytes = 4L * 1024 * 1024 * 1024;
/// <summary>
/// Wait time of the throttle loop (OnPostFrame) for the render thread to take a frame from ImageQueue.
/// When it elapses without a dequeue, AsyncGPUReadback.WaitAllRequests() is called once so the readbacks
/// still in flight complete (the render thread may be waiting for one of them to fill a gap in ImageQueue).
/// </summary>
private const int ThrottleWaitMilliseconds = 10;
/// <summary>Frame limit of ImageQueue applied by the last throttle check (MaxQueuedFrames or lower).</summary>
public int QueueFrameLimit { get; private set; } = MaxQueuedFrames;
// Statistics
public long framesSent;
@@ -213,6 +232,8 @@ internal class RenderManager : MonoBehaviour
// Thread and Queue State
internal SortedList<long, CapturedFrame> ImageQueue = new();
internal AutoResetEvent FrameAvailableEvent = new(false);
// Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it.
internal AutoResetEvent FrameDequeuedEvent = new(false);
public Thread? RenderThread;
public long passCount;
public volatile bool endRequested;
@@ -396,27 +417,74 @@ internal class RenderManager : MonoBehaviour
frameCount++;
}
// Floors that own a hold renderer, collected from scrLevelMaker.listFloors (see RefreshHoldFloors).
private readonly List<scrFloor> _holdFloors = new();
private List<scrFloor>? _holdFloorsSource;
private int _holdFloorsSourceCount = -1;
public void OnLastUpdate()
{
if (Instance.stage < RenderStage.Load) return;
if (!Configuration.AdvancedConfig.NonTile) return;
foreach (var floor in scrLevelMaker.instance.listFloors
.Where(floor => floor.holdRenderer && floor.holdRenderer.m_meshRenderer.enabled))
if (Configuration.AdvancedConfig.NonTile)
{
floor.holdRenderer.m_meshRenderer.enabled = false;
// scrHoldRenderer.UpdateColor sets m_meshRenderer.enabled = true every time it runs
// (scrHoldRenderer.cs, called from scrPlanet/scrPlayer), so the hold meshes are disabled again every frame.
RefreshHoldFloors();
foreach (var floor in _holdFloors)
{
if (!floor || !floor.holdRenderer) continue;
var meshRenderer = floor.holdRenderer.m_meshRenderer;
if (meshRenderer && meshRenderer.enabled) meshRenderer.enabled = false;
}
}
TriggerOnLastUpdate();
}
/// <summary>
/// Rebuilds the hold floor list when scrLevelMaker.listFloors is a different list or has a different count
/// (hold renderers are created together with the floors in scrLevelMaker).
/// </summary>
private void RefreshHoldFloors()
{
var floors = scrLevelMaker.instance.listFloors;
if (ReferenceEquals(floors, _holdFloorsSource) && floors.Count == _holdFloorsSourceCount) return;
_holdFloors.Clear();
foreach (var floor in floors)
{
if (floor && floor.holdRenderer) _holdFloors.Add(floor);
}
_holdFloorsSource = floors;
_holdFloorsSourceCount = floors.Count;
}
/// <summary>
/// Frame limit of ImageQueue for the given capture texture: MaxQueuedBytes divided by the readback bytes of
/// one frame (YUV 4:2:0 when the GPU conversion is active, RGBA otherwise), within 4 to MaxQueuedFrames.
/// </summary>
private int ComputeQueueFrameLimit(RenderTexture texture)
{
var frameBytes = _frameReader?.GetFrameBufferBytes(texture.width, texture.height)
?? (long)texture.width * texture.height * 4;
if (frameBytes <= 0) return MaxQueuedFrames;
return (int)Math.Min(MaxQueuedFrames, Math.Max(4, MaxQueuedBytes / frameBytes));
}
// ReSharper disable once Unity.IncorrectMethodSignature
public void OnPostFrame(RenderTexture? texture)
{
if (Instance.stage < RenderStage.Render || !texture) return;
// [Backpressure Throttling]
// The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread
// removes frames, and it signals FrameDequeuedEvent for each one. The wait has a timeout so that the loop
// re-checks the render thread state and pumps pending readback completions (see ThrottleWaitMilliseconds).
isThrottling = false;
var frameLimit = QueueFrameLimit = ComputeQueueFrameLimit(texture!);
while (Instance.stage >= RenderStage.Render)
{
if (RenderThread is not { IsAlive: true })
@@ -431,7 +499,7 @@ internal class RenderManager : MonoBehaviour
count = ImageQueue.Count;
}
if (count <= MaxQueuedFrames) break;
if (count <= frameLimit) break;
if (!isThrottling)
{
@@ -439,7 +507,10 @@ internal class RenderManager : MonoBehaviour
throttleCount++;
}
AsyncGPUReadback.WaitAllRequests();
if (!FrameDequeuedEvent.WaitOne(ThrottleWaitMilliseconds))
{
AsyncGPUReadback.WaitAllRequests();
}
}
isThrottling = false;
@@ -619,6 +690,7 @@ internal class RenderManager : MonoBehaviour
endRequested = false;
endSuccessful = false;
FrameAvailableEvent.Reset();
FrameDequeuedEvent.Reset();
var renderFPS = Configuration.RenderFPS;
var videoFPS = Configuration.VideoFPS;
@@ -682,7 +754,7 @@ internal class RenderManager : MonoBehaviour
throw;
}
var encoder = videoFFmpeg!;
var encoder = videoFFmpeg;
RenderThread = new Thread(() => RunRenderThread(encoder))
{
IsBackground = true,
@@ -734,14 +806,18 @@ internal class RenderManager : MonoBehaviour
/// <summary>
/// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio
/// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready.
/// Returns when the render is aborted or all requested frames have been sent after endRequested.
/// Returns when the render is aborted (checked before every frame, frames left in ImageQueue are released
/// by the cleanup after the thread ends) or all requested frames have been sent after endRequested.
/// </summary>
private void EncodeQueuedFrames(FFmpegNativeEncoder videoFFmpeg)
{
while (true)
{
if (IsRenderAborted) return;
if (TryDequeueNextFrame(out var frame))
{
FrameDequeuedEvent.Set();
SendVideoFrame(videoFFmpeg, frame);
WriteAudioForFrame(videoFFmpeg);
continue;
@@ -884,7 +960,7 @@ internal class RenderManager : MonoBehaviour
var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED";
Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources.");
// 1. Wait for render background thread to finish processing remaining queued frames and terminate
// 1. Wait (up to 2s) for the render background thread to see the abort flag and terminate
if (RenderThread is { IsAlive: true })
{
RenderThread.Join(2000);
+1 -1
View File
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine;
using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.4.0-BETA", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer;
+1 -1
View File
@@ -164,7 +164,7 @@ internal class DebugMonitorUI : MonoBehaviour
}
}
sb.AppendLine($"• Queue: <b>{queueCount}</b> / {RenderManager.MaxQueuedFrames} frames");
sb.AppendLine($"• Queue: <b>{queueCount}</b> / {(RenderManager.Instance ? RenderManager.Instance.QueueFrameLimit : RenderManager.MaxQueuedFrames)} frames");
sb.AppendLine($"• Rendered Frames: <b>{(RenderManager.Instance ? RenderManager.Instance.frameCount : 0)}</b>");
sb.AppendLine($"• Frames Sent: <b>{(RenderManager.Instance ? Interlocked.Read(ref RenderManager.Instance.framesSent) : 0)}</b>");
+60 -16
View File
@@ -9,6 +9,7 @@ using ADOFAIRenderer.Data;
using ADOFAIRenderer.Localization;
using ADOFAIRenderer.Util.OS;
using MelonLoader;
using Newtonsoft.Json;
using O5Kit.Control;
using O5Kit.Core;
using O5Kit.Factory;
@@ -32,6 +33,14 @@ internal static class RenderConfigWindow
private static Coroutine? _deleteResetCoroutine;
private static string? _lastLoadedPresetName;
/// <summary>
/// Configuration edited by this window. Kept for the whole game session; a render started from the window
/// receives a copy of it (RenderSession.Start), so values adjusted during the render are not written back here.
/// </summary>
private static RenderConfig? _uiConfig;
private static RenderConfig UiConfig => _uiConfig ??= new RenderConfig();
// Controls cache for validation & sync
private static O5Slider? _renderFpsSlider;
private static O5Slider? _videoFPSSlider;
@@ -111,8 +120,7 @@ internal static class RenderConfigWindow
ReloadPresetDropdown(GlobalSettings.Current.LastPresetName);
}
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
var config = UiConfig;
ApplyConfigToUI(config);
SetWindowActive(true);
@@ -125,8 +133,7 @@ internal static class RenderConfigWindow
var (_, scrollContent, _) =
O5Factory.ScrollView(ctx, content, spacing: 10f, padding: 8f);
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
var config = UiConfig;
EnsureDefaultConfig(config);
double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144,
@@ -744,11 +751,14 @@ internal static class RenderConfigWindow
{
EnsureDefaultConfig(config);
_renderFpsSlider?.Set((float) config.RenderFPS, false);
_videoFPSSlider?.Set((float) config.VideoFPS, false);
_renderWidthSlider?.Set(config.RenderWidth, false);
_renderHeightSlider?.Set(config.RenderHeight, false);
_delayAfterEndSlider?.Set((float) config.DelayAfterEnd, false);
// Sliders show the config values as they are (noFilter: true).
// O5Slider.Set applies the slider filter (snapping to the nearest snap value) unless noFilter is set;
// the filter is meant for dragging, and typed values are not snapped either (O5Slider.SetFromInput).
_renderFpsSlider?.Set(config.RenderFPS, false, noFilter: true);
_videoFPSSlider?.Set(config.VideoFPS, false, noFilter: true);
_renderWidthSlider?.Set(config.RenderWidth, false, noFilter: true);
_renderHeightSlider?.Set(config.RenderHeight, false, noFilter: true);
_delayAfterEndSlider?.Set(config.DelayAfterEnd, false, noFilter: true);
_showJudgementToggle?.Set(config.ShowJudgementText, false);
_codecDropdown?.Set(config.VideoConfig.Codec, false);
@@ -756,13 +766,13 @@ internal static class RenderConfigWindow
_bitrateInput?.Set(config.VideoConfig.Bitrate.ToString(), false);
_maxBitrateInput?.Set(config.VideoConfig.MaxBitrate.ToString(), false);
_qpInput?.Set(config.VideoConfig.QP.ToString(), false);
_keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false);
_keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false, noFilter: true);
_forceSoftwareToggle?.Set(config.VideoConfig.ForceSoftwareEncoder, false);
_renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false);
_sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false);
_channelsDropdown?.Set(config.AudioConfig.AudioChannels, false);
_audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false);
_audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false, noFilter: true);
_outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false);
@@ -789,17 +799,51 @@ internal static class RenderConfigWindow
UpdateVideoCodecUI(config.VideoConfig.Codec, config);
UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true);
if (!string.IsNullOrEmpty(GlobalSettings.Current.LastPresetName))
UpdatePresetLabel(config);
}
/// <summary>
/// Sets the preset dropdown from the config values.
/// Every value equals the last loaded preset (_lastLoadedPresetName): the preset name is shown.
/// Any value differs (edited, or the preset file changed): the Custom label is shown (MarkConfigModified),
/// the same state as after editing a value in the window.
/// </summary>
private static void UpdatePresetLabel(RenderConfig config)
{
if (_presetDropdown == null) return;
var presetName = _lastLoadedPresetName;
var preset = string.IsNullOrEmpty(presetName)
? null
: PresetManager.GetPresets().FirstOrDefault(p => string.Equals(p.Name, presetName, StringComparison.OrdinalIgnoreCase));
if (preset == null || !ConfigValuesEqual(config, preset.Config))
{
_presetDropdown?.Set(GlobalSettings.Current.LastPresetName, false);
_lastLoadedPresetName = GlobalSettings.Current.LastPresetName;
UpdateDeleteButtonState();
MarkConfigModified();
return;
}
if (_presetDropdown.Values.Contains(CustomPresetLabel))
{
_presetDropdown.SetValues(_presetDropdown.Values.Where(v => v != CustomPresetLabel).ToList());
}
_presetDropdown.Set(preset.Name, false);
UpdateDeleteButtonState();
}
/// <summary>
/// Compares two configs by their serialized JSON (the same form as the preset files), so every saved value is compared.
/// </summary>
private static bool ConfigValuesEqual(RenderConfig a, RenderConfig b)
{
var settings = new JsonSerializerSettings();
settings.Converters.Add(new ColorJsonConverter());
return JsonConvert.SerializeObject(a, settings) == JsonConvert.SerializeObject(b, settings);
}
private static void TryGoRender()
{
var config = RenderManager.Instance.Configuration;
var config = UiConfig;
SyncUItoConfig(config);
// Sync Output Directory
+3 -3
View File
@@ -1125,7 +1125,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// </summary>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
/// <param name="yStride">Row stride in bytes for Y plane.</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes).</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes).</param>
/// <param name="uvStride">Row stride in bytes for UV plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
@@ -1178,7 +1178,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = (w / 2) * 2;
int uvRowBytes = ((w + 1) / 2) * 2;
// Copy Y plane (row by row to safely skip linesize GPU padding)
for (int y = 0; y < h; y++)
@@ -1239,7 +1239,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = (w / 2) * 2;
int uvRowBytes = ((w + 1) / 2) * 2;
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
{
+147 -35
View File
@@ -201,6 +201,12 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
/// <summary>
/// Color range of the encoder input and output (codec context color_range of the selected encoder).
/// Full range (JPEG) for every encoder except libvvenc, limited range (MPEG) for libvvenc.
/// </summary>
private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG;
/// <summary>Pixel format of the finally selected and running encoder.</summary>
public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
@@ -255,6 +261,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Whether audio encoding is enabled.</summary>
private bool _audioEnabled;
/// <summary>Bytes of one input sample of every channel (interleaved input format × channel count).</summary>
private int _audioInputBytesPerSample;
/// <summary>Encoding settings passed at construction time (immutable).</summary>
private readonly EncoderSettings _settings;
@@ -267,26 +276,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Utility Methods
// ================================================================
/// <summary>
/// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg.
/// AVCodecID identifies "which codec it is (e.g., H.264, AV1)",
/// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately.
/// </summary>
// ReSharper disable once InconsistentNaming
// ReSharper disable once UnusedMember.Local
private AVCodecID GetAVCodecID()
{
return _settings.Codec switch
{
Codec.H264 => AVCodecID.AV_CODEC_ID_H264,
Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC,
Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1,
Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9,
Codec.VVC => AVCodecID.AV_CODEC_ID_VVC,
_ => throw new ArgumentOutOfRangeException()
};
}
/// <summary>
/// Returns a list of candidate encoders to try for the selected codec in priority order.
///
@@ -416,7 +405,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Basic video parameters configuration ──
_pCodecContext->width = _settings.Width;
_pCodecContext->height = _settings.Height;
_pCodecContext->color_range = AVColorRange.AVCOL_RANGE_JPEG;
// ── Output color tags ──
// Matrix: BT.709, the RGB→YUV matrix of both conversion paths
// (GPU: RGB2YUV_BT709_Full in ColorConversion.compute, CPU: ConfigureSwsColorspace).
// Players use this tag to convert the YUV frames back to RGB.
// Range: full (0-255) for every encoder except libvvenc, limited (16-235) for libvvenc.
// The FFmpeg libvvenc wrapper passes the matrix/primaries/transfer tags to VVenC but not the range,
// so a VVC stream is read as limited range; its input is converted to limited range to match.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_color_format)
_pCodecContext->color_range = encoder == VideoEncoder.LibVvenc
? AVColorRange.AVCOL_RANGE_MPEG
: AVColorRange.AVCOL_RANGE_JPEG;
_pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709;
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
// so setting 0 (auto) is required for parallel encoding utilizing all cores.
@@ -690,7 +691,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null)
{
// The codec context (and the frames context it holds) is released by Dispose in the catch block of Start.
ffmpeg.av_buffer_unref(&hwDeviceCtx);
throw new Exception("Failed to allocate QSV hardware frame structure.");
}
break;
}
@@ -769,6 +774,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage";
Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})");
_dstPixFmt = selectedPixFmt;
_colorRange = _pCodecContext->color_range;
openedSuccessfully = true;
break;
}
@@ -797,8 +803,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Copy parameters set in codec context (resolution, codec ID, profile, etc.)
// to stream's codecpar. Information to be written to container header.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0)
throw new Exception("Failed to copy codec parameters to stream.");
var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext);
if (paramResult < 0)
throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})");
// Stream time base: unit of the timestamps stored in the container.
// Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks.
@@ -823,15 +830,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
{
if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0)
throw new Exception($"Cannot open output file {outputPath}.");
var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE);
if (avioResult < 0)
throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
}
// Write file header (ftyp, moov, container metadata).
// Packets can be written with av_interleaved_write_frame after this point.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0)
throw new Exception("Failed to write file header.");
var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
if (headerResult < 0)
throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})");
// ── Step 6: Initialize frame buffers and colorspace converter ──
// av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet).
@@ -865,6 +874,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSwsContext == null)
throw new Exception("Failed to initialize software scaler.");
ConfigureSwsColorspace();
_isInitialized = true;
_loggedConversionMode = false;
}
@@ -1160,10 +1171,42 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock)
{
if (_disposed || !_isInitialized) return;
ValidateInputLength(buffer.Length, isColorConverted: false);
WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr());
}
}
/// <summary>
/// Number of bytes read from the input buffer of one frame.
/// RGBA input (isColorConverted = false): Width × Height × 4
/// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV)
/// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
/// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
/// </summary>
private long GetRequiredInputLength(bool isColorConverted)
{
long w = _settings.Width, h = _settings.Height;
if (!isColorConverted) return w * h * 4;
return _dstPixFmt switch
{
AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2),
AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2),
_ => 0
};
}
/// <summary>
/// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it
/// (GetRequiredInputLength), so that the copy does not read outside the buffer.
/// </summary>
private void ValidateInputLength(long length, bool isColorConverted)
{
var required = GetRequiredInputLength(isColorConverted);
if (length < required)
throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input.");
}
/// <summary>
/// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources.
@@ -1182,6 +1225,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
}
ValidateInputLength(length, isColorConverted);
if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
else WriteFrameFromBuffer((byte*)pointer.ToPointer());
}
@@ -1196,6 +1240,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock)
{
if (_disposed || !_isInitialized) return;
if (length > buffer.Length)
throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes).");
ValidateInputLength(length, isColorConverted);
fixed (byte* pBuffer = buffer)
{
if (!_loggedConversionMode)
@@ -1213,7 +1261,51 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
/// <summary>
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// Sets the RGB→YUV conversion of the CPU conversion path (_pSwsContext) to the BT.709 matrix
/// and the color range of the encoder (_colorRange), so that it produces the same colors as the GPU path
/// (RGB2YUV_BT709_Full in ColorConversion.compute) and as the tags of the output stream.
/// A new libswscale context uses the BT.601 matrix (SWS_CS_DEFAULT = SWS_CS_ITU601) and the limited range.
///
/// For RGB input, "table" holds the coefficients of the output YUV space and dstRange its range (1 = full, 0 = limited).
/// The source (RGB) values, brightness, contrast and saturation keep the values of the context.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails, sws_getCoefficients)
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libswscale/swscale.h (SWS_CS_ITU709, SWS_CS_DEFAULT)
/// </summary>
private void ConfigureSwsColorspace()
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
var ret = ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot read the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
return;
}
var coefficients = ffmpeg.sws_getCoefficients(ffmpeg.SWS_CS_ITU709);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = currentInvTable[i];
table[i] = coefficients[i];
}
var fullRange = _colorRange == AVColorRange.AVCOL_RANGE_JPEG;
ret = ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, srcRange, table, fullRange ? 1 : 0,
brightness, contrast, saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot set the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
}
}
/// <summary>
/// Sets the format, size, color range and matrix of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
/// Second argument 0 means default alignment.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
@@ -1223,7 +1315,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
_pDstFrame->color_range = _colorRange;
_pDstFrame->colorspace = AVColorSpace.AVCOL_SPC_BT709;
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
if (ret < 0)
@@ -1262,7 +1355,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
switch (_dstPixFmt)
{
case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV:
case AVPixelFormat.AV_PIX_FMT_NV12:
{
// Copy Y Plane
var pSrcY = buffer;
@@ -1485,7 +1578,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame);
if (response < 0)
{
throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})");
throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})");
}
// Pop all encoded packets and write to file.
@@ -1499,7 +1592,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// EOF: Encoder flush complete (all delayed frames outputted) → exit loop
if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
if (response < 0)
throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})");
throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})");
// Convert PTS/DTS from codec timebase → stream timebase
ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base);
@@ -1510,7 +1603,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// and written in time order. This function is used even when only video is present.
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
if (response < 0)
throw new Exception($"Error occurred while writing packet to file. (Code: {response})");
throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
// Unref packet data reference for reuse in next receive_packet.
ffmpeg.av_packet_unref(_pPacket);
@@ -1775,6 +1868,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
? audioSettings.InputSampleFormat
: AVSampleFormat.AV_SAMPLE_FMT_FLT;
// Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
_audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
// Allocate SwrContext
fixed (SwrContext** ppSwr = &_pSwrContext)
{
@@ -1838,6 +1935,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock)
{
if (_disposed || !_isInitialized || !_audioEnabled) return;
if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float))
throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats).");
fixed (float* pSamples = samples)
{
@@ -1872,6 +1971,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary>
/// <summary>
/// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
/// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
/// Throws an exception when the buffer cannot be allocated.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
/// </summary>
private void MakeAudioFrameWritable()
{
var ret = ffmpeg.av_frame_make_writable(_pAudioFrame);
if (ret < 0)
throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
}
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
{
// ── 1. Resampling: Input format → Encoder format ──
@@ -1910,7 +2022,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
{
ffmpeg.av_frame_make_writable(_pAudioFrame);
MakeAudioFrameWritable();
_pAudioFrame->nb_samples = frameSize;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize);
@@ -1989,7 +2101,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo);
if (remaining > 0)
{
ffmpeg.av_frame_make_writable(_pAudioFrame);
MakeAudioFrameWritable();
_pAudioFrame->nb_samples = remaining;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);
+42 -5
View File
@@ -73,6 +73,16 @@ namespace ADOFAIRenderer.Util
}
}
/// <summary>
/// Bytes of the readback buffer of one frame: Width x Height x 3 / 2 when the GPU conversion to NV12 / YUV420P
/// is used, Width x Height x 4 (RGBA) otherwise. The same size as ReadbackRequestState.Setup uses.
/// </summary>
public long GetFrameBufferBytes(int width, int height)
{
var pixels = (long)width * height;
return _useGPUConversion ? pixels * 3 / 2 : pixels * 4;
}
public void RequestFrameCapture(Texture texture, long currentFrame)
{
if (!RenderManager.IsCapturingActive) return;
@@ -154,14 +164,11 @@ namespace ADOFAIRenderer.Util
}
Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame.");
var byteSize = state.Width * state.Height * 3 / 2;
var blackFrame = ArrayPool<byte>.Shared.Rent(byteSize);
Array.Clear(blackFrame, 0, byteSize);
var blackFrame = CreateBlackFrame(state, out var byteSize);
var capFrame = new RenderManager.CapturedFrame
{
ManagedBuffer = blackFrame, Length = byteSize,
IsColorConverted = true
IsColorConverted = state.IsGPUConversion
};
PushToQueue(frame, capFrame);
RecycleState(state);
@@ -190,6 +197,36 @@ namespace ADOFAIRenderer.Util
}
}
/// <summary>
/// Creates a black frame in the same format as the frames of this reader, from ArrayPool&lt;byte&gt;.Shared
/// (returned to the pool by the render thread after encoding).
///
/// GPU conversion (NV12 / YUV420P, full range): Width × Height × 3 / 2 bytes.
/// Y plane (Width × Height bytes) = 0, U/V data (the rest) = 128.
/// 128 is the zero chroma value of the 8-bit full range: the U/V offset 0.5 of RGB2YUV_BT709_Full
/// in ColorConversion.compute, which the encoder input of this path uses.
/// Both NV12 (interleaved UV) and YUV420P (U plane, then V plane) store all chroma after the Y plane,
/// so the same fill applies to both.
/// No GPU conversion (RGBA input, converted by the encoder): Width × Height × 4 bytes, all 0.
/// </summary>
private static byte[] CreateBlackFrame(ReadbackRequestState state, out int byteSize)
{
var lumaSize = state.Width * state.Height;
byteSize = state.IsGPUConversion ? lumaSize * 3 / 2 : lumaSize * 4;
var frame = ArrayPool<byte>.Shared.Rent(byteSize);
if (state.IsGPUConversion)
{
Array.Clear(frame, 0, lumaSize);
frame.AsSpan(lumaSize, byteSize - lumaSize).Fill(128);
}
else
{
Array.Clear(frame, 0, byteSize);
}
return frame;
}
internal void RecycleState(ReadbackRequestState state)
{
lock (_poolLock)
+3 -3
View File
@@ -97,7 +97,7 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false);
_textureUV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.RG16, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureUV);
@@ -118,8 +118,8 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
_textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
_textureU = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
_textureV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureU);
ConfigureTexture(_textureV);