14 Commits
14 changed files with 658 additions and 263 deletions

No files matched your search

+3 -1
View File
@@ -520,7 +520,9 @@ public class RenderSession
} }
RenderManager.Instance.levelPath = levelPath; RenderManager.Instance.levelPath = levelPath;
RenderManager.Instance.Configuration = renderConfig; // The render uses its own copy of the configuration: values adjusted during the render
// (e.g. the audio sample rate in SetupPlay) stay in this copy and do not change the caller's object.
RenderManager.Instance.Configuration = renderConfig.Clone();
RenderManager.Instance.stage = RenderStage.Load; RenderManager.Instance.stage = RenderStage.Load;
+249 -132
View File
@@ -5,7 +5,6 @@ using System.IO;
using System.Threading; using System.Threading;
using System.Buffers; using System.Buffers;
using System.Diagnostics; using System.Diagnostics;
using System.Linq;
using ADOFAIRenderer.API; using ADOFAIRenderer.API;
using ADOFAIRenderer.Core.Data; using ADOFAIRenderer.Core.Data;
using ADOFAIRenderer.Data; using ADOFAIRenderer.Data;
@@ -81,7 +80,27 @@ internal class RenderManager : MonoBehaviour
public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene; public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene;
public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render; public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render;
/// <summary>
/// Maximum number of frames in ImageQueue. The main thread waits (throttles) while the queue holds more frames.
/// </summary>
public const int MaxQueuedFrames = 64; public const int MaxQueuedFrames = 64;
/// <summary>
/// Upper bound of the readback buffer bytes of the frames in ImageQueue (4 GiB).
/// The frame limit becomes min(MaxQueuedFrames, MaxQueuedBytes / bytes per frame) (see ComputeQueueFrameLimit),
/// so it only lowers the frame count at resolutions where MaxQueuedFrames frames exceed this size.
/// </summary>
public const long MaxQueuedBytes = 4L * 1024 * 1024 * 1024;
/// <summary>
/// Wait time of the throttle loop (OnPostFrame) for the render thread to take a frame from ImageQueue.
/// When it elapses without a dequeue, AsyncGPUReadback.WaitAllRequests() is called once so the readbacks
/// still in flight complete (the render thread may be waiting for one of them to fill a gap in ImageQueue).
/// </summary>
private const int ThrottleWaitMilliseconds = 10;
/// <summary>Frame limit of ImageQueue applied by the last throttle check (MaxQueuedFrames or lower).</summary>
public int QueueFrameLimit { get; private set; } = MaxQueuedFrames;
// Statistics // Statistics
public long framesSent; public long framesSent;
@@ -213,6 +232,8 @@ internal class RenderManager : MonoBehaviour
// Thread and Queue State // Thread and Queue State
internal SortedList<long, CapturedFrame> ImageQueue = new(); internal SortedList<long, CapturedFrame> ImageQueue = new();
internal AutoResetEvent FrameAvailableEvent = new(false); internal AutoResetEvent FrameAvailableEvent = new(false);
// Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it.
internal AutoResetEvent FrameDequeuedEvent = new(false);
public Thread? RenderThread; public Thread? RenderThread;
public long passCount; public long passCount;
public volatile bool endRequested; public volatile bool endRequested;
@@ -396,27 +417,74 @@ internal class RenderManager : MonoBehaviour
frameCount++; frameCount++;
} }
// Floors that own a hold renderer, collected from scrLevelMaker.listFloors (see RefreshHoldFloors).
private readonly List<scrFloor> _holdFloors = new();
private List<scrFloor>? _holdFloorsSource;
private int _holdFloorsSourceCount = -1;
public void OnLastUpdate() public void OnLastUpdate()
{ {
if (Instance.stage < RenderStage.Load) return; if (Instance.stage < RenderStage.Load) return;
if (!Configuration.AdvancedConfig.NonTile) return;
foreach (var floor in scrLevelMaker.instance.listFloors if (Configuration.AdvancedConfig.NonTile)
.Where(floor => floor.holdRenderer && floor.holdRenderer.m_meshRenderer.enabled))
{ {
floor.holdRenderer.m_meshRenderer.enabled = false; // scrHoldRenderer.UpdateColor sets m_meshRenderer.enabled = true every time it runs
// (scrHoldRenderer.cs, called from scrPlanet/scrPlayer), so the hold meshes are disabled again every frame.
RefreshHoldFloors();
foreach (var floor in _holdFloors)
{
if (!floor || !floor.holdRenderer) continue;
var meshRenderer = floor.holdRenderer.m_meshRenderer;
if (meshRenderer && meshRenderer.enabled) meshRenderer.enabled = false;
}
} }
TriggerOnLastUpdate(); TriggerOnLastUpdate();
} }
/// <summary>
/// Rebuilds the hold floor list when scrLevelMaker.listFloors is a different list or has a different count
/// (hold renderers are created together with the floors in scrLevelMaker).
/// </summary>
private void RefreshHoldFloors()
{
var floors = scrLevelMaker.instance.listFloors;
if (ReferenceEquals(floors, _holdFloorsSource) && floors.Count == _holdFloorsSourceCount) return;
_holdFloors.Clear();
foreach (var floor in floors)
{
if (floor && floor.holdRenderer) _holdFloors.Add(floor);
}
_holdFloorsSource = floors;
_holdFloorsSourceCount = floors.Count;
}
/// <summary>
/// Frame limit of ImageQueue for the given capture texture: MaxQueuedBytes divided by the readback bytes of
/// one frame (YUV 4:2:0 when the GPU conversion is active, RGBA otherwise), within 4 to MaxQueuedFrames.
/// </summary>
private int ComputeQueueFrameLimit(RenderTexture texture)
{
var frameBytes = _frameReader?.GetFrameBufferBytes(texture.width, texture.height)
?? (long)texture.width * texture.height * 4;
if (frameBytes <= 0) return MaxQueuedFrames;
return (int)Math.Min(MaxQueuedFrames, Math.Max(4, MaxQueuedBytes / frameBytes));
}
// ReSharper disable once Unity.IncorrectMethodSignature // ReSharper disable once Unity.IncorrectMethodSignature
public void OnPostFrame(RenderTexture? texture) public void OnPostFrame(RenderTexture? texture)
{ {
if (Instance.stage < RenderStage.Render || !texture) return; if (Instance.stage < RenderStage.Render || !texture) return;
// [Backpressure Throttling] // [Backpressure Throttling]
// The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread
// removes frames, and it signals FrameDequeuedEvent for each one. The wait has a timeout so that the loop
// re-checks the render thread state and pumps pending readback completions (see ThrottleWaitMilliseconds).
isThrottling = false; isThrottling = false;
var frameLimit = QueueFrameLimit = ComputeQueueFrameLimit(texture!);
while (Instance.stage >= RenderStage.Render) while (Instance.stage >= RenderStage.Render)
{ {
if (RenderThread is not { IsAlive: true }) if (RenderThread is not { IsAlive: true })
@@ -431,7 +499,7 @@ internal class RenderManager : MonoBehaviour
count = ImageQueue.Count; count = ImageQueue.Count;
} }
if (count <= MaxQueuedFrames) break; if (count <= frameLimit) break;
if (!isThrottling) if (!isThrottling)
{ {
@@ -439,7 +507,10 @@ internal class RenderManager : MonoBehaviour
throttleCount++; throttleCount++;
} }
AsyncGPUReadback.WaitAllRequests(); if (!FrameDequeuedEvent.WaitOne(ThrottleWaitMilliseconds))
{
AsyncGPUReadback.WaitAllRequests();
}
} }
isThrottling = false; isThrottling = false;
@@ -619,6 +690,7 @@ internal class RenderManager : MonoBehaviour
endRequested = false; endRequested = false;
endSuccessful = false; endSuccessful = false;
FrameAvailableEvent.Reset(); FrameAvailableEvent.Reset();
FrameDequeuedEvent.Reset();
var renderFPS = Configuration.RenderFPS; var renderFPS = Configuration.RenderFPS;
var videoFPS = Configuration.VideoFPS; var videoFPS = Configuration.VideoFPS;
@@ -682,129 +754,8 @@ internal class RenderManager : MonoBehaviour
throw; throw;
} }
RenderThread = new Thread(() => var encoder = videoFFmpeg;
{ RenderThread = new Thread(() => RunRenderThread(encoder))
try
{
while (true)
{
CapturedFrame frame = default;
bool hasFrame = false;
lock (ImageQueue)
{
if (ImageQueue.Count > 0)
{
var firstKey = ImageQueue.Keys[0];
if (firstKey > passCount && Instance.stage < RenderStage.Render)
{
passCount = firstKey;
}
if (firstKey == passCount)
{
frame = ImageQueue[firstKey];
ImageQueue.RemoveAt(0);
passCount++;
hasFrame = true;
}
}
}
if (hasFrame)
{
if (frame.ManagedBuffer != null)
{
if (frame.ManagedBuffer.Length > 0)
{
videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length, frame.IsColorConverted);
Interlocked.Increment(ref framesSent);
ArrayPool<byte>.Shared.Return(frame.ManagedBuffer);
}
}
else if (frame.NativePointer != IntPtr.Zero && frame.Length > 0)
{
videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length, frame.IsColorConverted);
Interlocked.Increment(ref framesSent);
if (frame.ReadbackState != null)
{
_frameReader?.RecycleState(frame.ReadbackState);
}
}
if (Configuration.AudioConfig.RenderAudio && SyncCalculator is { } syncCalculator)
{
var frameIdx = (int)(passCount - 1);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
var frameTime = frameIdx / effectiveVideoFps;
var channels = Configuration.AudioConfig.AudioChannels;
var samplesNeeded = syncCalculator.GetNextFrameSamples();
var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
try
{
AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
}
finally
{
ArrayPool<float>.Shared.Return(audioBuffer);
}
}
continue;
}
if (IsRenderAborted) break;
if (endRequested)
{
if (passCount > Volatile.Read(ref lastRequestedVideoFrameIndex)) break;
}
FrameAvailableEvent.WaitOne(50);
}
if (!IsRenderAborted)
{
// End throws when the file cannot be finished (e.g. the trailer cannot be written),
// which is handled as a render failure by the catch block below.
videoFFmpeg.End();
Volatile.Write(ref _incompleteOutputPath, null);
}
videoFFmpeg.Dispose();
lock (_allVideoFFmpegInstances)
{
_allVideoFFmpegInstances.Remove(videoFFmpeg);
}
_activeVideoFFmpeg = null;
endSuccessful = !IsRenderAborted;
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"FFmpeg rendering thread error: {ex}");
MarkFailed("encoder thread error. See the log for details.");
try
{
videoFFmpeg.Dispose();
}
catch (Exception disposeEx)
{
Melon<RenderMod>.Logger.Error($"Encoder dispose after thread error also failed: {disposeEx.Message}");
}
lock (_allVideoFFmpegInstances)
{
_allVideoFFmpegInstances.Remove(videoFFmpeg);
}
_activeVideoFFmpeg = null;
}
})
{ {
IsBackground = true, IsBackground = true,
Priority = ThreadPriority.AboveNormal Priority = ThreadPriority.AboveNormal
@@ -813,6 +764,172 @@ internal class RenderManager : MonoBehaviour
RenderThread.Start(); RenderThread.Start();
} }
/// <summary>
/// Body of the render thread. Sends queued frames and their audio to the encoder until the render ends,
/// finishes the output file, and releases the encoder. An exception is handled as a render failure.
/// </summary>
private void RunRenderThread(FFmpegNativeEncoder videoFFmpeg)
{
try
{
EncodeQueuedFrames(videoFFmpeg);
if (!IsRenderAborted)
{
// End throws when the file cannot be finished (e.g. the trailer cannot be written),
// which is handled as a render failure by the catch block below.
videoFFmpeg.End();
Volatile.Write(ref _incompleteOutputPath, null);
}
videoFFmpeg.Dispose();
UnregisterEncoder(videoFFmpeg);
endSuccessful = !IsRenderAborted;
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"FFmpeg rendering thread error: {ex}");
MarkFailed("encoder thread error. See the log for details.");
try
{
videoFFmpeg.Dispose();
}
catch (Exception disposeEx)
{
Melon<RenderMod>.Logger.Error($"Encoder dispose after thread error also failed: {disposeEx.Message}");
}
UnregisterEncoder(videoFFmpeg);
}
}
/// <summary>
/// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio
/// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready.
/// Returns when the render is aborted (checked before every frame, frames left in ImageQueue are released
/// by the cleanup after the thread ends) or all requested frames have been sent after endRequested.
/// </summary>
private void EncodeQueuedFrames(FFmpegNativeEncoder videoFFmpeg)
{
while (true)
{
if (IsRenderAborted) return;
if (TryDequeueNextFrame(out var frame))
{
FrameDequeuedEvent.Set();
SendVideoFrame(videoFFmpeg, frame);
WriteAudioForFrame(videoFFmpeg);
continue;
}
if (ShouldStopEncodeLoop()) return;
FrameAvailableEvent.WaitOne(50);
}
}
/// <summary>
/// Removes the frame whose key equals passCount from ImageQueue and advances passCount.
/// Before the Render stage, passCount jumps to the first queued key (the first captured frame index is not 0).
/// Returns false when the queue is empty or the next frame has not arrived yet.
/// </summary>
private bool TryDequeueNextFrame(out CapturedFrame frame)
{
frame = default;
lock (ImageQueue)
{
if (ImageQueue.Count == 0) return false;
var firstKey = ImageQueue.Keys[0];
if (firstKey > passCount && Instance.stage < RenderStage.Render)
{
passCount = firstKey;
}
if (firstKey != passCount) return false;
frame = ImageQueue[firstKey];
ImageQueue.RemoveAt(0);
passCount++;
return true;
}
}
/// <summary>
/// Sends one captured frame to the encoder and releases its buffer (managed: ArrayPool, native: readback state).
/// A frame without data is skipped.
/// </summary>
private void SendVideoFrame(FFmpegNativeEncoder videoFFmpeg, in CapturedFrame frame)
{
if (frame.ManagedBuffer != null)
{
if (frame.ManagedBuffer.Length == 0) return;
videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length, frame.IsColorConverted);
Interlocked.Increment(ref framesSent);
ArrayPool<byte>.Shared.Return(frame.ManagedBuffer);
return;
}
if (frame.NativePointer == IntPtr.Zero || frame.Length <= 0) return;
videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length, frame.IsColorConverted);
Interlocked.Increment(ref framesSent);
if (frame.ReadbackState != null)
{
_frameReader?.RecycleState(frame.ReadbackState);
}
}
/// <summary>
/// Mixes the audio of the frame that was just sent (index passCount - 1) and writes it to the encoder.
/// Does nothing when audio rendering is off.
/// </summary>
private void WriteAudioForFrame(FFmpegNativeEncoder videoFFmpeg)
{
if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return;
var frameIdx = (int)(passCount - 1);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
var frameTime = frameIdx / effectiveVideoFps;
var channels = Configuration.AudioConfig.AudioChannels;
var samplesNeeded = syncCalculator.GetNextFrameSamples();
var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
try
{
AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
}
finally
{
ArrayPool<float>.Shared.Return(audioBuffer);
}
}
/// <summary>
/// True when the render is aborted, or when the end was requested and every requested frame has been sent.
/// </summary>
private bool ShouldStopEncodeLoop()
{
if (IsRenderAborted) return true;
return endRequested && passCount > Volatile.Read(ref lastRequestedVideoFrameIndex);
}
private void UnregisterEncoder(FFmpegNativeEncoder videoFFmpeg)
{
lock (_allVideoFFmpegInstances)
{
_allVideoFFmpegInstances.Remove(videoFFmpeg);
}
_activeVideoFFmpeg = null;
}
public void CancelRender() public void CancelRender()
{ {
ResetRenderState(success: false); ResetRenderState(success: false);
@@ -843,7 +960,7 @@ internal class RenderManager : MonoBehaviour
var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED"; var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED";
Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources."); Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources.");
// 1. Wait for render background thread to finish processing remaining queued frames and terminate // 1. Wait (up to 2s) for the render background thread to see the abort flag and terminate
if (RenderThread is { IsAlive: true }) if (RenderThread is { IsAlive: true })
{ {
RenderThread.Join(2000); RenderThread.Join(2000);
+1 -21
View File
@@ -4,8 +4,6 @@
--- ---
## 1. Legal Terms (Official)
Linking this Program statically or dynamically with other modules is making a combined work based on this Program. Thus, the terms and conditions of the GNU General Public License cover the whole combination. Linking this Program statically or dynamically with other modules is making a combined work based on this Program. Thus, the terms and conditions of the GNU General Public License cover the whole combination.
As a special exception, the copyright holders of this Program give you permission to dynamically link this Program with independent modules that interact with or reference this Program solely through the public application programming interfaces (APIs), types, and data structures declared within the following namespaces: As a special exception, the copyright holders of this Program give you permission to dynamically link this Program with independent modules that interact with or reference this Program solely through the public application programming interfaces (APIs), types, and data structures declared within the following namespaces:
@@ -21,22 +19,4 @@ An **"independent module"** is a separate and distinct software module that is n
This exception does not grant any permission to modify the source code of this Program itself outside the terms of the GNU General Public License version 3. This exception does not grant any permission to modify the source code of this Program itself outside the terms of the GNU General Public License version 3.
In accordance with section 7 of the GNU GPL version 3, if you modify this Program, you may extend this exception to your version of the Program, but you are not obligated to do so. If you do not wish to provide this exception in your derivative work, you may remove this exception statement from your version. In accordance with section 7 of the GNU GPL version 3, if you modify this Program, you may extend this exception to your version of the Program, but you are not obligated to do so. If you do not wish to provide this exception in your derivative work, you may remove this exception statement from your version.
---
## 2. 조항 해설 (Korean Guide)
본 문서는 GNU General Public License 버전 3 (GPLv3) 제7조(추가적인 허가, Additional Permissions)에 따라 제공되는 공식 예외 조항입니다.
### 핵심 요약
- **동적 링킹(Dynamic Linking)만 허용**:
- 본 예외 조항은 **동적 링킹(런타임 참조 및 어셈블리 동적 로드)**에만 배타적으로 적용됩니다.
- ADOFAIRenderer 또는 그 일부 코드를 대상 모드/프로그램에 직접 병합하거나 포함(ILMerge, Costura, 바이너리 직접 내장/정적 컴파일 등)하는 **정적 링킹(Static Linking)은 본 예외 대상에서 제외**되며, 순수 GPLv3의 카피레프트(전염성) 의무가 그대로 적용됩니다.
- **외부 모드의 라이선스 자유**:
- 제3자 모드 또는 외부 프로그램이 동적 링킹을 통해 `ADOFAIRenderer.API` 및 `ADOFAIRenderer.Data` 네임스페이스의 공개 API, 클래스, 데이터 구조만을 참조·호출하여 연동하는 경우, 해당 모드는 **GPLv3의 카피레프트 의무를 적용받지 않습니다.**
- 외부 모드 개발자는 자신의 모드를 MIT, Apache-2.0, 독점/상용 라이선스 등 원하는 라이선스로 자유롭게 배포할 수 있습니다.
- **ADOFAIRenderer 코어 보호**:
- ADOFAIRenderer 자체의 소스 코드(렌더링 파이프라인, 코어 패치, 내부 유틸리티 등)를 수정하거나 재배포하는 경우에는 본 예외 조항이 적용되지 않으며, 순수 GPLv3 라이선스 조건이 그대로 적용됩니다.
+25
View File
@@ -0,0 +1,25 @@
using ADOFAIRenderer.Core;
using HarmonyLib;
namespace ADOFAIRenderer.Patch.Game;
[HarmonyPatch(typeof(DetailedResults))]
// ReSharper disable once InconsistentNaming
public static class DetailedResultsPatch
{
[HarmonyPrefix]
[HarmonyPatch("Show")]
// ReSharper disable once InconsistentNaming
public static bool PreventShow(DetailedResults __instance)
{
if (!RenderManager.IsActive) return true;
if (__instance)
{
__instance.enabled = false;
if (__instance.gameObject) __instance.gameObject.SetActive(false);
}
return false;
}
}
+34
View File
@@ -41,4 +41,38 @@ public static class scrControllerPatch
{ {
return !RenderManager.IsActive; return !RenderManager.IsActive;
} }
[HarmonyPostfix]
[HarmonyPatch("OnLandOnPortal")]
public static void PreventWinTextsPostfix(scrController __instance)
{
if (!RenderManager.IsActive || __instance == null) return;
if (__instance.txtCongrats)
{
__instance.txtCongrats.text = string.Empty;
__instance.txtCongrats.gameObject.SetActive(false);
}
if (__instance.txtAprilCongrats)
{
__instance.txtAprilCongrats.text = string.Empty;
__instance.txtAprilCongrats.gameObject.SetActive(false);
}
if (__instance.txtAllStrictClear)
{
__instance.txtAllStrictClear.text = string.Empty;
__instance.txtAllStrictClear.gameObject.SetActive(false);
}
if (__instance.detailedResults)
{
__instance.detailedResults.enabled = false;
if (__instance.detailedResults.gameObject != null)
{
__instance.detailedResults.gameObject.SetActive(false);
}
}
}
} }
+6
View File
@@ -14,6 +14,12 @@ public class scrPressToStartPatch
if (!RenderManager.Instance || !RenderManager.Instance.skipPressToStart) return true; if (!RenderManager.Instance || !RenderManager.Instance.skipPressToStart) return true;
RenderManager.Instance.skipPressToStart = false; RenderManager.Instance.skipPressToStart = false;
// scrPressToStart.ShowText ends with ShowDifficultyContainer(mode). Skipping ShowText skips that call too,
// which leaves the difficulty selector active and clickable. DontShow deactivates difficultyContainer
// and makes the keyboard difficulty shortcuts (difficultyUIMode != DontShow checks) do nothing.
if (scrUIController.instance)
scrUIController.instance.ShowDifficultyContainer(DifficultyUIMode.DontShow);
return false; return false;
} }
} }
+1 -1
View File
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine; using UnityEngine;
using Object = UnityEngine.Object; using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")] [assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.4.0-BETA", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")] [assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer; namespace ADOFAIRenderer;
+1 -1
View File
@@ -164,7 +164,7 @@ internal class DebugMonitorUI : MonoBehaviour
} }
} }
sb.AppendLine($"• Queue: <b>{queueCount}</b> / {RenderManager.MaxQueuedFrames} frames"); sb.AppendLine($"• Queue: <b>{queueCount}</b> / {(RenderManager.Instance ? RenderManager.Instance.QueueFrameLimit : RenderManager.MaxQueuedFrames)} frames");
sb.AppendLine($"• Rendered Frames: <b>{(RenderManager.Instance ? RenderManager.Instance.frameCount : 0)}</b>"); sb.AppendLine($"• Rendered Frames: <b>{(RenderManager.Instance ? RenderManager.Instance.frameCount : 0)}</b>");
sb.AppendLine($"• Frames Sent: <b>{(RenderManager.Instance ? Interlocked.Read(ref RenderManager.Instance.framesSent) : 0)}</b>"); sb.AppendLine($"• Frames Sent: <b>{(RenderManager.Instance ? Interlocked.Read(ref RenderManager.Instance.framesSent) : 0)}</b>");
+60 -16
View File
@@ -9,6 +9,7 @@ using ADOFAIRenderer.Data;
using ADOFAIRenderer.Localization; using ADOFAIRenderer.Localization;
using ADOFAIRenderer.Util.OS; using ADOFAIRenderer.Util.OS;
using MelonLoader; using MelonLoader;
using Newtonsoft.Json;
using O5Kit.Control; using O5Kit.Control;
using O5Kit.Core; using O5Kit.Core;
using O5Kit.Factory; using O5Kit.Factory;
@@ -32,6 +33,14 @@ internal static class RenderConfigWindow
private static Coroutine? _deleteResetCoroutine; private static Coroutine? _deleteResetCoroutine;
private static string? _lastLoadedPresetName; private static string? _lastLoadedPresetName;
/// <summary>
/// Configuration edited by this window. Kept for the whole game session; a render started from the window
/// receives a copy of it (RenderSession.Start), so values adjusted during the render are not written back here.
/// </summary>
private static RenderConfig? _uiConfig;
private static RenderConfig UiConfig => _uiConfig ??= new RenderConfig();
// Controls cache for validation & sync // Controls cache for validation & sync
private static O5Slider? _renderFpsSlider; private static O5Slider? _renderFpsSlider;
private static O5Slider? _videoFPSSlider; private static O5Slider? _videoFPSSlider;
@@ -111,8 +120,7 @@ internal static class RenderConfigWindow
ReloadPresetDropdown(GlobalSettings.Current.LastPresetName); ReloadPresetDropdown(GlobalSettings.Current.LastPresetName);
} }
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract var config = UiConfig;
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
ApplyConfigToUI(config); ApplyConfigToUI(config);
SetWindowActive(true); SetWindowActive(true);
@@ -125,8 +133,7 @@ internal static class RenderConfigWindow
var (_, scrollContent, _) = var (_, scrollContent, _) =
O5Factory.ScrollView(ctx, content, spacing: 10f, padding: 8f); O5Factory.ScrollView(ctx, content, spacing: 10f, padding: 8f);
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract var config = UiConfig;
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
EnsureDefaultConfig(config); EnsureDefaultConfig(config);
double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144, double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144,
@@ -744,11 +751,14 @@ internal static class RenderConfigWindow
{ {
EnsureDefaultConfig(config); EnsureDefaultConfig(config);
_renderFpsSlider?.Set((float) config.RenderFPS, false); // Sliders show the config values as they are (noFilter: true).
_videoFPSSlider?.Set((float) config.VideoFPS, false); // O5Slider.Set applies the slider filter (snapping to the nearest snap value) unless noFilter is set;
_renderWidthSlider?.Set(config.RenderWidth, false); // the filter is meant for dragging, and typed values are not snapped either (O5Slider.SetFromInput).
_renderHeightSlider?.Set(config.RenderHeight, false); _renderFpsSlider?.Set(config.RenderFPS, false, noFilter: true);
_delayAfterEndSlider?.Set((float) config.DelayAfterEnd, false); _videoFPSSlider?.Set(config.VideoFPS, false, noFilter: true);
_renderWidthSlider?.Set(config.RenderWidth, false, noFilter: true);
_renderHeightSlider?.Set(config.RenderHeight, false, noFilter: true);
_delayAfterEndSlider?.Set(config.DelayAfterEnd, false, noFilter: true);
_showJudgementToggle?.Set(config.ShowJudgementText, false); _showJudgementToggle?.Set(config.ShowJudgementText, false);
_codecDropdown?.Set(config.VideoConfig.Codec, false); _codecDropdown?.Set(config.VideoConfig.Codec, false);
@@ -756,13 +766,13 @@ internal static class RenderConfigWindow
_bitrateInput?.Set(config.VideoConfig.Bitrate.ToString(), false); _bitrateInput?.Set(config.VideoConfig.Bitrate.ToString(), false);
_maxBitrateInput?.Set(config.VideoConfig.MaxBitrate.ToString(), false); _maxBitrateInput?.Set(config.VideoConfig.MaxBitrate.ToString(), false);
_qpInput?.Set(config.VideoConfig.QP.ToString(), false); _qpInput?.Set(config.VideoConfig.QP.ToString(), false);
_keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false); _keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false, noFilter: true);
_forceSoftwareToggle?.Set(config.VideoConfig.ForceSoftwareEncoder, false); _forceSoftwareToggle?.Set(config.VideoConfig.ForceSoftwareEncoder, false);
_renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false); _renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false);
_sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false); _sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false);
_channelsDropdown?.Set(config.AudioConfig.AudioChannels, false); _channelsDropdown?.Set(config.AudioConfig.AudioChannels, false);
_audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false); _audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false, noFilter: true);
_outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false); _outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false);
@@ -789,17 +799,51 @@ internal static class RenderConfigWindow
UpdateVideoCodecUI(config.VideoConfig.Codec, config); UpdateVideoCodecUI(config.VideoConfig.Codec, config);
UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true); UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true);
if (!string.IsNullOrEmpty(GlobalSettings.Current.LastPresetName)) UpdatePresetLabel(config);
}
/// <summary>
/// Sets the preset dropdown from the config values.
/// Every value equals the last loaded preset (_lastLoadedPresetName): the preset name is shown.
/// Any value differs (edited, or the preset file changed): the Custom label is shown (MarkConfigModified),
/// the same state as after editing a value in the window.
/// </summary>
private static void UpdatePresetLabel(RenderConfig config)
{
if (_presetDropdown == null) return;
var presetName = _lastLoadedPresetName;
var preset = string.IsNullOrEmpty(presetName)
? null
: PresetManager.GetPresets().FirstOrDefault(p => string.Equals(p.Name, presetName, StringComparison.OrdinalIgnoreCase));
if (preset == null || !ConfigValuesEqual(config, preset.Config))
{ {
_presetDropdown?.Set(GlobalSettings.Current.LastPresetName, false); MarkConfigModified();
_lastLoadedPresetName = GlobalSettings.Current.LastPresetName; return;
UpdateDeleteButtonState();
} }
if (_presetDropdown.Values.Contains(CustomPresetLabel))
{
_presetDropdown.SetValues(_presetDropdown.Values.Where(v => v != CustomPresetLabel).ToList());
}
_presetDropdown.Set(preset.Name, false);
UpdateDeleteButtonState();
}
/// <summary>
/// Compares two configs by their serialized JSON (the same form as the preset files), so every saved value is compared.
/// </summary>
private static bool ConfigValuesEqual(RenderConfig a, RenderConfig b)
{
var settings = new JsonSerializerSettings();
settings.Converters.Add(new ColorJsonConverter());
return JsonConvert.SerializeObject(a, settings) == JsonConvert.SerializeObject(b, settings);
} }
private static void TryGoRender() private static void TryGoRender()
{ {
var config = RenderManager.Instance.Configuration; var config = UiConfig;
SyncUItoConfig(config); SyncUItoConfig(config);
// Sync Output Directory // Sync Output Directory
+19 -12
View File
@@ -23,7 +23,7 @@ namespace ADOFAIRenderer.Util.FFmpeg;
/// 2. Codec &amp; Hardware Device Discovery: /// 2. Codec &amp; Hardware Device Discovery:
/// a) avcodec_find_decoder: Locate codec implementation by codec ID. /// a) avcodec_find_decoder: Locate codec implementation by codec ID.
/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder. /// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder.
/// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox). /// c) HW device creation for the reported configurations, in the order of PreferredHwDeviceTypes.
/// d) Attach hw_device_ctx and register custom get_format callback. /// d) Attach hw_device_ctx and register custom get_format callback.
/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding /// e) If HW initialization fails or is unsupported, fall back to CPU software decoding
/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME). /// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME).
@@ -35,8 +35,10 @@ namespace ADOFAIRenderer.Util.FFmpeg;
/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data /// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data
/// into system RAM as NV12 (av_frame_copy_props preserves timestamps). /// into system RAM as NV12 (av_frame_copy_props preserves timestamps).
/// d) (Output Dispatch): /// d) (Output Dispatch):
/// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback. /// - TryReadNV12Frame: NV12 frames are copied row by row (Buffer.MemoryCopy) into the output buffers,
/// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback. /// other formats are converted with sws_scale.
/// - TryReadYUV420PFrame: YUV420P/YUVJ420P planes are copied row by row into the output buffers,
/// other formats are converted with sws_scale.
/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip). /// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
/// ///
/// 4. Seeking &amp; Scrubbing (SeekToTime / SeekToFrame): /// 4. Seeking &amp; Scrubbing (SeekToTime / SeekToFrame):
@@ -76,9 +78,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// <summary> /// <summary>
/// Preferred hardware device types in priority order. /// Preferred hardware device types in priority order.
/// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration, /// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config),
/// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA. /// sorted by this list; configurations not in the list follow in reported order.
/// Non-Windows systems fall back to VAAPI or VideoToolbox. /// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Other platforms: VAAPI or VideoToolbox.
/// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by
/// avcodec_find_decoder report DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan in the bundled FFmpeg 8 build;
/// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name.
/// </summary> /// </summary>
private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[] private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
{ {
@@ -661,9 +666,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_getFormatDelegate = GetFormat; _getFormatDelegate = GetFormat;
_pCodecContext->get_format = _getFormatDelegate; _pCodecContext->get_format = _getFormatDelegate;
// Hardware decoding requires single-threaded decoding (thread_count = 1). // Hardware decoding runs with one decoder thread (thread_count = 1).
// Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function // The GPU performs the decoding; the decoder thread only parses packets and submits them.
// GPU decoders and causes surface pool allocation exhaustion or crashes in get_format. // With frame threading, libavcodec adds one hardware surface per thread to the frame pool
// (avcodec_get_hw_frames_parameters). The software fallback in GetFormat keeps this thread count.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/decode.c
_pCodecContext->thread_count = 1; _pCodecContext->thread_count = 1;
// Attempt opening codec context with hardware configuration // Attempt opening codec context with hardware configuration
@@ -1118,7 +1125,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// </summary> /// </summary>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param> /// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
/// <param name="yStride">Row stride in bytes for Y plane.</param> /// <param name="yStride">Row stride in bytes for Y plane.</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes).</param> /// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes).</param>
/// <param name="uvStride">Row stride in bytes for UV plane.</param> /// <param name="uvStride">Row stride in bytes for UV plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param> /// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns> /// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
@@ -1171,7 +1178,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width; int w = Width;
int h = Height; int h = Height;
int uvH = (h + 1) / 2; int uvH = (h + 1) / 2;
int uvRowBytes = (w / 2) * 2; int uvRowBytes = ((w + 1) / 2) * 2;
// Copy Y plane (row by row to safely skip linesize GPU padding) // Copy Y plane (row by row to safely skip linesize GPU padding)
for (int y = 0; y < h; y++) for (int y = 0; y < h; y++)
@@ -1232,7 +1239,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width; int w = Width;
int h = Height; int h = Height;
int uvH = (h + 1) / 2; int uvH = (h + 1) / 2;
int uvRowBytes = (w / 2) * 2; int uvRowBytes = ((w + 1) / 2) * 2;
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH) if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
{ {
+205 -68
View File
@@ -18,12 +18,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// 2. Start(outputPath): /// 2. Start(outputPath):
/// a) Create AVFormatContext for output container (mp4, etc.) /// a) Create AVFormatContext for output container (mp4, etc.)
/// b) Add video stream (AVStream) /// b) Add video stream (AVStream)
/// c) Try hardware encoders (NVENC → AMF → QSV) → software encoders (libx264, etc.) /// c) Try the candidate encoders of the codec in order and select the first one that opens
/// in order to select an available encoder /// (encoders of the active GPU vendor → other hardware encoders → software encoders; see GetEncoderCandidates)
/// d) Write file header and initialize frame/scaler buffers /// d) Write file header and initialize frame/scaler buffers
/// ///
/// 3. WriteFrame*(): Called for every frame /// 3. WriteFrame*(): Called for every frame
/// a) Convert Unity RGBA original frame to NV12/YUV420P using sws_scale /// a) Pre-converted NV12/YUV420P frame (GPU compute shader output): copied into the encoder frame
/// RGBA frame: converted to the encoder pixel format with sws_scale (CPU)
/// b) (In QSV hw_frames mode) Upload to GPU surface /// b) (In QSV hw_frames mode) Upload to GPU surface
/// c) avcodec_send_frame → avcodec_receive_packet → Write to file /// c) avcodec_send_frame → avcodec_receive_packet → Write to file
/// ///
@@ -90,13 +91,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
public RateControlMode Mode; public RateControlMode Mode;
/// <summary>Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)</summary> /// <summary>Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)</summary>
public int QualityValue; // CRF public int QualityValue;
/// <summary>Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg.</summary> /// <summary>Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg.</summary>
public int TargetBitrate; // VBR public int TargetBitrate;
/// <summary>Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.</summary> /// <summary>Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.</summary>
public int MaxBitrate; // VBR public int MaxBitrate;
/// <summary>GOP (Group of Pictures) size (in seconds). Represents keyframe interval.</summary> /// <summary>GOP (Group of Pictures) size (in seconds). Represents keyframe interval.</summary>
public double GopSize; public double GopSize;
@@ -165,7 +166,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Accumulated index of encoded frames. Used as the PTS (Presentation Time Stamp) value.</summary> /// <summary>Accumulated index of encoded frames. Used as the PTS (Presentation Time Stamp) value.</summary>
private long _frameIndex; private long _frameIndex;
/// <summary>Whether Start() completed successfully and encoding is ready.</summary> /// <summary>
/// Whether the encoder accepts frames: set to true when Start() completes,
/// and to false when End() runs (End() is not repeated) or Dispose() releases the resources.
/// </summary>
private bool _isInitialized; private bool _isInitialized;
/// <summary>Whether Dispose() has already been called. Prevents duplicate cleanup.</summary> /// <summary>Whether Dispose() has already been called. Prevents duplicate cleanup.</summary>
@@ -182,9 +186,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
private bool _isQsvEncoder; private bool _isQsvEncoder;
/// <summary> /// <summary>
/// Target pixel format for the encoder. /// Pixel format of the frames passed to the encoder (output of the color conversion).
/// - General encoders (NVENC, AMF, Software): YUV420P (most common 4:2:0 chroma subsampling) /// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane)
/// - QSV encoder: NV12 (Y plane + UV interleaved plane, preferred format for Intel hardware) /// - libvvenc: YUV420P10LE (10-bit, 3 planes)
/// - Other software encoders: YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling)
/// ///
/// Reference - Major pixel format relationships: /// Reference - Major pixel format relationships:
/// RGBA (Unity source, 4-channel 8-bit) /// RGBA (Unity source, 4-channel 8-bit)
@@ -196,6 +201,12 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary> /// </summary>
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P; private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
/// <summary>
/// Color range of the encoder input and output (codec context color_range of the selected encoder).
/// Full range (JPEG) for every encoder except libvvenc, limited range (MPEG) for libvvenc.
/// </summary>
private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG;
/// <summary>Pixel format of the finally selected and running encoder.</summary> /// <summary>Pixel format of the finally selected and running encoder.</summary>
public AVPixelFormat SelectedPixelFormat => _dstPixFmt; public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
@@ -250,6 +261,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Whether audio encoding is enabled.</summary> /// <summary>Whether audio encoding is enabled.</summary>
private bool _audioEnabled; private bool _audioEnabled;
/// <summary>Bytes of one input sample of every channel (interleaved input format × channel count).</summary>
private int _audioInputBytesPerSample;
/// <summary>Encoding settings passed at construction time (immutable).</summary> /// <summary>Encoding settings passed at construction time (immutable).</summary>
private readonly EncoderSettings _settings; private readonly EncoderSettings _settings;
@@ -262,30 +276,14 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Utility Methods // Utility Methods
// ================================================================ // ================================================================
/// <summary>
/// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg.
/// AVCodecID identifies "which codec it is (e.g., H.264, AV1)",
/// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately.
/// </summary>
// ReSharper disable once InconsistentNaming
// ReSharper disable once UnusedMember.Local
private AVCodecID GetAVCodecID()
{
return _settings.Codec switch
{
Codec.H264 => AVCodecID.AV_CODEC_ID_H264,
Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC,
Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1,
Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9,
Codec.VVC => AVCodecID.AV_CODEC_ID_VVC,
_ => throw new ArgumentOutOfRangeException()
};
}
/// <summary> /// <summary>
/// Returns a list of candidate encoders to try for the selected codec in priority order. /// Returns a list of candidate encoders to try for the selected codec in priority order.
/// ///
/// Priority: Hardware encoders (NVENC → AMF → QSV) → Software encoders (libx264, etc.) /// Order (GPUUtil.GetPrioritizedEncoders):
/// 1. Hardware encoders of the active GPU vendor
/// 2. The remaining encoders in the declaration order of VideoEncoder.All
/// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has)
/// With ForceSoftware, only the software encoders are returned.
/// ///
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based) /// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework) /// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
@@ -407,7 +405,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Basic video parameters configuration ── // ── Basic video parameters configuration ──
_pCodecContext->width = _settings.Width; _pCodecContext->width = _settings.Width;
_pCodecContext->height = _settings.Height; _pCodecContext->height = _settings.Height;
_pCodecContext->color_range = AVColorRange.AVCOL_RANGE_JPEG;
// ── Output color tags ──
// Matrix: BT.709, the RGB→YUV matrix of both conversion paths
// (GPU: RGB2YUV_BT709_Full in ColorConversion.compute, CPU: ConfigureSwsColorspace).
// Players use this tag to convert the YUV frames back to RGB.
// Range: full (0-255) for every encoder except libvvenc, limited (16-235) for libvvenc.
// The FFmpeg libvvenc wrapper passes the matrix/primaries/transfer tags to VVenC but not the range,
// so a VVC stream is read as limited range; its input is converted to limited range to match.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_color_format)
_pCodecContext->color_range = encoder == VideoEncoder.LibVvenc
? AVColorRange.AVCOL_RANGE_MPEG
: AVColorRange.AVCOL_RANGE_JPEG;
_pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709;
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count. // thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count, // For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
// so setting 0 (auto) is required for parallel encoding utilizing all cores. // so setting 0 (auto) is required for parallel encoding utilizing all cores.
@@ -416,11 +426,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Pixel format configuration ── // ── Pixel format configuration ──
// Video encoders generally expect input in YUV420P or NV12 format. // Video encoders generally expect input in YUV420P or NV12 format.
// //
// • YUV420P: 3 independent planes Y/U/V. Used by most software encoders. // • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE).
// • NV12: Y plane + UV interleaved plane. Preferred by hardware encoders (NVENC, AMF, QSV). // • NV12: Y plane + UV interleaved plane. Used by the hardware encoders (NVENC, AMF, QSV).
// • QSV Special Note: pix_fmt of codec context is AV_PIX_FMT_QSV (hardware surface identifier), // • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is
// while actual sws_scale output is set to NV12 (software format). // AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode:
// FFmpeg internally handles mapping between NV12 ↔ QSV surfaces. // system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly.
// GPU frame pool input mode: pix_fmt stays AV_PIX_FMT_QSV, and SubmitDstFrame() uploads
// each NV12 frame to a QSV surface with av_hwframe_transfer_data.
// //
// Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html // Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html
var selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P; var selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
@@ -479,17 +491,21 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pCodecContext->gop_size = calculatedGop; _pCodecContext->gop_size = calculatedGop;
} }
// ── Disable B-frames ── // ── B-frame configuration ──
// B-frames (Bidirectional predictive frames): Increase compression ratio by referencing before/after frames, // B-frames (bidirectional predictive frames) reference both earlier and later frames.
// but increase encoding latency and may cause compatibility issues on some encoders. // They raise the compression ratio and make the encoder output packets out of presentation order
// In real-time recording scenarios, disable them to minimize latency. // (dts < pts for reordered frames).
// //
// Reference: has_b_frames is a read-only field internally set by the encoder, // Configuration per encoder:
// so do not set it directly. // SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure).
// Setting max_b_frames on AVCodecContext makes SVT-AV1 report a "bad parameter" error.
// libvvenc: max_b_frames = -1 (encoder default GOP structure).
// Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order).
// //
// SVT-AV1 Special Note: Setting max_b_frames directly on AVCodecContext // has_b_frames is set by the encoder itself and is not written here.
// causes a "bad parameter" error, so control via svtav1-params. //
// pred-struct=1: Low Delay (IPPP) structure disabling B-frames. // Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md
// (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2)
if (encoder == VideoEncoder.LibSvtAv1) if (encoder == VideoEncoder.LibSvtAv1)
{ {
ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0); ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0);
@@ -675,7 +691,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pHwFrame = ffmpeg.av_frame_alloc(); _pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null) if (_pHwFrame == null)
{
// The codec context (and the frames context it holds) is released by Dispose in the catch block of Start.
ffmpeg.av_buffer_unref(&hwDeviceCtx);
throw new Exception("Failed to allocate QSV hardware frame structure."); throw new Exception("Failed to allocate QSV hardware frame structure.");
}
break; break;
} }
@@ -696,8 +716,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
} }
} }
// Transfer ownership of hwDeviceCtx to codec context, // Release the local reference of hwDeviceCtx.
// so release local reference (reference count management). // The device stays alive through the references taken above:
// system memory input mode: codec context hw_device_ctx (av_buffer_ref)
// GPU frame pool input mode: the frames context (av_hwframe_ctx_alloc keeps a device reference),
// held by codec context hw_frames_ctx
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html
ffmpeg.av_buffer_unref(&hwDeviceCtx); ffmpeg.av_buffer_unref(&hwDeviceCtx);
} }
@@ -751,6 +774,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage"; var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage";
Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})"); Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})");
_dstPixFmt = selectedPixFmt; _dstPixFmt = selectedPixFmt;
_colorRange = _pCodecContext->color_range;
openedSuccessfully = true; openedSuccessfully = true;
break; break;
} }
@@ -779,8 +803,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Copy parameters set in codec context (resolution, codec ID, profile, etc.) // Copy parameters set in codec context (resolution, codec ID, profile, etc.)
// to stream's codecpar. Information to be written to container header. // to stream's codecpar. Information to be written to container header.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0) var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext);
throw new Exception("Failed to copy codec parameters to stream."); if (paramResult < 0)
throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})");
// Stream time base: unit of the timestamps stored in the container. // Stream time base: unit of the timestamps stored in the container.
// Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks. // Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks.
@@ -805,15 +830,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0) if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
{ {
if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0) var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE);
throw new Exception($"Cannot open output file {outputPath}."); if (avioResult < 0)
throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
} }
// Write file header (ftyp, moov, container metadata). // Write file header (ftyp, moov, container metadata).
// Packets can be written with av_interleaved_write_frame after this point. // Packets can be written with av_interleaved_write_frame after this point.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0) var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
throw new Exception("Failed to write file header."); if (headerResult < 0)
throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})");
// ── Step 6: Initialize frame buffers and colorspace converter ── // ── Step 6: Initialize frame buffers and colorspace converter ──
// av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet). // av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet).
@@ -833,8 +860,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Colorspace is converted without resolution change here, but sws_scale // Colorspace is converted without resolution change here, but sws_scale
// can also perform resizing simultaneously. // can also perform resizing simultaneously.
// //
// SWS_POINT: Nearest neighbor interpolation. Since only colorspace is converted without resolution change, // SWS_POINT: Nearest neighbor sampling. The luma plane keeps the source resolution (1:1).
// interpolation algorithm does not affect output quality, and SWS_POINT is fastest. // The chroma planes are subsampled to half width and height (4:2:0) with the same filter,
// so each chroma sample takes one source pixel instead of averaging the 2x2 block.
// This scaler is used only by the CPU conversion path (WriteFrame with RGBA input).
// //
// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html // Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
_pSwsContext = ffmpeg.sws_getContext( _pSwsContext = ffmpeg.sws_getContext(
@@ -845,6 +874,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSwsContext == null) if (_pSwsContext == null)
throw new Exception("Failed to initialize software scaler."); throw new Exception("Failed to initialize software scaler.");
ConfigureSwsColorspace();
_isInitialized = true; _isInitialized = true;
_loggedConversionMode = false; _loggedConversionMode = false;
} }
@@ -1140,10 +1171,42 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock) lock (_lock)
{ {
if (_disposed || !_isInitialized) return; if (_disposed || !_isInitialized) return;
ValidateInputLength(buffer.Length, isColorConverted: false);
WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr()); WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr());
} }
} }
/// <summary>
/// Number of bytes read from the input buffer of one frame.
/// RGBA input (isColorConverted = false): Width × Height × 4
/// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV)
/// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
/// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
/// </summary>
private long GetRequiredInputLength(bool isColorConverted)
{
long w = _settings.Width, h = _settings.Height;
if (!isColorConverted) return w * h * 4;
return _dstPixFmt switch
{
AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2),
AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2),
_ => 0
};
}
/// <summary>
/// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it
/// (GetRequiredInputLength), so that the copy does not read outside the buffer.
/// </summary>
private void ValidateInputLength(long length, bool isColorConverted)
{
var required = GetRequiredInputLength(isColorConverted);
if (length < required)
throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input.");
}
/// <summary> /// <summary>
/// Encodes a frame from IntPtr (native pointer). /// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources. /// Can be used from external native plugins or custom capture sources.
@@ -1162,6 +1225,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)"); : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
} }
ValidateInputLength(length, isColorConverted);
if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer()); if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
else WriteFrameFromBuffer((byte*)pointer.ToPointer()); else WriteFrameFromBuffer((byte*)pointer.ToPointer());
} }
@@ -1176,6 +1240,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock) lock (_lock)
{ {
if (_disposed || !_isInitialized) return; if (_disposed || !_isInitialized) return;
if (length > buffer.Length)
throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes).");
ValidateInputLength(length, isColorConverted);
fixed (byte* pBuffer = buffer) fixed (byte* pBuffer = buffer)
{ {
if (!_loggedConversionMode) if (!_loggedConversionMode)
@@ -1193,7 +1261,51 @@ namespace ADOFAIRenderer.Util.FFmpeg
} }
/// <summary> /// <summary>
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it. /// Sets the RGB→YUV conversion of the CPU conversion path (_pSwsContext) to the BT.709 matrix
/// and the color range of the encoder (_colorRange), so that it produces the same colors as the GPU path
/// (RGB2YUV_BT709_Full in ColorConversion.compute) and as the tags of the output stream.
/// A new libswscale context uses the BT.601 matrix (SWS_CS_DEFAULT = SWS_CS_ITU601) and the limited range.
///
/// For RGB input, "table" holds the coefficients of the output YUV space and dstRange its range (1 = full, 0 = limited).
/// The source (RGB) values, brightness, contrast and saturation keep the values of the context.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails, sws_getCoefficients)
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libswscale/swscale.h (SWS_CS_ITU709, SWS_CS_DEFAULT)
/// </summary>
private void ConfigureSwsColorspace()
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
var ret = ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot read the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
return;
}
var coefficients = ffmpeg.sws_getCoefficients(ffmpeg.SWS_CS_ITU709);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = currentInvTable[i];
table[i] = coefficients[i];
}
var fullRange = _colorRange == AVColorRange.AVCOL_RANGE_JPEG;
ret = ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, srcRange, table, fullRange ? 1 : 0,
brightness, contrast, saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot set the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
}
}
/// <summary>
/// Sets the format, size, color range and matrix of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers. /// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
/// Second argument 0 means default alignment. /// Second argument 0 means default alignment.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer) /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
@@ -1203,7 +1315,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pDstFrame->format = (int)_dstPixFmt; _pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width; _pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height; _pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG; _pDstFrame->color_range = _colorRange;
_pDstFrame->colorspace = AVColorSpace.AVCOL_SPC_BT709;
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0); var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
if (ret < 0) if (ret < 0)
@@ -1242,7 +1355,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
switch (_dstPixFmt) switch (_dstPixFmt)
{ {
case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV: case AVPixelFormat.AV_PIX_FMT_NV12:
{ {
// Copy Y Plane // Copy Y Plane
var pSrcY = buffer; var pSrcY = buffer;
@@ -1389,8 +1502,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool. // av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool.
// av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface. // av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface.
// //
// Through this process, the encoder can read frames directly from GPU memory, // The frame is copied from system memory to the GPU surface on every call;
// minimizing PCIe bandwidth usage. // the encoder then reads the surface from GPU memory.
// //
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
@@ -1465,7 +1578,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame); var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame);
if (response < 0) if (response < 0)
{ {
throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})"); throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})");
} }
// Pop all encoded packets and write to file. // Pop all encoded packets and write to file.
@@ -1479,7 +1592,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// EOF: Encoder flush complete (all delayed frames outputted) → exit loop // EOF: Encoder flush complete (all delayed frames outputted) → exit loop
if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break; if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
if (response < 0) if (response < 0)
throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})"); throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})");
// Convert PTS/DTS from codec timebase → stream timebase // Convert PTS/DTS from codec timebase → stream timebase
ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base); ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base);
@@ -1490,7 +1603,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// and written in time order. This function is used even when only video is present. // and written in time order. This function is used even when only video is present.
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket); response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
if (response < 0) if (response < 0)
throw new Exception($"Error occurred while writing packet to file. (Code: {response})"); throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
// Unref packet data reference for reuse in next receive_packet. // Unref packet data reference for reuse in next receive_packet.
ffmpeg.av_packet_unref(_pPacket); ffmpeg.av_packet_unref(_pPacket);
@@ -1755,6 +1868,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
? audioSettings.InputSampleFormat ? audioSettings.InputSampleFormat
: AVSampleFormat.AV_SAMPLE_FMT_FLT; : AVSampleFormat.AV_SAMPLE_FMT_FLT;
// Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
_audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
// Allocate SwrContext // Allocate SwrContext
fixed (SwrContext** ppSwr = &_pSwrContext) fixed (SwrContext** ppSwr = &_pSwrContext)
{ {
@@ -1818,6 +1935,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock) lock (_lock)
{ {
if (_disposed || !_isInitialized || !_audioEnabled) return; if (_disposed || !_isInitialized || !_audioEnabled) return;
if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float))
throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats).");
fixed (float* pSamples = samples) fixed (float* pSamples = samples)
{ {
@@ -1852,6 +1971,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html /// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html /// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary> /// </summary>
/// <summary>
/// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
/// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
/// Throws an exception when the buffer cannot be allocated.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
/// </summary>
private void MakeAudioFrameWritable()
{
var ret = ffmpeg.av_frame_make_writable(_pAudioFrame);
if (ret < 0)
throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
}
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples) private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
{ {
// ── 1. Resampling: Input format → Encoder format ── // ── 1. Resampling: Input format → Encoder format ──
@@ -1890,7 +2022,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize) while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
{ {
ffmpeg.av_frame_make_writable(_pAudioFrame); MakeAudioFrameWritable();
_pAudioFrame->nb_samples = frameSize; _pAudioFrame->nb_samples = frameSize;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize); var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize);
@@ -1953,8 +2085,13 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary> /// <summary>
/// Flushes the audio encoder. /// Flushes the audio encoder.
/// Pads and encodes remaining samples (less than frame_size) in FIFO, /// Sends the samples remaining in the FIFO (fewer than frame_size) as one last short frame,
/// then sends null frame to output all delayed frames in encoder. /// then sends a null frame to output all delayed packets of the encoder.
///
/// The short frame is handled by libavcodec (encode_send_frame_internal):
/// encoders with AV_CODEC_CAP_SMALL_LAST_FRAME or AV_CODEC_CAP_VARIABLE_FRAME_SIZE receive it as is,
/// other encoders receive it padded with silence up to frame_size (pad_last_frame).
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/encode.c
/// </summary> /// </summary>
private void FlushAudioEncoder() private void FlushAudioEncoder()
{ {
@@ -1964,7 +2101,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo); var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo);
if (remaining > 0) if (remaining > 0)
{ {
ffmpeg.av_frame_make_writable(_pAudioFrame); MakeAudioFrameWritable();
_pAudioFrame->nb_samples = remaining; _pAudioFrame->nb_samples = remaining;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining); var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);
+42 -5
View File
@@ -73,6 +73,16 @@ namespace ADOFAIRenderer.Util
} }
} }
/// <summary>
/// Bytes of the readback buffer of one frame: Width x Height x 3 / 2 when the GPU conversion to NV12 / YUV420P
/// is used, Width x Height x 4 (RGBA) otherwise. The same size as ReadbackRequestState.Setup uses.
/// </summary>
public long GetFrameBufferBytes(int width, int height)
{
var pixels = (long)width * height;
return _useGPUConversion ? pixels * 3 / 2 : pixels * 4;
}
public void RequestFrameCapture(Texture texture, long currentFrame) public void RequestFrameCapture(Texture texture, long currentFrame)
{ {
if (!RenderManager.IsCapturingActive) return; if (!RenderManager.IsCapturingActive) return;
@@ -154,14 +164,11 @@ namespace ADOFAIRenderer.Util
} }
Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame."); Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame.");
var byteSize = state.Width * state.Height * 3 / 2; var blackFrame = CreateBlackFrame(state, out var byteSize);
var blackFrame = ArrayPool<byte>.Shared.Rent(byteSize);
Array.Clear(blackFrame, 0, byteSize);
var capFrame = new RenderManager.CapturedFrame var capFrame = new RenderManager.CapturedFrame
{ {
ManagedBuffer = blackFrame, Length = byteSize, ManagedBuffer = blackFrame, Length = byteSize,
IsColorConverted = true IsColorConverted = state.IsGPUConversion
}; };
PushToQueue(frame, capFrame); PushToQueue(frame, capFrame);
RecycleState(state); RecycleState(state);
@@ -190,6 +197,36 @@ namespace ADOFAIRenderer.Util
} }
} }
/// <summary>
/// Creates a black frame in the same format as the frames of this reader, from ArrayPool&lt;byte&gt;.Shared
/// (returned to the pool by the render thread after encoding).
///
/// GPU conversion (NV12 / YUV420P, full range): Width × Height × 3 / 2 bytes.
/// Y plane (Width × Height bytes) = 0, U/V data (the rest) = 128.
/// 128 is the zero chroma value of the 8-bit full range: the U/V offset 0.5 of RGB2YUV_BT709_Full
/// in ColorConversion.compute, which the encoder input of this path uses.
/// Both NV12 (interleaved UV) and YUV420P (U plane, then V plane) store all chroma after the Y plane,
/// so the same fill applies to both.
/// No GPU conversion (RGBA input, converted by the encoder): Width × Height × 4 bytes, all 0.
/// </summary>
private static byte[] CreateBlackFrame(ReadbackRequestState state, out int byteSize)
{
var lumaSize = state.Width * state.Height;
byteSize = state.IsGPUConversion ? lumaSize * 3 / 2 : lumaSize * 4;
var frame = ArrayPool<byte>.Shared.Rent(byteSize);
if (state.IsGPUConversion)
{
Array.Clear(frame, 0, lumaSize);
frame.AsSpan(lumaSize, byteSize - lumaSize).Fill(128);
}
else
{
Array.Clear(frame, 0, byteSize);
}
return frame;
}
internal void RecycleState(ReadbackRequestState state) internal void RecycleState(ReadbackRequestState state)
{ {
lock (_poolLock) lock (_poolLock)
+9 -3
View File
@@ -9,8 +9,12 @@ namespace ADOFAIRenderer.Util.OS;
/// <summary> /// <summary>
/// Sends native desktop notifications on Windows, macOS, and Linux. /// Sends native desktop notifications on Windows, macOS, and Linux.
/// <para> /// <para>
/// - <b>Windows</b>: Uses Win32 <c>Shell_NotifyIcon</c> (system tray balloon) via P/Invoke. /// - <b>Windows</b>: Toast notification shown by a hidden <c>powershell.exe</c> process
/// No PowerShell, no UWP, no WinRT dependencies. /// (WinRT <c>ToastNotificationManager</c>, script passed with <c>-EncodedCommand</c>).
/// With a folder path, the script also registers the <c>adofai-renderer://</c> URL protocol under
/// <c>HKCU\Software\Classes\adofai-renderer</c> (open command: <c>explorer.exe "&lt;folder&gt;"</c>)
/// so the toast button opens the folder; the key is overwritten on each notification and stays registered.
/// When PowerShell fails, a Win32 <c>Shell_NotifyIcon</c> balloon is shown via P/Invoke.
/// </para> /// </para>
/// <para> /// <para>
/// - <b>macOS</b>: Uses <c>osascript</c> with AppleScript's <c>display notification</c>. /// - <b>macOS</b>: Uses <c>osascript</c> with AppleScript's <c>display notification</c>.
@@ -88,7 +92,9 @@ internal static class OSNotification
/// <summary> /// <summary>
/// Attempts to show a Windows Toast notification via PowerShell. /// Attempts to show a Windows Toast notification via PowerShell.
/// Returns true if the process launched successfully, false otherwise. /// With a folder path, registers the adofai-renderer:// protocol in HKCU first (see class summary).
/// Waits up to 5 seconds for the process. Returns true when it exits with code 0,
/// false when it exits with another code, does not exit within 5 seconds, or fails to start.
/// </summary> /// </summary>
private static bool TrySendToast(string title, string message, string? folderPath) private static bool TrySendToast(string title, string message, string? folderPath)
{ {
+3 -3
View File
@@ -97,7 +97,7 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA"); _kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false); _textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false); _textureUV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.RG16, false);
ConfigureTexture(_textureY); ConfigureTexture(_textureY);
ConfigureTexture(_textureUV); ConfigureTexture(_textureUV);
@@ -118,8 +118,8 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA"); _kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false); _textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false); _textureU = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
_textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false); _textureV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
ConfigureTexture(_textureY); ConfigureTexture(_textureY);
ConfigureTexture(_textureU); ConfigureTexture(_textureU);
ConfigureTexture(_textureV); ConfigureTexture(_textureV);