diff --git a/API/FrameHookContext.cs b/API/FrameHookContext.cs index cec727f..ac34e82 100644 --- a/API/FrameHookContext.cs +++ b/API/FrameHookContext.cs @@ -18,6 +18,21 @@ namespace ADOFAIRenderer.API; /// When RenderFPS exceeds VideoFPS, intermediate game rendering frames are skipped and the hook /// will not be invoked on every rendered game frame. /// +/// +/// Ping-pong model: the renderer owns two RenderTextures with the size and format of the game frame. +/// is the current frame, is the buffer the next pass writes. +/// A pass draws from into and then calls , which makes +/// the written buffer the new . , and by default +/// and , swap automatically. The texture in +/// after the last hook is encoded. When no hook swaps or replaces the frame, the game frame is +/// encoded unchanged. The first of every frame is the game's own frame texture +/// (); do not write into it. +/// +/// +/// Threading and lifetime: hooks run on the main Unity thread at the end of the frame, after the game has +/// rendered. The textures are valid only during the hook call: do not keep references to , +/// or the context. RenderTexture.active is restored after all hooks have run. +/// /// [SuppressMessage("ReSharper", "UnusedType.Global")] [SuppressMessage("ReSharper", "UnusedMember.Global")] @@ -25,12 +40,22 @@ namespace ADOFAIRenderer.API; [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] public class FrameHookContext { + /// Game frame of this capture (camera quad texture); never written by the context. private RenderTexture _originalSource = null!; + + /// First ping-pong buffer, owned by FrameHookPipeline. private RenderTexture _pingRT = null!; + + /// Second ping-pong buffer, owned by FrameHookPipeline. private RenderTexture _pongRT = null!; + /// Current frame: the original source, a ping-pong buffer, or a texture set by ReplaceCurrent. private RenderTexture _currentSource = null!; + + /// Buffer the next pass writes: always the ping or pong buffer. private RenderTexture _currentDestination = null!; + + /// True after the first Swap or ReplaceCurrent of this frame. private bool _hasModified; /// @@ -45,6 +70,7 @@ public class FrameHookContext /// /// The active representing the current state of the modified frame. + /// Same texture as ; after the last hook it is the frame that gets encoded. /// public RenderTexture Current => _currentSource; @@ -59,7 +85,7 @@ public class FrameHookContext public bool HasModified => _hasModified; /// - /// The sequential 0-based index of the video frame currently being encoded. + /// The sequential 0-based index of the video frame currently being captured (its position in the output video). /// public long FrameIndex { get; private set; } @@ -69,12 +95,13 @@ public class FrameHookContext public long VideoFrameIndex => FrameIndex; /// - /// The sequential 0-based index of the game render frame. + /// The sequential 0-based index of the game render frame, counted from the start of capture. + /// With RenderFPS above the video frame rate it advances by more than one between hook calls. /// public long RenderFrameIndex { get; private set; } /// - /// The timestamp of the current frame in seconds relative to video start. + /// The timestamp of the current frame in seconds relative to video start ( / ). /// public double VideoTime { get; private set; } @@ -84,30 +111,32 @@ public class FrameHookContext public double DeltaTime => VideoFPS > 0 ? 1.0 / VideoFPS : 0.0; /// - /// The width of the frame in pixels. + /// The width of the frame in pixels (width of the current ). /// public int Width => _currentSource.width; /// - /// The height of the frame in pixels. + /// The height of the frame in pixels (height of the current ). /// public int Height => _currentSource.height; /// - /// The configured render frames per second. + /// The configured render frames per second (game frames simulated per second of render time). /// public double RenderFPS { get; private set; } /// - /// The output video frames per second. + /// The output video frames per second: the effective rate, min(configured VideoFPS, ). /// public double VideoFPS { get; private set; } /// - /// The number of successful modification passes completed for the current frame. + /// The number of modification passes completed for the current frame (calls of and + /// , including the automatic swaps of the helper methods). 0 at the start of every frame. /// public int PassCount { get; private set; } + /// Creates a context for the first frame; see for the parameters. internal FrameHookContext( RenderTexture originalSource, RenderTexture pingRT, @@ -122,6 +151,10 @@ public class FrameHookContext Reset(originalSource, pingRT, pongRT, firstDestination, frameIndex, renderFrameIndex, videoTime, renderFps, videoFps); } + /// + /// Prepares the context for a new frame: Source = , Destination = + /// (ping or pong), not modified, PassCount 0, and the frame metadata. + /// internal void Reset( RenderTexture originalSource, RenderTexture pingRT, @@ -152,6 +185,11 @@ public class FrameHookContext /// /// Blits from to using the specified and automatically swaps ping-pong buffers. /// + /// + /// Uses Graphics.Blit, which binds as the material's _MainTex and changes + /// RenderTexture.active. + /// Reference: https://docs.unity3d.com/ScriptReference/Graphics.Blit.html + /// /// The material containing the shader to apply, or null for a standard copy. /// The shader pass index to use, or -1 to execute all passes. public void Blit(Material? material = null, int pass = -1) @@ -174,6 +212,10 @@ public class FrameHookContext /// /// Blits from a custom source to using the specified and automatically swaps ping-pong buffers. /// + /// + /// The current is not read; the result replaces the frame. is + /// stretched to the size of . + /// /// The custom source texture to blit from. /// The material containing the shader to apply, or null for a standard copy. /// The shader pass index to use, or -1 to execute all passes. @@ -200,6 +242,7 @@ public class FrameHookContext /// /// Copies the contents of into without swapping buffers. /// Call this before drawing custom overlays into if you wish to preserve the background. + /// Does not count as a modification; call after drawing to make the result the current frame. /// public void Copy() { @@ -207,7 +250,8 @@ public class FrameHookContext } /// - /// Clears with the specified color. + /// Clears with the specified color (color only; no depth buffer is cleared). + /// Does not swap; the previous RenderTexture.active is restored. /// /// The color to clear with, or null for . public void ClearDestination(Color? clearColor = null) @@ -261,6 +305,11 @@ public class FrameHookContext /// /// Executes a against the GPU and optionally swaps buffers. /// + /// + /// The context does not set a render target for the command buffer: the commands must render into + /// themselves (for example with SetRenderTarget or Blit(Source, Destination)), + /// so build or update the buffer inside the hook with the current and . + /// /// The command buffer to execute. /// If true (default), automatically calls after execution. /// Thrown when is null. @@ -278,6 +327,11 @@ public class FrameHookContext /// Swaps the active and ping-pong buffers. /// Call this after manually writing or rendering into so that subsequent hooks or the video encoder read the updated frame. /// + /// + /// After the call, is the buffer that was , and + /// is the other ping-pong buffer (its content is left over from an earlier pass or frame; the next pass overwrites it). + /// A texture set by is dropped by a swap. + /// public void Swap() { _hasModified = true; @@ -291,6 +345,12 @@ public class FrameHookContext /// /// Replaces the current modified frame texture with a custom . /// + /// + /// The texture becomes / ; is unchanged. + /// The texture is read back for encoding when it is still current after the last hook, so it must stay alive and + /// unchanged until the end of the hook chain, and it should have the frame's width, height and a format the capture + /// can read: the capture sizes its buffers from the texture, and the encoder expects the render resolution. + /// /// The custom texture to set as the active frame. /// Thrown when is null. public void ReplaceCurrent(RenderTexture customTexture) diff --git a/API/IFrameHook.cs b/API/IFrameHook.cs index 179547f..e3459a7 100644 --- a/API/IFrameHook.cs +++ b/API/IFrameHook.cs @@ -9,6 +9,8 @@ namespace ADOFAIRenderer.API; /// Execution Frequency Note: Frame hooks are only invoked for frames that are actually encoded into the final video (VideoFPS). /// When RenderFPS is higher than VideoFPS, intermediate game render frames are skipped and this hook /// will not be called on every game frame. +/// Register an implementation per render with or for every render +/// with . /// [SuppressMessage("ReSharper", "UnusedType.Global")] public interface IFrameHook @@ -18,6 +20,10 @@ public interface IFrameHook /// /// /// This method is not called on every rendered game frame if RenderFPS exceeds VideoFPS. + /// It is called on the main Unity thread at the end of the frame. Draw from + /// into and swap (see ); returning without + /// a swap leaves the frame unchanged. Do not keep the context or its textures after returning. + /// An exception thrown here is logged and the next hook still runs. /// /// The frame hook context providing access to ping-pong textures and frame metadata. void OnRenderFrame(FrameHookContext context); diff --git a/API/RenderResult.cs b/API/RenderResult.cs index d1ade45..2dbece4 100644 --- a/API/RenderResult.cs +++ b/API/RenderResult.cs @@ -7,6 +7,9 @@ namespace ADOFAIRenderer.API; /// /// Represents the result and metadata of a completed or canceled render session. /// +/// +/// Passed to the completion callback registered with . Immutable. +/// [SuppressMessage("ReSharper", "UnusedType.Global")] [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] public class RenderResult @@ -17,7 +20,10 @@ public class RenderResult public bool Success { get; } /// - /// The absolute output path of the rendered video file. Empty string if rendering failed or output was not created. + /// The output path of the rendered video file: the path given to as passed + /// (relative if it was relative), or the path of the time-stamped file in the output directory chosen in the mod + /// settings (by default <game root>/Render). + /// Empty string if rendering failed or was cancelled (the unfinished file is deleted). /// public string OutputPath { get; } @@ -27,12 +33,13 @@ public class RenderResult public long TotalFrames { get; } /// - /// The calculated video duration in seconds (based on TotalFrames and FPS). + /// The calculated video duration in seconds: divided by the effective video frame rate + /// (min of the configured VideoFPS and RenderFPS); 0 when the frame rate is not positive. /// public double VideoDuration { get; } /// - /// Total wall-clock time spent rendering. + /// Total wall-clock time spent rendering, from the end of the countdown (start of capture) to the end of the render. /// public TimeSpan RenderDuration { get; } @@ -43,6 +50,8 @@ public class RenderResult /// /// Error message or reason if rendering failed or was canceled; null on success. + /// A cancelled render has "Rendering was cancelled by user."; a failed render has "Rendering failed: " followed by + /// the reason. Use to test the outcome; the texts are meant for display. /// public string? ErrorMessage { get; } @@ -82,6 +91,7 @@ public class RenderResult /// The effective frames per second used to compute video duration. /// Total wall-clock time spent rendering. /// A new representing a successful render session. + /// The file size is read from disk; it is 0 when the file does not exist or cannot be read. internal static RenderResult CreateSuccess(string outputPath, long totalFrames, double fps, TimeSpan renderDuration) { var fileSizeBytes = 0L; @@ -103,6 +113,7 @@ public class RenderResult /// The effective frames per second used to compute video duration. /// Total wall-clock time spent rendering prior to termination. /// A new representing a failed or canceled render session. + /// OutputPath is empty and FileSizeBytes is 0. A null message is replaced by a generic one. internal static RenderResult CreateFailure(string? errorMessage, long totalFrames, double fps, TimeSpan renderDuration) { var videoDuration = fps > 0 ? totalFrames / fps : 0.0; diff --git a/API/RenderSession.cs b/API/RenderSession.cs index 56ad5da..fea8d2c 100644 --- a/API/RenderSession.cs +++ b/API/RenderSession.cs @@ -10,13 +10,41 @@ using UnityEngine; namespace ADOFAIRenderer.API; /// -/// Fluent builder for configuring and starting a render session. +/// Fluent builder for configuring and starting a render session, and the registry of global frame hooks. /// +/// +/// +/// Usage: create an instance, configure it with the With…, On… and AddFrameHook methods, +/// then call . The builder's values are copied into the render when is called; +/// changing the builder afterwards does not affect the running render. A builder can be reused for later renders. +/// +/// +/// Lifecycle of a render: loads the game scene of the level. When the scene has loaded, +/// a 3-second countdown runs, then capture starts and the level plays with the game clock driven by the render +/// frame rate (offline, not real time). The render ends successfully seconds +/// (render time) after the end condition first holds, or is cancelled by the user (ESC), or fails. In every case +/// the completion callback receives a and the game returns to the level editor. +/// +/// +/// Threading: every member must be called on the main Unity thread, except the global hook members, which are +/// thread-safe (they lock an internal object). All callbacks (end condition, completion, last update, frame hooks) +/// are invoked on the main Unity thread. +/// +/// +/// Frame hooks run on each captured video frame before it is encoded, in this order: session hooks in the order +/// they were added, then global hooks in registration order. +/// +/// [SuppressMessage("ReSharper", "UnusedType.Global")] [SuppressMessage("ReSharper", "UnusedMember.Global")] [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] public class RenderSession { + /// + /// Handle returned by AddGlobalFrameHook. Key is the object the caller registered (delegate, material or IFrameHook) + /// and is used by RemoveGlobalFrameHook; Action is the delegate the pipeline invokes. + /// Dispose removes this exact registration once; later calls do nothing. + /// private sealed class GlobalHookRegistration : IDisposable { public object Key { get; } @@ -37,6 +65,10 @@ public class RenderSession } } + /// + /// Session hook added to this builder. Key is the object the caller passed (delegate, material or IFrameHook) and + /// is used by RemoveFrameHook; Action is the delegate copied to the render by Start. + /// private sealed class SessionHookEntry { public object Key { get; } @@ -49,11 +81,22 @@ public class RenderSession } } + /// Global hooks in registration order. Guarded by GlobalHooksLock. private static readonly List GlobalFrameHooks = new(); + + /// Lock for GlobalFrameHooks and _globalHooksVersion (global hooks may be changed from any thread). private static readonly object GlobalHooksLock = new(); + + /// + /// Incremented on every change of GlobalFrameHooks. RenderManager compares it each captured frame to rebuild its + /// cached hook list only after a change. Guarded by GlobalHooksLock. + /// private static int _globalHooksVersion; + /// Session hooks of this builder in the order they were added. private readonly List _frameHooks = new(); + + /// Autoplay setting copied to the render by Start (default true). private bool _auto = true; /// @@ -63,27 +106,48 @@ public class RenderSession { /// /// Time-related patch controlling frame timing and clock synchronization. + /// While capturing, Time.time, Time.timeAsDouble, Time.unscaledTime, + /// Time.unscaledTimeAsDouble, Time.deltaTime, Time.unscaledDeltaTime, Time.frameCount + /// and AudioSettings.dspTime return values of the render clock (one frame = 1 / RenderFPS seconds), + /// so the game advances exactly one render frame per Unity frame regardless of how long a frame takes. + /// Disabling it leaves these values to Unity. /// Time, /// /// Screen-related patch controlling screen resolution. + /// While the render is active (from the countdown on), Screen.width and Screen.height return + /// and , so the game creates its + /// camera RenderTexture at the render resolution. /// Screen } + /// Time patch setting copied to the render by Start (default true). private bool _timePatch = true; + + /// Screen patch setting copied to the render by Start (default true). private bool _screenPatch = true; + + /// Custom output file path, or null for the default output directory. private string? _outputPath; + /// Notification setting copied to the render by Start (default true). private bool _notification = true; + /// Custom end condition, or null to keep the default (level cleared). private Func? _endCondition; + + /// Combined completion callbacks, or null to keep the default (notification). private Action? _onComplete; + + /// Combined last-update callbacks, or null for none. private Action? _onLastUpdate; /// - /// Sets whether auto mode is enabled. + /// Sets whether auto mode (the game's autoplay, RDC.auto) is enabled during the render. + /// With auto mode off, the level is not played automatically; the render still enables the game's no-fail + /// settings (scrController.noFail, scrController.noFailInfiniteMargin). /// /// true to enable auto mode; otherwise, false. Default is true. /// The current instance for method chaining. @@ -115,6 +179,13 @@ public class RenderSession /// /// Sets the custom condition function that determines when the render session should end. /// + /// + /// The condition is polled on the main Unity thread once per frame from the start of the countdown until the render ends. + /// When it first returns true, the render continues for seconds of + /// render time and then finishes successfully; it is not required to stay true. The default condition is + /// that the level is cleared (scrController.instance.state == States.Won). Calling this method again + /// replaces the previous condition. Exceptions thrown by the condition are not caught. + /// /// A delegate returning true when the session end condition is met. /// The current instance for method chaining. public RenderSession WithEndCondition(Func endCondition) @@ -126,6 +197,13 @@ public class RenderSession /// /// Sets the custom output path for the rendered file. /// + /// + /// The directory of the path is created by when it does not exist. An existing file at the path + /// is overwritten. The container format follows the file extension. With null, the file is named + /// yyyy-MM-dd HH-mm-ss.<ext> (local start time, extension from ) + /// in the output directory chosen in the mod settings. When the render is cancelled or fails, the unfinished file + /// is deleted. + /// /// The file path where the output will be saved, or null to use the default capture directory. /// The current instance for method chaining. public RenderSession WithOutputPath(string? outputPath) @@ -137,6 +215,10 @@ public class RenderSession /// /// Sets whether desktop notifications should be displayed when rendering completes. /// + /// + /// The notification is shown by the default completion callback only. When a callback is added with + /// , it replaces the default callback and no notification is shown. Default is true. + /// /// true to enable desktop notifications; otherwise, false. /// The current instance for method chaining. public RenderSession WithNotification(bool enabled) @@ -148,6 +230,13 @@ public class RenderSession /// /// Adds a callback action to be invoked when the render session completes or fails. /// + /// + /// Called once per render on the main Unity thread: on success after the output file has been finished, on cancel or + /// failure after the unfinished file has been deleted, and before the game returns to the level editor. + /// It is not called when the render stops because the game quits. Multiple calls combine the callbacks (invoked in + /// the order added). The combined callback replaces the default one, which shows the desktop notification. + /// An exception thrown by a callback is logged and stops the callbacks added after it. + /// /// The callback action that receives the resulting . /// The current instance for method chaining. public RenderSession OnComplete(Action callback) @@ -159,6 +248,11 @@ public class RenderSession /// /// Adds a callback action to be invoked on the last update of each frame in the render session. /// + /// + /// Called on the main Unity thread every Unity frame after all other LateUpdate calls, from the moment the game scene + /// of the render has loaded until the render ends (including the countdown, before capture starts). + /// Multiple calls combine the callbacks. An exception thrown by a callback is logged. + /// /// The callback action to execute. /// The current instance for method chaining. public RenderSession OnLastUpdate(Action callback) @@ -172,9 +266,19 @@ public class RenderSession /// The hook can modify the rendered frame using GPU ping-pong double buffering. /// /// + /// /// Execution Frequency Note: This hook is invoked only for frames that are captured for the output video. /// When RenderFPS is higher than VideoFPS, intermediate game render frames are skipped, /// so this hook will not be called on every game frame. + /// + /// + /// The hook is called on the main Unity thread at the end of the frame, after the game has rendered it and before + /// the GPU readback for encoding is requested. Write the result into and + /// swap (or use , which swaps); a hook that does not swap leaves + /// the frame unchanged. An exception thrown by the hook is logged and the next hook still runs. + /// Session hooks apply only to the render started by and are discarded when it ends. + /// The same delegate can be added more than once; it then runs once per addition. + /// /// /// The callback action that receives a . /// The current instance for method chaining. @@ -190,6 +294,13 @@ public class RenderSession /// Adds a material-based post-processing pass to be executed for each encoded video frame. /// Automatically performs a GPU blit from source to destination and swaps buffers. /// + /// + /// Equivalent to a hook calling : the current frame is bound as the + /// material's _MainTex by Graphics.Blit. The material is referenced, not copied, so property changes + /// made to it later apply to the following frames. Execution rules are the same as + /// . + /// Reference: https://docs.unity3d.com/ScriptReference/Graphics.Blit.html + /// /// The material containing the shader to apply. /// The shader pass index to use, or -1 to execute all passes. /// The current instance for method chaining. @@ -204,6 +315,10 @@ public class RenderSession /// /// Adds a custom frame modification hook implementing . /// + /// + /// is invoked with the same rules as + /// . Remove it with . + /// /// The hook instance to execute. /// The current instance for method chaining. /// Thrown when is null. @@ -239,6 +354,11 @@ public class RenderSession /// /// Removes a previously added frame modification hook callback from this session. /// + /// + /// Removes the first entry (in the order added) whose registered object is or whose delegate + /// equals it. When the same delegate was added several times, one call removes one addition. Does nothing when no + /// entry matches. Affects only renders started after this call. + /// /// The hook action to remove. /// The current instance for method chaining. public RenderSession RemoveFrameHook(Action hook) @@ -251,6 +371,10 @@ public class RenderSession /// /// Removes a previously added material-based post-processing pass from this session. /// + /// + /// Removes every pass added with (any pass index). Does nothing when the material is + /// null or destroyed. Affects only renders started after this call. + /// /// The material to remove. /// The current instance for method chaining. public RenderSession RemoveFrameHook(Material material) @@ -271,6 +395,9 @@ public class RenderSession /// /// Removes a previously added from this session. /// + /// + /// Removes the first entry added with this instance (reference equality). Affects only renders started after this call. + /// /// The hook instance to remove. /// The current instance for method chaining. public RenderSession RemoveFrameHook(IFrameHook hook) @@ -294,9 +421,19 @@ public class RenderSession /// Registers a global frame modification hook that will execute on every render session. /// /// + /// /// Execution Frequency Note: Global frame hooks are executed only for frames that are captured for the output video. /// When RenderFPS is higher than VideoFPS, intermediate game render frames are skipped, /// so this hook will not be called on every game frame. + /// + /// + /// The registration stays active for every following render (and a render already running, from its next + /// captured frame) until the returned handle is disposed or the hook is removed with + /// or . + /// Global hooks run after the session hooks, in registration order, on the main Unity thread; an exception thrown + /// by the hook is logged and the next hook still runs. This method is thread-safe. Registering the same delegate + /// twice creates two registrations. + /// /// /// The callback action that receives a . /// An registration handle that unregisters the hook when disposed. @@ -316,6 +453,10 @@ public class RenderSession /// /// Registers a global material-based post-processing pass that will execute on every render session. /// + /// + /// Same lifetime and ordering as ; the pass is applied as + /// . Thread-safe. + /// /// The material containing the shader to apply. /// The shader pass index to use, or -1 to execute all passes. /// An registration handle that unregisters the hook when disposed. @@ -335,6 +476,9 @@ public class RenderSession /// /// Registers a global frame modification hook implementing . /// + /// + /// Same lifetime and ordering as . Thread-safe. + /// /// The hook instance to execute. /// An registration handle that unregisters the hook when disposed. /// Thrown when is null. @@ -353,6 +497,10 @@ public class RenderSession /// /// Unregisters a previously registered global frame modification hook. /// + /// + /// Removes the first registration (in registration order) whose registered object is or whose + /// delegate equals it. Thread-safe; a running render stops calling the hook from its next captured frame. + /// /// The hook action to remove. /// true if the hook was successfully removed; otherwise, false. public static bool RemoveGlobalFrameHook(Action hook) @@ -373,6 +521,7 @@ public class RenderSession /// /// Unregisters a previously registered global material-based post-processing pass. /// + /// Removes every registration made with (any pass index). Thread-safe. /// The material to remove. /// true if any hooks matching were removed; otherwise, false. public static bool RemoveGlobalFrameHook(Material material) @@ -399,6 +548,7 @@ public class RenderSession /// /// Unregisters a previously registered global frame modification hook implementing . /// + /// Removes the first registration made with this instance (reference equality). Thread-safe. /// The hook instance to remove. /// true if the hook was successfully removed; otherwise, false. public static bool RemoveGlobalFrameHook(IFrameHook hook) @@ -416,6 +566,7 @@ public class RenderSession } } + /// Removes one registration (called by its Dispose). Does nothing when it was already removed. private static void RemoveGlobalRegistration(GlobalHookRegistration registration) { lock (GlobalHooksLock) @@ -430,6 +581,10 @@ public class RenderSession /// /// Clears all registered global frame modification hooks. /// + /// + /// Removes the registrations of every mod, not only the caller's. Handles returned earlier stay valid; disposing them + /// does nothing. Thread-safe. + /// public static void ClearGlobalFrameHooks() { lock (GlobalHooksLock) @@ -443,7 +598,7 @@ public class RenderSession } /// - /// Indicates whether any global frame modification hooks are currently registered. + /// Indicates whether any global frame modification hooks are currently registered (by any mod). Thread-safe. /// public static bool HasGlobalFrameHooks { @@ -457,7 +612,7 @@ public class RenderSession } /// - /// The number of currently registered global frame modification hooks. + /// The number of currently registered global frame modification hooks (by any mod). Thread-safe. /// public static int GlobalFrameHookCount { @@ -470,6 +625,7 @@ public class RenderSession } } + /// Change counter of the global hook list (see _globalHooksVersion). Thread-safe. internal static int GlobalHooksVersion { get @@ -481,6 +637,7 @@ public class RenderSession } } + /// Appends the delegates of every global hook, in registration order, to . Thread-safe. internal static void CopyGlobalFrameHooksTo(List> destination) { lock (GlobalHooksLock) @@ -495,11 +652,29 @@ public class RenderSession /// /// Applies the configured flags and initiates the render session. /// + /// + /// + /// Must be called on the main Unity thread while no render is running; this method does not check for a running + /// render. It copies the builder's options and hooks into the render, copies + /// (later changes to the caller's object do not affect the render), creates the output directory, and starts + /// loading the game scene (scnGame) with the level. It returns immediately; the render runs over the following + /// frames and reports its end through the callback. + /// + /// + /// Options that were not set keep their defaults: auto mode on, both patches on, default output path, notification + /// on, end condition "level cleared", default completion callback (notification), no last-update callback. + /// + /// /// The configuration settings for the render session. - /// The file path of the level to render. + /// The file path of the level to render (the .adofai file). /// Thrown when the output directory cannot be determined from the output path. + /// + /// Thrown by the directory creation when the output path has no directory part (for example a bare file name). + /// public void Start(RenderConfig renderConfig, string levelPath) { + // Callbacks not set on this builder keep the values RenderContext holds; RenderContext.Reset at the end of every + // render restores the defaults. RenderContext.Auto = _auto; RenderContext.TimePatch = _timePatch; RenderContext.ScreenPatch = _screenPatch; diff --git a/Core/Data/GlobalSettings.cs b/Core/Data/GlobalSettings.cs index 398f88e..f4bb28a 100644 --- a/Core/Data/GlobalSettings.cs +++ b/Core/Data/GlobalSettings.cs @@ -6,29 +6,59 @@ using Newtonsoft.Json; namespace ADOFAIRenderer.Core.Data; +/// +/// Mod-wide settings that are not part of a render preset, stored as JSON in +/// <game root>/UserData/ADOFAIRenderer/global_settings.json (see ). +/// internal class GlobalConfig { + /// + /// Output directory chosen in the render configuration window; null or empty selects + /// GlobalSettings.DefaultOutputDirectory. Ignored while the directory does not exist (see EffectiveOutputDirectory). + /// public string? OutputDirectory { get; set; } + /// Name of the preset last selected or saved in the render configuration window; selected again when it opens. public string? LastPresetName { get; set; } + /// + /// True once PresetManager.EnsureDefaultPresetsCreated has written the built-in preset files. Built-in presets the user + /// deleted afterwards are not written again. + /// public bool DefaultPresetsInitialized { get; set; } } +/// +/// Loads and saves . The file is read once by the static constructor and again by +/// RenderMod.OnInitializeMelon (Load), and written by Save. Read and write errors are logged as warnings and never +/// thrown; a file that cannot be read yields the default settings. Main thread. +/// internal static class GlobalSettings { + /// MelonLoader's game root directory, or the application base directory when it is empty. private static string GetGameRootDir() { var root = MelonEnvironment.GameRootDirectory; return string.IsNullOrEmpty(root) ? AppDomain.CurrentDomain.BaseDirectory : root; } + /// <game root>/UserData/ADOFAIRenderer/global_settings.json. private static string SettingsFilePath => Path.Combine(GetGameRootDir(), "UserData", "ADOFAIRenderer", "global_settings.json"); + /// + /// Settings in memory. Changes are kept in memory until is called; replaces the + /// object. + /// public static GlobalConfig Current { get; private set; } = new(); + /// Output directory used when none is configured: <game root>/Render. public static string DefaultOutputDirectory => Path.Combine(GetGameRootDir(), "Render"); + /// + /// Directory where renders without an explicit output path are written: Current.OutputDirectory when it is set and + /// exists, otherwise . The default directory is not created here + /// (RenderManager.SetupCaptureDirectory creates it). + /// public static string EffectiveOutputDirectory { get @@ -47,6 +77,10 @@ internal static class GlobalSettings Load(); } + /// + /// Replaces with the content of the settings file. A missing, empty or unreadable file gives the + /// default settings (an unreadable file is logged as a warning). + /// public static void Load() { try @@ -66,6 +100,10 @@ internal static class GlobalSettings Current = new GlobalConfig(); } + /// + /// Writes as indented JSON to the settings file, creating the directory when needed. + /// Errors are logged as warnings. + /// public static void Save() { try diff --git a/Core/FirstUpdateHook.cs b/Core/FirstUpdateHook.cs index 1f1debb..185e518 100644 --- a/Core/FirstUpdateHook.cs +++ b/Core/FirstUpdateHook.cs @@ -4,14 +4,30 @@ using UnityEngine; namespace ADOFAIRenderer.Core; +/// +/// MonoBehaviour that runs before every other script's Update (execution order int.MinValue; scripts run from the +/// lowest order to the highest). Lives on a DontDestroyOnLoad object created by RenderMod. +/// It performs the start-up checks (FFmpeg libraries present, NVIDIA driver version) and then advances the render +/// frame counter through RenderManager.OnPreUpdate once per Unity frame, so every other script of that frame sees +/// the same render time. +/// Reference: https://docs.unity3d.com/ScriptReference/DefaultExecutionOrder.html +/// [DefaultExecutionOrder(int.MinValue)] internal class FirstUpdateHook : MonoBehaviour { + /// True once the FFmpeg libraries were found in UserLibs (after the download UI, if it was needed). private bool _checkedFFmpeg; + + /// True once the NVIDIA driver version check ran (it runs once per game session). private bool _checkedNvidiaDriver; + /// + /// Main thread, every frame. The checks are done in order and each one blocks the following steps while its dialog + /// is open: the frame counter is not advanced until both checks are done. + /// private void Update() { + // 1. FFmpeg libraries: while the download UI is open, wait. When the libraries are missing, open the download UI. if (!_checkedFFmpeg) { if (FFmpegDownloadUI.Instance) return; @@ -24,6 +40,7 @@ internal class FirstUpdateHook : MonoBehaviour _checkedFFmpeg = true; } + // 2. NVIDIA driver version: waits until no other warning dialog is open, then runs once. if (!_checkedNvidiaDriver) { if (CommonWarningDialogUI.Instance) return; @@ -32,12 +49,19 @@ internal class FirstUpdateHook : MonoBehaviour CheckNvidiaDriverWarning(); } + // 3. Render frame counter (only counts while a render is in the OnScene stage or later). if (RenderManager.Instance) { RenderManager.Instance.OnPreUpdate(); } } + /// + /// Shows a warning dialog when the active GPU is an NVIDIA GPU with a driver version below 610.00. + /// No dialog is shown when the GPU is not NVIDIA or the version cannot be determined (GPUUtil returns null). + /// Buttons: open the NVIDIA driver download page and quit the game, disable the render feature for this session + /// (RenderMod.IsFeatureDisabled), or continue. + /// private void CheckNvidiaDriverWarning() { var driverVer = Util.GPUUtil.GetNvidiaDriverVersion(); diff --git a/Core/FrameHookPipeline.cs b/Core/FrameHookPipeline.cs index cf59da4..d25d9f4 100644 --- a/Core/FrameHookPipeline.cs +++ b/Core/FrameHookPipeline.cs @@ -6,16 +6,58 @@ using UnityEngine; namespace ADOFAIRenderer.Core; +/// +/// Runs the frame hooks (session and global, see RenderSession) on a captured frame before it is read back. +/// Owns two RenderTextures (ping and pong) with the size and format of the source frame; each hook pass reads the +/// current frame and writes the other buffer (see FrameHookContext). One instance per render, created in +/// RenderManager.SetupPlay and disposed when the render ends. Main thread only. +/// internal class FrameHookPipeline : IDisposable { + /// First ping-pong buffer. Created by EnsureBuffers, released by ReleaseBuffers. private RenderTexture? _pingRT; + + /// Second ping-pong buffer. Created by EnsureBuffers, released by ReleaseBuffers. private RenderTexture? _pongRT; + + /// + /// Buffer returned as the output of the previous Process call (ping or pong), or null when the previous frame was + /// not modified by any hook. Its readback may still be in flight, so the next frame starts writing the other buffer. + /// private RenderTexture? _lastOutputRT; + + /// Context object passed to the hooks; reset and reused for every frame (no allocation per frame). private FrameHookContext? _reusableContext; + + /// Width of the current buffers in pixels (0 when no buffers exist). private int _width; + + /// Height of the current buffers in pixels (0 when no buffers exist). private int _height; + + /// Format of the source texture the current buffers were created for. private RenderTextureFormat _format; + /// + /// Runs in order on and returns the resulting texture. + /// An exception thrown by a hook is logged and the remaining hooks still run. + /// RenderTexture.active is restored after the hooks. + /// + /// Captured game frame. Hooks do not write into it (they write the ping-pong buffers). + /// Video frame index (0-based) of this frame. + /// + /// Render frame index (0-based, RenderManager.frameCountForVideo): counts every game frame since capture started, + /// including game frames that were not captured because they fall in an already captured video frame. + /// + /// Time of this video frame in seconds from the start of the video. + /// Render frame rate (game frames per second). + /// Effective video frame rate (min(VideoFPS, RenderFPS)). + /// Hooks to run, in order. Session hooks come before global hooks. + /// + /// when there are no hooks, the source is not created, the buffers could not be + /// created, or no hook modified the frame; otherwise the texture holding the final frame (a ping-pong buffer, or a + /// texture a hook passed to FrameHookContext.ReplaceCurrent). + /// public RenderTexture? Process( RenderTexture sourceTexture, long frameIndex, @@ -81,6 +123,8 @@ internal class FrameHookPipeline : IDisposable } } + // Current is the source when nothing was modified. A texture set by ReplaceCurrent is stored in + // _lastOutputRT too; it then matches neither buffer and the next frame starts with ping. var result = _reusableContext.Current; _lastOutputRT = _reusableContext.HasModified ? result : null; return result; @@ -91,6 +135,12 @@ internal class FrameHookPipeline : IDisposable } } + /// + /// Creates the ping and pong buffers when they do not exist, were released, or the source size or format changed. + /// The buffers copy the source descriptor without depth, MSAA and mipmaps, and enable random write (UAV) when the + /// GPU supports compute shaders and random write for that format, so compute shaders can write them. + /// When creating them from the descriptor throws, ARGB32 buffers of the same size are created. + /// private void EnsureBuffers(RenderTexture sourceTexture) { var width = sourceTexture.width; @@ -160,6 +210,9 @@ internal class FrameHookPipeline : IDisposable } } + /// + /// Releases and destroys both buffers and drops the reusable context. The next Process call creates them again. + /// public void ReleaseBuffers() { if (_pingRT) @@ -182,6 +235,7 @@ internal class FrameHookPipeline : IDisposable _reusableContext = null; } + /// Same as . Main thread (Object.Destroy is used). public void Dispose() { ReleaseBuffers(); diff --git a/Core/PostRenderHook.cs b/Core/PostRenderHook.cs index e8e49a9..b6ac16c 100644 --- a/Core/PostRenderHook.cs +++ b/Core/PostRenderHook.cs @@ -5,10 +5,22 @@ using UnityEngine; namespace ADOFAIRenderer.Core; +/// +/// Per-frame hook attached to the scrCamera GameObject by RenderManager when the game scene of a render has loaded, +/// and destroyed again when a render is cancelled or fails. +/// Execution order int.MaxValue: its LateUpdate runs after every other script's LateUpdate of the frame. +/// Each frame it calls RenderManager.OnLastUpdate (LateUpdate) and, after the frame has been rendered +/// (end of frame), RenderManager.OnPostFrame with the texture shown on scrCamera's camera quad, which is the frame +/// that gets captured. +/// Reference: https://docs.unity3d.com/ScriptReference/DefaultExecutionOrder.html +/// [DefaultExecutionOrder(int.MaxValue)] internal class PostRenderHook : MonoBehaviour { + /// Private field scrCamera.camQuadMesh: the MeshRenderer of the quad that displays the camera RenderTexture. private FieldInfo? _camQuadMeshField; + + /// Cached value of camQuadMesh for the current scrCamera instance. Looked up again while it is null or destroyed. private MeshRenderer? _camQuadMeshRenderer; private void Awake() @@ -16,6 +28,10 @@ internal class PostRenderHook : MonoBehaviour _camQuadMeshField = AccessTools.Field(typeof(scrCamera), "camQuadMesh"); } + /// + /// Main thread, once per frame after all other LateUpdate calls. Runs the last-update work of RenderManager + /// (OnLastUpdate callbacks of the session) and schedules the end-of-frame capture. + /// private void LateUpdate() { if (RenderManager.Instance) @@ -24,6 +40,11 @@ internal class PostRenderHook : MonoBehaviour PostFrameAsync().Forget(); } + /// + /// Waits until the end of the current frame (after the cameras have rendered) and passes the camera quad's main + /// texture to RenderManager.OnPostFrame. The wait is cancelled when this component is destroyed. + /// The texture is null when the quad's material has no RenderTexture; OnPostFrame then only updates the audio. + /// private async UniTaskVoid PostFrameAsync() { await UniTask.WaitForEndOfFrame(this, destroyCancellationToken); diff --git a/Core/PresetManager.cs b/Core/PresetManager.cs index eaa7e6a..97725a7 100644 --- a/Core/PresetManager.cs +++ b/Core/PresetManager.cs @@ -11,11 +11,16 @@ using Newtonsoft.Json.Linq; namespace ADOFAIRenderer.Core; +/// A named render configuration, stored as one JSON file in the presets directory. public class RenderPreset { + /// Display name of the preset. A file without a name uses the file name (without extension). public string Name { get; set; } = string.Empty; + + /// Render configuration of the preset. public RenderConfig Config { get; set; } = null!; + /// Returns a copy with a copied configuration. public RenderPreset Clone() { return new RenderPreset @@ -26,8 +31,18 @@ public class RenderPreset } } +/// +/// Loads, saves and deletes render presets. Presets are JSON files (one per preset, file name derived from the preset +/// name) in <game root>/UserData/ADOFAIRenderer/Presets. The built-in presets are written once, on first use +/// (GlobalConfig.DefaultPresetsInitialized), so the user can edit or delete them. Main thread. +/// public static class PresetManager { + /// + /// Built-in presets (H.264 CBR, 48 kHz stereo AAC at 256 kbps, mkv): 1080p, 1440p and 2160p at 60 FPS video. + /// "(4x)" presets render 240 game frames per second (4 per video frame); "(1x)" renders 60. + /// Video bitrates are in kbps. + /// private static readonly List DefaultPresets = new() { new RenderPreset @@ -96,6 +111,10 @@ public static class PresetManager } }; + /// + /// <root>/UserData/ADOFAIRenderer/Presets, where root is MelonLoader's game root directory, or the application + /// base directory when that is unavailable. + /// private static string GetPresetsDirectory() { string? root = null; @@ -125,6 +144,10 @@ public static class PresetManager return Path.GetInvalidFileNameChars().Aggregate(name, (current, c) => current.Replace(c, '_')); } + /// + /// Creates the presets directory and, once per installation (GlobalConfig.DefaultPresetsInitialized), writes the + /// built-in presets whose files do not exist yet, then saves the flag. Errors are logged, not thrown. + /// public static void EnsureDefaultPresetsCreated() { try @@ -164,6 +187,12 @@ public static class PresetManager } } + /// + /// Saves a copy of as preset , overwriting the file of the same + /// safe file name (see GetSafeFileName). + /// + /// The name is empty or white space. + /// The directory or file cannot be written (not caught). public static void SavePreset(string name, RenderConfig config) { if (string.IsNullOrWhiteSpace(name)) @@ -195,6 +224,13 @@ public static class PresetManager File.WriteAllText(filePath, json); } + /// + /// Loads every *.json preset file (after creating the built-in presets on first use). Files written by older + /// versions are migrated in memory (old key names and flat properties moved into the nested configs); the files are + /// not rewritten. A file that cannot be read is logged and skipped. When two files have the same preset name + /// (case-insensitive), the first one in directory order is used. Returns copies of the built-in presets when the + /// directory does not exist. + /// public static List GetPresets() { EnsureDefaultPresetsCreated(); @@ -332,6 +368,10 @@ public static class PresetManager return presets; } + /// + /// Renames key to : a non-null old value overwrites the target + /// key, and the old key is always removed. + /// private static void TryChangeKey(JObject configObj, string keyBefore, string targetKey) { var legacyToken = configObj[keyBefore]; @@ -341,6 +381,12 @@ public static class PresetManager configObj.Remove(keyBefore); } + /// + /// Deletes the file of preset : first the file with the safe file name; otherwise the first + /// file whose base name or stored "Name" matches (case-insensitive). Does nothing for an empty name or when no file + /// matches. + /// + /// The file with the safe file name exists but cannot be deleted (not caught). public static void DeletePreset(string name) { if (string.IsNullOrWhiteSpace(name)) return; diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index bb41f58..43ced1e 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -25,24 +25,61 @@ using ThreadPriority = System.Threading.ThreadPriority; namespace ADOFAIRenderer.Core; +/// +/// Progress of the current render (RenderManager.stage). Values are ordered: checks use comparisons such as +/// stage >= OnScene. +/// +/// None: no render. Set at start-up and whenever a render ends, is cancelled or fails (ResetRenderState). +/// Load: RenderSession.Start was called; the game scene (scnGame) is being loaded. +/// OnScene: the game scene has loaded (RenderManager.Update set up the scene): the frame counter runs, the +/// 3-second countdown is shown, input is ignored. Nothing is captured yet. RenderManager.IsActive is true from this stage. +/// Render: the encoder and render thread run and frames are captured (set by SetupPlay after the countdown). +/// RenderManager.IsCapturingActive is true; the time patches replace Unity's clock with the render clock. +/// +/// internal enum RenderStage { None, Load, OnScene, Render } +/// +/// Options of the current render, copied from a by RenderSession.Start and read by +/// RenderManager and the Harmony patches. restores the defaults when the render environment is +/// restored (RenderManager.RestoreRenderEnvironment). Main thread only. +/// internal static class RenderContext { + /// Value assigned to RDC.auto (the game's autoplay flag) when the game scene of the render has loaded. public static bool Auto = true; + + /// + /// Enables the time patches (Time.time / unscaledTime / deltaTime / frameCount, AudioSettings.dspTime) while capturing, + /// which drive the game by the render clock (frameCount / RenderFPS) instead of real time. + /// public static bool TimePatch = true; + + /// Enables the Screen.width / Screen.height patches (report the render resolution) while the render is active. public static bool ScreenPatch = true; + + /// Output file path of the render, or null for a time-stamped file in GlobalSettings.EffectiveOutputDirectory. public static string? OutputPath; + /// Whether the default OnComplete callback shows an OS notification after a successful render. public static bool Notification = true; + + /// Frame hooks of the current session, run before the global hooks (see RenderSession.AddFrameHook). public static readonly List> SessionFrameHooks = new(); + /// Default end condition: the level is cleared (scrController state Won). private static readonly Func EndConditionDefault = () => scrController.instance.state == States.Won; + + /// + /// Polled by RenderManager.Update every frame from the OnScene stage. When it first returns true, the render + /// continues for RenderConfig.DelayAfterEnd seconds of render time and then ends successfully. + /// public static Func EndCondition = EndConditionDefault; + /// Default completion callback: shows an OS notification for a successful render when Notification is true. private static readonly Action OnCompleteDefault = result => { if (result.Success && Notification) @@ -54,11 +91,23 @@ internal static class RenderContext OSNotification.Show("Rendering Complete!", $"{fileString} saved successfully.", renderFolder); } }; + /// + /// Called once on the main Unity thread when the render finishes (success) or is cancelled or fails. + /// A session callback replaces the default one, so no notification is shown when a session sets OnComplete. + /// Exceptions are caught and logged (RenderManager.TriggerOnComplete). + /// public static Action OnComplete = OnCompleteDefault; private static readonly Action OnLastUpdateDefault = () => { }; + + /// + /// Called on the main Unity thread every frame after all other LateUpdate calls, through PostRenderHook (attached when the + /// game scene of the render has loaded) while the stage is Load or later. + /// Exceptions are caught and logged (RenderManager.TriggerOnLastUpdate). + /// public static Action OnLastUpdate = OnLastUpdateDefault; + /// Restores every option to its default and clears the session frame hooks. public static void Reset() { Auto = true; @@ -74,15 +123,38 @@ internal static class RenderContext } } +/// +/// Drives a render from scene setup to the finished video file. Singleton MonoBehaviour on a DontDestroyOnLoad object +/// created by RenderMod. +/// +/// Flow: RenderSession.Start (stage Load) → SetupGameScene loads scnGame → Update sets up the loaded scene +/// (stage OnScene, 3-second countdown) → SetupPlay starts the encoder and the render thread (stage Render) → +/// every frame OnPreUpdate advances frameCount, OnPostFrame queues the audio commands and requests the GPU capture of +/// the frame (FrameReader, AsyncGPUReadback) → readback completions add frames to ImageQueue → the render thread +/// (RunRenderThread) encodes them in frame order together with the audio of each frame. +/// +/// End: success (EndCondition plus DelayAfterEnd) → StopFFmpeg(true) → StopRenderCoroutine waits for the render +/// thread to finish the file → EndRender. Cancel (ESC) or failure → StopFFmpeg(false) → EndRender, which deletes the +/// unfinished file. Encoder initialization failure → StopFFmpeg(false) → RestoreRenderEnvironment and an error dialog. +/// +/// Threads: the main Unity thread runs every public method unless noted otherwise. The render thread reads +/// ImageQueue (lock on ImageQueue), passCount (render thread only while it runs), endRequested / endSuccessful +/// (volatile), the abort state (volatile, written under _abortStateLock) and lastRequestedVideoFrameIndex +/// (Volatile.Read/Write). FrameAvailableEvent wakes the render thread, FrameDequeuedEvent wakes the throttle loop. +/// internal class RenderManager : MonoBehaviour { - // Singleton Accessors + /// The single instance, set in Awake. Unity null checks (if (Instance)) are used before access. public static RenderManager Instance { get; private set; } = null!; + + /// True from the OnScene stage (scene set up, countdown running) until the render ends. public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene; + + /// True while frames are captured and encoded (Render stage). The time patches apply only then. public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render; /// - /// Maximum number of frames in ImageQueue. The main thread waits (throttles) while the queue holds more frames. + /// Maximum number of frames in ImageQueue. The main Unity thread waits (throttles) while the queue holds more frames. /// public const int MaxQueuedFrames = 64; @@ -104,26 +176,66 @@ internal class RenderManager : MonoBehaviour public int QueueFrameLimit { get; private set; } = MaxQueuedFrames; // Statistics + + /// + /// Number of video frames sent to the encoder in this render. Incremented by the render thread (Interlocked), + /// read with Interlocked.Read by the main Unity thread (result, debug monitor). Reset in SetupPlay. + /// public long framesSent; + + /// Wall-clock time of the render from the end of the countdown (SetupPlay) to BuildResult. private readonly Stopwatch _renderStopwatch = new(); // Core Render Configuration and Path + + /// + /// Seconds left in the countdown before capture starts (3 when the scene is set up, decremented once per real-time + /// second by SetupPlay). Shown by scrShowIfDebugPatch and read by scrPlayerManagerPatch. + /// public int countdown; + + /// + /// Render frame counter: incremented once per Unity frame by OnPreUpdate from the OnScene stage and reset to 0 when + /// capture starts (SetupPlay). The render clock is frameCount / RenderFPS seconds (AudioTime, time patches). + /// Main thread only. + /// public long frameCount; + + /// + /// Set by SetupGameScene; scrPressToStartPatch consumes it (sets it back to false) to skip the press-to-start + /// screen once. Cleared by StopFFmpeg. + /// public bool skipPressToStart; + + /// Path of the level file to render (ADOBase.levelPath of the editor, or the path given to RenderSession.Start). public string levelPath = ""; + + /// Serializes changes of the abort state (_isRenderCancelled, _renderFailed, _failureReason). private readonly object _abortStateLock = new(); + + /// The render was cancelled (ESC, or a non-successful stop). Written under _abortStateLock, read by any thread. private volatile bool _isRenderCancelled; + + /// The render failed (MarkFailed). Written under _abortStateLock, read by any thread. private volatile bool _renderFailed; + + /// Reason given to the first MarkFailed call of this render; null when the render has not failed. private volatile string? _failureReason; + /// True when the render was cancelled. Thread-safe read. // ReSharper disable once MemberCanBePrivate.Global public bool IsRenderCancelled => _isRenderCancelled; + + /// True when the render failed. Thread-safe read. public bool IsRenderFailed => _renderFailed; - // Cancelled and failed are mutually exclusive; this is true for either (render must stop). + /// + /// True when the render was cancelled or failed (the two are mutually exclusive) and must stop. + /// Read by the render thread before each frame and by readback completions. Thread-safe read. + /// public bool IsRenderAborted => _isRenderCancelled || _renderFailed; + /// Marks the render as cancelled unless it has already failed (a failure keeps its reason). private void MarkCancelled() { lock (_abortStateLock) @@ -132,7 +244,10 @@ internal class RenderManager : MonoBehaviour } } - // The first failure reason wins; ignored once the render is already cancelled or failed. + /// + /// Marks the render as failed with (shown in the result message as "Rendering failed: reason"). + /// The first failure reason wins; ignored once the render is already cancelled or failed. Callable from any thread. + /// private void MarkFailed(string reason) { lock (_abortStateLock) @@ -143,6 +258,7 @@ internal class RenderManager : MonoBehaviour } } + /// Clears cancelled, failed and the failure reason. Called when a new encoder starts (StartFFmpegThread). private void ClearAbortState() { lock (_abortStateLock) @@ -153,30 +269,70 @@ internal class RenderManager : MonoBehaviour } } + /// Error message of the RenderResult of an aborted render (failure reason, or the cancel message). private string BuildAbortMessage() => IsRenderFailed ? $"Rendering failed: {_failureReason ?? "unknown error"}" : "Rendering was cancelled by user."; + + /// + /// Configuration of the current render: a copy made by RenderSession.Start, so values adjusted during the render + /// (the audio sample rate in SetupPlay) do not change the caller's object. Read by the main Unity thread, the render + /// thread (audio settings) and the patches. Null until the first render starts. + /// public RenderConfig Configuration = null!; // Render Sessions State (OOP Instance Fields) + + /// Current stage of the render. Written on the main Unity thread; read by the render thread in TryDequeueNextFrame. public RenderStage stage = RenderStage.None; + + /// Time.timeAsDouble when capture started. The time patch returns timeBase + frameCount / RenderFPS. public double timeBase; + + /// Time.unscaledTimeAsDouble when capture started. The time patch returns unscaledTimeBase + frameCount / RenderFPS. public double unscaledTimeBase; + + /// + /// AudioSettings.dspTime when capture started (seconds). The dspTime patch returns dspTimeBase + frameCount / RenderFPS; + /// audio command times are stored relative to it (scheduled dspTime - dspTimeBase). + /// public double dspTimeBase; + + /// Time.frameCount when capture started. The frameCount patch returns frameCountBase + frameCount. public int frameCountBase; + /// + /// Offline mixer of the song and sound effects of this render; null when audio is not rendered or no render runs. + /// Created in SetupPlay. Commands are added on the main Unity thread (OnPostFrame, audio patches), mixing runs on the + /// render thread (WriteAudioForFrame); the mixer synchronizes the two internally. + /// public OfflineAudioMixer? AudioMixer { get; private set; } /// Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands. public double AudioTime => frameCount / Configuration.RenderFPS; + /// + /// Number of audio samples (per channel) to write after each video frame, so that the audio stays aligned with + /// the video frame times at the effective video frame rate. Created in SetupPlay when audio is rendered; + /// used by the render thread only. + /// private AudioSyncCalculator? SyncCalculator { get; set; } + /// GPU capture of frames (color conversion and AsyncGPUReadback). Created in SetupPlay after the encoder started. private FrameReader? _frameReader; + + /// Frame hook runner of this render. Created in SetupPlay, disposed by ResetFrameHookState. private FrameHookPipeline? _frameHookPipeline; + + /// Session hooks followed by global hooks, rebuilt when the global hook version changes. private readonly List> _activeHooksCache = new(); + + /// RenderSession.GlobalHooksVersion the cache was built for; -1 forces a rebuild. private int _cachedGlobalHooksVersion = -1; + + /// True when _activeHooksCache is not empty. private bool _hasActiveHooksCached; + /// True when at least one session or global frame hook is registered. Main thread. public bool HasActiveFrameHooks { get @@ -186,6 +342,11 @@ internal class RenderManager : MonoBehaviour } } + /// + /// Rebuilds the active hook list when the global hook registrations changed since the last build. + /// Session hooks are only read on a rebuild; SetupPlay forces one (version -1) at the start of capture, so changes to + /// RenderContext.SessionFrameHooks after that are picked up only together with a global hook change. + /// private void EnsureActiveHooksUpToDate() { var globalVersion = RenderSession.GlobalHooksVersion; @@ -201,12 +362,14 @@ internal class RenderManager : MonoBehaviour _hasActiveHooksCached = _activeHooksCache.Count > 0; } + /// Returns the cached active hook list (session hooks first, then global hooks). The list is reused. private List> GetActiveFrameHooks() { EnsureActiveHooksUpToDate(); return _activeHooksCache; } + /// Disposes the frame hook pipeline (ping-pong buffers) and clears the hook cache. private void ResetFrameHookState() { _frameHookPipeline?.Dispose(); @@ -216,46 +379,132 @@ internal class RenderManager : MonoBehaviour _hasActiveHooksCached = false; } + /// + /// Number of game frames offered for capture since capture started (OnPostFrame with a texture, after throttling). + /// The value before the increment is the render frame index of the current frame. Main thread only. + /// public long frameCountForVideo; + + /// + /// Video frame index of the last captured frame (-1 before the first). A game frame is captured only when its video + /// frame index (render frame index × effective video FPS / RenderFPS, rounded down) is greater. Main thread only. + /// public long lastVideoFrameIndex = -1; - // long cannot be volatile: access via Volatile.Read/Write (written on main thread, read on render thread) + + /// + /// Video frame index of the last capture request (-1 before the first). Written on the main Unity thread, read by the + /// render thread to know when every requested frame has been sent after endRequested. + /// + // long cannot be volatile: access via Volatile.Read/Write (written on main Unity thread, read on render thread) public long lastRequestedVideoFrameIndex = -1; + + /// + /// Number of GPU readbacks that failed and were replaced by a black frame (shown by the debug monitor). + /// Incremented in the readback completion callback. + /// public int readbackFailureCount; + + /// Number of frames on which the main Unity thread had to wait for the render thread (throttle loop entries). public int throttleCount; + + /// True while the main Unity thread is inside the throttle wait of OnPostFrame. public bool isThrottling; + /// + /// One frame waiting in ImageQueue: either a native readback buffer (owned by ReadbackState until it is recycled) + /// or a managed buffer rented from ArrayPool<byte>.Shared (black frame for a failed readback). + /// The data is YUV 4:2:0 in the encoder's pixel format when IsColorConverted, RGBA otherwise. + /// internal struct CapturedFrame { + /// Start of the native frame data (ReadbackState.NativeBuffer); zero for a managed frame. public IntPtr NativePointer; + + /// Number of valid bytes of the frame data. public int Length; + + /// Managed frame data rented from ArrayPool<byte>.Shared (may be longer than Length); null for a native frame. public byte[]? ManagedBuffer; // Non-null if it's a managed buffer (e.g. black frame) + + /// Readback state owning NativePointer; returned to the FrameReader pool after the frame is encoded or dropped. public ReadbackRequestState? ReadbackState; // Associated state to recycle + + /// True when the GPU converted the frame to the encoder's YUV format; false for RGBA data. public bool IsColorConverted; } // Thread and Queue State + + /// + /// Captured frames waiting for the render thread, keyed by video frame index. Readbacks may complete in any order; + /// the render thread takes them in index order (passCount). Every access locks the list itself. + /// Frames are added by readback completions (main Unity thread) and removed by the render thread, or released by the + /// cleanup in StopFFmpeg / RestoreRenderEnvironment. + /// internal SortedList ImageQueue = new(); + + /// Set when a frame is added to ImageQueue or the end is requested; the render thread waits on it. internal AutoResetEvent FrameAvailableEvent = new(false); - // Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it. + + /// Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it. internal AutoResetEvent FrameDequeuedEvent = new(false); + + /// The render thread of the current render (background, AboveNormal priority); null before the first render. public Thread? RenderThread; + + /// + /// Video frame index the render thread sends next. Reset by StartFFmpegThread before the thread starts; afterwards + /// read and written only by the render thread, under the ImageQueue lock. + /// public long passCount; + + /// Set by StopFFmpeg: no more frames will be captured; the render thread drains the queue and stops. public volatile bool endRequested; + + /// Set by the render thread after the output file was finished and the encoder released without an abort. public volatile bool endSuccessful; + + /// + /// frameCount at which EndCondition first returned true; -1 until then. The render ends DelayAfterEnd seconds of + /// render time later. + /// public long endFrame = -1; // Private Instance Fields + + /// Encoder of the current render; null when none runs. Cleared when the render thread releases it. private FFmpegNativeEncoder? _activeVideoFFmpeg; + + /// + /// Every encoder created and not yet released, disposed by CleanUpActiveRender on exit. Locked on the list itself + /// (the render thread removes its encoder when it finishes). + /// private readonly List _allVideoFFmpegInstances = new(); + + /// The user's input offset (scrConductor.currentPreset.inputOffset), set to 0 during the render and restored afterwards. private int _originalInputOffset; + + /// Debug monitor overlay object, created when the encoder starts and destroyed when the render ends. private GameObject? _debugMonitorObj; + + /// ESC handler overlay object (cancels the render), created when the scene is set up and destroyed by StopFFmpeg. private GameObject? _escapeHandlerObj; + + /// + /// Output file path of the current or last render. Set when the encoder starts; set to null when an aborted render's + /// file was deleted. + /// internal string? CurrentOutPath; // Output file of the current render that is not finished yet. // Set just before the encoder opens the file, cleared by the render thread after the encoder finished the file (End). // Only this file is deleted when a render is cancelled, fails or the game exits (DeleteIncompleteOutputFile). private string? _incompleteOutputPath; + + /// + /// True from SetupGameScene until Update has set up the loaded game scene (or the render stops). While true, Update + /// hides the difficulty UI and waits for scnGame to finish loading. + /// private bool _renderSetup; private void Awake() @@ -268,6 +517,10 @@ internal class RenderManager : MonoBehaviour CleanUpActiveRender(); } + /// + /// Sets the stage to None. true clears the cancelled flag; false marks the render as + /// cancelled (unless it already failed). Called at every end path before StopFFmpeg. + /// public void ResetRenderState(bool success) { if (success) _isRenderCancelled = false; @@ -275,6 +528,11 @@ internal class RenderManager : MonoBehaviour stage = RenderStage.None; } + /// + /// Emergency cleanup when the game quits or this object is destroyed: marks a running render as failed, releases + /// the audio, the frame hooks and every encoder, waits up to 200 ms for the render thread, and deletes the + /// unfinished output file. Does not restore the game scene or settings. Main thread. + /// public void CleanUpActiveRender() { if (stage != RenderStage.None) @@ -346,8 +604,16 @@ internal class RenderManager : MonoBehaviour } } + /// + /// Main thread, every frame. + /// 1. While the game scene of a render loads: hides the difficulty UI of the press-to-start screen. + /// 2. Once scnGame has finished loading: sets up the scene for rendering and starts SetupPlay (stage OnScene). + /// 3. From the OnScene stage: fails the render when the render thread died, the level failed or the controller is + /// gone; ends it successfully DelayAfterEnd seconds (render time) after EndCondition first returned true. + /// private void Update() { + // scrPressToStart reads lm.hideDifficultyUI to decide whether the difficulty UI is shown (scrPressToStart.cs). if (_renderSetup && ADOBase.lm) { ADOBase.lm.hideDifficultyUI = true; @@ -356,18 +622,24 @@ internal class RenderManager : MonoBehaviour if (_renderSetup && scnGame.instance && !scnGame.instance.isLoading) { _renderSetup = false; + // scrCamera.Awake calls SetupRTCam(enable: true) when forceRTCam is set (scrCamera.cs), so the flag applies to + // the next scrCamera.Awake. scnGame also calls SetupRTCam(enable: true) when it starts playing (scnGame.cs). scrCamera.instance.forceRTCam = true; if (!scrCamera.instance.gameObject.GetComponent()) scrCamera.instance.gameObject.AddComponent(); + // The input offset shifts the song position (calibration_i = inputOffset / 1000 in scrConductor); + // the render runs with 0 so audio and visuals follow the level timing exactly. Restored by RestoreRenderEnvironment. _originalInputOffset = scrConductor.currentPreset.inputOffset; scrConductor.currentPreset.inputOffset = 0; + // Autoplay; disableCongratsMessage clears the "congratulations" text shown on the win screen (scrController.cs). RDC.auto = RenderContext.Auto; AccessTools.Field(typeof(scrController), "disableCongratsMessage") .SetValue(scrController.instance, true); + // No-fail with an infinite hit margin, so a missed hit does not fail the level during the render. if (scrController.instance) { scrController.instance.noFail = true; @@ -384,6 +656,9 @@ internal class RenderManager : MonoBehaviour DontDestroyOnLoad(_escapeHandlerObj); } + // While non-zero, Unity advances Time.time by exactly this interval per frame regardless of real time, + // so the game simulation steps 1 / RenderFPS seconds per frame. Reset to 0 by RestoreRenderEnvironment. + // Reference: https://docs.unity3d.com/ScriptReference/Time-captureDeltaTime.html Time.captureDeltaTime = (float)(1 / Configuration.RenderFPS); scrController.instance!.StartCoroutine(SetupPlay()); } @@ -404,6 +679,8 @@ internal class RenderManager : MonoBehaviour } else if (RenderContext.EndCondition()) { + // The first frame the end condition holds starts the DelayAfterEnd period; afterwards the end condition is + // still polled every frame, but endFrame is kept. if (endFrame == -1) { endFrame = frameCount; @@ -417,6 +694,9 @@ internal class RenderManager : MonoBehaviour } } + /// + /// Called by FirstUpdateHook before every other Update of the frame. Advances frameCount by one from the OnScene stage. + /// public void OnPreUpdate() { if (Instance.stage < RenderStage.OnScene) return; @@ -428,6 +708,10 @@ internal class RenderManager : MonoBehaviour private List? _holdFloorsSource; private int _holdFloorsSourceCount = -1; + /// + /// Called by PostRenderHook after every other LateUpdate of the frame, from the Load stage. + /// Hides the hold meshes when AdvancedConfig.NonTile is set, then runs RenderContext.OnLastUpdate. + /// public void OnLastUpdate() { if (Instance.stage < RenderStage.Load) return; @@ -479,6 +763,15 @@ internal class RenderManager : MonoBehaviour return (int)Math.Min(MaxQueuedFrames, Math.Max(4, MaxQueuedBytes / frameBytes)); } + /// + /// Called by PostRenderHook at the end of every frame (after rendering) with the frame texture. In the Render stage: + /// 1. Audio: updates the song voice, commits the audio commands of this frame, and deactivates the effect + /// AudioSources that were mapped to mixer voices. + /// 2. Throttle: blocks while ImageQueue holds more than QueueFrameLimit frames (render thread behind). + /// 3. Capture: when this game frame starts a new video frame, runs the frame hooks and requests the GPU readback. + /// With RenderFPS above the effective video FPS, only some game frames are captured. + /// + /// Frame shown by the camera quad; null skips throttling and capture. // ReSharper disable once Unity.IncorrectMethodSignature public void OnPostFrame(RenderTexture? texture) { @@ -495,7 +788,7 @@ internal class RenderManager : MonoBehaviour if (!texture) return; // [Backpressure Throttling] - // The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread + // The main Unity thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread // removes frames, and it signals FrameDequeuedEvent for each one. The wait has a timeout so that the loop // re-checks the render thread state and pumps pending readback completions (see ThrottleWaitMilliseconds). isThrottling = false; @@ -532,10 +825,13 @@ internal class RenderManager : MonoBehaviour var currentFrame = frameCountForVideo++; + // The video cannot have more frames per second than the game renders: effective video FPS = min(VideoFPS, RenderFPS). var renderFPS = Configuration.RenderFPS; var videoFps = Configuration.VideoFPS; var effectiveVideoFps = Math.Min(videoFps, renderFPS); + // Video frame that contains this game frame's time (currentFrame / renderFPS seconds), rounded down. + // Only the first game frame of each video frame is captured. var videoFrameIndex = (long)(currentFrame * effectiveVideoFps / renderFPS); if (videoFrameIndex > lastVideoFrameIndex) @@ -562,6 +858,13 @@ internal class RenderManager : MonoBehaviour } } + /// + /// Handler of the render button in the level editor (added by scnEditorPatch). + /// When the level has no file path, scnEditor.TryQuitToMenu is called and no render starts. + /// Otherwise, after the editor's unsaved-changes prompt, removes scnEditor.TryApplicationQuit from + /// Application.wantsToQuit (it would block quitting with an unsaved-changes prompt; scnEditor registers it again + /// when the editor starts) and opens the render configuration window. Main thread. + /// public static void StartRender() { var targetMethod = typeof(scnEditor).GetMethod("TryApplicationQuit", @@ -585,12 +888,18 @@ internal class RenderManager : MonoBehaviour }); } + /// Creates the default output directory (GlobalSettings.EffectiveOutputDirectory) when it does not exist. public void SetupCaptureDirectory() { var renderDir = GlobalSettings.EffectiveOutputDirectory; FileUtil.CreateDirectory(renderDir); } + /// + /// Loads scnGame with levelPath as the only custom level (checkpoint 0, speed trial multiplier 1) and arms the + /// press-to-start skip and the scene setup in Update. Seeds UnityEngine.Random when AdvancedConfig.RandomSeed is set, + /// so random effects repeat between renders. Called by RenderSession.Start. + /// public void SetupGameScene() { GCS.customLevelPaths = new[] { levelPath }; @@ -605,8 +914,18 @@ internal class RenderManager : MonoBehaviour _renderSetup = true; } + /// + /// Coroutine started on scrController when the game scene is set up (stage OnScene). + /// Waits the countdown (real-time seconds), sets the time bases, starts the encoder, the render thread and the + /// frame reader, switches to the Render stage, starts the level without the press-to-start wait, and creates the + /// offline audio mixer and spectrum generator. Stops without starting when the render was aborted during the + /// countdown. On an encoder initialization error, the render fails, the environment is restored and an error + /// dialog returns to the editor. + /// private IEnumerator SetupPlay() { + // The audio sample rate must be one the audio codec paired with the video codec supports + // (CodecUtil.AudioCodecMatch, CodecUtil.AvailableSampleRate); otherwise 48000 Hz is used. if (Configuration.AudioConfig.RenderAudio) { if (!CodecUtil.AvailableSampleRate[CodecUtil.AudioCodecMatch[Configuration.VideoConfig.Codec]].Contains(Configuration.AudioConfig.AudioSampleRate)) @@ -624,6 +943,8 @@ internal class RenderManager : MonoBehaviour frameCount = 0; framesSent = 0; _renderStopwatch.Restart(); + // Clock values at the start of capture. The time patches are not active yet (stage OnScene), so these are + // Unity's own values; from the Render stage the patched clocks continue from them at frameCount / RenderFPS. timeBase = Time.timeAsDouble; unscaledTimeBase = Time.unscaledTimeAsDouble; dspTimeBase = AudioSettings.dspTime; @@ -635,6 +956,7 @@ internal class RenderManager : MonoBehaviour _frameHookPipeline = new FrameHookPipeline(); _cachedGlobalHooksVersion = -1; EnsureActiveHooksUpToDate(); + // The frame reader converts on the GPU to the pixel format of the encoder that was actually opened. try { StartFFmpegThread(); @@ -663,6 +985,7 @@ internal class RenderManager : MonoBehaviour } Instance.stage = RenderStage.Render; + // levelWasSkipped ends the press-to-start wait loop of scrController (scrController.cs, WaitForStartCo). scrController.instance.paused = scrController.instance.audioPaused = false; scrController.instance.levelWasSkipped = true; @@ -681,10 +1004,18 @@ internal class RenderManager : MonoBehaviour AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer); } + // The song is played by the offline mixer only: the clip is removed from the game's song AudioSource so Unity + // does not play it. scrConductor.instance.song.clip = null; scrConductor.instance.song.time = -1000; } + /// + /// Resets the queue and render thread state, opens the encoder on the output file (RenderContext.OutputPath, or + /// "yyyy-MM-dd HH-mm-ss.ext" in the output directory with the configured extension), and starts the render thread. + /// The output path is registered as incomplete before the encoder opens the file. + /// + /// The encoder could not be created or started; it is disposed before the exception is rethrown. private void StartFFmpegThread() { if (!_debugMonitorObj) @@ -770,6 +1101,7 @@ internal class RenderManager : MonoBehaviour throw; } + // Background thread: it does not keep the process alive when the game quits. var encoder = videoFFmpeg; RenderThread = new Thread(() => RunRenderThread(encoder)) { @@ -858,8 +1190,10 @@ internal class RenderManager : MonoBehaviour } /// - /// Removes the frame whose key equals passCount from ImageQueue and advances passCount. - /// Before the Render stage, passCount jumps to the first queued key (the first captured frame index is not 0). + /// Removes the frame whose key equals passCount from ImageQueue and advances passCount. Render thread. + /// While the stage is below Render, passCount jumps forward to the first queued key when that key is greater: + /// frames are only captured in the Render stage, so this happens after the stage was set back to None at the end + /// of the render while the thread drains the queue, and a frame whose readback has not arrived yet is skipped. /// Returns false when the queue is empty or the next frame has not arrived yet. /// private bool TryDequeueNextFrame(out CapturedFrame frame) @@ -886,7 +1220,7 @@ internal class RenderManager : MonoBehaviour /// /// Sends one captured frame to the encoder and releases its buffer (managed: ArrayPool, native: readback state). - /// A frame without data is skipped. + /// A frame without data is skipped (not counted in framesSent). Render thread. /// private void SendVideoFrame(FFmpegNativeEncoder videoFFmpeg, in CapturedFrame frame) { @@ -914,8 +1248,11 @@ internal class RenderManager : MonoBehaviour /// /// Mixes the audio of the frame that was just sent (index passCount - 1) and writes it to the encoder. - /// Does nothing when audio rendering is off. + /// The sample count per frame comes from SyncCalculator (sample rate / effective video FPS, with the remainder + /// carried to later frames). Mixing waits until the main Unity thread has committed the audio commands up to that time. + /// Does nothing when audio rendering is off. Render thread. /// + /// The mixer was cancelled (render aborted) while waiting. private void WriteAudioForFrame(FFmpegNativeEncoder videoFFmpeg) { if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return; @@ -945,6 +1282,7 @@ internal class RenderManager : MonoBehaviour return endRequested && passCount > Volatile.Read(ref lastRequestedVideoFrameIndex); } + /// Removes the encoder from the cleanup list and clears _activeVideoFFmpeg. Called by the render thread when it finishes. private void UnregisterEncoder(FFmpegNativeEncoder videoFFmpeg) { lock (_allVideoFFmpegInstances) @@ -954,6 +1292,11 @@ internal class RenderManager : MonoBehaviour _activeVideoFFmpeg = null; } + /// + /// Stops the render without finishing the file (cancelled, or failed when MarkFailed was called first), deletes + /// the unfinished file, reports the result through OnComplete, restores the environment and returns to the editor. + /// Called by the ESC handler and on failures. Main thread. + /// public void CancelRender() { ResetRenderState(success: false); @@ -962,6 +1305,15 @@ internal class RenderManager : MonoBehaviour EndRender(); } + /// + /// Requests the end of encoding. Main thread. + /// true: completes the audio commands and starts StopRenderCoroutine, which waits for + /// the render thread to send the remaining frames and finish the file. + /// false: marks the render as cancelled (unless failed), cancels the audio waits, waits up + /// to 2 s for the render thread, disposes the encoder, releases the queued frames, the frame reader, the hooks and + /// the debug monitor, restores the camera and no-fail flags, deletes the unfinished file and calls OnComplete with + /// a failure result. Does not restore time, input offset or the scene (EndRender / RestoreRenderEnvironment do). + /// private void StopFFmpeg(bool success) { endRequested = true; @@ -993,7 +1345,8 @@ internal class RenderManager : MonoBehaviour RenderThread.Join(2000); } - // 2. After ensuring thread termination, safely release FFmpeg native encoder resources + // 2. Release the encoder. The render thread normally released it already (_activeVideoFFmpeg is then null). + // When the thread is still running after the 2 s wait, the encoder is disposed while that thread may still use it. if (_activeVideoFFmpeg != null) { _activeVideoFFmpeg.Dispose(); @@ -1055,6 +1408,11 @@ internal class RenderManager : MonoBehaviour } } + /// + /// Successful end: waits (one check per frame) until the render thread reports endSuccessful or has died. + /// On success calls OnComplete with a success result and EndRender; when the thread died without finishing + /// the file, the render fails and is cancelled (file deleted). + /// private IEnumerator StopRenderCoroutine() { yield return new WaitUntil(() => endSuccessful || RenderThread is not { IsAlive: true }); @@ -1071,6 +1429,7 @@ internal class RenderManager : MonoBehaviour EndRender(); } + /// Restores the render environment and loads the level editor (scnEditor). private void EndRender() { RestoreRenderEnvironment(); @@ -1079,7 +1438,11 @@ internal class RenderManager : MonoBehaviour Cursor.visible = true; } - // Undoes what the render setup changed (time, input offset, auto, queues, session hooks), without leaving the scene. + /// + /// Undoes what the render setup changed (time scale and capture delta time, input offset, autoplay, custom level + /// paths, queued frames, frame reader, frame hooks, audio mixer and spectrum, RenderContext options), without + /// leaving the scene. + /// private void RestoreRenderEnvironment() { Time.timeScale = 1f; @@ -1122,6 +1485,10 @@ internal class RenderManager : MonoBehaviour } + /// + /// Stops the render stopwatch and creates the result: frames sent, video duration (frames / effective video FPS), + /// wall-clock render time, and the output path and file size on success. + /// private RenderResult BuildResult(bool success, string? errorMessage = null) { _renderStopwatch.Stop(); @@ -1133,6 +1500,7 @@ internal class RenderManager : MonoBehaviour : RenderResult.CreateFailure(errorMessage, sent, effectiveFps, _renderStopwatch.Elapsed); } + /// Invokes RenderContext.OnComplete; an exception from the callback is logged. private static void TriggerOnComplete(RenderResult result) { try @@ -1145,6 +1513,7 @@ internal class RenderManager : MonoBehaviour } } + /// Invokes RenderContext.OnLastUpdate; an exception from the callback is logged. private static void TriggerOnLastUpdate() { try diff --git a/Data/Codec.cs b/Data/Codec.cs index d9ee105..346785d 100644 --- a/Data/Codec.cs +++ b/Data/Codec.cs @@ -4,14 +4,22 @@ using FFmpeg.AutoGen; namespace ADOFAIRenderer.Data; +/// +/// Video codec of the output. The encoder implementation (hardware or software) is chosen at run time from the +/// encoders available for the codec (VideoEncoder, GPUUtil.GetPrioritizedEncoders). +/// Values: H264 (H.264 / AVC), H265 (H.265 / HEVC), VP9, AV1, VVC (H.266). +/// The audio codec paired with each value is given by CodecUtil.AudioCodecMatch. +/// [SuppressMessage("ReSharper", "InconsistentNaming")] public enum Codec { H264, H265, VP9, AV1, VVC } +/// Audio codec choice and supported sample rates per video codec. internal static class CodecUtil { + /// Audio codec encoded together with each video codec (AAC for H.264/H.265, Opus for the others). public static readonly Dictionary AudioCodecMatch = new() { [Codec.H264] = AVCodecID.AV_CODEC_ID_AAC, @@ -21,6 +29,10 @@ internal static class CodecUtil [Codec.VVC] = AVCodecID.AV_CODEC_ID_OPUS }; + /// + /// Output sample rates (Hz) the mod allows per audio codec. RenderManager.SetupPlay replaces a configured rate that + /// is not in the list with 48000 Hz. + /// public static readonly Dictionary> AvailableSampleRate = new() { [AVCodecID.AV_CODEC_ID_AAC] = new List { 44100, 48000, 96000 }, diff --git a/Data/ColorJsonConverter.cs b/Data/ColorJsonConverter.cs index 374ff85..28c501d 100644 --- a/Data/ColorJsonConverter.cs +++ b/Data/ColorJsonConverter.cs @@ -10,13 +10,20 @@ namespace ADOFAIRenderer.Data; /// Converts Unity and nullable to and from hex string (#RRGGBBAA). /// Pure C# implementation to completely avoid Unity internal native ECalls (such as Color.linear or ColorUtility.TryParseHtmlString). /// +/// +/// Reading accepts "#RRGGBB" (alpha 1), "#RRGGBBAA" (the '#' is optional), an object {"r","g","b","a"} with 0–1 floats +/// (missing r/g/b = 0, missing a = 1), or null. Invalid or unsupported input reads as null for Color? and as +/// default(Color) (all components 0) for Color; no exception is thrown for an unparsable string. +/// public class ColorJsonConverter : JsonConverter { + /// True for and . public override bool CanConvert(Type objectType) { return objectType == typeof(Color) || objectType == typeof(Color?); } + /// Writes the color as "#RRGGBBAA" (see ), or a JSON null for a null value. public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) { if (value == null) @@ -29,6 +36,7 @@ public class ColorJsonConverter : JsonConverter writer.WriteValue(ToHex(color)); } + /// Reads a color in one of the accepted forms (see the class remarks). public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) { bool isNullable = Nullable.GetUnderlyingType(objectType) != null || !objectType.IsValueType; @@ -62,6 +70,10 @@ public class ColorJsonConverter : JsonConverter return isNullable ? null : (object)default(Color); } + /// + /// Formats the color as "#RRGGBBAA" with uppercase hex digits. Each component is scaled to 0–255, rounded and clamped. + /// The components are written as stored (no gamma conversion). + /// public static string ToHex(Color color) { byte r = (byte)Math.Clamp((int)Math.Round(color.r * 255f), 0, 255); @@ -71,6 +83,12 @@ public class ColorJsonConverter : JsonConverter return $"#{r:X2}{g:X2}{b:X2}{a:X2}"; } + /// + /// Parses "RRGGBB" or "RRGGBBAA" (optional leading '#', surrounding white space ignored, case-insensitive). + /// + /// Text to parse. + /// Parsed color (components / 255; alpha 1 for the 6-digit form), or default when parsing fails. + /// True when the text was parsed. public static bool TryParseHex(string? hex, out Color color) { color = default; diff --git a/Data/RateControlMode.cs b/Data/RateControlMode.cs index 5e646fe..d4ea7cf 100644 --- a/Data/RateControlMode.cs +++ b/Data/RateControlMode.cs @@ -2,10 +2,19 @@ namespace ADOFAIRenderer.Data; +/// Video bitrate control mode (see VideoEncodingConfig and FFmpegNativeEncoder.ApplyQualitySettings). [SuppressMessage("ReSharper", "InconsistentNaming")] public enum RateControlMode { + /// Constant bitrate at VideoEncodingConfig.Bitrate (kbps). CBR, + + /// + /// Variable bitrate averaging VideoEncodingConfig.Bitrate (kbps), peaking at MaxBitrate (kbps) when it is larger, + /// otherwise at 1.5 × Bitrate. + /// VBR, + + /// Constant quality with VideoEncodingConfig.QP (lower is better; range depends on the encoder). CQP } \ No newline at end of file diff --git a/Data/RenderConfig.cs b/Data/RenderConfig.cs index 832db43..397c5ea 100644 --- a/Data/RenderConfig.cs +++ b/Data/RenderConfig.cs @@ -4,22 +4,50 @@ using UnityEngine; namespace ADOFAIRenderer.Data; +/// +/// Settings of one render: resolution, frame rates, encoding and advanced options. Serialized to JSON in the preset +/// files (PresetManager). renders with a copy (). +/// [SuppressMessage("ReSharper", "RedundantDefaultMemberInitializer")] public class RenderConfig { + /// + /// Game frames simulated per second of render time (the game clock advances 1 / RenderFPS seconds per frame). + /// Values above render extra frames between video frames, which are not captured + /// (the game state then advances in smaller time steps). + /// public double RenderFPS { get; set; } + + /// + /// Frame rate of the output video. The effective rate is min(VideoFPS, RenderFPS). + /// public double VideoFPS { get; set; } + /// Width of the rendered frames and the output video in pixels (also reported by the Screen.width patch). public int RenderWidth { get; set; } + + /// Height of the rendered frames and the output video in pixels (also reported by the Screen.height patch). public int RenderHeight { get; set; } + /// Video encoder settings. public VideoEncodingConfig VideoConfig { get; set; } = new(); + + /// Audio rendering and encoder settings. public AudioEncodingConfig AudioConfig { get; set; } = new(); + + /// Advanced options (random seed, spectrum processing, tile hiding, container, planet colors). public AdvancedConfig AdvancedConfig { get; set; } = new(); + /// Seconds of render time recorded after the end condition first holds (default 15). public double DelayAfterEnd { get; set; } = 15; + + /// Whether the judgement texts (hit accuracy texts) are shown during the render (scrHitTextManagerPatch). Default false. public bool ShowJudgementText { get; set; } = false; + /// + /// Copies every value of into this object; the nested configs are copied value by value + /// into the existing nested objects. Does nothing when is null. + /// public void CopyFrom(RenderConfig? source) { if (source == null) return; @@ -35,6 +63,7 @@ public class RenderConfig AdvancedConfig.CopyFrom(source.AdvancedConfig); } + /// Returns a deep copy (nested configs are new objects). public RenderConfig Clone() { var clone = new RenderConfig(); @@ -43,23 +72,48 @@ public class RenderConfig } } +/// Advanced render options. [SuppressMessage("ReSharper", "RedundantDefaultMemberInitializer")] public class AdvancedConfig { + /// + /// Seed for UnityEngine.Random set before the game scene loads, so random visual effects repeat between renders; + /// null leaves the random state unchanged. + /// public int? RandomSeed { get; set; } = null; + + /// + /// When true, AudioSource.GetSpectrumData is answered from the offline song audio at the render time + /// (AudioSpectrumGenerator), so audio-reactive visuals follow the rendered audio. Default true. + /// public bool ProcessAudio { get; set; } = true; + + /// + /// When true, the level's floor tiles (including hold meshes and the portal particles of the level's floors) and the + /// planets are hidden during the render, so only the background and decorations remain (scnGamePatch, scrFloorPatch, + /// RenderManager.OnLastUpdate). Default false. + /// public bool NonTile { get; set; } = false; + + /// File extension of the default output file name, without the dot (e.g. "mkv", "mp4"); selects the container. public string FileExtension { get; set; } = "mkv"; + /// Color of the fire (red) planet during the render; null keeps the user's setting. Serialized as "#RRGGBBAA". [JsonConverter(typeof(ColorJsonConverter))] public Color? firePlanetColor = null; + /// Color of the ice (blue) planet during the render; null keeps the user's setting. Serialized as "#RRGGBBAA". [JsonConverter(typeof(ColorJsonConverter))] public Color? icePlanetColor = null; + /// + /// Color of the third (wind, planetGreen) planet during the render (PlanetarySystemPatch); null keeps the game's color. + /// Serialized as "#RRGGBBAA". + /// [JsonConverter(typeof(ColorJsonConverter))] public Color? windPlanetColor = null; + /// Copies every value of ; does nothing when it is null. public void CopyFrom(AdvancedConfig? source) { if (source == null) return; @@ -72,6 +126,7 @@ public class AdvancedConfig windPlanetColor = source.windPlanetColor; } + /// Returns a copy of this object. public AdvancedConfig Clone() { var clone = new AdvancedConfig(); @@ -80,18 +135,35 @@ public class AdvancedConfig } } +/// Video encoder settings (passed to FFmpegNativeEncoder.EncoderSettings). public class VideoEncodingConfig { + /// Video codec (default H.264). Also selects the audio codec (CodecUtil.AudioCodecMatch). public Codec Codec { get; set; } = Codec.H264; + + /// Bitrate control mode (default CBR). public RateControlMode RateControlMode { get; set; } = RateControlMode.CBR; + /// Target video bitrate in kbps for CBR and VBR (default 30000 = 30 Mbps). public int Bitrate { get; set; } = 30000; + + /// Peak video bitrate in kbps for VBR, used when larger than Bitrate (otherwise 1.5 × Bitrate). Default 50000. public int MaxBitrate { get; set; } = 50000; + + /// + /// Quality value for CQP: lower is higher quality. Clamped to the range of the selected encoder; a negative value + /// selects 18. Default 18. + /// // ReSharper disable once InconsistentNaming public int QP { get; set; } = 18; + + /// Keyframe interval in seconds (GOP length = round(effective video FPS × interval) frames, at least 1). Default 2. public double KeyframeInterval { get; set; } = 2.0; + + /// When true, only software (CPU) encoders are tried; hardware encoders are skipped. public bool ForceSoftwareEncoder { get; set; } + /// Copies every value of ; does nothing when it is null. public void CopyFrom(VideoEncodingConfig? source) { if (source == null) return; @@ -104,6 +176,7 @@ public class VideoEncodingConfig ForceSoftwareEncoder = source.ForceSoftwareEncoder; } + /// Returns a copy of this object. public VideoEncodingConfig Clone() { var clone = new VideoEncodingConfig(); @@ -112,32 +185,44 @@ public class VideoEncodingConfig } } +/// +/// Audio rendering and encoder settings. Non-positive values of the numeric settings read back as their defaults +/// (48000 Hz, 2 channels, 256000 bps). +/// public class AudioEncodingConfig { private int _audioSampleRate = 48000; private int _audioChannels = 2; private int _audioBitrate = 256000; + /// When true, the song and sound effects are mixed offline and encoded as an audio stream. Default true. public bool RenderAudio { get; set; } = true; + /// + /// Output sample rate in Hz (default 48000). A rate the audio codec does not support in the mod + /// (CodecUtil.AvailableSampleRate) is replaced with 48000 when the render starts. + /// public int AudioSampleRate { get => _audioSampleRate <= 0 ? 48000 : _audioSampleRate; set => _audioSampleRate = value <= 0 ? 48000 : value; } + /// Number of output audio channels (default 2). public int AudioChannels { get => _audioChannels <= 0 ? 2 : _audioChannels; set => _audioChannels = value <= 0 ? 2 : value; } + /// Audio bitrate in bits per second (default 256000 = 256 kbps). Unit differs from the video bitrate (kbps). public int AudioBitrate { get => _audioBitrate <= 0 ? 256000 : _audioBitrate; set => _audioBitrate = value <= 0 ? 256000 : value; } + /// Copies every value of ; does nothing when it is null. public void CopyFrom(AudioEncodingConfig? source) { if (source == null) return; @@ -147,6 +232,7 @@ public class AudioEncodingConfig AudioBitrate = source.AudioBitrate; } + /// Returns a copy of this object. public AudioEncodingConfig Clone() { var clone = new AudioEncodingConfig(); diff --git a/Localization/LocalizationManager.cs b/Localization/LocalizationManager.cs index 15bb674..e3c1db3 100644 --- a/Localization/LocalizationManager.cs +++ b/Localization/LocalizationManager.cs @@ -9,29 +9,49 @@ namespace ADOFAIRenderer.Localization; /// /// Provides access to localized strings and handles language synchronization with the game. +/// +/// Locale files are flat JSON objects ("key": "text") under Localization/Locale/, embedded in the mod +/// assembly as resources; the file name without extension is the locale code (en.json -> "en"). +/// The active locale follows the game language: LanguageSyncPatch calls after the game's +/// RDString.Setup and RDString.ChangeLanguage. Only Korean maps to its own locale ("ko"); every other +/// language uses English ("en"). +/// +/// +/// Lookup order in : active locale, then English, then the key itself. +/// Intended for the main Unity thread; the dictionaries are filled once during initialization and only read afterwards. +/// /// internal static class LocalizationManager { + /// Loaded locales: locale code (case-insensitive) -> key -> text. Keys are case-sensitive. private static readonly Dictionary> Locales = new(StringComparer.OrdinalIgnoreCase); + /// Locale used when the language has no locale file and for keys missing in the active locale. private static readonly string FallbackLocaleCode = "en"; /// /// Event triggered when the active language changes. + /// Raised by with notify: true when the language or the resolved locale differs from + /// the current one; the argument is the new game language. Exceptions from handlers are caught and logged + /// (an exception stops the remaining handlers of that invocation). /// public static event Action? OnLanguageChanged; /// - /// Currently active SystemLanguage. + /// Currently active SystemLanguage: the game language as passed to , also when it has no + /// locale file of its own (e.g. Japanese, which displays English text). /// public static SystemLanguage CurrentLanguage { get; private set; } = SystemLanguage.English; /// - /// Currently active locale code (e.g. "en", "ko"). + /// Currently active locale code (e.g. "en", "ko"): the locale whose strings returns first. + /// Always a loaded locale or the fallback "en". /// public static string CurrentLocaleCode { get; private set; } = "en"; + /// Set by the first call; later calls return immediately. private static bool _initialized; + /// Initializes on first access to any member, so lookups work before RenderMod calls . static LocalizationManager() { Initialize(); @@ -39,6 +59,8 @@ internal static class LocalizationManager /// /// Initializes the localization system and loads embedded language dictionaries. + /// Runs once; further calls do nothing. Selects the initial language without raising + /// . /// public static void Initialize() { @@ -47,19 +69,26 @@ internal static class LocalizationManager LoadEmbeddedLocales(); - // Detect initial language from game settings + // Detect initial language from game settings. + // At mod initialization RDString.Setup has usually not run yet, so this reads the field's initial value + // (English); LanguageSyncPatch applies the real language when the game calls RDString.Setup. var initialLang = DetectGameLanguage(); SetLanguage(initialLang, notify: false); LogMsg($"[Localization] Initialized. Active language: {CurrentLanguage} ({CurrentLocaleCode})"); } + /// + /// Logs through the MelonLoader logger of the mod, or to the console when that logger is not available + /// (e.g. when this class is used outside the running mod). + /// private static void LogMsg(string msg) { try { Melon.Logger.Msg(msg); } catch { Console.WriteLine(msg); } } + /// Error counterpart of ; falls back to standard error. private static void LogError(string msg) { try { Melon.Logger.Error(msg); } @@ -68,6 +97,9 @@ internal static class LocalizationManager /// /// Loads all *.json locale files embedded in this assembly. + /// A resource is used when its name ends with ".json" and contains "Localization.Locale" (the manifest name built + /// from the folder path, see the EmbeddedResource item in the .csproj). A file that fails to read or parse is + /// logged and skipped; a later resource with the same locale code replaces an earlier one. /// private static void LoadEmbeddedLocales() { @@ -109,7 +141,13 @@ internal static class LocalizationManager /// /// Sets the active language and notifies subscribers if changed. + /// The locale is chosen by ; when that locale is not loaded, "en" is used. /// + /// Game language (as used by RDString.language). + /// + /// Whether to raise when the language or locale changes; false is used for + /// the initial selection. + /// public static void SetLanguage(SystemLanguage language, bool notify = true) { var localeCode = LanguageToLocaleCode(language); @@ -140,7 +178,10 @@ internal static class LocalizationManager /// /// Retrieves a localized string for the specified key. + /// Order: active locale, then English, then the key itself (so a missing key is visible in the UI). /// + /// Localization key, e.g. "ui.render_config.window_title"; case-sensitive. + /// The localized text, the key when no locale has it, or an empty string for a null/empty key. public static string Get(string key) { if (string.IsNullOrEmpty(key)) return string.Empty; @@ -166,7 +207,12 @@ internal static class LocalizationManager /// /// Retrieves a formatted localized string. + /// The text from is used as a template + /// ({0}, {1}, ...), formatted with the current culture. When the template does not match the + /// arguments (), the unformatted template is returned. /// + /// Localization key. + /// Format arguments; with none, the template is returned as is. public static string Get(string key, params object[] args) { var template = Get(key); @@ -184,6 +230,7 @@ internal static class LocalizationManager /// /// Maps Unity's SystemLanguage enum to an internal locale code. + /// Korean -> "ko"; every other language -> "en". A new locale file also needs an entry here. /// private static string LanguageToLocaleCode(SystemLanguage language) { @@ -196,19 +243,22 @@ internal static class LocalizationManager /// /// Attempts to read the game's current language from RDString or Persistence. + /// Falls back to English when both reads throw. /// private static SystemLanguage DetectGameLanguage() { try { - // First attempt: RDString.language + // First attempt: RDString.language, a public static field of the game (initial value English, + // assigned from Persistence.language in RDString.Setup) return RDString.language; } catch { try { - // Fallback attempt: Persistence.language + // Fallback attempt: Persistence.language, the saved "language" preference (defaults to + // Application.systemLanguage) return Persistence.language; } catch @@ -221,6 +271,7 @@ internal static class LocalizationManager /// /// Convenient shorthand alias for . +/// Every member forwards to the member of the same name in . /// // ReSharper disable once InconsistentNaming internal static class I18n diff --git a/Patch/Engine/AsyncInputPatches.cs b/Patch/Engine/AsyncInputPatches.cs index 0e9540a..a8944ab 100644 --- a/Patch/Engine/AsyncInputPatches.cs +++ b/Patch/Engine/AsyncInputPatches.cs @@ -3,10 +3,31 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Engine; +/// +/// Makes the async input clock offset follow the render clock immediately. +/// +/// Patched member: AsyncInputUtils.UpdateOffsetTime(long fixDivider = 1). The async input system maps wall-clock ticks +/// (AsyncInputManager.currFrameTick = DateTime.Now.Ticks, 100 ns units) to conductor time with +/// AsyncInputManager.offsetTick = currFrameTick - scrConductor.dspTime * 10^7. With fixDivider 1 the +/// offset is set directly; otherwise it moves by 1 / fixDivider of the difference per call. scrConductor calls it every +/// frame with divider 100 (damped); scrController calls it with divider 1. +/// +/// +/// Active while (stage >= OnScene). From stage Render the conductor time follows the +/// patched AudioSettings.dspTime (offline render clock, a fixed step per frame independent of wall-clock time), so the +/// difference changes every frame; the damped update would lag behind it. The prefix always applies the divider-1 update. +/// +/// Reference: https://learn.microsoft.com/en-us/dotnet/api/system.datetime.ticks +/// [HarmonyPatch(typeof(AsyncInputUtils), "UpdateOffsetTime")] // ReSharper disable once InconsistentNaming public static class AsyncInputPatches { + /// + /// Prefix of AsyncInputUtils.UpdateOffsetTime. While rendering, sets offsetTick to + /// currFrameTick - dspTime * 10^7 (10^7 = 100 ns ticks per second) and offsetTickUpdated to true, for any divider. + /// + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix() diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 9d9c2f1..5f33e9a 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -18,6 +18,7 @@ namespace ADOFAIRenderer.Patch.Engine; [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.FindOrLoadAudioClipExternal))] public static class AudioManagerFindOrLoadAudioClipExternalPatch { + /// Records in when its audioLib key is not loaded yet. Runs the original always. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static void Prefix(AudioManager __instance, string path) @@ -28,20 +29,33 @@ public static class AudioManagerFindOrLoadAudioClipExternalPatch } } +/// +/// AudioManager._Play (private; AudioManager.Play forwards to it): registers each scheduled sound effect (hit sounds, PlaySound +/// events, hold sounds without an end time) as an effect voice of the offline mixer and maps the returned AudioSource to it. +/// Active while capturing (RenderManager.IsCapturingActive) with RenderAudio on. The original always runs: it creates the +/// AudioSource the game keeps using (its PlayScheduled call is blocked during rendering by another patch), and the game sets +/// time/pitch/SetScheduledEndTime on it afterwards, which the patches forward to the mapped state. +/// The game volume is passed through; the start pitch is 1 because _Play sets pitch = 1 before PlayScheduled. +/// [HarmonyPatch(typeof(AudioManager), "_Play")] public static class AudioManagerPlayPatch { + /// Opens the clip's source so its first chunk starts loading. Returns true (the original runs). [HarmonyPrefix] public static bool Prefix(string snd, double time, AudioMixerGroup group, float volume, int priority) { if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) { - // Cache the clip on the main thread + // Cache the clip on the main Unity thread RenderManager.Instance.AudioMixer?.CacheClip(snd); } return true; // Let the original method execute to prevent NullReferenceException } + /// + /// Registers the sound at (absolute DSP time, converted to render time) as a non-looping effect voice, + /// maps to it and queues the source's game object for deactivation at the end of the frame. + /// [HarmonyPostfix] // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __result, string snd, double time, AudioMixerGroup group, float volume, int priority) @@ -63,6 +77,10 @@ public static class AudioManagerPlayPatch /// /// scrConductor.PlayWithEndTime: registers a looping effect voice until the end time. The AudioSource it creates with /// AudioManager.MakeSource is mapped to that voice (). +/// Active while capturing (RenderManager.IsCapturingActive) with RenderAudio on; the original always runs. The game calls it for +/// hold sounds with an end time above 0 (scrConductor hold sound scheduling); it sets loop = true, pitch = 1, PlayScheduled(time) and +/// SetScheduledEndTime(endTime) on the new source. pitch and SetScheduledEndTime reach the mapped state through the +/// patches; PlayScheduled is blocked during rendering by another patch. /// [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] public static class PlayWithEndTimePatch @@ -70,9 +88,14 @@ public static class PlayWithEndTimePatch /// State registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. internal static SourceState? PendingState; + /// Clears when PlayWithEndTime returns or throws, so it cannot map a later MakeSource call. [HarmonyFinalizer] public static void Finalizer() => PendingState = null; + /// + /// Opens the clip's source and registers a looping voice from to + /// (absolute DSP times, converted to render time); the state waits in . Returns true (the original runs). + /// [HarmonyPrefix] public static bool Prefix(string snd, double time, double endTime, float volume, int priority) { @@ -81,7 +104,7 @@ public static class PlayWithEndTimePatch var mixer = RenderManager.Instance.AudioMixer; if (mixer != null) { - // Cache the clip on the main thread + // Cache the clip on the main Unity thread mixer.CacheClip(snd); // Convert absolute DSP times to relative times in seconds from render start @@ -96,10 +119,18 @@ public static class PlayWithEndTimePatch } } -/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (). +/// +/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (). +/// Active only while a PlayWithEndTime call registered a state (capturing with RenderAudio on); every other MakeSource call +/// (including the one inside AudioManager._Play) is left unchanged. +/// [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))] public static class AudioManagerMakeSourcePatch { + /// + /// Takes the pending state, maps to it and queues the source's game object for deactivation at + /// the end of the frame. + /// [HarmonyPostfix] // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __result) @@ -173,6 +204,7 @@ internal static class EffectSource [HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)] public static class AudioSourceIsPlayingPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref bool __result) @@ -187,6 +219,7 @@ public static class AudioSourceIsPlayingPatch [HarmonyPatch(typeof(AudioSource), "time", MethodType.Getter)] public static class AudioSourceTimeGetterPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref float __result) @@ -204,6 +237,7 @@ public static class AudioSourceTimeGetterPatch [HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)] public static class AudioSourceTimePatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, float value) @@ -214,10 +248,14 @@ public static class AudioSourceTimePatch } } -/// AudioSource.timeSamples getter (): clip position in samples of the clip. +/// +/// AudioSource.timeSamples getter (): clip position in samples of the clip while playing; while not playing, +/// the last value set through time or timeSamples (0 after Stop()). +/// [HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Getter)] public static class AudioSourceTimeSamplesGetterPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref int __result) @@ -232,6 +270,7 @@ public static class AudioSourceTimeSamplesGetterPatch [HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Setter)] public static class AudioSourceTimeSamplesPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, int value) @@ -246,6 +285,7 @@ public static class AudioSourceTimeSamplesPatch [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Getter)] public static class AudioSourcePitchGetterPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref float __result) @@ -260,6 +300,7 @@ public static class AudioSourcePitchGetterPatch [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] public static class AudioSourcePitchPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, float value) @@ -274,6 +315,7 @@ public static class AudioSourcePitchPatch [HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))] public static class AudioSourceSetScheduledEndTimePatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, double time) @@ -292,6 +334,7 @@ public static class AudioSourceSetScheduledEndTimePatch [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])] public static class AudioSourceStopPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance) @@ -306,11 +349,13 @@ public static class AudioSourceStopPatch /// AudioManager.StopAllSounds destroys every source in liveSources (all AudioManager._Play and scrConductor.PlayWithEndTime sources), /// so every effect voice stops at the current render time, including ones scheduled to start later. /// Callers include scrConductor.PlayHitTimes (from its second call), ScrubMusicToTile/Time, KillAllSoundsCoroutine, -/// scrController.FailAction and scnGame.Play. +/// scrController.FailAction, scrController.DebugUpdate, scnGame.Play and PauseMenu.Choose. +/// Active while capturing (RenderManager.IsCapturingActive) with RenderAudio on; the original always runs (it destroys the sources). /// [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.StopAllSounds))] public static class AudioManagerStopAllSoundsPatch { + /// Stops every effect voice while capturing with RenderAudio on. The original always runs (no return value). [HarmonyPrefix] public static void Prefix() { diff --git a/Patch/Engine/RenderEnvironmentPatches.cs b/Patch/Engine/RenderEnvironmentPatches.cs index 02856c4..d515ef9 100644 --- a/Patch/Engine/RenderEnvironmentPatches.cs +++ b/Patch/Engine/RenderEnvironmentPatches.cs @@ -6,13 +6,40 @@ using UnityEngine; namespace ADOFAIRenderer.Patch.Engine; +/// +/// Unity engine patches that put the game on the offline render clock and the render resolution. +/// +/// Render clock: while capturing (, stage >= Render) and +/// RenderContext.TimePatch is on (default on; a RenderSession can turn it off), the Unity time getters return +/// the value captured at the capture start (RenderManager.timeBase, unscaledTimeBase, dspTimeBase, +/// frameCountBase, read in SetupPlay) plus the render time . Each rendered frame then +/// advances game time by exactly 1 / RenderFPS seconds, independent of how long the frame took to render and encode, and the +/// clocks continue from their real values without a jump. The patches only affect managed callers of these getters +/// (game and mod scripts). +/// +/// +/// All prefixes run on the thread that calls the getter (normally the main Unity thread) and read RenderManager state +/// without locks. +/// +/// public static class RenderEnvironmentPatches { + /// + /// Render time of the current frame in seconds: RenderManager.frameCount / RenderFPS. + /// frameCount is reset to 0 when capture starts and incremented at the start of every frame (FirstUpdateHook). + /// private static double TimeSpent => RenderManager.Instance.frameCount / RenderManager.Instance.Configuration.RenderFPS; + /// + /// Time.time (scaled game time in seconds, float): returns timeBase + while capturing with + /// the time patch on. Time.timeScale is not applied to the render time. + /// [HarmonyPatch(typeof(Time), "time", MethodType.Getter)] public static class TimePatch { + /// Prefix of the Time.time getter. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref float __result) { @@ -22,9 +49,13 @@ public static class RenderEnvironmentPatches } } + /// Time.timeAsDouble (double precision of Time.time): timeBase + . [HarmonyPatch(typeof(Time), "timeAsDouble", MethodType.Getter)] public static class TimeAsDoublePatch { + /// Prefix of the getter; returns the render clock value while capturing with the time patch on. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref double __result) { @@ -34,9 +65,18 @@ public static class RenderEnvironmentPatches } } + /// + /// Time.unscaledTime (time in seconds independent of Time.timeScale, float): unscaledTimeBase + + /// . Unscaled time is not advanced by Time.captureDeltaTime (it follows real time), so it is + /// replaced as well. + /// Reference: https://docs.unity3d.com/ScriptReference/Time-captureDeltaTime.html + /// [HarmonyPatch(typeof(Time), "unscaledTime", MethodType.Getter)] public static class UnscaledTimePatch { + /// Prefix of the getter; returns the render clock value while capturing with the time patch on. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref float __result) { @@ -46,9 +86,15 @@ public static class RenderEnvironmentPatches } } + /// + /// Time.unscaledTimeAsDouble (double precision of Time.unscaledTime): unscaledTimeBase + . + /// [HarmonyPatch(typeof(Time), "unscaledTimeAsDouble", MethodType.Getter)] public static class UnscaledTimeAsDoublePatch { + /// Prefix of the getter; returns the render clock value while capturing with the time patch on. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref double __result) { @@ -58,9 +104,17 @@ public static class RenderEnvironmentPatches } } + /// + /// AudioSettings.dspTime (audio system clock in seconds, double): dspTimeBase + . + /// The game schedules the song and computes the song position from this clock (scrConductor), so the level advances + /// by one render frame per frame. The offline audio mixer uses the same base (RenderManager.dspTimeBase). + /// [HarmonyPatch(typeof(AudioSettings), "dspTime", MethodType.Getter)] public static class DspTimePatch { + /// Prefix of the getter; returns the render clock value while capturing with the time patch on. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref double __result) { @@ -70,9 +124,19 @@ public static class RenderEnvironmentPatches } } + /// + /// AudioSource.PlayScheduled(double time) (start playback at a dspTime): blocked for every AudioSource while + /// (stage >= OnScene), so Unity plays no scheduled sound in real time. During + /// capture the song and the sound effects are produced offline by OfflineAudioMixer; sound effects reach it through + /// the AudioManager._Play and scrConductor.PlayWithEndTime patches (AudioPatches.cs). + /// [HarmonyPatch(typeof(AudioSource), "PlayScheduled")] public static class PlayScheduledPatch { + /// Prefix of AudioSource.PlayScheduled. + /// The AudioSource (unused). + /// Scheduled start in dspTime seconds (unused). + /// True to run the original method (not rendering); false to skip it (rendering). // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, double time) { @@ -80,9 +144,16 @@ public static class RenderEnvironmentPatches } } + /// + /// Time.frameCount (frames since startup): frameCountBase + RenderManager.frameCount while capturing + /// (stage >= Render) with RenderContext.TimePatch on. + /// [HarmonyPatch(typeof(Time), "frameCount", MethodType.Getter)] public static class FrameCountPatch { + /// Prefix of the getter; returns the replacement value under the conditions in the type summary. + /// Replacement value. + /// True to run the original getter; false when the replacement value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref int __result) { @@ -92,9 +163,13 @@ public static class RenderEnvironmentPatches } } + /// Time.deltaTime (seconds since the previous frame, float): 1 / RenderFPS. [HarmonyPatch(typeof(Time), "deltaTime", MethodType.Getter)] public static class DeltaTimePatch { + /// Prefix of the getter; returns the render clock value while capturing with the time patch on. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref float __result) { @@ -104,9 +179,13 @@ public static class RenderEnvironmentPatches } } + /// Time.unscaledDeltaTime (timeScale-independent frame time in seconds, float): 1 / RenderFPS. [HarmonyPatch(typeof(Time), "unscaledDeltaTime", MethodType.Getter)] public static class UnscaledDeltaTimePatch { + /// Prefix of the getter; returns the render clock value while capturing with the time patch on. + /// Replacement value in seconds. + /// True to run the original getter; false when the render clock value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref float __result) { @@ -116,9 +195,18 @@ public static class RenderEnvironmentPatches } } + /// + /// Screen.width (window width in pixels): RenderConfig.RenderWidth while + /// (stage >= OnScene) and RenderContext.ScreenPatch is on (default on). Game code that sizes render textures from + /// the screen size (for example scrCamera recreates camRT at Screen.width x Screen.height when the size + /// differs) then works at the output resolution independent of the window size. + /// [HarmonyPatch(typeof(Screen), "width", MethodType.Getter)] public static class ScreenWidthPatch { + /// Prefix of the getter; returns the replacement value under the conditions in the type summary. + /// Replacement value. + /// True to run the original getter; false when the replacement value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref int __result) { @@ -128,9 +216,15 @@ public static class RenderEnvironmentPatches } } + /// + /// Screen.height (window height in pixels): RenderConfig.RenderHeight; same conditions as . + /// [HarmonyPatch(typeof(Screen), "height", MethodType.Getter)] public static class ScreenHeightPatch { + /// Prefix of the getter; returns the replacement value under the conditions in the type summary. + /// Replacement value. + /// True to run the original getter; false when the replacement value is returned. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(ref int __result) { @@ -140,9 +234,22 @@ public static class RenderEnvironmentPatches } } + /// + /// AudioSource.GetSpectrumData(float[] samples, int channel, FFTWindow window) (FFT magnitudes of the source's current + /// output): while capturing (stage >= Render) with AdvancedConfig.ProcessAudio on, fills samples with + /// (spectrum of the song at the render time) for any AudioSource. + /// No AudioSource plays during capture, so the original would return silence. The game calls it from + /// scrConductor, scrBarMaker and the volume trackers for audio-reactive visuals. + /// [HarmonyPatch(typeof(AudioSource), "GetSpectrumData", typeof(float[]), typeof(int), typeof(FFTWindow))] public static class GetSpectrumDataPatch { + /// Prefix of AudioSource.GetSpectrumData. + /// The AudioSource queried (not used; the song spectrum is returned for every source). + /// Output array; its length is the number of spectrum bins. + /// Channel index requested by the caller. + /// FFT window type requested by the caller. + /// True to run the original method; false when the generated spectrum was written. [SuppressMessage("ReSharper", "InconsistentNaming")] public static bool Prefix(AudioSource __instance, float[] samples, int channel, FFTWindow window) { @@ -152,9 +259,19 @@ public static class RenderEnvironmentPatches } } + /// + /// AudioManager.FindOrLoadAudioClipExternal(string path, bool mp3Streaming, float length, bool stream) (coroutine + /// that loads an external audio file of the level into audioLib): from stage Load with + /// AdvancedConfig.ProcessAudio on, forces stream to false. The game passes stream to + /// DownloadHandlerAudioClip.streamAudio for ogg/wav/aif/aiff files, so these clips are loaded as non-streaming + /// clips; the MP3 path uses mp3Streaming and ignores stream. + /// Reference: https://docs.unity3d.com/ScriptReference/Networking.DownloadHandlerAudioClip-streamAudio.html + /// [HarmonyPatch(typeof(AudioManager), "FindOrLoadAudioClipExternal", typeof(string), typeof(bool), typeof(float), typeof(bool))] public static class FindOrLoadAudioClipExternalPatch { + /// Prefix of AudioManager.FindOrLoadAudioClipExternal; the original always runs. + /// Streaming flag of the load; set to false under the conditions above. [SuppressMessage("ReSharper", "InconsistentNaming")] public static void Prefix(ref bool stream) { diff --git a/Patch/Game/DetailedResultsPatch.cs b/Patch/Game/DetailedResultsPatch.cs index d250ee2..9b6dcda 100644 --- a/Patch/Game/DetailedResultsPatch.cs +++ b/Patch/Game/DetailedResultsPatch.cs @@ -3,10 +3,27 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Hides the detailed results panel (per-judgement hit counts) that the game shows on the clear screen. +/// +/// Patched member: DetailedResults.Show(). The game calls it from scrController.OnLandOnPortal when a level is won +/// (and detailed results are enabled); it builds the result text per player, enables the component and activates +/// scrController.detailedResults. +/// +/// +/// Active while (stage >= OnScene), so the panel does not appear in the rendered video +/// during the delay after the level end. +/// +/// [HarmonyPatch(typeof(DetailedResults))] // ReSharper disable once InconsistentNaming public static class DetailedResultsPatch { + /// + /// Prefix of DetailedResults.Show. While rendering, disables the component and deactivates its GameObject. + /// + /// The DetailedResults component being shown. + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("Show")] // ReSharper disable once InconsistentNaming diff --git a/Patch/Game/EndscreenLanternsPatch.cs b/Patch/Game/EndscreenLanternsPatch.cs index 86e69aa..7193995 100644 --- a/Patch/Game/EndscreenLanternsPatch.cs +++ b/Patch/Game/EndscreenLanternsPatch.cs @@ -4,10 +4,29 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Suppresses the end-screen lanterns (level complete / accuracy / speed trial lamps) shown after a clear. +/// +/// Patched members: EndscreenLanterns.Show(scrPlayer, float, int, bool), which activates the lantern set, sets the +/// lantern texts and starts DOTween sequences (chainSequence for the chain drop, lampSequences[3] for the lamp +/// effects), returning the number of lanterns shown; and EndscreenLanterns.Hide(), which deactivates the GameObject. +/// +/// +/// Active while (stage >= OnScene), so the lanterns and their tweens do not appear in +/// the rendered video during the delay after the level end. See also . +/// +/// [HarmonyPatch(typeof(EndscreenLanterns))] // ReSharper disable once InconsistentNaming public static class EndscreenLanternsPatch { + /// + /// Prefix of EndscreenLanterns.Show. While rendering, hides the lantern set (which also runs + /// ) and reports zero lanterns shown. + /// + /// The lantern set being shown. + /// Return value of Show: number of lanterns shown; set to 0 when the original is skipped. + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("Show")] [SuppressMessage("ReSharper", "InconsistentNaming")] @@ -22,6 +41,21 @@ public static class EndscreenLanternsPatch return false; // Skip original Show method } + /// + /// Prefix of EndscreenLanterns.Hide. While rendering, releases the lantern tweens started by an earlier Show + /// before the original deactivates the GameObject. + /// + /// The private fields lampSequences (DOTween Sequence[]) and chainSequence (Sequence) are read + /// through Harmony . For each sequence, a public instance method Kill(bool) is looked up on the + /// object's runtime type by reflection and invoked with false when found; then the field slot is set to null. + /// In the game's DOTween.dll, Kill(this Tween, bool complete = false) is an extension method of + /// DG.Tweening.TweenExtensions, so the lookup on Sequence returns null and no Kill is invoked; the effect is + /// that the fields stop referencing the sequences, while the sequences themselves keep running in DOTween. + /// Reflection failures on a single sequence (including a null array element) are ignored. The original Hide always runs + /// afterwards. + /// + /// + /// The lantern set being hidden. [HarmonyPrefix] [HarmonyPatch("Hide")] [SuppressMessage("ReSharper", "InconsistentNaming")] diff --git a/Patch/Game/LanguageSyncPatch.cs b/Patch/Game/LanguageSyncPatch.cs index 88daa71..407509c 100644 --- a/Patch/Game/LanguageSyncPatch.cs +++ b/Patch/Game/LanguageSyncPatch.cs @@ -5,11 +5,19 @@ using UnityEngine; namespace ADOFAIRenderer.Patch.Game; /// -/// Synchronizes mod localization with ADOFAI's in-game language settings. +/// Keeps the mod UI language () equal to the game language. +/// +/// Patched members: RDString.Setup(), which initializes the game's string table and sets RDString.language from +/// Persistence.language (it returns immediately when already initialized); and RDString.ChangeLanguage(SystemLanguage), +/// which stores the language in Persistence.language, clears the initialized flag and calls Setup again. +/// +/// Always active (independent of the render stage). Runs on the main Unity thread. /// [HarmonyPatch(typeof(RDString))] internal static class LanguageSyncPatch { + /// Postfix of RDString.ChangeLanguage: applies the selected language to the mod UI. + /// Language passed to ChangeLanguage. [HarmonyPatch(nameof(RDString.ChangeLanguage))] [HarmonyPostfix] public static void OnChangeLanguage(SystemLanguage language) @@ -17,6 +25,10 @@ internal static class LanguageSyncPatch LocalizationManager.SetLanguage(language); } + /// + /// Postfix of RDString.Setup: applies RDString.language (the language resolved by the game) to the mod UI. + /// Runs on every Setup call, including the early-return calls made when the string table is already initialized. + /// [HarmonyPatch(nameof(RDString.Setup))] [HarmonyPostfix] public static void OnSetup() diff --git a/Patch/Game/PersistencePatch.cs b/Patch/Game/PersistencePatch.cs index e27f468..15c1534 100644 --- a/Patch/Game/PersistencePatch.cs +++ b/Patch/Game/PersistencePatch.cs @@ -3,9 +3,23 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Overrides two player settings read from Persistence during a render: the "skip intro" option and the planet colors. +/// [HarmonyPatch(typeof(Persistence))] public static class PersistencePatch { + /// + /// Postfix of the Persistence.skipIntroBehavior getter (setting values Off / AfterFirstTry / Always). + /// + /// scrConductor.StartMusicCo reads this setting when the song is scheduled and, for AfterFirstTry (after a death) or + /// Always (long intro, no checkpoint), sets skipOffset, which moves dspTimeSong back by the level offset and + /// starts the song part-way in. While (stage >= OnScene) and the song has not + /// started yet (scrConductor.hasSongStarted false), the getter returns Off, so skipOffset stays false and the + /// song always plays from its start. The mod's offline BGM mixing, spectrum and video background timing assume this. + /// + /// + /// Value returned by the getter; replaced with SkipIntroBehavior.Off under the conditions above. [HarmonyPostfix] [HarmonyPatch("skipIntroBehavior", MethodType.Getter)] // ReSharper disable once InconsistentNaming @@ -19,6 +33,18 @@ public static class PersistencePatch } } + /// + /// Postfix of Persistence.GetPlayerColor(bool red), which returns the player's saved planet color + /// (preset name or hex color from the general preferences; default red/blue otherwise). PlanetRenderer uses it to + /// color the fire (red) and ice (blue) planets. + /// + /// Active from stage Load (render started, scene loading) until the stage returns to None. When the render configuration + /// sets AdvancedConfig.firePlanetColor (red) or AdvancedConfig.icePlanetColor (blue), that color replaces the + /// saved color; an unset (null) override keeps the game value. The saved preference itself is not changed. + /// + /// + /// True for the fire (red) planet, false for the ice (blue) planet. + /// Color returned by the game; replaced by the configured override when set. [HarmonyPostfix] [HarmonyPatch("GetPlayerColor")] // ReSharper disable once InconsistentNaming diff --git a/Patch/Game/PlanetarySystemPatch.cs b/Patch/Game/PlanetarySystemPatch.cs index 1189c27..70dc598 100644 --- a/Patch/Game/PlanetarySystemPatch.cs +++ b/Patch/Game/PlanetarySystemPatch.cs @@ -3,9 +3,21 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Applies the configured color of the third (green, "wind") planet used by multi-planet levels. +/// +/// Patched members: PlanetarySystem.Init(), which creates the planets including planetGreen (default color +/// RGB 0.3, 0.7, 0.0); and PlanetarySystem.ApplyMultiplanetColors(), which recolors the extra planets +/// (green, yellow, purple, ...) from the red/blue planet colors (skipped by the game when GCS.staticPlanetColors is set). +/// Both postfixes reapply the override after the game has set its color. +/// +/// Active from stage Load until the stage returns to None. The fire/ice colors are handled by . +/// [HarmonyPatch(typeof(PlanetarySystem))] public static class PlanetarySystemPatch { + /// Postfix of PlanetarySystem.Init: applies the wind planet color override to the new planets. + /// The planetary system that was initialized. [HarmonyPostfix] [HarmonyPatch("Init")] // ReSharper disable once InconsistentNaming @@ -14,6 +26,8 @@ public static class PlanetarySystemPatch ApplyWindPlanetColor(__instance); } + /// Postfix of PlanetarySystem.ApplyMultiplanetColors: reapplies the wind planet color override. + /// The planetary system whose extra planet colors were updated. [HarmonyPostfix] [HarmonyPatch("ApplyMultiplanetColors")] // ReSharper disable once InconsistentNaming @@ -22,6 +36,11 @@ public static class PlanetarySystemPatch ApplyWindPlanetColor(__instance); } + /// + /// Sets planetGreen's renderer color to AdvancedConfig.windPlanetColor when a render is in progress + /// (stage != None) and the override is set. Does nothing when the override is null or the planet does not exist. + /// + /// The planetary system to update. private static void ApplyWindPlanetColor(PlanetarySystem planetarySystem) { if (RenderManager.Instance == null || RenderManager.Instance.stage == RenderStage.None) return; diff --git a/Patch/Game/scnEditorPatch.cs b/Patch/Game/scnEditorPatch.cs index 8698c0e..4092f3b 100644 --- a/Patch/Game/scnEditorPatch.cs +++ b/Patch/Game/scnEditorPatch.cs @@ -6,10 +6,28 @@ using UnityEngine.UI; namespace ADOFAIRenderer.Patch.Game; +/// +/// Adds the "Render" button to the level editor toolbar. +/// +/// Patched member: scnEditor.Start(), the editor scene setup (it wires the toolbar buttons, including +/// buttonExit). The postfix clones the exit button next to it and makes the clone start a render. +/// +/// +/// Always active except when the mod features are disabled (, set from the +/// driver warning dialog). Runs on the main Unity thread each time the editor scene starts. +/// +/// [HarmonyPatch(typeof(scnEditor))] // ReSharper disable once InconsistentNaming public static class scnEditorPatch { + /// + /// Postfix of scnEditor.Start: instantiates a copy of buttonExit under the same parent, named + /// buttonRender, whose click handler is , and replaces its icon with the + /// editor's MoveCamera event sprite (24 x 24 units). Logs a message and adds nothing when the editor or exit button is missing. + /// When the icon child or the sprite is not found, the button keeps the exit icon. + /// + /// The editor scene component. [HarmonyPostfix] [HarmonyPatch("Start")] // ReSharper disable once InconsistentNaming @@ -38,8 +56,9 @@ public static class scnEditorPatch var images = buttonObj.GetComponentsInChildren(); if (images.Length <= 1) return; + // images[0] is the background on the root, images[1] is the icon on the child. + // Sprite path: the icon of the editor's MoveCamera level event, loaded from the game's Resources. // ReSharper disable once Unity.UnknownResource - // images[0] is the background on the root, images[1] is the icon on the child var sprite = Resources.Load("LevelEditor/LevelEvents/MoveCamera"); if (sprite == null) return; diff --git a/Patch/Game/scnGamePatch.cs b/Patch/Game/scnGamePatch.cs index 3d2d943..5861eb8 100644 --- a/Patch/Game/scnGamePatch.cs +++ b/Patch/Game/scnGamePatch.cs @@ -6,10 +6,26 @@ using UnityEngine; namespace ADOFAIRenderer.Patch.Game; +/// +/// Patches of scnGame (custom level scene controller, also used by the editor's play mode): hides tiles and planets +/// for the "non-tile" render option and replaces the background video player during a render. +/// [HarmonyPatch(typeof(scnGame))] // ReSharper disable once InconsistentNaming public static class scnGamePatch { + /// + /// Postfix of scnGame.Play(int seqID, bool remakeFloors), which starts (or restarts) play of the custom level: + /// it resets the conductor, camera and controller state for the given start tile. + /// + /// Active from stage Load when AdvancedConfig.NonTile is enabled. Disables the renderers of every floor in + /// scrLevelMaker.listFloors (floor mesh, legacy sprite, icon, outline, top/bottom glow, multi-planet line, + /// hold mesh, dummy planets) and of every planet of every player (sprite mesh, core/tail/sparks particles, ring, glow, + /// face, samurai sprite), so only the background and decorations remain in the rendered video. The renderers are only + /// disabled; the objects keep running. Hold meshes re-enable themselves, so RenderManager.OnLastUpdate disables + /// them again every frame. + /// + /// [HarmonyPostfix] [HarmonyPatch("Play")] public static void ProcessFloors() @@ -47,6 +63,7 @@ public static class scnGamePatch } return; + // Disables every renderer that draws one planet (fire, ice, extra planets and floor dummy planets). void DisablePlanetRenderer(PlanetRenderer planet) { if (planet.sprite) DisableRenderer(planet.sprite.meshRenderer); @@ -62,12 +79,29 @@ public static class scnGamePatch foreach (var renderer in renderers) DisableRenderer(renderer); } + // Sets Renderer.enabled = false; null or destroyed renderers are skipped (Unity null check). void DisableRenderer(Renderer? renderer) { if (renderer) renderer.enabled = false; } } + /// + /// Prefix of scnGame.UpdateVideo(), which sets up the level's background video on the native Unity + /// VideoPlayer (scnGame.videoBG): it reads miscSettings.bgVideo, loads a VideoClip for internal + /// (Resources / Addressables) and bundle levels or sets videoBG.url to the file next to the custom level, then calls + /// Stop(), Prepare() and sets isLooping from loopVideo. The game calls it from + /// scnGame.FinishCustomLevelLoading and from scnGame.ReloadAssets when force is true. + /// + /// While (stage >= OnScene), an external video file of a custom level is played by + /// (FFmpeg decoder), which shows the frame for the render time of each frame. The native + /// VideoPlayer plays in real time and would drift from the offline render clock, so it is disabled for that case. + /// Internal and bundle levels (built-in VideoClip assets, not files) and a missing file use the original method. + /// + /// Outside a render, any left on the video object is destroyed and the native player re-enabled. + /// + /// The scene controller. + /// True to run the original UpdateVideo; false when the mod handled the video (or there is none). [HarmonyPrefix] [HarmonyPatch("UpdateVideo")] // ReSharper disable once InconsistentNaming @@ -109,7 +143,8 @@ public static class scnGamePatch return true; // Let the original print error } - // Disable native VideoPlayer component (to prevent it from attempting real-time playback and sync errors) + // Camera for the CameraFarPlane render mode: the VideoPlayer's target camera, or the "BGStaticCam" scene object + // when none is assigned. var targetCamera = __instance.videoBG.targetCamera; if (!targetCamera) { @@ -120,6 +155,7 @@ public static class scnGamePatch } } + // Disable the native VideoPlayer so it does not play the video in real time alongside VideoBGPlayer. __instance.videoBG.enabled = false; // Add or retrieve our custom FFmpeg video background player diff --git a/Patch/Game/scrCameraPatch.cs b/Patch/Game/scrCameraPatch.cs index 3ba2c4d..a065741 100644 --- a/Patch/Game/scrCameraPatch.cs +++ b/Patch/Game/scrCameraPatch.cs @@ -3,16 +3,37 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Protects the camera render texture scrCamera.camRT when the custom frame rate effect is turned off while it is +/// already off. +/// +/// Patched member: scrCamera.SetCustomFrameRate(bool enable, float frame) (level effect that updates the screen at a +/// lower frame rate). Enabling assigns a new RenderTexture to the camera quad material and refreshes it at frame FPS. +/// Disabling calls Release() on the quad material's current mainTexture and assigns camRT back. When +/// the effect was never enabled, mainTexture is camRT itself, so the original releases the live camera +/// render texture. +/// +/// Always active (independent of the render stage). +/// [HarmonyPatch(typeof(scrCamera))] // ReSharper disable once InconsistentNaming public class scrCameraPatch { + /// + /// Prefix of scrCamera.SetCustomFrameRate. For a disable request while enableCustomFPS is already false, + /// stores the arguments in enableCustomFPS / frameRate and skips the original (no texture is released). + /// Enable requests and disable requests while the effect is on run the original. + /// + /// The camera. + /// True to turn the custom frame rate on, false to turn it off. + /// Custom frame rate in frames per second. + /// True to run the original method; false to skip it. [HarmonyPrefix] [HarmonyPatch("SetCustomFrameRate")] [SuppressMessage("ReSharper", "InconsistentNaming")] private static bool FixCamRT(scrCamera __instance, bool enable, float frame) { - // There is an issue that `mainTexture.Release` triggers even if `mainTexture` is `camRT`, so mod tries to fix it + // Disable request while the effect is off: the quad material's mainTexture is camRT, which the original would Release(). if (enable || __instance.enableCustomFPS != enable) return true; __instance.enableCustomFPS = enable; diff --git a/Patch/Game/scrControllerPatch.cs b/Patch/Game/scrControllerPatch.cs index cbd98a9..74ed665 100644 --- a/Patch/Game/scrControllerPatch.cs +++ b/Patch/Game/scrControllerPatch.cs @@ -3,10 +3,25 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Keeps the game controller (scrController) in uninterrupted play during a render: no pause, no failure, +/// no scene change after the clear, and no clear texts. +/// +/// All patches are active while (stage >= OnScene). RenderManager also sets +/// noFail, noFailInfiniteMargin and disableCongratsMessage on the controller when the scene is ready. +/// +/// [HarmonyPatch(typeof(scrController))] // ReSharper disable once InconsistentNaming public static class scrControllerPatch { + /// + /// Prefix of scrController.TogglePauseGame() (pause menu toggle; sets Time.timeScale to 0 or 1 and shows the + /// pause menu; also called from the pause button, the pause key and OnApplicationPauseCallback). Skipped while + /// rendering, so the level cannot be paused. The skipped call returns + /// false (Harmony default for the unset bool result). + /// + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("TogglePauseGame")] public static bool PreventPause() @@ -14,6 +29,12 @@ public static class scrControllerPatch return !RenderManager.IsActive; } + /// + /// Prefix of scrController.PortalTravelAction(Portal), which leaves the level through a portal (next level, + /// level select, main menu, ...); the game calls it on entering the Won state for non-world levels and on input after a win. + /// Skipped while rendering, so the scene stays loaded for the delay after the level end. + /// + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("PortalTravelAction")] public static bool PreventPortalTravel() @@ -21,6 +42,11 @@ public static class scrControllerPatch return !RenderManager.IsActive; } + /// + /// Prefix of scrController.FailAction(...), which enters the Fail state: stops the song and all sounds, pauses the + /// VFX video and shows the failure texts. Skipped while rendering. + /// + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("FailAction")] public static bool PreventFailAction() @@ -28,6 +54,11 @@ public static class scrControllerPatch return !RenderManager.IsActive; } + /// + /// Prefix of scrController.Fail2Action(), which moves from Fail to Fail2 (accuracy calculation, saving of the + /// best result for boss/custom boss levels, death counter). Skipped while rendering. + /// + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("Fail2Action")] public static bool PreventFail2Action() @@ -35,6 +66,11 @@ public static class scrControllerPatch return !RenderManager.IsActive; } + /// + /// Prefix of scrController.OnPlayerDied(...), which kills the remaining players and calls FailAction when no other + /// player is alive. Skipped while rendering. + /// + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("OnPlayerDied")] public static bool PreventPlayerDied() @@ -42,6 +78,17 @@ public static class scrControllerPatch return !RenderManager.IsActive; } + /// + /// Postfix of scrController.OnLandOnPortal(...), the win handler on the last tile: it sets and activates + /// txtCongrats / txtAprilCongrats / txtAllStrictClear, may show the detailed results and end-screen + /// lanterns, saves the result and changes the state to Won. + /// + /// While rendering, clears and deactivates the three clear texts and disables the detailed results panel after the + /// original has run, so they do not appear in the rendered video. The Won state itself is kept: the default render end + /// condition (RenderContext.EndCondition) waits for it. + /// + /// + /// The game controller. [HarmonyPostfix] [HarmonyPatch("OnLandOnPortal")] public static void PreventWinTextsPostfix(scrController __instance) diff --git a/Patch/Game/scrFloorPatch.cs b/Patch/Game/scrFloorPatch.cs index 56c9be8..c95e26c 100644 --- a/Patch/Game/scrFloorPatch.cs +++ b/Patch/Game/scrFloorPatch.cs @@ -3,10 +3,25 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Prevents the end portal from being spawned on level tiles for the "non-tile" render option. +/// +/// Patched member: scrFloor.SpawnPortalParticles(), which instantiates the animated portal prefab +/// (scrPortalParticles: glow, particles, cap and icon) as a child of the floor. The game calls it from +/// scrLevelMaker, scrController and scrFloor for the portal floor. +/// +/// +/// Active from stage Load when AdvancedConfig.NonTile is enabled, and only for floors that belong to +/// scrLevelMaker.listFloors (the level's tiles). See for the other tile renderers. +/// +/// [HarmonyPatch(typeof(scrFloor))] // ReSharper disable once InconsistentNaming public class scrFloorPatch { + /// Prefix of scrFloor.SpawnPortalParticles. + /// The floor that would receive the portal. + /// True to run the original method (spawn the portal); false to skip it (non-tile render, level floor). [HarmonyPrefix] [HarmonyPatch("SpawnPortalParticles")] // ReSharper disable once InconsistentNaming diff --git a/Patch/Game/scrHitTextManagerPatch.cs b/Patch/Game/scrHitTextManagerPatch.cs index fa52f66..2228ec0 100644 --- a/Patch/Game/scrHitTextManagerPatch.cs +++ b/Patch/Game/scrHitTextManagerPatch.cs @@ -3,10 +3,20 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Hides the judgement texts (e.g. Perfect, Early, Late) during a render unless the render configuration enables them. +/// +/// Patched member: scrHitTextManager.ShowHitText(HitMargin, scrPlanet, float, int?), which shows the hit margin +/// text at the planet for each hit (the game itself skips it for no-HUD modes and hidden margins). +/// +/// Active while (stage >= OnScene) and RenderConfig.ShowJudgementText is false. +/// [HarmonyPatch(typeof(scrHitTextManager), "ShowHitText")] // ReSharper disable once InconsistentNaming public class scrHitTextManagerPatch { + /// Prefix of scrHitTextManager.ShowHitText. + /// True to run the original method (not rendering, or judgement texts enabled); false to skip it. public static bool Prefix() { return !RenderManager.IsActive || RenderManager.Instance is { Configuration: { ShowJudgementText: true } }; diff --git a/Patch/Game/scrMistakesManagerPatch.cs b/Patch/Game/scrMistakesManagerPatch.cs index 99e70e3..b0d5043 100644 --- a/Patch/Game/scrMistakesManagerPatch.cs +++ b/Patch/Game/scrMistakesManagerPatch.cs @@ -3,10 +3,25 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Keeps a render from writing custom level results to the player's save data. +/// +/// Patched member: scrMistakesManager.SaveCustom(string hash, bool wonLevel, float multiplier), which compares the +/// run with the stored completion, accuracy, speed trial and death records of the custom level (keyed by the level hash), +/// stores new bests through Persistence and returns an EndLevelInfo. The game calls it from +/// scrController.OnLandOnPortal (custom level win) and Fail2Action (custom boss level failure). +/// +/// +/// Active while (stage >= OnScene); the run of a render is not stored as a player +/// record. The skipped call returns a default EndLevelInfo. +/// +/// [HarmonyPatch(typeof(scrMistakesManager))] // ReSharper disable once InconsistentNaming public static class scrMistakesManagerPatch { + /// Prefix of scrMistakesManager.SaveCustom. + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("SaveCustom")] public static bool PreventSaveCustom() diff --git a/Patch/Game/scrPlayerManagerPatch.cs b/Patch/Game/scrPlayerManagerPatch.cs index 5689d63..499b6cb 100644 --- a/Patch/Game/scrPlayerManagerPatch.cs +++ b/Patch/Game/scrPlayerManagerPatch.cs @@ -3,10 +3,27 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Replaces the "any valid input" check with a render-controlled value, so player input cannot affect the level during a render. +/// +/// Patched member: scrPlayerManager.AnyValidInputWasTriggered(), true when any player pressed a valid key this frame. +/// scrController uses it to leave the press-to-start wait, to restart after a failure (Fail2_Update) and to +/// leave the level after a win (Won_Update). +/// +/// +/// Active while (stage >= OnScene). The result is true only while +/// RenderManager.countdown is 0 and RenderManager.frameCount is 0, that is, after the 3 second countdown in +/// SetupPlay has ended and reset the frame counter and before the next frame increments it; it is false otherwise. +/// SetupPlay also sets scrController.levelWasSkipped, which ends the press-to-start wait independently of input. +/// +/// [HarmonyPatch(typeof(scrPlayerManager))] // ReSharper disable once InconsistentNaming public static class scrPlayerManagerPatch { + /// Prefix of scrPlayerManager.AnyValidInputWasTriggered. + /// Return value used when the original is skipped (see class summary). + /// True to run the original method (not rendering); false to skip it (rendering). [HarmonyPrefix] [HarmonyPatch("AnyValidInputWasTriggered")] // ReSharper disable once InconsistentNaming diff --git a/Patch/Game/scrPortalParticlesPatch.cs b/Patch/Game/scrPortalParticlesPatch.cs index 4cec411..fc559ba 100644 --- a/Patch/Game/scrPortalParticlesPatch.cs +++ b/Patch/Game/scrPortalParticlesPatch.cs @@ -4,10 +4,26 @@ using UnityEngine; namespace ADOFAIRenderer.Patch.Game; +/// +/// Makes the end portal particle systems run on scaled game time during a render. +/// +/// Patched member: scrPortalParticles.Start() (the animated portal spawned by scrFloor.SpawnPortalParticles; +/// Start pre-simulates its particle system for 1 second and plays it). +/// +/// +/// Active from stage Load. RenderManager sets Time.captureDeltaTime to 1 / RenderFPS, which advances scaled time by a +/// fixed step per frame; unscaled time is not affected by captureDeltaTime and follows real time. A particle system with +/// main.useUnscaledTime would therefore play at the real (slower or faster) frame rate of the offline render, so the +/// postfix sets it to false on every particle system under the portal (including inactive children). +/// Reference: https://docs.unity3d.com/ScriptReference/Time-captureDeltaTime.html +/// +/// [HarmonyPatch(typeof(scrPortalParticles))] // ReSharper disable once InconsistentNaming public static class scrPortalParticlesPatch { + /// Postfix of scrPortalParticles.Start: sets useUnscaledTime = false on all child particle systems. + /// The portal that started. [HarmonyPostfix] [HarmonyPatch("Start")] // ReSharper disable once InconsistentNaming diff --git a/Patch/Game/scrPressToStartPatch.cs b/Patch/Game/scrPressToStartPatch.cs index 122b264..2849351 100644 --- a/Patch/Game/scrPressToStartPatch.cs +++ b/Patch/Game/scrPressToStartPatch.cs @@ -3,10 +3,26 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Hides the "press any key to begin" text of the first level start of a render. +/// +/// Patched member: scrPressToStart.ShowText(), which sets the press/tap-to-begin text and shows the difficulty +/// selector (scrUIController.ShowDifficultyContainer). scrController calls it before waiting for the start input. +/// +/// +/// Active once per render: RenderManager.skipPressToStart is set when the render loads scnGame and cleared by +/// the first ShowText call (or when the render stops). Later ShowText calls run the original. +/// +/// [HarmonyPatch(typeof(scrPressToStart))] // ReSharper disable once InconsistentNaming public class scrPressToStartPatch { + /// + /// Prefix of scrPressToStart.ShowText. When skipPressToStart is set, clears it, hides the difficulty + /// selector and skips the original. + /// + /// True to run the original method; false to skip it (first ShowText of a render). [HarmonyPrefix] [HarmonyPatch("ShowText")] private static bool PreventShowText() diff --git a/Patch/Game/scrShowIfDebugPatch.cs b/Patch/Game/scrShowIfDebugPatch.cs index c6e996e..c82ffcb 100644 --- a/Patch/Game/scrShowIfDebugPatch.cs +++ b/Patch/Game/scrShowIfDebugPatch.cs @@ -6,10 +6,31 @@ using UnityEngine.UI; namespace ADOFAIRenderer.Patch.Game; +/// +/// Replaces the game's status text (normally "Autoplay", "Auto tile" or "Debug Mode") with the render status. +/// +/// Patched member: scrShowIfDebug.Update(), which each frame sets its UI Text (txt) to the autoplay / +/// auto-tile / debug label or hides it (no-HUD modes, recording mode), in defaultColor (green) or orange for auto tiles. +/// +/// +/// Active while (stage >= OnScene). The postfix overwrites whatever the original set, +/// every frame, so the text is shown even in the game's no-HUD modes. +/// +/// [HarmonyPatch(typeof(scrShowIfDebug))] // ReSharper disable once InconsistentNaming public static class scrShowIfDebugPatch { + /// + /// Postfix of scrShowIfDebug.Update. Shows, in the default (green) color: + /// + /// "Render in Ns.." during the countdown (N = RenderManager.countdown, seconds), otherwise "Render"; + /// after the end condition is met (endFrame != -1), a second line "Waiting for X.Xs.." with the remaining + /// part of DelayAfterEnd in render seconds ((frameCount - endFrame) / RenderFPS elapsed), clamped at 0. + /// + /// + /// Private field txt: the status Text component. + /// Private field defaultColor: the game's status text color. [HarmonyPostfix] [HarmonyPatch("Update")] [SuppressMessage("ReSharper", "InconsistentNaming")] diff --git a/Patch/Game/scrUIControllerPatch.cs b/Patch/Game/scrUIControllerPatch.cs index faf5c72..a10c1e8 100644 --- a/Patch/Game/scrUIControllerPatch.cs +++ b/Patch/Game/scrUIControllerPatch.cs @@ -3,10 +3,26 @@ using HarmonyLib; namespace ADOFAIRenderer.Patch.Game; +/// +/// Hides the end-screen lanterns again right after the game shows them. +/// +/// Patched member: scrUIController.ShowEndscreenLanterns(), which calls EndscreenLanterns.Show for each player's +/// lantern set (and fades the end-screen flash in co-op). scrController.OnLandOnPortal calls it on a win. +/// +/// +/// Active while (stage >= OnScene). Works together with , +/// which already skips EndscreenLanterns.Show during a render. +/// +/// [HarmonyPatch(typeof(scrUIController))] // ReSharper disable once InconsistentNaming public class scrUIControllerPatch { + /// + /// Postfix of scrUIController.ShowEndscreenLanterns: calls HideEndscreenLanterns(), which runs + /// EndscreenLanterns.Hide on every lantern set. + /// + /// The UI controller. [HarmonyPostfix] [HarmonyPatch("ShowEndscreenLanterns")] // ReSharper disable once InconsistentNaming diff --git a/RenderMod.cs b/RenderMod.cs index 5a6020b..9d8f0e7 100644 --- a/RenderMod.cs +++ b/RenderMod.cs @@ -14,12 +14,30 @@ using Object = UnityEngine.Object; namespace ADOFAIRenderer; +/// +/// MelonLoader entry point of the mod. Loads the settings and localization, points FFmpeg.AutoGen at the native +/// FFmpeg libraries, and creates the persistent manager objects (RenderManager, FirstUpdateHook, RendererUIManager). +/// internal class RenderMod : MelonMod { + /// + /// When true, the render button is not added to the level editor (scnEditorPatch checks it in scnEditor.Start). + /// Set by the user from the NVIDIA driver warning dialog (FirstUpdateHook). Main thread only. + /// // ReSharper disable once MemberCanBePrivate.Global public static bool IsFeatureDisabled = false; + + /// + /// Expiration time of this build (UTC). After it, the render configuration window shows an "expired" note next to + /// the version text. It does not block rendering. + /// public static DateTime ExpirationTime = new(2026, 11, 21, 0, 0, 0, DateTimeKind.Utc); + /// + /// Called once by MelonLoader after the mod assembly is loaded. + /// The native FFmpeg libraries are loaded from MelonLoader's UserLibs directory. + /// The three manager GameObjects are marked DontDestroyOnLoad so they survive scene changes. + /// public override void OnInitializeMelon() { GlobalSettings.Load(); @@ -39,6 +57,10 @@ internal class RenderMod : MelonMod ModAssets.Load(); } + /// + /// Called by MelonLoader when the game quits. Aborts an active render (encoders disposed, render thread joined, + /// unfinished output file deleted) through RenderManager.CleanUpActiveRender. + /// public override void OnApplicationQuit() { if (RenderManager.Instance == null) return; diff --git a/UI/CommonWarningDialogUI.cs b/UI/CommonWarningDialogUI.cs index 86a8b8b..7c30c77 100644 --- a/UI/CommonWarningDialogUI.cs +++ b/UI/CommonWarningDialogUI.cs @@ -11,14 +11,29 @@ using Image = UnityEngine.UI.Image; namespace ADOFAIRenderer.UI; +/// +/// Description of one button of a dialog: label, click action and optional color. +/// internal class DialogButton { // ReSharper disable FieldCanBeMadeReadOnly.Global + /// Button label, shown as given (callers pass already localized text). public string Text; + + /// + /// Action run on the main Unity thread when the button is clicked. The dialog is destroyed after the action + /// returns, also when the action throws (the exception is logged). + /// public Action OnClick; + + /// Normal (idle) background color of the button; null keeps the O5Kit theme button color. public Color? BackgroundColor; // ReSharper restore FieldCanBeMadeReadOnly.Global + /// Creates a button description. + /// Button label. + /// Action run on click, before the dialog closes. + /// Optional normal background color; null uses the theme color. public DialogButton(string text, Action onClick, Color? backgroundColor = null) { Text = text; @@ -27,10 +42,29 @@ internal class DialogButton } } +/// +/// Modal message dialog with a title, a description and one or more buttons. +/// +/// Each call creates its own GameObject (kept across scene loads) with a screen-space overlay +/// canvas: a translucent full-screen backdrop that blocks input to everything below, and an O5Kit window on top of it. +/// While the dialog exists, every enabled UI Toolkit root () is disabled so that it cannot +/// receive input; the roots are enabled again in . Clicking any button runs its action and then +/// destroys the dialog. All members run on the main Unity thread. +/// +/// internal class CommonWarningDialogUI : MonoBehaviour { + /// + /// The most recently created dialog, or null when that dialog has been destroyed. + /// FirstUpdateHook checks it to wait until a startup dialog is closed. + /// If several dialogs are open at once, this refers to the newest one only. + /// public static CommonWarningDialogUI? Instance { get; private set; } + + /// UI Toolkit root elements disabled by this dialog, enabled again when the dialog is destroyed. private readonly List _disabledElements = new(); + + /// O5Kit window holding the title, description and buttons; disposed in . private O5Window? _window; private void Awake() @@ -38,6 +72,10 @@ internal class CommonWarningDialogUI : MonoBehaviour Instance = this; } + /// + /// Disposes the window, enables the UI Toolkit roots disabled in and clears + /// if it still refers to this dialog. + /// private void OnDestroy() { _window?.Dispose(); @@ -58,6 +96,15 @@ internal class CommonWarningDialogUI : MonoBehaviour } } + /// + /// Creates and shows a new dialog. Must be called on the main Unity thread. + /// + /// Window title text. + /// Body text, centered and word-wrapped; TextMeshPro rich text tags are allowed. + /// + /// Buttons laid out left to right with equal widths. With no buttons nothing is shown, because the dialog + /// has no close button and could not be dismissed. + /// public static void ShowDialog(string title, string description, params DialogButton[] buttons) { if (buttons.Length == 0) return; @@ -68,8 +115,14 @@ internal class CommonWarningDialogUI : MonoBehaviour dialog.Initialize(title, description, buttons); } + /// + /// Builds the canvas, backdrop, window and contents. Layout sizes are in canvas units at the 1920x1080 reference + /// resolution of the . + /// private void Initialize(string title, string description, DialogButton[] buttons) { + // uGUI buttons need an EventSystem to receive clicks. When the current scene has none, a child EventSystem + // is created; it is destroyed together with the dialog. if (!UnityEngine.EventSystems.EventSystem.current) { var eventSystemObj = new GameObject("EventSystem"); @@ -78,6 +131,8 @@ internal class CommonWarningDialogUI : MonoBehaviour eventSystemObj.AddComponent(); } + // Disable every enabled UI Toolkit root so that it ignores input while the dialog is open. + // Roots that are already disabled are left alone and are not re-enabled in OnDestroy. foreach (var doc in FindObjectsByType(FindObjectsSortMode.None)) { if (doc && doc.rootVisualElement is { enabledSelf: true }) @@ -87,14 +142,17 @@ internal class CommonWarningDialogUI : MonoBehaviour } } - // Set UI layer (5) + // Set UI layer (5, Unity's built-in "UI" layer) gameObject.layer = 5; - // Canvas setup + // Canvas setup. + // Sorting order 32766 draws the dialog above the RendererUIManager window canvas (32000), the render overlays + // (DebugMonitorUI 9999, EscapeHandlerUI 10000) and below FFmpegDownloadUI (32767, the largest sorting order). var canvas = gameObject.AddComponent(); canvas.renderMode = RenderMode.ScreenSpaceOverlay; canvas.sortingOrder = 32766; + // Scale with screen size against a 1920x1080 reference; match 0.5 blends width and height scaling equally. var scaler = gameObject.AddComponent(); scaler.uiScaleMode = CanvasScaler.ScaleMode.ScaleWithScreenSize; scaler.referenceResolution = new Vector2(1920, 1080); @@ -105,7 +163,8 @@ internal class CommonWarningDialogUI : MonoBehaviour var ctx = RendererUIManager.Context; - // 1. Create full screen backdrop (scrim) to block background input + // 1. Create full screen backdrop (scrim) to block background input. + // Black at 65% opacity, stretched over the whole canvas; raycastTarget makes it swallow clicks. var bgPanelObj = new GameObject("Backdrop"); bgPanelObj.transform.SetParent(transform, false); bgPanelObj.layer = 5; @@ -120,7 +179,8 @@ internal class CommonWarningDialogUI : MonoBehaviour bgRect.offsetMin = Vector2.zero; bgRect.offsetMax = Vector2.zero; - // 2. Create actual native O5Window + // 2. Create the O5Kit window as a child of the backdrop: 540x240 units, draggable, fixed size, + // no close button (the dialog closes only through its buttons), 16-unit inner padding. _window = O5Window.Create(ctx, bgPanelObj.transform, new O5WindowOptions { Id = "RendererMod_WarningDialog", @@ -133,7 +193,8 @@ internal class CommonWarningDialogUI : MonoBehaviour Padding = new RectOffset(16, 16, 16, 16) }); - // 3. Build content inside window.Content + // 3. Build content inside window.Content: a vertical stack of the description (takes the remaining height) + // and the button row (40 units high), 16 units apart. var content = _window.Content; var vLayout = content.gameObject.AddComponent(); @@ -144,7 +205,7 @@ internal class CommonWarningDialogUI : MonoBehaviour vLayout.childForceExpandWidth = true; vLayout.childForceExpandHeight = false; - // Description text + // Description text: 18 pt, centered, word-wrapped, light gray (88% white) var descText = O5Factory.ControlText(ctx, content, 18f, noPad: true); descText.text = description; descText.alignment = TextAlignmentOptions.Center; @@ -157,7 +218,7 @@ internal class CommonWarningDialogUI : MonoBehaviour descLE.flexibleWidth = 1f; descLE.flexibleHeight = 1f; - // Button action row + // Button action row: buttons share the row width equally, 10 units apart var btnRowObj = new GameObject("ButtonRow"); btnRowObj.transform.SetParent(content, false); var btnRowRect = btnRowObj.AddComponent(); @@ -176,6 +237,7 @@ internal class CommonWarningDialogUI : MonoBehaviour for (var i = 0; i < buttons.Length; i++) { var btnData = buttons[i]; + // The click handler runs the caller's action, logs any exception from it, and always destroys the dialog. var btn = O5Factory.Button( ctx: ctx, parent: btnRowRect, diff --git a/UI/DebugMonitorUI.cs b/UI/DebugMonitorUI.cs index 0079c42..52e5009 100644 --- a/UI/DebugMonitorUI.cs +++ b/UI/DebugMonitorUI.cs @@ -8,39 +8,73 @@ using UnityEngine.UI; namespace ADOFAIRenderer.UI; +/// +/// On-screen overlay shown while a render is running: a statistics panel in the top-right corner and a level progress +/// bar near the bottom of the screen. +/// +/// Created by RenderManager.StartFFmpegThread on a GameObject kept across scene loads, and reused for later +/// renders. The overlay is drawn on the game screen only; it is not part of the encoded video frames. +/// rebuilds the text every frame on the main Unity thread. +/// +/// internal class DebugMonitorUI : MonoBehaviour { // --- Progress Bar UI Configuration Constants --- + // Sizes and positions are canvas units at the 1920x1080 reference resolution of the CanvasScaler. + + /// Width of the progress bar; the fill width is BarWidth * progress. private const float BarWidth = 800f; + /// Height of the progress bar. private const float BarHeight = 28f; + /// Horizontal offset of the bar center from the screen center (0 = centered). private const float BarPosX = 0f; + /// Vertical offset of the bar center from the screen center; -420 is 120 units above the bottom edge (540 - 420). private const float BarPosY = -420f; // Y position relative to screen center (Reference: 1920x1080) + /// Font size of the percentage text drawn on the bar. private const int BarTextFontSize = 16; // Increased font size for visibility + /// Unfilled bar color: opaque near-black with a slight blue tint. private static readonly Color BarBackgroundColor = new(0.08f, 0.08f, 0.1f, 1.0f); + /// Filled part color: opaque cyan (#00CCFF). private static readonly Color BarFillColor = new(0.0f, 0.8f, 1.0f, 1.0f); // Sleek cyan/blue + /// Percentage text color: opaque white. private static readonly Color BarTextColor = new(1.0f, 1.0f, 1.0f, 1.0f); // --- Progress Bar Outline Configuration Constants --- + + /// Color of the uGUI drawn around the bar background. private static readonly Color BarOutlineColor = Color.white; // Contrasting white outline + /// + /// Outline.effectDistance of the bar outline: the background is redrawn shifted by +-3 units on both axes, + /// which shows as a border about 3 units wide. + /// private static readonly Vector2 BarOutlineDistance = new(3.0f, -3.0f); // Outline thickness + /// Statistics text inside the top-right panel. private TextMeshProUGUI _text = null!; + /// Progress bar background object (parent of the fill and the percentage text). private GameObject _progressBarObj = null!; + /// Progress bar fill object. private GameObject _progressBarFillObj = null!; + /// Fill rect, anchored to the left edge of the bar; its width is set every frame. private RectTransform _fillRect = null!; + /// Text on the bar: percentage and "current floor / last floor index". private TextMeshProUGUI _progressText = null!; + /// + /// Builds the overlay canvas, the statistics panel and the progress bar. Layout values are canvas units at the + /// 1920x1080 reference resolution. + /// private void Start() { - // Set UI layer (5) + // Set UI layer (5, Unity's built-in "UI" layer) gameObject.layer = 5; - // Canvas setup + // Canvas setup. Sorting order 9999 is below EscapeHandlerUI (10000) and the dialogs/windows of the mod. var canvas = gameObject.AddComponent(); canvas.renderMode = RenderMode.ScreenSpaceOverlay; canvas.sortingOrder = 9999; - // Dynamic resolution scaling setup (Reference: 1920x1080) + // Dynamic resolution scaling against a 1920x1080 reference resolution; match 0.5 blends width and height var scaler = gameObject.AddComponent(); scaler.uiScaleMode = CanvasScaler.ScaleMode.ScaleWithScreenSize; scaler.referenceResolution = new Vector2(1920, 1080); @@ -49,7 +83,7 @@ internal class DebugMonitorUI : MonoBehaviour gameObject.AddComponent(); - // Create background panel + // Create background panel: 360x220 units, pivot and anchor at the top-right corner, inset 30 units from both edges var panelObj = new GameObject("BackgroundPanel"); panelObj.transform.SetParent(transform, false); panelObj.layer = 5; @@ -64,7 +98,7 @@ internal class DebugMonitorUI : MonoBehaviour panelRect.anchoredPosition = new Vector2(-30, -30); panelRect.sizeDelta = new Vector2(360, 220); - // Create text + // Create text: 16 pt rich text, top-left aligned, filling the panel minus a 20-unit margin var textObj = new GameObject("MonitorText"); textObj.transform.SetParent(panelObj.transform, false); textObj.layer = 5; @@ -74,11 +108,11 @@ internal class DebugMonitorUI : MonoBehaviour _text.fontSize = 16; _text.alignment = TextAlignmentOptions.TopLeft; - // Prevent TMPro text clipping + // Wrap long lines and let text that exceeds the panel height overflow instead of being truncated _text.textWrappingMode = TextWrappingModes.Normal; _text.overflowMode = TextOverflowModes.Overflow; - // Load and apply font + // Apply the mod's bundled font (loaded by ModAssets.Load at mod initialization) _text.font = ModAssets.NanumSquareNeoNormal; var textRect = textObj.GetComponent(); @@ -88,7 +122,7 @@ internal class DebugMonitorUI : MonoBehaviour textRect.offsetMax = new Vector2(-20, -20); // --- Progress Bar UI Setup --- - // 1. Progress Bar Container + // 1. Progress Bar Container: centered anchor/pivot, placed at (BarPosX, BarPosY) from the screen center _progressBarObj = new GameObject("ProgressBar"); _progressBarObj.transform.SetParent(transform, false); _progressBarObj.layer = 5; @@ -108,7 +142,8 @@ internal class DebugMonitorUI : MonoBehaviour barRect.anchoredPosition = new Vector2(BarPosX, BarPosY); barRect.sizeDelta = new Vector2(BarWidth, BarHeight); - // 2. Progress Bar Fill + // 2. Progress Bar Fill: anchored to the left edge and stretched vertically (anchor x 0..0, y 0..1). + // sizeDelta.x is the fill width in units; Update sets it to BarWidth * progress. _progressBarFillObj = new GameObject("ProgressBarFill"); _progressBarFillObj.transform.SetParent(_progressBarObj.transform, false); _progressBarFillObj.layer = 5; @@ -123,7 +158,7 @@ internal class DebugMonitorUI : MonoBehaviour _fillRect.anchoredPosition = Vector2.zero; _fillRect.sizeDelta = new Vector2(0, 0); // width will be updated dynamically in Update() - // 3. Progress Bar Text + // 3. Progress Bar Text: centered over the whole bar var progressTextObj = new GameObject("ProgressBarText"); progressTextObj.transform.SetParent(_progressBarObj.transform, false); progressTextObj.layer = 5; @@ -135,7 +170,7 @@ internal class DebugMonitorUI : MonoBehaviour _progressText.font = ModAssets.NanumSquareNeoNormal; _progressText.color = BarTextColor; - // Add progress bar text outline for readability + // Black outline (90% opacity, 1.5-unit offset) keeps the white text readable over both the fill and the background var textOutline = progressTextObj.AddComponent(); textOutline.effectColor = new Color(0f, 0f, 0f, 0.9f); textOutline.effectDistance = new Vector2(1.5f, -1.5f); @@ -147,10 +182,22 @@ internal class DebugMonitorUI : MonoBehaviour textRectBar.offsetMax = Vector2.zero; } + /// + /// Refreshes the statistics text and the progress bar from and the game state. + /// Values fall back to 0 (and the queue limit to RenderManager.MaxQueuedFrames) when no RenderManager exists. + /// private void Update() { if (!_text) return; + // Statistics lines: + // Queue - frames waiting in ImageQueue for the render thread / queue frame limit + // Rendered Frames - RenderManager.frameCount (frames produced by the game) + // Frames Sent - RenderManager.framesSent (frames handed to the encoder; written by the render thread) + // Throttling - RenderManager.throttleCount (times the main Unity thread started waiting because ImageQueue + // held more frames than its limit) + // Readback failures- RenderManager.readbackFailureCount (AsyncGPUReadback requests that completed with an error) + // Mono heap - GC.GetTotalMemory(false), managed heap size in MiB without forcing a collection var sb = new StringBuilder(); sb.AppendLine("ADOFAI Renderer Realtime Monitor"); sb.AppendLine(); @@ -158,6 +205,7 @@ internal class DebugMonitorUI : MonoBehaviour var queueCount = 0; if (RenderManager.Instance) { + // ImageQueue is shared with the render thread, so its count is read under the same lock lock (RenderManager.Instance.ImageQueue) { queueCount = RenderManager.Instance.ImageQueue.Count; @@ -177,7 +225,9 @@ internal class DebugMonitorUI : MonoBehaviour _text.text = sb.ToString(); - // Update Progress Bar + // Update Progress Bar. + // progress = seqID of the current floor / index of the last floor (listFloors.Count - 1), clamped to 0..1. + // The text shows the percentage with one decimal and "(current / last)". var progress = 0f; var currentFloor = 0; var totalFloors = 0; diff --git a/UI/EscapeHandlerUI.cs b/UI/EscapeHandlerUI.cs index 5d41ddd..470e950 100644 --- a/UI/EscapeHandlerUI.cs +++ b/UI/EscapeHandlerUI.cs @@ -6,15 +6,33 @@ using UnityEngine.UI; namespace ADOFAIRenderer.UI; +/// +/// Hold-Escape-to-cancel handler shown during a render. +/// +/// Created by RenderManager when the render reaches the OnScene stage, on a GameObject kept across scene loads. +/// While Escape is held, a centered panel shows a countdown from 3 seconds; holding it for the full 3 seconds +/// (measured in real time with a , independent of the patched game time) calls +/// RenderManager.CancelRender. Releasing Escape earlier hides the panel and resets the countdown. +/// The component destroys its GameObject as soon as RenderManager.IsActive is false (stage below OnScene). +/// Runs on the main Unity thread. +/// +/// internal class EscapeHandlerUI : MonoBehaviour { + /// Centered countdown panel (background and text); inactive while Escape is not held. private GameObject _containerObj = null!; + /// Countdown text inside . private TextMeshProUGUI _textComponent = null!; + /// Real-time hold duration of the current Escape press; null while Escape is not held. private Stopwatch? _escapeStopwatch; + /// + /// Builds the overlay canvas and the hidden countdown panel. Layout values are canvas units at the 1920x1080 + /// reference resolution. + /// private void Start() { - // Set UI layer (5) + // Set UI layer (5, Unity's built-in "UI" layer) gameObject.layer = 5; // Canvas setup @@ -22,7 +40,7 @@ internal class EscapeHandlerUI : MonoBehaviour canvas.renderMode = RenderMode.ScreenSpaceOverlay; canvas.sortingOrder = 10000; // Placed above debug monitor - // Responsive Canvas Scaler setup + // Canvas Scaler: 1920x1080 reference resolution, width and height matched equally (0.5) var scaler = gameObject.AddComponent(); scaler.uiScaleMode = CanvasScaler.ScaleMode.ScaleWithScreenSize; scaler.referenceResolution = new Vector2(1920, 1080); @@ -31,7 +49,7 @@ internal class EscapeHandlerUI : MonoBehaviour gameObject.AddComponent(); - // 1. Create container and background panel object + // 1. Create container and background panel object: 460x160 units at the screen center _containerObj = new GameObject("EscapeContainer"); _containerObj.transform.SetParent(transform, false); _containerObj.layer = 5; @@ -46,7 +64,7 @@ internal class EscapeHandlerUI : MonoBehaviour containerRect.anchoredPosition = Vector2.zero; // Screen center containerRect.sizeDelta = new Vector2(460, 160); - // 2. Create text object + // 2. Create text object: 60 pt centered rich text, no wrapping, 10-unit margin inside the panel var textObj = new GameObject("EscapeText"); textObj.transform.SetParent(_containerObj.transform, false); textObj.layer = 5; @@ -57,7 +75,8 @@ internal class EscapeHandlerUI : MonoBehaviour _textComponent.textWrappingMode = TextWrappingModes.NoWrap; _textComponent.richText = true; - // Apply game's default font (RDString.fontData.fontTMP) + // Apply the game's TextMeshPro font for the current language (RDString.fontData.fontTMP, assigned in + // RDString.Setup). It is null before RDString.Setup has run; TextMeshPro's default font is kept then. if (RDString.fontData.fontTMP != null) _textComponent.font = RDString.fontData.fontTMP; @@ -71,9 +90,13 @@ internal class EscapeHandlerUI : MonoBehaviour _containerObj.SetActive(false); } + /// + /// Tracks the Escape key every frame: shows and updates the countdown while it is held, cancels the render after + /// 3 seconds of holding, and hides the panel when it is released. + /// private void Update() { - // Destroy self if rendering is no longer active + // Destroy self once the render is inactive (stage below OnScene: finished, cancelled or failed) if (!RenderManager.IsActive) { Destroy(gameObject); @@ -82,6 +105,7 @@ internal class EscapeHandlerUI : MonoBehaviour if (Input.GetKey(KeyCode.Escape)) { + // First frame of the press: start timing and show the panel if (_escapeStopwatch == null) { _escapeStopwatch = new Stopwatch(); @@ -92,13 +116,14 @@ internal class EscapeHandlerUI : MonoBehaviour var elapsed = (float)_escapeStopwatch.Elapsed.TotalSeconds; var remaining = Mathf.Max(0f, 3f - elapsed); - // Update remaining time text + // Update remaining time text (seconds, one decimal). The text is fixed English, not localized. _textComponent.text = $"Cancelling in {remaining:F1}s..\nHold Esc to cancel render"; - // Micro pulse animation (container smoothly scales up while holding) + // Grow animation: the panel scales linearly from 1.0 to 1.15 over the 3-second hold var scale = 1.0f + elapsed / 3.0f * 0.15f; _containerObj.transform.localScale = new Vector3(scale, scale, 1f); + // Held for the full 3 seconds: reset the panel, then cancel if (elapsed < 3f) return; _escapeStopwatch.Stop(); @@ -106,12 +131,14 @@ internal class EscapeHandlerUI : MonoBehaviour _containerObj.transform.localScale = Vector3.one; // Reset scale _containerObj.SetActive(false); - // Cancel rendering and exit scene + // Cancel rendering: RenderManager.CancelRender resets the render state, stops the encoder as cancelled and + // ends the render. StopFFmpeg also destroys this handler's GameObject. if (RenderManager.Instance) RenderManager.Instance.CancelRender(); } else { + // Escape released before 3 seconds: stop timing and hide the panel if (_escapeStopwatch == null) return; _escapeStopwatch.Stop(); _escapeStopwatch = null; @@ -120,6 +147,7 @@ internal class EscapeHandlerUI : MonoBehaviour } } + /// Stops the hold stopwatch if a press is in progress. private void OnDestroy() { if (_escapeStopwatch == null) return; diff --git a/UI/FFmpegDownloadUI.cs b/UI/FFmpegDownloadUI.cs index 440198a..6760d47 100644 --- a/UI/FFmpegDownloadUI.cs +++ b/UI/FFmpegDownloadUI.cs @@ -15,27 +15,71 @@ using UnityEngine.UI; namespace ADOFAIRenderer.UI; +/// +/// Modal dialog that installs the FFmpeg shared libraries into MelonLoader's UserLibs folder. +/// +/// Shown by FirstUpdateHook when reports missing libraries. On Windows and Linux +/// the "Download" button downloads a prebuilt BtbN FFmpeg build (GPL, shared libraries) and extracts the libraries +/// into UserLibs; on macOS the button opens https://ffmpeg.org, because the dialog text tells the user to build FFmpeg +/// manually there. The libraries are found by FFmpeg.AutoGen through ffmpeg.RootPath (set to UserLibs in +/// RenderMod.OnInitializeMelon). +/// +/// +/// Threads: UI construction and all Unity object access happen on the main Unity thread. The download and extraction +/// run on thread-pool threads (); they post UI updates to +/// , which drains on the main Unity thread. +/// +/// +/// Flow: buttons -> (Download) progress bar + status text -> download to a temporary archive in UserLibs -> extract +/// libraries -> "complete" for 1 second -> dialog destroyed. On failure the status shows the error and the buttons +/// return so the user can retry or cancel. Cancel destroys the dialog at once and cancels a running download. +/// +/// internal class FFmpegDownloadUI : MonoBehaviour { + /// Row with the Download/Open Website and Cancel buttons; hidden while a download runs. private GameObject _buttonContainerObj = null!; + /// Progress bar background (rounded, theme colors); shown when the download starts. private GameObject _progressBarObj = null!; + /// Progress bar fill object. private GameObject _progressBarFillObj = null!; + /// Fill rect; its anchorMax.x is the progress fraction 0..1 (see ). private RectTransform _fillRect = null!; + /// Text centered on the progress bar (percentage or "Extracting..."). private TextMeshProUGUI _progressText = null!; + /// Status line below the progress bar; shown when the download starts. private GameObject _statusTextObj = null!; + /// Status message, colored by state (gray starting, cyan downloading, yellow extracting, green done, red failed). private TextMeshProUGUI _statusText = null!; + /// O5Kit window holding the dialog contents; disposed in . private O5Window? _window; + /// True from the Download click until the download fails; prevents a second concurrent download. private bool _isDownloading; + + /// + /// Archive URL for the current platform: a fixed BtbN FFmpeg-Builds autobuild (2026-05-31, FFmpeg master + /// N-124714-g49a77d37be, GPL, shared libraries). Windows gets the win64 .zip; every other platform gets the + /// linux64 .tar.xz (macOS never downloads, see ). + /// private string DownloadUrl => Application.platform switch { RuntimePlatform.WindowsPlayer => "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2026-05-31-13-22/ffmpeg-N-124714-g49a77d37be-win64-gpl-shared.zip", _ => "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2026-05-31-13-22/ffmpeg-N-124714-g49a77d37be-linux64-gpl-shared.tar.xz" }; + /// Cancellation for the running download; created per download, cancelled by Cancel, disposed in OnDestroy. private System.Threading.CancellationTokenSource? _cts; + /// Actions posted from background threads, run in order on the main Unity thread by . private readonly System.Collections.Concurrent.ConcurrentQueue _mainThreadQueue = new(); + /// + /// Whether UserLibs contains at least one file for each of the five FFmpeg libraries the mod uses + /// (avcodec, avformat, avutil, swscale, swresample), matched by platform-specific file name patterns: + /// avcodec-*.dll on Windows, libavcodec.*.dylib on macOS, libavcodec.so* elsewhere. + /// Only file presence is checked, not the library version or whether it loads. + /// + /// false when UserLibs does not exist or any library is missing. public static bool CheckFFmpegExists() { var userLibs = MelonEnvironment.UserLibsDirectory; @@ -68,6 +112,10 @@ internal class FFmpegDownloadUI : MonoBehaviour return true; } + /// + /// Creates the dialog on a new GameObject kept across scene loads, unless the libraries are already present. + /// Main thread only. + /// public static void ShowDownloadUI() { @@ -78,8 +126,13 @@ internal class FFmpegDownloadUI : MonoBehaviour obj.AddComponent(); } + /// + /// The open dialog, or null after it is destroyed. FirstUpdateHook waits while it is set and checks + /// the libraries again after it closes. + /// public static FFmpegDownloadUI? Instance { get; private set; } + /// Registers ; destroys the dialog again when the libraries already exist. private void Awake() { Instance = this; @@ -89,9 +142,13 @@ internal class FFmpegDownloadUI : MonoBehaviour } } + /// + /// Builds the canvas, backdrop and window: description, button row, and the hidden progress bar and status line. + /// Layout values are canvas units at the 1920x1080 reference resolution. + /// private void Start() { - // Set UI layer (5) + // Set UI layer (5, Unity's built-in "UI" layer) gameObject.layer = 5; // Canvas setup @@ -99,7 +156,7 @@ internal class FFmpegDownloadUI : MonoBehaviour canvas.renderMode = RenderMode.ScreenSpaceOverlay; canvas.sortingOrder = 32767; // Topmost - // Responsive Canvas Scaler setup + // Canvas Scaler: 1920x1080 reference resolution, width and height matched equally (0.5) var scaler = gameObject.AddComponent(); scaler.uiScaleMode = CanvasScaler.ScaleMode.ScaleWithScreenSize; scaler.referenceResolution = new Vector2(1920, 1080); @@ -110,7 +167,7 @@ internal class FFmpegDownloadUI : MonoBehaviour var ctx = RendererUIManager.Context; - // 1. Create full screen backdrop + // 1. Create full screen backdrop: black at 65% opacity, blocks clicks to everything below var bgPanelObj = new GameObject("Backdrop"); bgPanelObj.transform.SetParent(transform, false); bgPanelObj.layer = 5; @@ -125,7 +182,8 @@ internal class FFmpegDownloadUI : MonoBehaviour bgRect.offsetMin = Vector2.zero; bgRect.offsetMax = Vector2.zero; - // 2. Create actual native O5Window + // 2. Create the O5Kit window: 580x270 units, draggable, fixed size, no close button (Cancel closes it). + // The title is localized once here; the dialog does not follow later language changes. _window = O5Window.Create(ctx, bgPanelObj.transform, new O5WindowOptions { Id = "RendererMod_FFmpegDownloadDialog", @@ -138,6 +196,8 @@ internal class FFmpegDownloadUI : MonoBehaviour Padding = new RectOffset(16, 16, 16, 16) }); + // Vertical stack: description (takes the remaining height), buttons (40), progress bar (24), status (22), + // 14 units apart var content = _window.Content; var vLayout = content.gameObject.AddComponent(); @@ -148,7 +208,7 @@ internal class FFmpegDownloadUI : MonoBehaviour vLayout.childForceExpandWidth = true; vLayout.childForceExpandHeight = false; - // 3. Description text + // 3. Description text: "required" line, then either the macOS build/website lines or the download/size lines var descText = O5Factory.ControlText(ctx, content, 18f, noPad: true); var descStrBuilder = new StringBuilder(); descStrBuilder.AppendLine(LocalizationManager.Get("ui.ffmpeg_download.desc_required")); @@ -172,7 +232,7 @@ internal class FFmpegDownloadUI : MonoBehaviour descLE.flexibleWidth = 1f; descLE.flexibleHeight = 1f; - // 4. Button container + // 4. Button container: two equal-width 40-unit buttons, 10 units apart _buttonContainerObj = new GameObject("ButtonContainer"); _buttonContainerObj.transform.SetParent(content, false); _buttonContainerObj.layer = 5; @@ -189,6 +249,7 @@ internal class FFmpegDownloadUI : MonoBehaviour btnLayout.childForceExpandWidth = true; btnLayout.childForceExpandHeight = true; + // Primary button: download and install (Windows/Linux), or open the FFmpeg website and close the dialog (macOS) Action onDownloadClick = Application.platform != RuntimePlatform.OSXPlayer ? StartDownloadProcess : () => { Application.OpenURL("https://ffmpeg.org"); Destroy(gameObject); }; @@ -220,7 +281,8 @@ internal class FFmpegDownloadUI : MonoBehaviour cancelBtnLE.flexibleWidth = 1f; cancelBtnLE.preferredHeight = 40f; - // 5. Progress bar container + // 5. Progress bar container (hidden until the download starts): rounded background in the theme object color + // with a rounded outline drawn over it _progressBarObj = new GameObject("ProgressBarContainer"); _progressBarObj.transform.SetParent(content, false); _progressBarObj.layer = 5; @@ -251,7 +313,8 @@ internal class FFmpegDownloadUI : MonoBehaviour barOutlineRect.offsetMin = Vector2.zero; barOutlineRect.offsetMax = Vector2.zero; - // Progress bar fill image + // Progress bar fill image: anchored at the left edge with zero size offset, so its width is set entirely by + // anchorMax.x (0 = empty, 1 = full) in UpdateProgress _progressBarFillObj = new GameObject("ProgressBarFill"); _progressBarFillObj.transform.SetParent(_progressBarObj.transform, false); _progressBarFillObj.layer = 5; @@ -268,7 +331,7 @@ internal class FFmpegDownloadUI : MonoBehaviour _fillRect.anchoredPosition = Vector2.zero; _fillRect.sizeDelta = Vector2.zero; - // Progress text + // Progress text: 13 pt white, centered over the bar var progressTextObj = new GameObject("ProgressBarText"); progressTextObj.transform.SetParent(_progressBarObj.transform, false); progressTextObj.layer = 5; @@ -286,7 +349,7 @@ internal class FFmpegDownloadUI : MonoBehaviour progressTextRect.offsetMin = Vector2.zero; progressTextRect.offsetMax = Vector2.zero; - // 6. Status text + // 6. Status text (hidden until the download starts): 14 pt, centered _statusTextObj = new GameObject("StatusText"); _statusTextObj.transform.SetParent(content, false); _statusTextObj.layer = 5; @@ -310,20 +373,29 @@ internal class FFmpegDownloadUI : MonoBehaviour statusRect.offsetMax = Vector2.zero; } + /// Runs every action posted by the background download task, in posting order. private void Update() { - // Execute queued actions on main thread + // Execute queued actions on main Unity thread while (_mainThreadQueue.TryDequeue(out var action)) { action?.Invoke(); } } + /// + /// Posts an action to run on the main Unity thread during the next . Safe to call from any thread. + /// Actions posted after the dialog is destroyed are never run. + /// private void ExecuteOnMainThread(Action action) { _mainThreadQueue.Enqueue(action); } + /// + /// Download button handler (Windows/Linux): switches the dialog to progress mode and starts the background task + /// that downloads the archive, extracts the libraries and closes the dialog. Ignored while a download runs. + /// private void StartDownloadProcess() { if (_isDownloading) return; @@ -336,7 +408,9 @@ internal class FFmpegDownloadUI : MonoBehaviour _cts = new System.Threading.CancellationTokenSource(); - // Start download in background thread + // Start download in background thread. Everything inside runs on thread-pool threads: Unity objects are only + // touched through UpdateStatus/UpdateProgress/ExecuteOnMainThread. The task is not awaited; its outcome is + // reported through the status text. Task.Run(async () => { string tempZipPath = ""; @@ -351,6 +425,8 @@ internal class FFmpegDownloadUI : MonoBehaviour Directory.CreateDirectory(userLibsDir); } + // Temporary archive path in UserLibs: ffmpeg_temp.zip (Windows, macOS) or ffmpeg_temp.tar.xz (others). + // It is deleted after extraction (ExtractDllsAsync) or on cancellation. string extension = Application.platform switch { RuntimePlatform.WindowsPlayer => ".zip", @@ -359,6 +435,8 @@ internal class FFmpegDownloadUI : MonoBehaviour }; tempZipPath = Path.Combine(userLibsDir, "ffmpeg_temp" + extension); + // percent is the downloaded fraction 0..1. Progress created on a thread-pool thread (no + // SynchronizationContext) invokes this callback on thread-pool threads; it only posts UI updates. var progressReporter = new Progress(percent => { UpdateProgress(percent, $"{percent * 100f:F1}%"); @@ -367,7 +445,7 @@ internal class FFmpegDownloadUI : MonoBehaviour await DownloadFileAsync(DownloadUrl, tempZipPath, progressReporter, _cts.Token); - // Extract after download complete + // Extract after download complete; the bar is held at 99% while extracting UpdateStatus(LocalizationManager.Get("ui.ffmpeg_download.status_extracting"), new Color(0.9f, 0.9f, 0.2f)); UpdateProgress(0.99f, LocalizationManager.Get("ui.ffmpeg_download.progress_extracting")); @@ -377,9 +455,10 @@ internal class FFmpegDownloadUI : MonoBehaviour UpdateStatus(LocalizationManager.Get("ui.ffmpeg_download.status_complete"), new Color(0.2f, 0.9f, 0.2f)); UpdateProgress(1f, "100%"); + // Keep the "complete" message visible for 1000 ms before closing await Task.Delay(1000); - // Destroy UI on main thread + // Destroy UI on main Unity thread. FirstUpdateHook then sees Instance == null and checks the libraries again. ExecuteOnMainThread(() => { Destroy(gameObject); @@ -387,7 +466,8 @@ internal class FFmpegDownloadUI : MonoBehaviour } catch (OperationCanceledException) { - // Clean up temporary file on cancellation + // Clean up temporary file on cancellation. Cancellation comes from CancelProcess, which has already + // destroyed the dialog, so the status update below is posted but never shown. if (!string.IsNullOrEmpty(tempZipPath) && File.Exists(tempZipPath)) { try { File.Delete(tempZipPath); } @@ -403,7 +483,9 @@ internal class FFmpegDownloadUI : MonoBehaviour Melon.Logger.Error($"Error during FFmpeg download/installation: {ex}"); UpdateStatus(LocalizationManager.Get("ui.ffmpeg_download.status_failed", ex.Message), new Color(1f, 0.3f, 0.3f)); - // Reset buttons on failure + // Reset buttons on failure so the user can retry (Download) or close the dialog (Cancel). + // The label is set to "Retry" only if a child at "DownloadButton/DownloadText" exists. The button made by + // O5Factory.Button in Start is a GameObject named "O5Button", which this path does not match. ExecuteOnMainThread(() => { _isDownloading = false; @@ -418,6 +500,10 @@ internal class FFmpegDownloadUI : MonoBehaviour }); } + /// + /// Cancel button handler: requests cancellation of a running download and destroys the dialog immediately. + /// An extraction already in progress is not cancellable and finishes in the background. + /// private void CancelProcess() { if (_isDownloading) @@ -427,6 +513,7 @@ internal class FFmpegDownloadUI : MonoBehaviour Destroy(gameObject); } + /// Posts a status text and color change to the main Unity thread. Callable from any thread. private void UpdateStatus(string message, Color color) { ExecuteOnMainThread(() => @@ -439,6 +526,9 @@ internal class FFmpegDownloadUI : MonoBehaviour }); } + /// Posts a progress bar update to the main Unity thread. Callable from any thread. + /// Filled fraction 0..1 (not 0..100); becomes the fill's anchorMax.x. + /// Text shown on the bar. private void UpdateProgress(float percent, string text) { ExecuteOnMainThread(() => @@ -454,15 +544,28 @@ internal class FFmpegDownloadUI : MonoBehaviour }); } + /// + /// Streams the HTTP response body of to (created or + /// overwritten) in 8 KiB chunks. Runs on a thread-pool thread. + /// + /// Archive URL. + /// Output file path. + /// + /// Receives the downloaded fraction 0..1 after every chunk; not reported when the server sends no Content-Length. + /// + /// Cancels the request and the copy loop (). + /// The server returned a non-success status code. private async Task DownloadFileAsync(string url, string destinationPath, IProgress progress, System.Threading.CancellationToken cancellationToken) { using var client = new HttpClient(); - // Add User-Agent header to prevent blocking + // Add User-Agent header to prevent blocking by servers that reject requests without one client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) ADOFAIRenderer/1.0"); + // ResponseHeadersRead returns after the headers, so the body is streamed instead of buffered in memory using var response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken); response.EnsureSuccessStatusCode(); + // Total size from Content-Length, or -1 when unknown. File stream: 8192-byte buffer, asynchronous I/O. var totalBytes = response.Content.Headers.ContentLength ?? -1L; await using var contentStream = await response.Content.ReadAsStreamAsync(); await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, 8192, true); @@ -483,12 +586,28 @@ internal class FFmpegDownloadUI : MonoBehaviour } } + /// + /// Extracts the shared libraries from the downloaded archive into , overwriting + /// files of the same name, and deletes the archive afterwards (also on failure; a failed delete is logged as a + /// warning). Runs on thread-pool threads and is not cancellable. + /// + /// Windows/macOS (.zip): every entry ending in .dll (Windows) or .dylib (macOS) is extracted flat, by its file + /// name only, ignoring its folder inside the archive. + /// Other platforms (.tar.xz): the archive is unpacked with the system tar command into the temporary + /// folder "ffmpeg_temp_extracted" in ; every file matching *.so* in any + /// subfolder is copied flat to , then the temporary folder is deleted. + /// + /// + /// Downloaded archive path (.zip or .tar.xz despite the parameter name). + /// UserLibs folder. + /// tar exited with a non-zero code (message includes its standard error). private async Task ExtractDllsAsync(string zipPath, string outputDirectory) { try { if (Application.platform == RuntimePlatform.OSXPlayer || Application.platform == RuntimePlatform.WindowsPlayer) { + // .zip: extract matching library entries flat into outputDirectory await Task.Run(() => { var targetExtension = Application.platform == RuntimePlatform.OSXPlayer ? ".dylib" : ".dll"; @@ -522,6 +641,7 @@ internal class FFmpegDownloadUI : MonoBehaviour } Directory.CreateDirectory(tempExtractDir); + // tar receives forward-slash paths, quoted for spaces: tar -xf "" -C "" var archiveUnixPath = zipPath.Replace('\\', '/'); var tempExtractUnixDir = tempExtractDir.Replace('\\', '/'); @@ -545,7 +665,8 @@ internal class FFmpegDownloadUI : MonoBehaviour } } - // Find .so files and copy to outputDirectory + // Find .so files (including versioned names such as libavcodec.so.) and copy them flat to + // outputDirectory var soFiles = Directory.GetFiles(tempExtractDir, "*.so*", SearchOption.AllDirectories); foreach (var soFile in soFiles) { @@ -582,6 +703,10 @@ internal class FFmpegDownloadUI : MonoBehaviour } } + /// + /// Disposes the window and the cancellation source and clears . A download that was not + /// cancelled keeps running in the background; its UI updates are posted but no longer drained. + /// private void OnDestroy() { _window?.Dispose(); diff --git a/UI/LocalizedText.cs b/UI/LocalizedText.cs index 96bf389..cc601aa 100644 --- a/UI/LocalizedText.cs +++ b/UI/LocalizedText.cs @@ -8,15 +8,25 @@ namespace ADOFAIRenderer.UI; /// /// Component that binds a TextMeshProUGUI component to a localization key or dynamic string provider, /// automatically updating the displayed text whenever the active language changes. +/// +/// The text is set when the binding is made (), when the component is enabled, +/// and on every event. A key and a dynamic provider are mutually +/// exclusive: setting one clears the other. With neither set, leaves the text unchanged. +/// Runs on the main Unity thread. +/// /// internal class LocalizedText : MonoBehaviour { + /// Text component that receives the string; looked up on the same GameObject if not assigned by Bind. private TextMeshProUGUI? _textComponent; + /// Localization key resolved through ; null when a provider is used. private string? _key; + /// Function that returns the full text (already localized/formatted); null when a key is used. private Func? _dynamicProvider; /// - /// Gets or sets the localization key. Setting this will clear any dynamic provider. + /// Gets or sets the localization key. Setting this will clear any dynamic provider and refresh the text immediately. + /// A null or empty key leaves the current text as it is. /// public string? Key { @@ -30,8 +40,11 @@ internal class LocalizedText : MonoBehaviour } /// - /// Sets a dynamic string provider function. + /// Sets a dynamic string provider function, clears the key and refreshes the text immediately. + /// The provider is called on every refresh (enable and language change), so it can combine several keys or + /// runtime values, e.g. a label that depends on UI state. /// + /// Function returning the text to display; called on the main Unity thread. public void SetDynamic(Func provider) { _dynamicProvider = provider; @@ -39,27 +52,35 @@ internal class LocalizedText : MonoBehaviour Refresh(); } + /// + /// Finds the text component and subscribes to language changes. The subscription lasts until , + /// also while the component is disabled (a refresh of a disabled object only changes its text). + /// private void Awake() { EnsureComponent(); LocalizationManager.OnLanguageChanged += HandleLanguageChanged; } + /// Refreshes the text each time the component is enabled. private void OnEnable() { Refresh(); } + /// Unsubscribes from . private void OnDestroy() { LocalizationManager.OnLanguageChanged -= HandleLanguageChanged; } + /// Language change handler; the new language is read by . private void HandleLanguageChanged(SystemLanguage language) { Refresh(); } + /// Assigns from the same GameObject if it is not set yet. private void EnsureComponent() { if (_textComponent == null) @@ -70,6 +91,8 @@ internal class LocalizedText : MonoBehaviour /// /// Refreshes the text using either the dynamic provider or the static localization key. + /// The provider takes precedence; with neither set, or without a text component, nothing changes. + /// A key that is missing in every locale is displayed as the key itself (see ). /// public void Refresh() { @@ -88,7 +111,13 @@ internal class LocalizedText : MonoBehaviour /// /// Binds a TextMeshProUGUI component to a localization key. + /// Reuses the already on the target's GameObject or adds one, then sets the key, + /// which replaces any earlier binding and updates the text immediately. /// + /// Text to localize. + /// Localization key, e.g. "ui.render_config.codec". + /// The binding component. + /// is null. public static LocalizedText Bind(TextMeshProUGUI target, string key) { if (target == null) throw new ArgumentNullException(nameof(target)); @@ -100,7 +129,12 @@ internal class LocalizedText : MonoBehaviour /// /// Binds a TextMeshProUGUI component to a dynamic string provider function. + /// Reuses or adds the component like and updates the text immediately. /// + /// Text to localize. + /// Function returning the text; called on every refresh. + /// The binding component. + /// is null. public static LocalizedText Bind(TextMeshProUGUI target, Func provider) { if (target == null) throw new ArgumentNullException(nameof(target)); diff --git a/UI/RendererUIManager.cs b/UI/RendererUIManager.cs index 1bd224f..267d963 100644 --- a/UI/RendererUIManager.cs +++ b/UI/RendererUIManager.cs @@ -8,9 +8,20 @@ namespace ADOFAIRenderer.UI; using UnityEngine; +/// +/// Assets loaded from the mod's embedded asset bundles (fonts and the color conversion compute shader). +/// +/// is called once from RenderMod.OnInitializeMelon; the fields stay null until then. +/// Bundles are obtained from BundleLoader.GetBundle by name ("fonts", "shaders"). Main thread only. +/// +/// [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] internal static class ModAssets { + /// + /// Loads the font and shader bundles and the assets inside them, logging after each bundle. + /// Exceptions from bundle or asset loading are not caught here and propagate to the caller. + /// public static void Load() { FontBundle = BundleLoader.GetBundle("fonts"); @@ -23,24 +34,49 @@ internal static class ModAssets Melon.Logger.Msg("Shader loaded successfully."); } + /// Asset bundle "fonts" holding the TextMeshPro font assets. public static AssetBundle FontBundle = null!; + /// Asset bundle "shaders" holding the compute shaders. public static AssetBundle ShaderBundle = null!; + /// + /// NanumSquare Neo, asset "NanumSquareNeoOTF-Bd SDF" (Bd = Bold weight). Used as the regular UI font of the mod + /// (O5Kit , DebugMonitorUI text). + /// public static TMP_FontAsset NanumSquareNeoNormal = null!; + /// + /// NanumSquare Neo, asset "NanumSquareNeoOTF-Eb SDF" (Eb = ExtraBold weight). Used as the O5Kit medium font + /// (). + /// public static TMP_FontAsset NanumSquareNeoBold = null!; - + + /// + /// Compute shader "ColorConversion" that converts captured frames to the encoder's YUV pixel format on the GPU + /// (used by FrameReader). + /// public static ComputeShader ColorConversionShader = null!; } +/// +/// O5Kit font provider for the mod's UI: the bundled NanumSquare Neo fonts for regular and medium text, and O5Kit's +/// default monospace font. +/// internal sealed class RendererFontProvider : IFontProvider { + /// O5Kit default provider, used only for . private static readonly DefaultFontProvider DefaultFontProvider = new(); private bool _disposed; + /// Regular text font: . public TMP_FontAsset Regular => ModAssets.NanumSquareNeoNormal; + /// Medium (emphasis) text font: . public TMP_FontAsset Medium => ModAssets.NanumSquareNeoBold; + /// Monospace font (e.g. hex color input): O5Kit's default monospace font. public TMP_FontAsset Monospace => DefaultFontProvider.Monospace; + /// + /// Marks the provider disposed. The font assets belong to and are not unloaded. + /// public void Dispose() { if (_disposed) return; @@ -48,19 +84,39 @@ internal sealed class RendererFontProvider : IFontProvider } } +/// +/// Owner of the mod's O5Kit UI: the shared (config, theme, fonts) and the window manager +/// with its canvas. +/// +/// Added once by RenderMod.OnInitializeMelon to a GameObject kept across scene loads. Other UI code reads +/// and through the static . +/// drives O5Kit's per-frame work (object ticks, tooltip, keyboard shortcuts). Main thread only. +/// +/// internal class RendererUIManager : MonoBehaviour { + /// The single instance, assigned in . public static RendererUIManager Instance = null!; + /// Shared O5Kit context used to build every mod window and dialog. public static O5Context Context => Instance._context; + /// Window manager whose canvas holds the mod windows (e.g. RenderConfigWindow). public O5WindowManager WindowManager => _windowManager; private O5Context _context = null!; private O5WindowManager _windowManager = null!; + /// + /// Creates the O5Kit context with the mod theme and fonts, then the window manager and its canvas + /// ("RendererMod_Canvas"), and lowers that canvas's sorting order to . + /// private void Awake() { Instance = this; - + + // O5Kit config: UI scale 1.0, middle click resets a control to its default value, slider drag sensitivity 1.0. + // Theme: O5Kit's dark theme with the mod colors. Colors are RGBA 0..1; the accent color + // (1.000, 0.294, 0.079) is orange-red (#FF4B14 approx.). Sizes are canvas units: corner radius 12, + // outline width 2, control height 40, body font 18, H1 font 48. _context = new O5Context( config: new O5Config { UIScale = 1.0f, @@ -101,14 +157,20 @@ internal class RendererUIManager : MonoBehaviour if (windowCanvas != null) windowCanvas.sortingOrder = WindowCanvasSortingOrder; } + /// + /// Sorting order of the window canvas: below CommonWarningDialogUI (32766) and FFmpegDownloadUI (32767), above the + /// render overlays (DebugMonitorUI 9999, EscapeHandlerUI 10000). + /// private const int WindowCanvasSortingOrder = 32000; + /// Runs O5Kit's per-frame updates: object ticks, tooltip, and keyboard shortcut handling. private void Update() { O5Object.TickAll(); Context.Tooltip.Tick(); O5Kit.Input.O5ShortcutManager.HandleUpdate(); } + /// Disposes the window manager, then the context. private void OnDestroy() { _windowManager.Dispose(); _context.Dispose(); diff --git a/UI/Window/RenderConfigWindow.cs b/UI/Window/RenderConfigWindow.cs index 22a252d..8a2ef1a 100644 --- a/UI/Window/RenderConfigWindow.cs +++ b/UI/Window/RenderConfigWindow.cs @@ -18,19 +18,70 @@ using UnityEngine.UI; namespace ADOFAIRenderer.UI.Window; +/// +/// The "Render Config" window: edits a and the global output directory, manages presets, +/// and starts a render ("Go Render"). +/// +/// Opened from the level editor through RenderManager (after the unsaved-changes check) with +/// . The window is built once and then hidden/shown; its state lives in static fields for the +/// whole game session. All members run on the main Unity thread. +/// +/// +/// Data flow: +/// +/// Each control's change callback writes its value into right away and calls +/// (the preset dropdown then shows the Custom label). +/// writes a config back into every control without invoking the callbacks +/// (preset selection, window reopen). +/// reads every control into the config again before saving a preset or rendering, +/// and converts NTSC frame rates (29.97 etc.) to their exact fractions. +/// The output directory is stored in GlobalSettings.Current.OutputDirectory (null = default folder), +/// not in the render config, and is saved to the global settings file. +/// +/// +/// +/// File layout: fields; ; (cards in display order: Presets, top section, +/// Video Encoding, Audio Encoding, Global Settings, Advanced Settings, Go Render, version label); config/UI sync and +/// render start; preset handling (Custom label, select, save, delete, language change); layout helpers +/// (columns, field slots, cards); control factory helpers; enable/disable rules for dependent fields; window visibility. +/// +/// +/// Layout sizes are O5Kit canvas units (the canvas scales with the screen); label rows are 18 units, field rows use +/// 3 units of spacing between label and control, dropdown controls are 50 units high. +/// +/// internal static class RenderConfigWindow { + // =================================================================================================== + // Fields + // =================================================================================================== + + /// The window, created by the first call and reused (hidden, not destroyed, on close). private static O5Window? _configWindow; // Presets controls + /// Preset selector: preset names, plus the Custom label at index 0 while the values differ from the preset. private static O5Dropdown? _presetDropdown; + /// Name used by the Save button; set to the selected preset's name when a preset is loaded. private static O5InputField? _presetNameInput; + /// Saves the current values under the name in . private static O5Button? _savePresetButton; + /// + /// Two-step delete button (red theme): the first click arms it ("Confirm?", wider, brighter red) for 3 seconds, + /// a second click within that time deletes the preset. + /// private static O5Button? _deletePresetButton; + /// Layout element of the delete button; preferred width 90 units normally, 110 while armed. // ReSharper disable once InconsistentNaming private static LayoutElement? _deletePresetButtonLE; + /// True while the delete button is armed and waiting for the confirming click. private static bool _isDeletePending; + /// Coroutine (on RendererUIManager) that disarms the delete button after 3 seconds. private static Coroutine? _deleteResetCoroutine; + /// + /// Name of the preset whose values were last loaded or saved. The Custom label is compared against this preset, + /// and Delete targets it when the dropdown shows the Custom label. null when no presets exist. + /// private static string? _lastLoadedPresetName; /// @@ -39,9 +90,15 @@ internal static class RenderConfigWindow /// private static RenderConfig? _uiConfig; + /// + /// , created on first access with the initializer values + /// (RenderFPS, VideoFPS, RenderWidth and RenderHeight start at 0; fills them). + /// private static RenderConfig UIConfig => _uiConfig ??= new RenderConfig(); - // Controls cache for validation & sync + // Controls cache for validation & sync. + // Each field holds the control bound to the RenderConfig property of the same name; ApplyConfigToUI writes them, + // SyncUItoConfig reads them. They stay null until BuildUI runs. private static O5Slider? _renderFpsSlider; private static O5Slider? _videoFPSSlider; private static O5Slider? _renderWidthSlider; @@ -51,6 +108,8 @@ internal static class RenderConfigWindow private static O5Dropdown? _codecDropdown; private static O5Dropdown? _rateControlDropdown; + // Bitrate / max bitrate / QP inputs keep their label and the running label color tween, so UpdateRateControlUI + // can dim the label together with the blocked input (see SetFieldBlocked). private static O5InputField? _bitrateInput; private static TMPro.TextMeshProUGUI? _bitrateLabel; private static ITweenHandle? _bitrateLabelTween; @@ -71,6 +130,9 @@ internal static class RenderConfigWindow private static O5Dropdown? _channelsDropdown; private static O5Slider? _audioBitrateSlider; + /// + /// Output directory input. Bound to GlobalSettings.Current.OutputDirectory, not to a RenderConfig property. + /// private static O5InputField? _outputDirInput; private static O5Dropdown? _fileExtensionDropdown; @@ -78,6 +140,8 @@ internal static class RenderConfigWindow private static O5Toggle? _processAudioToggle; private static O5Toggle? _nonTileToggle; + // Planet color overrides: the toggle enables the override, the picker holds the color. A disabled toggle stores + // null in AdvancedConfig (no override is applied by PersistencePatch) and blocks the picker. private static O5Toggle? _firePlanetToggle; private static O5ColorPicker? _firePlanetPicker; private static O5Toggle? _icePlanetToggle; @@ -85,10 +149,26 @@ internal static class RenderConfigWindow private static O5Toggle? _windPlanetToggle; private static O5ColorPicker? _windPlanetPicker; + // Colors shown in a planet picker when its override is off (and its middle-click default value) private static readonly Color DefaultFirePlanetColor = new(1f, 0f, 0f, 1f); // #FF0000 private static readonly Color DefaultIcePlanetColor = new(0f, 0f, 1f, 1f); // #0000FF private static readonly Color DefaultWindPlanetColor = new(0.3f, 0.7f, 0f, 1f); // #4DB300 + // =================================================================================================== + // Window creation + // =================================================================================================== + + /// + /// Shows the window, building it on the first call. + /// + /// First call: creates a 900x600 resizable, draggable window with a close button, builds the controls + /// () and subscribes . + /// Later calls: reloads the preset list from disk and selects GlobalSettings.Current.LastPresetName + /// (or the first preset). + /// + /// Then writes into the controls, shows the window and brings it to the front. + /// The close button hides the window (closing open color pickers and disarming the delete button first). + /// public static void CreateWindow() { if (_configWindow == null) @@ -127,6 +207,24 @@ internal static class RenderConfigWindow _configWindow.BringToFront(); } + // =================================================================================================== + // UI construction + // =================================================================================================== + + /// + /// Builds every control of the window inside a vertical scroll view (10 units between items, 8 units padding). + /// + /// The initial preset is GlobalSettings.Current.LastPresetName if it exists, otherwise the first preset whose + /// name starts with "QHD", otherwise the first preset. Its values are copied into only when + /// RenderFPS is 0 or less at that point (see the comment at that check). Each control is created with the config + /// value and a change callback that writes back into the config. + /// + /// + /// Sliders use a sliderFilter that snaps dragged values to the nearest entry of a snap list (common frame + /// rates, resolutions, bitrates); values typed into the slider's input box are not snapped. + /// + /// + /// Content rect of the window. private static void BuildUI(RectTransform content) { var ctx = RendererUIManager.Context; @@ -136,6 +234,10 @@ internal static class RenderConfigWindow var config = UIConfig; EnsureDefaultConfig(config); + // Snap lists for slider dragging (Snap picks the entry nearest to the dragged value; order does not matter). + // Render FPS: standard video rates, then multiples of 120 and of 288 up to the 3000 maximum. + // Video FPS: standard video rates, then multiples of 60 and of 72 up to the 600 maximum. + // Audio bitrate: kbps values. Width/height: common resolutions grouped by aspect ratio. double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144, 120, 240, 360, 480, 600, 720, // +120 288, 576, 864, 1152, 1440, 1728, // +288 @@ -161,6 +263,9 @@ internal static class RenderConfigWindow 864, 960, 1536, 2048, 3072 // 4:3 }; + // --------------------------------------------------------------------------------------------- + // Section 1: Presets card - preset dropdown, then a row with name input, Save and Delete buttons + // --------------------------------------------------------------------------------------------- // Presets (Collapsible Card) var (_, presetContent) = CreateCard( ctx, scrollContent, @@ -169,6 +274,8 @@ internal static class RenderConfigWindow var presets = PresetManager.GetPresets(); var presetNames = presets.Select(p => p.Name).ToList(); + // Initial preset: last used preset if it still exists, else the first "QHD..." preset, else the first preset, + // else "" (no presets on disk) var initialPresetName = GlobalSettings.Current.LastPresetName; if (string.IsNullOrEmpty(initialPresetName) || !presetNames.Contains(initialPresetName)) { @@ -177,6 +284,10 @@ internal static class RenderConfigWindow } _lastLoadedPresetName = initialPresetName; + // Copies the initial preset into a config that has not been filled yet (RenderFPS <= 0). + // EnsureDefaultConfig at the start of BuildUI sets RenderFPS to 60 when it is 0 or less, so on the first build + // this condition is false and the config keeps the defaults (EnsureDefaultConfig values and RenderConfig + // initializers). UpdatePresetLabel then shows the Custom label unless those defaults equal the preset. if (config.RenderFPS <= 0) { var initialPreset = presets.FirstOrDefault(p => string.Equals(p.Name, initialPresetName, StringComparison.OrdinalIgnoreCase)) ?? presets.FirstOrDefault(); @@ -187,6 +298,7 @@ internal static class RenderConfigWindow } EnsureDefaultConfig(config); + // Selecting a preset loads it into the config and all controls (OnPresetSelected) _presetDropdown = CreateDropdownField( ctx, presetContent, "ui.render_config.preset_select", @@ -199,6 +311,8 @@ internal static class RenderConfigWindow } ); + // Preset actions row (40 units high): name input takes the remaining width, Save and Delete are 90 units wide, + // 8 units apart var actionsRowObj = new GameObject("PresetActionsRow"); actionsRowObj.transform.SetParent(presetContent, false); var actionsRowRect = actionsRowObj.AddComponent(); @@ -248,6 +362,8 @@ internal static class RenderConfigWindow saveLE.flexibleWidth = 0f; saveLE.preferredHeight = 40f; + // The Delete button is built with a copy of the context whose button color is the theme's SoftRed + // (hover: SoftRed at 80% brightness, alpha also scaled by the Color multiplication) var redTheme = ctx.Theme with { ObjectButton = ctx.Theme.SoftRed, @@ -267,6 +383,7 @@ internal static class RenderConfigWindow onClick: () => OnDeletePresetClicked(config), height: 40f ); + // Delete label follows both the language and the armed state ("Delete" / "Confirm?") if (_deletePresetButton.Label != null) { LocalizedText.Bind(_deletePresetButton.Label, () => @@ -281,9 +398,17 @@ internal static class RenderConfigWindow UpdateDeleteButtonState(); - // 1-3. Top section: 좌/우 컨테이너 1개씩으로 3행을 쌓는다 (중앙선 1개) + // --------------------------------------------------------------------------------------------- + // Section 2: Top section (no card) - two columns, three rows: + // Render FPS | Video FPS + // Render Width | Render Height + // Delay After End| Show Judgement Text + // One left and one right column container hold all three rows, so the column boundary is the same on every row. + // --------------------------------------------------------------------------------------------- var (topLeft, topRight) = CreateTwoColumnPanel(scrollContent, spacing: 8f, columnSpacing: 10f); + // Render FPS (RenderConfig.RenderFPS): game frames simulated per second of render time (the game clock advances + // 1 / RenderFPS seconds per frame), 0.01..3000, 3 decimals _renderFpsSlider = CreateSlider( ctx: ctx, parent: CreateFieldSlot(topLeft), labelKey: "ui.render_config.render_fps", id: "slider_render_fps", @@ -293,6 +418,7 @@ internal static class RenderConfigWindow onChanged: val => { config.RenderFPS = val; MarkConfigModified(); } ); + // Video FPS (RenderConfig.VideoFPS): frame rate of the output video, 0.01..600, 3 decimals _videoFPSSlider = CreateSlider( ctx: ctx, parent: CreateFieldSlot(topRight), labelKey: "ui.render_config.video_fps", id: "slider_video_fps", @@ -302,6 +428,8 @@ internal static class RenderConfigWindow onChanged: val => { config.VideoFPS = val; MarkConfigModified(); } ); + // Render width/height in pixels (RenderConfig.RenderWidth/RenderHeight): slider range 0..15360 / 0..8640, + // typed values are raised to at least 2 (inputFilter). Odd values are rejected later by TryGoRender. _renderWidthSlider = CreateSlider( ctx: ctx, parent: CreateFieldSlot(topLeft), labelKey: "ui.render_config.render_width", id: "slider_render_width", @@ -320,6 +448,8 @@ internal static class RenderConfigWindow onChanged: val => { config.RenderHeight = (int)Math.Round(val); MarkConfigModified(); } ); + // Delay After End (RenderConfig.DelayAfterEnd): seconds of render time recorded after the end condition is met, + // 0..9999.999 _delayAfterEndSlider = CreateSlider( ctx: ctx, parent: CreateFieldSlot(topLeft), labelKey: "ui.render_config.delay_after_end", id: "slider_delay_after_end", @@ -328,6 +458,7 @@ internal static class RenderConfigWindow onChanged: val => { config.DelayAfterEnd = val; MarkConfigModified(); } ); + // Show Judgement Text (RenderConfig.ShowJudgementText): keeps the hit judgement texts visible during the render _showJudgementToggle = CreateToggleField( ctx, CreateFieldSlot(topRight), labelKey: "", @@ -339,6 +470,14 @@ internal static class RenderConfigWindow generateLabel: false ); + // --------------------------------------------------------------------------------------------- + // Section 3: Video Encoding card (RenderConfig.VideoConfig) + // Codec | Rate Control Mode + // Bitrate | Max Bitrate + // QP | Force Software Encoder + // Keyframe Interval (full width) + // Bitrate, Max Bitrate and QP are enabled or blocked by the rate control mode (UpdateRateControlUI). + // --------------------------------------------------------------------------------------------- // 4. Video Encoding Config (Collapsible Card) var (_, videoContent) = CreateCard( ctx, scrollContent, @@ -346,6 +485,8 @@ internal static class RenderConfigWindow ); var (videoLeft, videoRight) = CreateTwoColumnPanel(videoContent); + // Every Codec enum value is offered. Changing the codec also replaces the sample rate choices with those of the + // codec's audio codec (UpdateVideoCodecUI). var codecs = (Codec[])Enum.GetValues(typeof(Codec)); _codecDropdown = CreateDropdownField( ctx, CreateFieldSlot(videoLeft), @@ -360,6 +501,8 @@ internal static class RenderConfigWindow MarkConfigModified(); }); + // Bitrate / Max Bitrate in kbps (label text) and QP: integer inputs. A value that does not parse, or is out of + // range (bitrate <= 0, QP < 0), leaves the config value unchanged; the input box keeps showing the typed text. _bitrateInput = CreateInputField( ctx, CreateFieldSlot(videoLeft), "ui.render_config.bitrate", @@ -413,6 +556,7 @@ internal static class RenderConfigWindow generateLabel: true ); + // Keyframe interval in seconds (VideoConfig.KeyframeInterval), 0.1..10; full card width _keyframeIntervalSlider = CreateSlider( ctx: ctx, parent: videoContent, labelKey: "ui.render_config.keyframe_interval", id: "slider_keyframe_interval", @@ -421,6 +565,11 @@ internal static class RenderConfigWindow onChanged: val => { config.VideoConfig.KeyframeInterval = val; MarkConfigModified(); } ); + // --------------------------------------------------------------------------------------------- + // Section 4: Audio Encoding card (RenderConfig.AudioConfig) + // Render Audio | Audio Bitrate (kbps) + // Channels | Sample Rate + // --------------------------------------------------------------------------------------------- // 5. Audio Encoding Config (Collapsible Card) var (_, audioContent) = CreateCard( ctx, scrollContent, @@ -439,6 +588,7 @@ internal static class RenderConfigWindow generateLabel: false ); + // Audio bitrate: the slider shows kbps (64..384), the config stores bits per second (kbps * 1000) _audioBitrateSlider = CreateSlider( ctx: ctx, parent: CreateFieldSlot(audioRight), labelKey: "ui.render_config.audio_bitrate", id: "slider_audio_bitrate", @@ -448,6 +598,7 @@ internal static class RenderConfigWindow onChanged: val => { config.AudioConfig.AudioBitrate = (int)Math.Round(val) * 1000; MarkConfigModified(); } ); + // Output channel count: 1 (mono) or 2 (stereo) var channels = new[] { 1, 2 }; _channelsDropdown = CreateDropdownField( ctx, CreateFieldSlot(audioLeft), @@ -458,6 +609,8 @@ internal static class RenderConfigWindow c => { config.AudioConfig.AudioChannels = c; MarkConfigModified(); } ); + // Sample rate in Hz, shown as kHz. Choices: the rates supported by the audio codec paired with the selected video + // codec (CodecUtil.AudioCodecMatch -> CodecUtil.AvailableSampleRate). _sampleRateDropdown = CreateDropdownField( ctx, CreateFieldSlot(audioRight), "ui.render_config.sample_rate", @@ -467,12 +620,20 @@ internal static class RenderConfigWindow r => { config.AudioConfig.AudioSampleRate = r; MarkConfigModified(); } ); + // --------------------------------------------------------------------------------------------- + // Section 5: Global Settings card - output directory (GlobalSettings, shared by all presets) + // Output Directory (full width) + // Browse... | Reset to Default + // --------------------------------------------------------------------------------------------- // 6. Global Settings (Collapsible Card) var (_, globalContent) = CreateCard( ctx, scrollContent, "ui.render_config.global_settings" ); + // Typing updates GlobalSettings.Current.OutputDirectory in memory only (saved by Go Render, Browse or Reset). + // Empty text or the default folder (case-insensitive) stores null, meaning "use the default folder". + // The input shows EffectiveOutputDirectory: the stored folder if it exists, else the default. _outputDirInput = CreateFullWidthInputField( ctx, globalContent, "ui.render_config.output_dir", @@ -495,6 +656,9 @@ internal static class RenderConfigWindow ); var (globalLeft, globalRight) = CreateTwoColumnPanel(globalContent); + // Browse: opens the OS folder picker (FolderDialogUtil) at the current folder; a selected folder is written to + // the input and saved to the global settings immediately. FolderDialogUtil runs the callback through the + // caller's SynchronizationContext, i.e. on the main Unity thread; cancel or failure passes null (ignored here). CreateButton( ctx, globalLeft, "ui.render_config.browse", @@ -517,6 +681,7 @@ internal static class RenderConfigWindow } ); + // Reset to Default: shows the default folder and saves null (default) to the global settings immediately CreateButton( ctx, globalRight, "ui.render_config.reset_default", @@ -529,6 +694,12 @@ internal static class RenderConfigWindow } ); + // --------------------------------------------------------------------------------------------- + // Section 6: Advanced Settings card (RenderConfig.AdvancedConfig) + // File Extension | Random Seed + // Process Audio | Non-Tile Mode + // "Planet Colors" label, then three columns: Fire | Ice | Wind (toggle + color picker each) + // --------------------------------------------------------------------------------------------- // 7. Advanced Settings (Collapsible Card) var (_, advContent) = CreateCard( ctx, scrollContent, @@ -536,6 +707,7 @@ internal static class RenderConfigWindow ); var (advLeft, advRight) = CreateTwoColumnPanel(advContent); + // Output container, chosen by file extension (AdvancedConfig.FileExtension, without the dot) var extensions = new[] { "mkv", "mp4", "mov", "webm", "avi" }; _fileExtensionDropdown = CreateDropdownField( ctx, CreateFieldSlot(advLeft), @@ -550,6 +722,8 @@ internal static class RenderConfigWindow } ); + // Process Audio (AdvancedConfig.ProcessAudio): answers the game's AudioSource.GetSpectrumData queries from the + // offline song audio at the render time (AudioSpectrumGenerator), so audio-reactive visuals follow the rendered audio _processAudioToggle = CreateToggleField( ctx, CreateFieldSlot(advLeft), labelKey: "", @@ -565,6 +739,8 @@ internal static class RenderConfigWindow generateLabel: false ); + // Random Seed (AdvancedConfig.RandomSeed): integer passed to UnityEngine.Random.InitState before the game scene + // loads; empty text stores null (no seeding). Text that is neither empty nor an integer keeps the previous value. _randomSeedInput = CreateInputField( ctx, CreateFieldSlot(advRight), "ui.render_config.random_seed", @@ -579,6 +755,8 @@ internal static class RenderConfigWindow } ); + // Non-Tile Mode (AdvancedConfig.NonTile): hides the floor tiles (including hold meshes and the end portal) and the + // planets during the render, leaving the background and decorations _nonTileToggle = CreateToggleField( ctx, CreateFieldSlot(advRight), labelKey: "", @@ -594,7 +772,9 @@ internal static class RenderConfigWindow generateLabel: false ); - // 7-1. Planet Color Settings (Optional) + // 7-1. Planet Color Settings (Optional). + // For each planet: the toggle turns the override on/off (off stores null and blocks/collapses the picker); + // the picker writes its color into the config only while the toggle is on. var planetColorsLabel = CreateLabel(ctx, advContent, "ui.render_config.planet_colors"); // ReSharper disable once InconsistentNaming var planetLabelLE = planetColorsLabel.gameObject.AddComponent(); @@ -635,6 +815,7 @@ internal static class RenderConfigWindow } } ); + // No override stored: start with the picker blocked (second argument: no animation) if (!config.AdvancedConfig.firePlanetColor.HasValue) { _firePlanetPicker.SetBlocked(true, true); @@ -712,6 +893,9 @@ internal static class RenderConfigWindow _windPlanetPicker.SetBlocked(true, true); } + // --------------------------------------------------------------------------------------------- + // Section 7: Go Render button (48 units high) and version label + // --------------------------------------------------------------------------------------------- // 8. Go Render Button CreateButton( ctx, scrollContent, @@ -721,7 +905,8 @@ internal static class RenderConfigWindow height: 48f ); - // 9. Version Label + // 9. Version Label: 12 pt dim centered text, "ADOFAI Renderer v" plus "(Expired)" after + // RenderMod.ExpirationTime (GetVersionText) var versionLabel = O5Factory.ControlText(ctx, scrollContent, 12f, noPad: true); LocalizedText.Bind(versionLabel, GetVersionText); versionLabel.alignment = TMPro.TextAlignmentOptions.Center; @@ -734,10 +919,19 @@ internal static class RenderConfigWindow return; + // Returns the entry of array closest to value (first one on ties in array order) double Snap(double[] array, double value) => array.OrderBy(n => Math.Abs(n - value)).First(); } + // =================================================================================================== + // Config <-> UI synchronization and render start + // =================================================================================================== + + /// + /// Replaces unset or invalid top-level values with defaults: 2560x1440 when width/height is 0 or less, 60 FPS when + /// RenderFPS/VideoFPS is 0 or less, 15 seconds when DelayAfterEnd is negative. Other values are not checked. + /// private static void EnsureDefaultConfig(RenderConfig config) { if (config.RenderWidth <= 0) config.RenderWidth = 2560; @@ -747,6 +941,11 @@ internal static class RenderConfigWindow if (config.DelayAfterEnd < 0) config.DelayAfterEnd = 15.0; } + /// + /// Writes every value of (after ) into its control without + /// invoking the change callbacks, refreshes the dependent fields (sample rates, rate control fields) and updates the + /// preset dropdown label. The output directory input is set from GlobalSettings.EffectiveOutputDirectory. + /// private static void ApplyConfigToUI(RenderConfig config) { EnsureDefaultConfig(config); @@ -772,6 +971,7 @@ internal static class RenderConfigWindow _renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false); _sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false); _channelsDropdown?.Set(config.AudioConfig.AudioChannels, false); + // Config stores bits per second, the slider shows kbps _audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false, noFilter: true); _outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false); @@ -781,6 +981,8 @@ internal static class RenderConfigWindow _processAudioToggle?.Set(config.AdvancedConfig.ProcessAudio, false); _nonTileToggle?.Set(config.AdvancedConfig.NonTile, false); + // Planet colors: toggle on when an override is stored; the picker shows the stored color or the default and is + // blocked without an override var fireColor = config.AdvancedConfig.firePlanetColor; _firePlanetToggle?.Set(fireColor.HasValue, false); _firePlanetPicker?.Set(fireColor ?? DefaultFirePlanetColor, false); @@ -796,6 +998,9 @@ internal static class RenderConfigWindow _windPlanetPicker?.Set(windColor ?? DefaultWindPlanetColor, false); _windPlanetPicker?.SetBlocked(!windColor.HasValue, true); + // The sample rate list depends on the codec. UpdateVideoCodecUI writes the dropdown's resulting value back into + // the config: O5Dropdown.SetValues selects the first listed rate when the current one is not in the new list, + // so a rate the codec does not support becomes the codec's first rate. UpdateVideoCodecUI(config.VideoConfig.Codec, config); UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true); @@ -841,6 +1046,15 @@ internal static class RenderConfigWindow return JsonConvert.SerializeObject(a, settings) == JsonConvert.SerializeObject(b, settings); } + /// + /// Go Render button handler: reads the controls into , validates, saves the global settings, + /// hides the window and starts a for RenderManager.Instance.levelPath. + /// + /// Validation: width and height must be even, and a non-default output directory must be a valid path; it is + /// converted to a full path and created if missing. On a validation failure an error dialog is shown, the error is + /// logged and the window stays open. The session receives a copy of the config (RenderSession.Start). + /// + /// private static void TryGoRender() { var config = UIConfig; @@ -854,7 +1068,8 @@ internal static class RenderConfigWindow return; } - // Sync Output Directory + // Sync Output Directory: empty or default folder -> null (default); otherwise the full path, created if needed. + // The error message here is fixed English text (the locale key "ui.render_config.invalid_path" is not used). if (_outputDirInput != null) { var path = _outputDirInput.Value.Trim(); @@ -888,6 +1103,12 @@ internal static class RenderConfigWindow new RenderSession().Start(config, RenderManager.Instance.levelPath); } + /// + /// Reads every control into , applying the same parsing rules as the change callbacks + /// (unparsable or out-of-range numbers keep the previous value; empty seed = null; planet color = picker value only + /// while its toggle is on). Render and video FPS values near 23.976/23.98, 29.97 and 59.94 are replaced by the exact + /// NTSC fractions 24000/1001, 30000/1001 and 60000/1001. The output directory is not handled here. + /// private static void SyncUItoConfig(RenderConfig target) { if (_renderFpsSlider != null) target.RenderFPS = _renderFpsSlider.Value; @@ -954,6 +1175,8 @@ internal static class RenderConfigWindow return; + // Maps rounded NTSC rates to their exact values (N * 1000 / 1001). A value within 0.001 of 29.97, 59.94, + // 23.976 or 23.98 is replaced; any other value is returned unchanged. // ReSharper disable once InconsistentNaming double AlignNTSCFramerate(double framerate) { @@ -967,6 +1190,10 @@ internal static class RenderConfigWindow } } + // =================================================================================================== + // Preset handling: Custom label, select, save, delete, dropdown reload, language change + // =================================================================================================== + /// /// Whether the text is the Custom label of any language. A preset with such a name could not be told apart /// from the Custom entry of the dropdown after a language change. @@ -974,8 +1201,17 @@ internal static class RenderConfigWindow private static bool IsCustomPresetLabel(string text) => text == CustomPresetLabel || text == "(Modified)" || text == "(수정됨)"; + /// + /// Dropdown entry shown while the values differ from the last loaded preset, in the current language + /// ("ui.render_config.preset_custom": "(Modified)" in en.json, "(수정됨)" in ko.json). + /// private static string CustomPresetLabel => LocalizationManager.Get("ui.render_config.preset_custom"); + /// + /// Shows the Custom label in the preset dropdown: inserts it at index 0 if missing and selects it (without invoking + /// the dropdown callback). Called by every value change callback. Does nothing before the dropdown exists or when the + /// Custom label is already selected. + /// private static void MarkConfigModified() { if (_presetDropdown == null) return; @@ -994,6 +1230,11 @@ internal static class RenderConfigWindow UpdateDeleteButtonState(); } + /// + /// Preset dropdown callback: loads the selected preset into and all controls, remembers it + /// as the last preset (saved to the global settings), puts its name in the name input and removes the Custom label. + /// Selecting the Custom label, or a name that is no longer on disk, only disarms the delete button. + /// private static void OnPresetSelected(string? selectedName, RenderConfig config) { ResetDeletePendingState(); @@ -1020,6 +1261,16 @@ internal static class RenderConfigWindow UpdateDeleteButtonState(); } + /// + /// Save button handler: reads the controls into and saves it under the trimmed name from + /// the name input. + /// + /// Empty name, or a name equal to a Custom label: a validation dialog is shown, nothing is saved. + /// The name differs from the last loaded preset and an existing preset has the same name or the same file name: + /// an overwrite confirmation dialog is shown; Confirm saves. + /// Otherwise the preset is saved directly (re-saving the loaded preset needs no confirmation). + /// + /// private static void OnSavePresetClicked(RenderConfig config) { SyncUItoConfig(config); @@ -1064,6 +1315,10 @@ internal static class RenderConfigWindow } } + /// + /// Writes the preset file (PresetManager.SavePreset), stores the name as the last preset in the global + /// settings, and reloads the dropdown with that preset selected (which also removes the Custom label). + /// private static void DoSavePreset(string name, RenderConfig config) { PresetManager.SavePreset(name, config); @@ -1073,6 +1328,12 @@ internal static class RenderConfigWindow ReloadPresetDropdown(selectName: name); } + /// + /// Delete button handler with a two-click confirmation. + /// The target is the selected preset, or the last loaded preset while the Custom label is selected. + /// First click: arms the button (label "Confirm?", width 110 units, color (0.95, 0.25, 0.25)) and starts a + /// 3-second coroutine that disarms it. Second click while armed: disarms and deletes the preset. + /// private static void OnDeletePresetClicked(RenderConfig config) { var targetPresetName = _presetDropdown?.Value; @@ -1111,12 +1372,20 @@ internal static class RenderConfigWindow DoDeletePreset(targetPresetName, config); } + /// + /// Disarms the delete button after 3 seconds. Uses , i.e. scaled game time, run on + /// RendererUIManager. + /// private static System.Collections.IEnumerator ResetDeletePendingAfterDelay() { yield return new WaitForSeconds(3f); ResetDeletePendingState(); } + /// + /// Disarms the delete button: stops the disarm coroutine, and if the button is armed (or still shows a label other + /// than "Delete") restores the "Delete" label, the 90-unit width and the SoftRed color. + /// private static void ResetDeletePendingState() { if (_deleteResetCoroutine != null) @@ -1143,6 +1412,11 @@ internal static class RenderConfigWindow } } + /// + /// Deletes the preset file, then selects the first remaining preset (or none): stores it as the last preset, + /// reloads the dropdown, and loads its values into , the controls and the name input. + /// With no presets left the config and controls keep their values. + /// private static void DoDeletePreset(string name, RenderConfig config) { PresetManager.DeletePreset(name); @@ -1163,6 +1437,11 @@ internal static class RenderConfigWindow } } + /// + /// Rebuilds the dropdown from the presets on disk and selects (or the first preset + /// when it is null or missing), recording the selection as . With no presets the + /// dropdown holds only the Custom label and the last preset is cleared. Does not load preset values into the config. + /// private static void ReloadPresetDropdown(string? selectName = null) { ResetDeletePendingState(); @@ -1190,6 +1469,12 @@ internal static class RenderConfigWindow UpdateDeleteButtonState(); } + /// + /// Enables the delete button only when the target preset exists on disk. The target is + /// , else the dropdown value, and the last loaded preset while the Custom label is + /// selected. + /// + /// Explicit target; every current caller passes none. private static void UpdateDeleteButtonState(string? presetName = null) { var targetName = presetName ?? _presetDropdown?.Value; @@ -1205,6 +1490,10 @@ internal static class RenderConfigWindow _deletePresetButton?.SetBlocked(!canDelete); } + /// + /// handler: rebuilds the preset dropdown so that the Custom label + /// appears in the new language. All other texts follow the language through their bindings. + /// private static void OnLanguageChanged(SystemLanguage language) { if (_configWindow == null || _presetDropdown == null) return; @@ -1234,6 +1523,10 @@ internal static class RenderConfigWindow UpdateDeleteButtonState(); } + /// + /// Version label text: "ADOFAI Renderer v" + the MelonInfo version, followed by the localized "(Expired)" once the + /// current UTC time reaches RenderMod.ExpirationTime. + /// private static string GetVersionText() { var versionText = $"ADOFAI Renderer v{Melon.Instance.Info.Version}"; @@ -1242,6 +1535,10 @@ internal static class RenderConfigWindow return versionText; } + /// + /// Logs as an error and shows it in a dialog titled "Invalid Settings" (localized) with a + /// single "OK" button (fixed English text). + /// private static void ShowValidationError(string message) { Melon.Logger.Error(message); @@ -1252,12 +1549,29 @@ internal static class RenderConfigWindow ); } - // 카드(또는 상단 섹션)당 하나의 다열 컨테이너를 만들어 - // 가로 한 줄 각 필드 사이 중앙이 행마다 어긋나지 않도록 한다. - // - 패널 1개 = 균등 분할 (행마다 HLG를 만들던 기존 방식의 어긋남 방지) - // - 각 컬럼은 preferredWidth=0 + flexibleWidth=1 로 항상 1:1:...:1 균등 분할 - // - 각 필드는 Field 슬롯(label 18 + 간격 3 + 컨트롤 50)으로 높이를 고정해 - // 좌/우 스택을 써도 행 단위 Y 정렬이 유지되도록 한다. + // =================================================================================================== + // Layout helpers: column panels, field slots, cards + // =================================================================================================== + + /// + /// Creates one multi-column container for a card (or the top section) and returns its columns. Fields are stacked + /// vertically inside each column, so one panel holds all rows of a card. + /// + /// The panel is a that stretches to the parent width; the columns are + /// units apart. + /// Every column has preferredWidth 0 and flexibleWidth 1 (see ), so the width is + /// always split equally (1:1:...:1) regardless of the label or value text lengths, and the column boundaries are at + /// the same x position on every row. + /// Rows line up across columns when the fields in the same row have the same height: put each field in a + /// (label 18 + spacing 3 + control; dropdown controls are 50 units high). + /// + /// + /// Card content or scroll content. + /// Number of columns, at least 1. + /// Horizontal gap between columns, in units. + /// Vertical gap between fields inside a column, in units. + /// The column rects, left to right. + /// is less than 1. private static RectTransform[] CreateMultiColumnPanel( Transform parent, int columnCount, float spacing = 8f, float columnSpacing = 8f) { @@ -1290,6 +1604,7 @@ internal static class RenderConfigWindow return columns; } + /// Two-column form of ; returns the left and right column. private static (RectTransform left, RectTransform right) CreateTwoColumnPanel( Transform parent, float spacing = 8f, float columnSpacing = 8f) { @@ -1297,6 +1612,10 @@ internal static class RenderConfigWindow return (cols[0], cols[1]); } + /// + /// Creates one column: a (children stretched to the column width, + /// units apart) with a layout element of min/preferred width 0 and flexible width 1. + /// private static RectTransform CreateColumn(Transform parent, string name, float spacing) { var colObj = new GameObject(name); @@ -1312,9 +1631,9 @@ internal static class RenderConfigWindow colLayout.childForceExpandHeight = false; // ReSharper disable once InconsistentNaming - // 핵심: 좌/우 선호 너비를 0으로 강제해 항상 정확히 절반씩 나누도록 한다. - // (기존 행마다 HLG 방식은 라벨/값 텍스트 길이에 따라 preferred가 달라져 - // 중앙이 (W + P1 - P2) / 2 로 행마다 틀어졌음) + // Preferred width 0 with flexible width 1: the HorizontalLayoutGroup distributes the whole width through the + // flexible weights, so every column gets an exact equal share. With non-zero preferred widths (which depend on + // the label and value text lengths) the split would differ from row to row. var colLE = colObj.AddComponent(); colLE.minWidth = 0f; colLE.preferredWidth = 0f; @@ -1323,6 +1642,10 @@ internal static class RenderConfigWindow return colRect; } + /// + /// Creates a field slot ("Field") in a column: a vertical stack with 3 units between its children (typically a label + /// and a control), stretched to the column width. The slot's height is the sum of its children's heights. + /// private static RectTransform CreateFieldSlot(Transform column) { var slotObj = new GameObject("Field"); @@ -1344,6 +1667,12 @@ internal static class RenderConfigWindow return slotRect; } + /// + /// Creates a collapsible O5Kit card (O5Factory.Card) with a localized title. The title text + /// (child "Header/TitleTrigger/TitleText", or any descendant named "TitleText") is set to 18 pt and bound to + /// so it follows language changes. The remaining parameters are passed to O5Kit unchanged. + /// + /// The card root and the content rect that receives the card's fields. // ReSharper disable once UnusedTupleComponentInReturnValue private static (RectTransform card, RectTransform content) CreateCard( O5Context ctx, @@ -1377,6 +1706,13 @@ internal static class RenderConfigWindow return (card, content); } + // =================================================================================================== + // Control factory helpers (each binds its texts to localization keys through LocalizedText) + // =================================================================================================== + + /// + /// Creates a localized text label (default 13 pt, theme dim text color unless is given). + /// private static TMPro.TextMeshProUGUI CreateLabel( O5Context ctx, Transform parent, @@ -1392,6 +1728,24 @@ internal static class RenderConfigWindow return label; } + /// + /// Creates an O5Kit slider with a localized label (the slider's own label; no separate label row). + /// + /// O5Kit context. + /// Field slot or column that receives the slider. + /// Localization key of the slider label. + /// O5Kit control id. + /// Value restored by middle click. + /// Minimum value. + /// Maximum value. + /// Initial value. + /// Numeric display format, e.g. "F0", "F3". + /// How values outside min..max are clamped (O5Kit ). + /// Applied to dragged values (used for snapping); typed values are not filtered by it. + /// Applied to values typed into the slider's input box. + /// Called with the new value on every change. + /// Called when an edit is completed. + /// Whether the filled part of the track is drawn. private static O5Slider CreateSlider( O5Context ctx, Transform parent, @@ -1435,6 +1789,9 @@ internal static class RenderConfigWindow return slider; } + /// + /// Creates an O5Kit button with a localized label. null uses the theme control height. + /// private static O5Button CreateButton( O5Context ctx, Transform parent, @@ -1460,6 +1817,7 @@ internal static class RenderConfigWindow return button; } + /// Same as the CreateInputField overload with an out label parameter, without returning the label. private static O5InputField CreateInputField( O5Context ctx, Transform colParent, @@ -1473,6 +1831,20 @@ internal static class RenderConfigWindow return CreateInputField(ctx, colParent, labelKey, defaultValue, value, id, out _, characterValidation, onChanged); } + /// + /// Creates a labeled text input in (normally a field slot): an 18-unit label row and an + /// O5Kit input below it. The label and the placeholder are both bound to . + /// + /// O5Kit context. + /// Field slot or column that receives the label and the input. + /// Localization key of the label and the placeholder. + /// Value restored by middle click; null for none. + /// Initial text. + /// O5Kit control id. + /// Receives the label text component (used to dim the label of a blocked field). + /// TextMeshPro character filter for typed text, e.g. Integer. + /// Called with the raw text on every change; parsing and validation are up to the caller. + /// The input field. private static O5InputField CreateInputField( O5Context ctx, Transform colParent, @@ -1484,6 +1856,8 @@ internal static class RenderConfigWindow TMPro.TMP_InputField.CharacterValidation characterValidation = TMPro.TMP_InputField.CharacterValidation.None, Action? onChanged = null) { + // Labeled text input: an 18-unit label row (13 pt, dim) above an O5Kit input. The label key also serves as the + // placeholder. characterValidation restricts typed characters (e.g. Integer). onChanged receives the raw text. label = O5Factory.ControlText(ctx, colParent, 13f, noPad: true); label.text = LocalizationManager.Get(labelKey); label.color = ctx.Theme.TextDim; @@ -1514,6 +1888,20 @@ internal static class RenderConfigWindow return inputField; } + // =================================================================================================== + // Dependent field rules (rate control, codec) + // =================================================================================================== + + /// + /// Blocks or unblocks an input and dims its label: a blocked label uses the theme dim text color at 40% of its + /// alpha. The label color changes with a 0.2-second tween (any running tween for that label is killed first), or + /// immediately with . + /// + /// Input to block; may be null. + /// Label above the input; may be null. + /// Stored tween handle of this label, replaced by the new tween. + /// True to block (disable) the field. + /// True to apply the change without animation. private static void SetFieldBlocked( O5InputField? input, TMPro.TextMeshProUGUI? label, @@ -1542,6 +1930,10 @@ internal static class RenderConfigWindow } } + /// + /// Enables only the video fields used by the rate control mode: + /// CBR - Bitrate; VBR - Bitrate and Max Bitrate; CQP - QP. Values of blocked fields stay in the config. + /// private static void UpdateRateControlUI(RateControlMode mode, bool noAnimate = false) { bool blockBitrate = mode is RateControlMode.CQP; @@ -1553,6 +1945,11 @@ internal static class RenderConfigWindow SetFieldBlocked(_qpInput, _qpLabel, ref _qpLabelTween, blockQp, noAnimate); } + /// + /// Replaces the sample rate choices with those supported by the audio codec paired with . + /// O5Dropdown.SetValues keeps the current rate when it is in the new list and otherwise selects the first entry; + /// the resulting rate is written to when one is given. + /// private static void UpdateVideoCodecUI(Codec codec, RenderConfig? config = null) { if (_sampleRateDropdown == null) return; @@ -1565,6 +1962,14 @@ internal static class RenderConfigWindow } } + // =================================================================================================== + // More control factory helpers + // =================================================================================================== + + /// + /// Creates a labeled text input that spans the full width of : its own vertical container + /// with an 18-unit label row, 3 units of spacing and the input. The label key also serves as the placeholder. + /// private static O5InputField CreateFullWidthInputField( O5Context ctx, Transform parent, @@ -1614,6 +2019,11 @@ internal static class RenderConfigWindow return input; } + /// + /// Creates an O5Kit toggle whose text is localized with . With + /// an 18-unit label row is added above it (text from , + /// empty when the key is empty), which keeps the toggle aligned with labeled fields in the neighboring column. + /// private static O5Toggle CreateToggleField( O5Context ctx, Transform colParent, @@ -1656,6 +2066,11 @@ internal static class RenderConfigWindow return toggle; } + /// + /// Creates a labeled dropdown: an 18-unit label row and a container of preferred height 50 units holding the O5Kit + /// dropdown. converts values to their shown text. The change callback is not invoked for + /// a null value. + /// private static O5Dropdown CreateDropdownField( O5Context ctx, Transform colParent, @@ -1697,6 +2112,11 @@ internal static class RenderConfigWindow ); } + /// + /// Creates an O5Kit color picker in a container one control row high (theme control height for the body font). + /// The picker's popup is positioned on the window manager canvas. is used both while + /// editing and on completion. The header is re-proportioned for narrow (three-column) layouts, see below. + /// private static O5ColorPicker CreateColorPickerField( O5Context ctx, Transform colParent, @@ -1728,9 +2148,10 @@ internal static class RenderConfigWindow id: id ); - // 3열 등 다열 레이아웃 환경에 맞춰 O5ColorPicker 헤더 내부 비율 및 폰트 크기 조정 - // O5Kit 기본값(Preview: 0~0.5, HexInput: 0.62~1.0, fontSize: 19)은 너비가 좁을 때 - // Hex 문자열(8자리 RGBA) 앞부분이 잘려 보이는 현상이 발생하므로 공간을 재분배합니다. + // Header proportions and font size for multi-column (e.g. three-column) layouts. + // O5Kit's header layout (anchor x ranges: Preview 0..0.5, ToggleArea 0..0.6, HexInput 0.62..1.0; hex font size 19) + // leaves too little width in a narrow column, and the start of the 8-digit RGBA hex text is cut off. + // Here: Preview 0..0.28, ToggleArea 0..0.30, HexInput 0.32..1.0, hex font size 15. if (picker.Rect.Find("Header") is RectTransform header) { if (header.Find("Preview") is RectTransform preview) @@ -1757,6 +2178,11 @@ internal static class RenderConfigWindow return picker; } + // =================================================================================================== + // Window visibility + // =================================================================================================== + + /// Shows or hides the window; hiding first collapses all open color pickers. private static void SetWindowActive(bool enabled) { if (!enabled) @@ -1766,6 +2192,7 @@ internal static class RenderConfigWindow _configWindow?.Rect.gameObject.SetActive(enabled); } + /// Collapses the three planet color pickers without animation. private static void CloseAllPickers() { _firePlanetPicker?.SetExpanded(false, true); diff --git a/Util/Audio/AudioSpectrumGenerator.cs b/Util/Audio/AudioSpectrumGenerator.cs index 58d1c4c..669761c 100644 --- a/Util/Audio/AudioSpectrumGenerator.cs +++ b/Util/Audio/AudioSpectrumGenerator.cs @@ -28,14 +28,31 @@ internal static class AudioSpectrumGenerator /// private const double MonoGain = 0.70710678118654752; + /// Store created by this class when the mixer store cannot be shared (disposed by ); otherwise null. private static AudioClipStore? _ownStore; + + /// Store the song source belongs to: the mixer store or . private static AudioClipStore? _store; + + /// Song source at the output sample rate, or null when not initialized or the song could not be loaded. private static AudioClipSource? _source; + + /// Cursor on moved to each analysis window, so the decoder thread loads the chunks there. private static AudioCursor? _cursor; + + /// Chunk currently held for (invalid when none). private static ChunkView _view; + + /// True after a chunk loading error was logged (logged once per render). private static bool _loadErrorLogged; + /// RenderManager.frameCount of the spectra in ; -1 when empty. private static long _cachedFrame = -1; + + /// + /// Spectra computed in the current render frame, by spectrum length and source channel. Several game objects read the + /// spectrum every frame; within one frame the render time does not change, so they receive the same values. + /// private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new(); /// Registers the song clip. Call after the is created (null when audio is not rendered). @@ -93,6 +110,17 @@ internal static class AudioSpectrumGenerator CachedSpectrums.Clear(); } + /// + /// Fills with the magnitude spectrum of the song at the current render time + /// (called by the AudioSource.GetSpectrumData patch while capturing with ProcessAudio on, for any AudioSource). + /// Bin k covers frequency k * outputRate / (2 * samples.Length) Hz in output time; values are 0 ~ 1. + /// Zeros are written when the song is not available, the conductor has not started the song, or the length is not a power of two. + /// is not read: the Blackman-Harris window is always used (every GetSpectrumData call in the game + /// passes FFTWindow.BlackmanHarris). Main thread only. + /// + /// Destination; its length is the spectrum length (power of two). + /// Requested channel (0 = left, 1 = right), clamped to the song's channels; ignored for mono songs. + /// Requested window (not used). public static void GetSpectrumData(float[] samples, int channel, FFTWindow window) { var n = samples.Length; @@ -127,6 +155,8 @@ internal static class AudioSpectrumGenerator var now = render.dspTimeBase + render.AudioTime; var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown; + // Window of 2n output samples centered on the current time: output sample i reads source frame first + i * pitch, + // so the window starts n output samples (n * pitch source frames) before the song position. var fftSize = 2 * n; var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0); var first = songSeconds * _store.OutputRate - n * pitch; @@ -134,6 +164,7 @@ internal static class AudioSpectrumGenerator var windowSamples = new float[fftSize]; try { + // Keep the cursor at the window start so the decoder thread loads (and keeps) the chunks the window reads. _cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1); for (var i = 0; i < fftSize; i++) { @@ -178,6 +209,10 @@ internal static class AudioSpectrumGenerator return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel]; } + /// + /// Magnitude spectrum of (length n, a power of two) after the Blackman-Harris window: + /// writes min(n / 2, spectrum.Length) bins, bin k = min(|X[k]| * 2.5 / n, 1). + /// private static void CalculateFFT(float[] samples, float[] spectrum) { var n = samples.Length; @@ -199,10 +234,10 @@ internal static class AudioSpectrumGenerator im[i] = 0f; } - // Cooley-Tukey Radix-2 FFT + // Cooley-Tukey Radix-2 FFT (iterative, in place, decimation in time); m = log2(n) stages. var m = (int)Math.Round(Math.Log(n, 2)); - // Bit-reversal + // Bit-reversal: reorder the input so element i moves to the index with the bits of i reversed (j tracks that index). var j = 0; for (var i = 0; i < n - 1; i++) { @@ -225,7 +260,8 @@ internal static class AudioSpectrumGenerator j += k; } - // FFT stages + // FFT stages: stage s combines pairs of transforms of length le/2 into length le = 2^s with butterflies. + // (ur, ui) is the twiddle factor exp(-i * pi * sub / le2), advanced by multiplying with (wr, wi) = exp(-i * pi / le2). for (var stage = 1; stage <= m; stage++) { var le = 1 << stage; diff --git a/Util/Audio/AudioSyncCalculator.cs b/Util/Audio/AudioSyncCalculator.cs index 21e7bd4..f6a7c31 100644 --- a/Util/Audio/AudioSyncCalculator.cs +++ b/Util/Audio/AudioSyncCalculator.cs @@ -2,29 +2,50 @@ using System; namespace ADOFAIRenderer.Util.Audio; +/// +/// Splits the audio sample stream into per-video-frame sample counts so that audio and video stay aligned without drift. +/// +/// The frame rate is converted to an exact fraction Numerator / Denominator (). +/// Frame k receives floor((k + 1) * sampleRate * Denominator / Numerator) - floor(k * sampleRate * Denominator / Numerator) samples: +/// the remainder of each division is carried into the next frame, so the sum over any number of frames equals the exact +/// sample count of that duration rounded down (for example 44100 Hz at 60 FPS gives 735 every frame; at 144 FPS three frames of 306 +/// are followed by one of 307, 306.25 on average). +/// Used by the render thread only (RenderManager.WriteAudioForFrame, once per frame sent to the encoder). +/// internal class AudioSyncCalculator { + /// Audio sample rate in Hz (samples per channel per second). private readonly long _sampleRate; + + /// Video frame rate as a fraction (frames per second = Numerator / Denominator). private readonly (long Numerator, long Denominator) _fpsRational; + /// Total samples handed out since construction or (kept for debugging, not read). // ReSharper disable once NotAccessedField.Local private long _currentSample; + + /// Remainder of the last division, in units of 1 / Numerator samples (0 ~ Numerator - 1). private long _remainder; + /// Audio sample rate in Hz. + /// Video frame rate in frames per second (the encoded frame rate). public AudioSyncCalculator(long sampleRate, double fps) { _sampleRate = sampleRate; _fpsRational = RationalUtil.Double2MaxPrecisionInt(fps); } + /// Restarts the sequence at frame 0 (the remainder is cleared). public void Reset() { _currentSample = 0; _remainder = 0; } + /// Returns the sample count (frames per channel) of the next video frame and advances to the following frame. public int GetNextFrameSamples() { + // samplesNeeded = floor((sampleRate * Denominator + remainder) / Numerator); the new remainder carries the fraction. var numerator = _sampleRate * _fpsRational.Denominator + _remainder; var samplesNeeded = numerator / _fpsRational.Numerator; diff --git a/Util/Audio/Clip/AudioClipPaths.cs b/Util/Audio/Clip/AudioClipPaths.cs index 1900d18..701d311 100644 --- a/Util/Audio/Clip/AudioClipPaths.cs +++ b/Util/Audio/Clip/AudioClipPaths.cs @@ -20,6 +20,7 @@ internal static class AudioClipPaths /// Suffix of the game's audioLib keys for external clips. public const string ExternalSuffix = "*external"; + /// File path by audioLib key; also the lock object. Entries are kept for the whole game session. private static readonly Dictionary Paths = new(); /// audioLib key of an external clip loaded from . diff --git a/Util/Audio/Clip/AudioClipResolver.cs b/Util/Audio/Clip/AudioClipResolver.cs index d62ee06..bcc7010 100644 --- a/Util/Audio/Clip/AudioClipResolver.cs +++ b/Util/Audio/Clip/AudioClipResolver.cs @@ -16,6 +16,10 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// internal static class AudioClipResolver { + /// + /// Returns the source registered under in , opening it on first use + /// (file first, then the Unity clip; see the class summary). A key that could not be opened is remembered as null. + /// /// Store of the current render. /// audioLib key of the clip (for example "song.ogg*external" or "sndKick"). /// Loaded Unity clip for the key, used when the clip has no file; may be null when the key has a file. @@ -38,6 +42,9 @@ internal static class AudioClipResolver Melon.Logger.Warning($"[AudioClipResolver] FFmpeg could not open '{path}' for '{key}', using the Unity clip: {e.Message}"); } + // The reader opened here becomes the first decode stream; additional streams reopen the file with OpenFile. + // When no source results (store disposed, or the source could not be created), the reader is disposed here + // (readers allow Dispose more than once). if (fileReader != null) { var source = store.GetOrAdd(key, () => fileReader, () => OpenFile(path)); diff --git a/Util/Audio/Clip/AudioClipSource.cs b/Util/Audio/Clip/AudioClipSource.cs index 7f856f2..d545bb0 100644 --- a/Util/Audio/Clip/AudioClipSource.cs +++ b/Util/Audio/Clip/AudioClipSource.cs @@ -13,9 +13,13 @@ namespace ADOFAIRenderer.Util.Audio.Clip; internal sealed class AudioCursor { private readonly AudioClipSource _source; + + /// Backing value of , accessed with Interlocked (64-bit reads are not atomic on every platform). private long _position; + private volatile bool _loop; + /// Created by . internal AudioCursor(AudioClipSource source, long position, bool loop) { _source = source; @@ -39,7 +43,10 @@ internal sealed class AudioCursor } } - /// When true, the chunk after the last chunk is chunk 0. + /// + /// When true, the chunk after the last chunk is chunk 0 (the window wraps to the clip start), and a position past the end + /// wraps into the clip. Setting it wakes the decoder thread. + /// public bool Loop { get => _loop; @@ -70,8 +77,10 @@ internal readonly struct ChunkView /// Channel count of the stored samples (same as the source clip). public readonly int Channels; + /// Index of the chunk in its source (used by ). public readonly int ChunkIndex; + /// Created by ; default(ChunkView) is the invalid view. internal ChunkView(float[] data, long startFrame, int frameCount, int channels, int chunkIndex) { Data = data; @@ -81,10 +90,13 @@ internal readonly struct ChunkView ChunkIndex = chunkIndex; } + /// True for a view that holds a chunk; false for default(ChunkView). public bool IsValid => Data != null; + /// Source frame after the last valid frame (exclusive end). public long EndFrame => StartFrame + FrameCount; + /// True when source frame is one of the valid frames of this chunk. public bool Contains(long frame) => frame >= StartFrame && frame < StartFrame + FrameCount; /// @@ -163,7 +175,7 @@ internal sealed class AudioClipSource /// /// Most decode streams (reader + resampler) a windowed source opens. A load continues the stream positioned at the chunk start, /// otherwise opens another stream up to this count, otherwise restarts the least recently used stream. - /// Each stream holds one open reader (file handle and decoder state, measured 0.3 ~ 2.7 MiB for ogg/flac/mp3 with the clip harness). + /// Each stream holds one open reader (file handle and decoder state, measured 0.3 ~ 2.7 MiB for ogg/flac/MP3 with the clip harness). /// public const int MaxDecodeStreams = 4; @@ -171,26 +183,45 @@ internal sealed class AudioClipSource internal int DecodeStreamLimit { get; set; } = MaxDecodeStreams; private readonly AudioClipStore _store; + + /// Reader passed to the constructor (the first decode stream); disposed again in . private readonly IPCMReader _firstReader; + + /// Opens another reader for an additional decode stream; null when only the first reader can be used. private readonly Func? _openReader; // Guarded by _sync: chunk table, hold counts, cursors, demand set, loaded set. private readonly object _sync = new(); + + /// Sample array of each chunk, null while not loaded. private readonly float[]?[] _chunks; + + /// Number of consumers holding each chunk (TryAcquire minus Release); a held chunk is not freed. private readonly int[] _holdCounts; + private readonly List _cursors = new(); + + /// Chunks a consumer asked for while they were not loaded; an index is removed when the chunk is acquired. private readonly HashSet _demanded = new(); + + /// Indices of the loaded chunks (non-null entries of ). private readonly HashSet _loaded = new(); + private int _residentLoadedCount; // Resident: chunks [0, _residentLoadedCount) are loaded // Guarded by _loadLock: decode streams. private readonly object _loadLock = new(); private readonly List _streams = new(); + + /// Incremented on every load; stored in for least-recently-used selection. private long _streamUseCounter; + + /// True after opening an additional reader failed; no further readers are opened for this source. private bool _openFailed; private volatile bool _isFailed; + /// Key the source is registered under in the store (the game's audioLib key / AudioClip.name). public string Key { get; } /// Channel count of the stored samples (same as the clip). @@ -205,12 +236,19 @@ internal sealed class AudioClipSource /// Clip length in seconds: clip frames / clip rate. public double Duration { get; } + /// Frames per chunk: / (rounded down). public int FramesPerChunk { get; } + /// Number of chunks: ceil( / ); 0 for an empty clip. public int ChunkCount { get; } + /// + /// for forward-only readers and clips of at most one chunk, otherwise + /// . Fixed at construction. + /// public ChunkPolicy Policy { get; } + /// True when is . public bool IsResident => Policy == ChunkPolicy.Resident; /// True after loading failed. A failed source returns no chunks. @@ -279,6 +317,7 @@ internal sealed class AudioClipSource return cursor; } + /// Unregisters ; its window chunks become freeable on the decoder thread's next pass. public void RemoveCursor(AudioCursor cursor) { lock (_sync) @@ -288,6 +327,7 @@ internal sealed class AudioClipSource _store.RequestService(); } + /// Wakes the decoder thread after a cursor changed chunk or loop flag (resident sources ignore cursors). internal void OnCursorMoved() { if (!IsResident) _store.RequestService(); @@ -325,6 +365,7 @@ internal sealed class AudioClipSource return false; } + /// Ends one hold of the chunk of taken by . An invalid view is ignored. public void Release(in ChunkView view) { if (!view.IsValid) return; @@ -335,6 +376,7 @@ internal sealed class AudioClipSource } } + /// View of chunk ; the last chunk's frame count stops at . private ChunkView CreateView(int index, float[] data) { var start = (long)index * FramesPerChunk; @@ -406,6 +448,8 @@ internal sealed class AudioClipSource return true; } + // For each cursor, the first missing chunk of its window; the lowest slot over all cursors wins + // (ties go to the earlier cursor). foreach (var cursor in _cursors) { if (!TryGetWindowStart(cursor, out var first, out var loop)) continue; @@ -441,6 +485,10 @@ internal sealed class AudioClipSource return false; } + /// + /// First chunk of the cursor's window: the chunk of its position (negative positions count as 0; positions past the end + /// wrap when looping). False when a non-looping cursor is past the end (no window). Caller holds _sync. + /// private bool TryGetWindowStart(AudioCursor cursor, out int first, out bool loop) { var position = cursor.Position; @@ -456,6 +504,7 @@ internal sealed class AudioClipSource return true; } + /// Chunk index of window slot after ; wraps when looping, false past the end otherwise. private bool TryWindowIndex(int first, int slot, bool loop, out int index) { index = first + slot; @@ -493,6 +542,7 @@ internal sealed class AudioClipSource stream.LastUse = ++_streamUseCounter; stream.Fill(start, frames, data); } + // Out of memory fails the whole render (store fatal error); any other error makes only this source silent. catch (OutOfMemoryException e) { ReturnArray(data); @@ -568,12 +618,14 @@ internal sealed class AudioClipSource return oldest!; } + /// Marks the source failed (no more chunks are returned or loaded); a resident source also closes its readers now. Caller holds _loadLock. private void Fail() { _isFailed = true; if (IsResident) DisposeStreams(); } + /// Disposes every decode stream (reader and resampler) and clears the list. Caller holds _loadLock. private void DisposeStreams() { foreach (var stream in _streams) @@ -581,6 +633,7 @@ internal sealed class AudioClipSource _streams.Clear(); } + /// Gives a chunk array back to the store: exact arrays (resident) are uncounted, pooled arrays (windowed) are returned to the pool. private void ReturnArray(float[]? data) { if (data == null) return; @@ -609,5 +662,6 @@ internal sealed class AudioClipSource } } + /// ceil(a / b) for a ≥ 0, b > 0. private static long CeilDiv(long a, long b) => (a + b - 1) / b; } diff --git a/Util/Audio/Clip/AudioClipStore.cs b/Util/Audio/Clip/AudioClipStore.cs index 631c41e..35152ea 100644 --- a/Util/Audio/Clip/AudioClipStore.cs +++ b/Util/Audio/Clip/AudioClipStore.cs @@ -12,7 +12,7 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// - Decoder thread (): the only thread that decodes, resamples, loads and frees chunks. /// Each step loads one chunk, the one with the lowest priority value over all sources /// (a chunk a consumer waits for, then cursor windows by slot, then resident chunks in order). -/// - Consumers (render thread, main thread for the spectrum): waits for the decoder thread and never loads. +/// - Consumers (render thread, main Unity thread for the spectrum): waits for the decoder thread and never loads. /// - Main thread: creates readers (for Unity clips this copies the clip data). /// /// Memory has no limit; current and peak usage are tracked and logged by . @@ -29,18 +29,34 @@ internal sealed class AudioClipStore : IDisposable /// Longest wait for the decoder thread to stop in , in milliseconds. private const int DecoderStopTimeoutMs = 3000; + /// Sources by key; a null value marks a key whose reader could not be created. Also the lock of the source tables. private readonly Dictionary _sources = new(); + + /// Non-null sources in creation order, scanned by the decoder thread. Guarded by the lock. private readonly List _sourceList = new(); + + /// Free windowed chunk arrays ( floats each), guarded by . private readonly Stack _chunkPool = new(); + private readonly object _poolLock = new(); + + /// Monitor the decoder thread pulses after each load (or failure); consumers in wait on it. private readonly object _signal = new(); + + /// Wakes the decoder thread when a cursor, demand or source changes (). private readonly AutoResetEvent _serviceRequested = new(false); private Thread? _decoderThread; private volatile bool _stopRequested; + + /// Bytes of allocated sample arrays (see ), updated with Interlocked. private long _allocatedBytes; + private long _peakBytes; + + /// First fatal error (out of memory, or an exception that stopped the decoder thread); null while none. private volatile Exception? _fatalError; + private volatile bool _disposed; /// Sample rate every source is resampled to, in Hz. @@ -52,9 +68,12 @@ internal sealed class AudioClipStore : IDisposable /// Highest value of since creation. public long PeakBytes => Interlocked.Read(ref _peakBytes); - /// Out-of-memory error raised while loading, or null. + /// + /// Out-of-memory error raised while opening or loading a clip, or the exception that stopped the decoder thread; null while none. + /// public Exception? FatalError => _fatalError; + /// Sample rate in Hz every source is resampled to (positive). Call to begin decoding. public AudioClipStore(int outputRate) { if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate)); @@ -75,6 +94,11 @@ internal sealed class AudioClipStore : IDisposable _decoderThread.Start(); } + /// + /// Decoder thread body: loads one chunk per iteration, or sleeps until (at most + /// ms) when nothing is missing. An unexpected exception ends the thread and is kept as the + /// fatal error, so waiting consumers fail instead of waiting forever. + /// private void DecoderLoop() { try @@ -124,6 +148,7 @@ internal sealed class AudioClipStore : IDisposable return true; } + /// Wakes the decoder thread to re-evaluate what to free and load. Any thread. internal void RequestService() => _serviceRequested.Set(); // ── Sources ── @@ -169,6 +194,8 @@ internal sealed class AudioClipStore : IDisposable Melon.Logger.Error($"[AudioClipStore] Failed to open clip '{key}': {e}"); } + // The reader is created outside the lock (it may decode or copy a whole clip). If another caller registered the + // same key meanwhile, its entry wins and this source is discarded. lock (_sources) { if (_disposed || _sources.TryGetValue(key, out var raced)) @@ -188,6 +215,10 @@ internal sealed class AudioClipStore : IDisposable return source; } + /// + /// True when has been registered; is then the source, or null when its reader + /// could not be created. Any thread. + /// public bool TryGet(string key, out AudioClipSource? source) { lock (_sources) @@ -214,6 +245,8 @@ internal sealed class AudioClipStore : IDisposable if (source.IsFailed || _disposed) return default; if (source.TryAcquire(frame, out var view)) return view; + // Retry under the signal lock so a load completed between the first try and the wait is not missed; + // the timed wait also rechecks cancellation and failure states periodically. lock (_signal) { if (source.TryAcquire(frame, out view)) return view; @@ -224,6 +257,7 @@ internal sealed class AudioClipStore : IDisposable } } + /// Throws wrapping when one is set. private void ThrowIfFatal() { var error = _fatalError; @@ -231,6 +265,7 @@ internal sealed class AudioClipStore : IDisposable throw new InvalidOperationException("Audio clip loading failed: " + error.Message, error); } + /// Wakes every consumer waiting in (after a load, a failure, or on dispose). internal void NotifyChunkLoaded() { lock (_signal) @@ -239,12 +274,14 @@ internal sealed class AudioClipStore : IDisposable } } + /// Logs a load error of one source (the source is already marked failed and plays silence) and wakes waiting consumers. internal void ReportSourceFailure(AudioClipSource source, Exception error) { Melon.Logger.Error($"[AudioClipStore] Failed to load clip '{source.Key}', it will be silent: {error}"); NotifyChunkLoaded(); } + /// Logs an out-of-memory error, keeps it as (first one wins) and wakes waiting consumers. internal void ReportFatal(AudioClipSource source, OutOfMemoryException error) { Melon.Logger.Error($"[AudioClipStore] Out of memory while loading clip '{source.Key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {error}"); @@ -254,6 +291,7 @@ internal sealed class AudioClipStore : IDisposable // ── Arrays ── + /// Takes a pooled array of floats, or allocates (and counts) a new one. Contents are undefined. internal float[] RentChunkArray() { lock (_poolLock) @@ -265,6 +303,7 @@ internal sealed class AudioClipStore : IDisposable return data; } + /// Puts a rented array back into the pool (it stays counted); after dispose it is dropped and uncounted. internal void ReturnChunkArray(float[] data) { if (_disposed) @@ -278,6 +317,7 @@ internal sealed class AudioClipStore : IDisposable } } + /// Allocates a counted array of exactly floats (resident chunks, copied Unity clip blocks). internal float[] AllocateExactArray(int floats) { var data = new float[floats]; @@ -285,8 +325,10 @@ internal sealed class AudioClipStore : IDisposable return data; } + /// Uncounts an array from ; the array itself is left to the garbage collector. internal void FreeExactArray(float[] data) => AddAllocated(-data.LongLength * sizeof(float)); + /// Adds (negative to subtract) to the allocation counter and raises the peak lock-free. private void AddAllocated(long bytes) { var current = Interlocked.Add(ref _allocatedBytes, bytes); @@ -297,6 +339,7 @@ internal sealed class AudioClipStore : IDisposable } } + /// Logs the source count and the current and peak sample memory in MiB. public void LogMemoryUsage() { int sourceCount; diff --git a/Util/Audio/Clip/DecodeStream.cs b/Util/Audio/Clip/DecodeStream.cs index 9ff5bf8..a962fc1 100644 --- a/Util/Audio/Clip/DecodeStream.cs +++ b/Util/Audio/Clip/DecodeStream.cs @@ -18,18 +18,34 @@ internal sealed class DecodeStream : IDisposable /// Input frames read from the reader per resampler call. private const int ReadBlockFrames = 4096; + /// Resampler from the reader's rate to the output rate; null when the rates are equal (samples are copied). private readonly PCMResampler? _resampler; + private readonly int _channels; private readonly int _outputRate; + + /// Input block of frames, interleaved. private readonly float[] _readBuffer; + + /// Scratch output for frames discarded after a restart; allocated on first use. private float[]? _discardBuffer; + + /// True once the reader returned no more input and the resampler is being drained with Flush. private bool _flushing; + + /// True once the input and the resampler tail are exhausted; the stream must restart before producing more. private bool _ended; + /// Input frames per alignment step: input rate / gcd (1 without resampling). private readonly long _inputStep; + + /// Output frames per alignment step: output rate / gcd (1 without resampling). private readonly long _outputStep; + + /// Input frames fed before the requested position on a restart, so the filter history matches a continuous run. private readonly long _prerollInputFrames; + /// Reader of this stream (owned; disposed with the stream). public IPCMReader Reader { get; } /// Output frame the stream continues at, or -1 when the stream is not positioned. @@ -41,6 +57,8 @@ internal sealed class DecodeStream : IDisposable /// Number of times the stream started anew (seek + resampler reset). public int RestartCount { get; private set; } + /// Reader of the clip; ownership passes to the stream. + /// Output sample rate in Hz. public DecodeStream(IPCMReader reader, int outputRate) { Reader = reader; @@ -58,6 +76,7 @@ internal sealed class DecodeStream : IDisposable // History fed before the first kept output frame when a stream starts mid-clip. // The filter spans about filter_size input frames per output frame at ratios up to 1:1 // and filter_size * (input rate / output rate) input frames when downsampling. + // The preroll is four times that span plus one alignment step (the restart position is rounded down to a step). var ratio = Math.Max(1L, (reader.SampleRate + outputRate - 1L) / outputRate); _prerollInputFrames = 4L * _resampler.FilterSize * ratio + _inputStep; } @@ -77,6 +96,7 @@ internal sealed class DecodeStream : IDisposable var channels = _channels; var filled = 0; + // Same rate: output frame = input frame; seek only when the reader is not already at the start. if (_resampler == null) { if (Reader.Position != start) @@ -115,6 +135,7 @@ internal sealed class DecodeStream : IDisposable fixed (float* pIn = _readBuffer) fixed (float* pOut = data) { + // Produce and drop the output frames between the aligned restart position and the requested start. while (discard > 0 && !_ended) { _discardBuffer ??= new float[ReadBlockFrames * channels]; @@ -167,12 +188,14 @@ internal sealed class DecodeStream : IDisposable return 0; } + /// Releases the resampler and the reader. public void Dispose() { _resampler?.Dispose(); Reader.Dispose(); } + /// Greatest common divisor of two positive integers (Euclid). private static int GCD(int a, int b) { while (b != 0) (a, b) = (b, a % b); diff --git a/Util/Audio/Clip/FFmpegPCMReader.cs b/Util/Audio/Clip/FFmpegPCMReader.cs index 3690b6a..504b7de 100644 --- a/Util/Audio/Clip/FFmpegPCMReader.cs +++ b/Util/Audio/Clip/FFmpegPCMReader.cs @@ -31,12 +31,20 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader /// private const double SeekMarginSeconds = 0.25; + // FFmpeg objects owned by this reader, freed by Dispose (null after). private AVFormatContext* _format; private AVCodecContext* _codec; + + /// Sample format converter: decoder format to interleaved float, same rate and layout. private SwrContext* _swr; + private AVPacket* _packet; private AVFrame* _frame; + + /// Index of the decoded audio stream in the container; every other stream is discarded by the demuxer. private readonly int _streamIndex; + + /// Time base of the stream's timestamps (seconds per tick). private readonly AVRational _timeBase; private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary) @@ -49,14 +57,27 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader private long _nextDecodedFrame; // Frame index of the next decoded frame when counting sequentially private bool _syncToTimestamp; // True until the first frame after open/seek sets _nextDecodedFrame from its timestamp + // _flushSent: the null packet (drain request) was sent at the end of the file. + // _endOfStream: the decoder returned EOF, or nothing more after the drain request. private bool _flushSent; private bool _endOfStream; + /// File path the reader was opened with. public string Path { get; } + + /// public int Channels { get; } + + /// public int SampleRate { get; } + + /// public long TotalFrames { get; } + + /// False: the reader seeks within the file. public bool IsForwardOnly => false; + + /// public long Position { get; private set; } /// True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary). @@ -65,6 +86,12 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader /// Number of demuxer seeks performed (for diagnostics). public int SeekCount { get; private set; } + /// + /// Opens , selects the best audio stream and opens its decoder. For MP3 files the whole file is + /// scanned once to count frames (see the class summary). + /// + /// The file cannot be opened, has no decodable audio stream, has no duration + /// (non-MP3), or an FFmpeg setup call fails. Everything allocated so far is freed. public FFmpegPCMReader(string path) { Path = path; @@ -181,6 +208,11 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}"); } + /// + /// Frame count from the stream duration, or from the container duration (AV_TIME_BASE units, microseconds) when the stream + /// has none. + /// + /// Neither duration is known. private long EstimateTotalFrames(AVStream* stream) { if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0) @@ -190,6 +222,13 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader throw new InvalidOperationException($"Audio file '{Path}' has no duration."); } + /// + /// Moves to (0 ~ ). A target within the pending block or up to the seek margin + /// after it is reached by decoding forward; otherwise the demuxer seeks backward (AVSEEK_FLAG_BACKWARD) to target - margin + /// (or to the stream start if that fails) and the decoder is flushed. Decoded samples before the target are discarded by Read. + /// + /// is outside 0 ~ TotalFrames. + /// The demuxer cannot seek. public void Seek(long frame) { if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); @@ -205,6 +244,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader return; } + // Target timestamp in stream ticks; the lead (silent) frames have no packets, so they are subtracted first. var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) @@ -222,6 +262,12 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader Position = frame; } + /// + /// + /// Frames before the first decoded frame (MP3 lead silence) and gaps between decoded timestamps are returned as zeros. + /// Returns fewer frames than requested when decoding ends before . + /// + /// Reading, decoding or sample conversion fails. public int Read(float[] buffer, int frameCount) { var limit = (int)Math.Min(frameCount, TotalFrames - Position); @@ -264,6 +310,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader } /// Decodes the next audio frame into the pending buffer. Returns false at the end of the stream. + /// + /// Send/receive loop: receive a frame; on EAGAIN read and send the next packet of the stream; at the end of the file send a + /// null packet to drain the decoder, then receive until EOF. + /// private bool DecodeNextFrame() { while (!_endOfStream) @@ -315,6 +365,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader return false; } + /// + /// Converts the decoded frame to interleaved float into the pending buffer and assigns its frame position: from its timestamp + /// for the first frame after open/seek (the current position when it has none), sequential afterwards. + /// private void StoreFrame() { var samples = _frame->nb_samples; @@ -347,6 +401,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader _nextDecodedFrame += samples; } + /// Frees every FFmpeg object and closes the file. Safe to call more than once and on a partly constructed reader. public void Dispose() { if (_frame != null) @@ -381,6 +436,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader } } + /// FFmpeg error text for (av_strerror) followed by the numeric code. private static string ErrorString(int error) { var buffer = stackalloc byte[1024]; diff --git a/Util/Audio/Clip/MP3HeaderFrame.cs b/Util/Audio/Clip/MP3HeaderFrame.cs index fe91fb8..deae053 100644 --- a/Util/Audio/Clip/MP3HeaderFrame.cs +++ b/Util/Audio/Clip/MP3HeaderFrame.cs @@ -37,8 +37,13 @@ internal static class MP3HeaderFrame return SilentFrames(file); } + /// + /// for a seekable stream positioned anywhere (the stream is read from offset 0). + /// Only the first valid frame header within after the ID3v2 tags is examined. + /// public static int SilentFrames(Stream stream) { + // Skip consecutive ID3v2 tags: 10-byte header + syncsafe size (+ 10-byte footer when flagged). var tag = new byte[10]; long offset = 0; while (true) @@ -65,6 +70,14 @@ internal static class MP3HeaderFrame return 0; } + /// + /// Parses the 4-byte frame header at . False for a missing sync or a reserved version, layer, bitrate + /// index (15) or sample rate index (3). + /// + /// Bytes to parse. + /// Offset of the first header byte in . + /// Samples per channel in the frame (384, 576 or 1152). + /// Offset of a Xing/Info tag from the header start: 4 + CRC (2) + Layer III side information (0 for Layers I/II). private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset) { samplesPerFrame = 0; @@ -91,6 +104,7 @@ internal static class MP3HeaderFrame return true; } + /// True when the ASCII string is at within the first bytes. private static bool HasTag(byte[] data, int length, int at, string tag) { if (at < 0 || at + tag.Length > length) return false; @@ -99,6 +113,7 @@ internal static class MP3HeaderFrame return true; } + /// Reads until bytes or the end of the stream; returns the bytes read. private static int ReadFully(Stream stream, byte[] buffer, int count) { var total = 0; diff --git a/Util/Audio/Clip/MP3MonoLayoutReader.cs b/Util/Audio/Clip/MP3MonoLayoutReader.cs index b1e35ab..c1c0c8e 100644 --- a/Util/Audio/Clip/MP3MonoLayoutReader.cs +++ b/Util/Audio/Clip/MP3MonoLayoutReader.cs @@ -16,6 +16,8 @@ namespace ADOFAIRenderer.Util.Audio.Clip; internal sealed class MP3MonoLayoutReader : IPCMReader { private readonly IPCMReader _inner; + + /// Reused buffer for inner frames, grown to the largest request. private float[] _innerBuffer = Array.Empty(); /// Mono reader of the MP3 file with the game's MP3 timing (). Owned by this reader. @@ -26,10 +28,19 @@ internal sealed class MP3MonoLayoutReader : IPCMReader Position = inner.Position * 2; } + /// Always 1. public int Channels => 1; + + /// The inner reader's sample rate (the game plays the doubled data at this rate). public int SampleRate => _inner.SampleRate; + + /// Twice the inner frame count. public long TotalFrames => _inner.TotalFrames * 2; + + /// public bool IsForwardOnly => _inner.IsForwardOnly; + + /// public long Position { get; private set; } /// Moves to ; the inner reader moves to the inner frame of the next even position. @@ -39,12 +50,14 @@ internal sealed class MP3MonoLayoutReader : IPCMReader Position = frame; } + /// public int Read(float[] buffer, int frameCount) { var limit = (int)Math.Min(frameCount, TotalFrames - Position); var written = 0; while (written < limit) { + // Odd position: the zero slot after an inner sample (also the first frame after a seek to an odd frame). if ((Position & 1) == 1) { buffer[written++] = 0f; @@ -52,6 +65,7 @@ internal sealed class MP3MonoLayoutReader : IPCMReader continue; } + // Even position: read enough inner frames for the remaining output, then interleave each with a zero. var want = (limit - written + 1) / 2; if (_innerBuffer.Length < want) _innerBuffer = new float[want]; var got = _inner.Read(_innerBuffer, want); @@ -69,5 +83,6 @@ internal sealed class MP3MonoLayoutReader : IPCMReader return written; } + /// Disposes the inner reader. public void Dispose() => _inner.Dispose(); } diff --git a/Util/Audio/Clip/MemoryPCMReader.cs b/Util/Audio/Clip/MemoryPCMReader.cs index 4d62840..16b42a0 100644 --- a/Util/Audio/Clip/MemoryPCMReader.cs +++ b/Util/Audio/Clip/MemoryPCMReader.cs @@ -11,19 +11,38 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// internal sealed class MemoryPCMReader : IPCMReader { + /// Block arrays; an entry is set to null when the block is released. private readonly float[]?[] _blocks; + + /// Called with each released block (for example to uncount its memory in the store); may be null. private readonly Action? _onBlockReleased; + private int _releasedCount; // Blocks [0, _releasedCount) are released + /// public int Channels { get; } + + /// public int SampleRate { get; } + + /// public long TotalFrames { get; } + + /// True: blocks are released as they are read. public bool IsForwardOnly => true; + + /// public long Position { get; private set; } /// Frames per block (all blocks except the last). public int FramesPerBlock { get; } + /// Interleaved blocks in frame order; the reader takes ownership of the arrays (the list is copied). + /// Frames in every block except possibly the last (positive). + /// Channel count (positive). + /// Sample rate in Hz. + /// Frame count; the blocks must hold at least this many frames. + /// Called with each block when it is released; may be null. public MemoryPCMReader(IReadOnlyList blocks, int framesPerBlock, int channels, int sampleRate, long totalFrames, Action? onBlockReleased) { @@ -40,6 +59,8 @@ internal sealed class MemoryPCMReader : IPCMReader TotalFrames = totalFrames; } + /// + /// lies in a block that was already released. public void Seek(long frame) { if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); @@ -48,6 +69,8 @@ internal sealed class MemoryPCMReader : IPCMReader Position = frame; } + /// + /// Blocks whose end the position has passed are released after each copy. public int Read(float[] buffer, int frameCount) { var total = (int)Math.Min(frameCount, TotalFrames - Position); @@ -68,6 +91,7 @@ internal sealed class MemoryPCMReader : IPCMReader return written; } + /// Releases every block with an index below that is not released yet, in order. private void ReleaseBlocksBefore(int blockIndex) { while (_releasedCount < blockIndex && _releasedCount < _blocks.Length) @@ -79,5 +103,6 @@ internal sealed class MemoryPCMReader : IPCMReader } } + /// Releases every remaining block. Safe to call more than once. public void Dispose() => ReleaseBlocksBefore(_blocks.Length); } diff --git a/Util/Audio/Clip/PCMResampler.cs b/Util/Audio/Clip/PCMResampler.cs index c9d4ef0..31040c6 100644 --- a/Util/Audio/Clip/PCMResampler.cs +++ b/Util/Audio/Clip/PCMResampler.cs @@ -11,16 +11,27 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// internal sealed unsafe class PCMResampler : IDisposable { + /// Resampler context owned by this instance; null after . private SwrContext* _swr; + + /// True after until . private bool _flushed; + /// Channel count of input and output (interleaved). public int Channels { get; } + + /// Input sample rate in Hz. public int InputRate { get; } + + /// Output sample rate in Hz. public int OutputRate { get; } /// Value of the SwrContext "filter_size" option (filter length per phase in taps at a 1:1 ratio). public int FilterSize { get; } + /// Creates and initializes a float-to-float resampler with the default channel layout for . + /// A count or rate is not positive. + /// libswresample cannot allocate or initialize the context. public PCMResampler(int channels, int inputRate, int outputRate) { if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); @@ -45,6 +56,8 @@ internal sealed unsafe class PCMResampler : IDisposable throw new InvalidOperationException($"swr_alloc_set_opts2 failed ({ret})."); _swr = swr; + // Falls back to 32, the libswresample default of "filter_size", when the option cannot be read. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libswresample/options.c (filter_size) long filterSize; FilterSize = ffmpeg.av_opt_get_int(_swr, "filter_size", 0, &filterSize) >= 0 ? (int)filterSize : 32; @@ -96,6 +109,7 @@ internal sealed unsafe class PCMResampler : IDisposable return ret; } + /// Frees the context. Safe to call more than once. public void Dispose() { if (_swr == null) return; diff --git a/Util/Audio/Clip/UnityClipLoader.cs b/Util/Audio/Clip/UnityClipLoader.cs index c9acbe5..3ec2953 100644 --- a/Util/Audio/Clip/UnityClipLoader.cs +++ b/Util/Audio/Clip/UnityClipLoader.cs @@ -7,7 +7,7 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// /// Copies a Unity into memory for clips without a file on disk /// (built-in hit sounds from Resources, official/DLC level audio). -/// runs on the main thread only, so the whole clip is copied once on the main thread +/// runs on the main Unity thread only, so the whole clip is copied once on the main Unity thread /// and the decoder thread reads the copy through a . /// Streaming clips () are rejected because GetData does not read them. /// Reference: https://docs.unity3d.com/ScriptReference/AudioClip.GetData.html @@ -25,6 +25,9 @@ internal static class UnityClipLoader if (clip.loadType == AudioClipLoadType.Streaming) throw new NotSupportedException($"AudioClip '{clip.name}' is a streaming clip; GetData cannot read it."); + // Clips with preloadAudioData off start unloaded; LoadAudioData loads the data synchronously, or asynchronously for + // clips with loadInBackground (GetData below then fails while the data is still loading). + // Reference: https://docs.unity3d.com/ScriptReference/AudioClip.LoadAudioData.html if (clip.loadState == AudioDataLoadState.Unloaded) clip.LoadAudioData(); diff --git a/Util/Audio/Filter/Gain.cs b/Util/Audio/Filter/Gain.cs index bafa79c..62384d7 100644 --- a/Util/Audio/Filter/Gain.cs +++ b/Util/Audio/Filter/Gain.cs @@ -2,22 +2,31 @@ using System; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Constant gain: every sample is multiplied by 10^( / 20). Stateless and without latency. +/// internal class Gain : IAudioFilter { + /// Gain in dB (0 = unchanged, -1 ≈ ×0.891, -6 ≈ ×0.501). Read on every sample, so changes apply immediately. public double Level = 0; + /// public void Initialize(int sampleRate, int channels) { } + /// Returns × 10^( / 20); is not used. public double ProcessSample(double sample, int channel) => sample * Math.Pow(10, Level / 20); + /// public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); } + /// public void Reset() { } + /// public double Latency => 0.0; } \ No newline at end of file diff --git a/Util/Audio/Filter/HardClip.cs b/Util/Audio/Filter/HardClip.cs index 2e20f6c..8a361da 100644 --- a/Util/Audio/Filter/HardClip.cs +++ b/Util/Audio/Filter/HardClip.cs @@ -3,15 +3,22 @@ using System.Runtime.CompilerServices; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Hard clipper: limits every sample to [-, +]. Stateless and without latency. +/// In the chain of it catches the peaks the RMS-based limiter lets through before the final gain. +/// internal class HardClip : IAudioFilter { + /// Clip level as a linear amplitude (1.0 = full scale). // ReSharper disable once MemberCanBePrivate.Global // ReSharper disable once FieldCanBeMadeReadOnly.Global // ReSharper disable once ConvertToConstant.Global public double Threshold = 1.0; + /// public void Initialize(int sampleRate, int channels) { } + /// Returns clamped to ±; is not used. [MethodImpl(MethodImplOptions.AggressiveInlining)] public double ProcessSample(double sample, int channel) { @@ -20,13 +27,16 @@ internal class HardClip : IAudioFilter return sample; } + /// public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); } + /// public void Reset() { } + /// public double Latency => 0.0; } \ No newline at end of file diff --git a/Util/Audio/Filter/IAudioFilter.cs b/Util/Audio/Filter/IAudioFilter.cs index ebf9a00..26460bf 100644 --- a/Util/Audio/Filter/IAudioFilter.cs +++ b/Util/Audio/Filter/IAudioFilter.cs @@ -2,14 +2,34 @@ using System; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Stage of the output filter chain of . Samples are doubles at nominal full scale ±1.0. +/// A filter keeps its own state between calls, so consecutive blocks form one continuous signal. +/// Used by one thread at a time (the render thread during rendering). +/// internal interface IAudioFilter { + /// + /// Sets the sample rate (Hz) and interleaved channel count of the signal and allocates per-channel state. + /// Called once before the first sample; the state is cleared as by . + /// void Initialize(int sampleRate, int channels); + /// Processes one sample of channel (0 ~ channels - 1) and returns the output sample. double ProcessSample(double sample, int channel); + + /// + /// Processes in place. The buffer holds interleaved frames starting at channel 0 + /// (length = frames * channels of ). + /// void ProcessBlock(Span buffer); + /// Clears the internal state (envelopes, delay lines) without changing the settings. void Reset(); + /// + /// Delay of the output relative to the input, in seconds (0 for filters without look-ahead). + /// sums the latencies of the chain and compensates them. + /// double Latency { get; } } \ No newline at end of file diff --git a/Util/Audio/Filter/RMSCompressor.cs b/Util/Audio/Filter/RMSCompressor.cs index ea07d27..3e82f85 100644 --- a/Util/Audio/Filter/RMSCompressor.cs +++ b/Util/Audio/Filter/RMSCompressor.cs @@ -3,6 +3,24 @@ using System.Runtime.CompilerServices; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Feed-forward RMS compressor with soft knee, attack/release ballistics in the dB domain and optional look-ahead. +/// Each channel is detected and gain-reduced independently (no stereo link). +/// +/// Per sample of channel c: +/// 1. Mean square: ms[n] = r * ms[n-1] + (1 - r) * x[n]², r = exp(-1 / (sampleRate * )); +/// level L = 10 * log10(ms + 1e-12) dB (the 1e-12 floor keeps silence finite, about -120 dB). +/// 2. Static curve (gain reduction G ≤ 0 dB, T = , R = , W = ): +/// G = (1/R - 1) * (L - T) above T + W/2; G = (1/R - 1) * (L - T + W/2)² / (2W) inside the knee; 0 below T - W/2. +/// With W = 0 the knee is hard (G = (1/R - 1) * (L - T) above T). R = infinity gives G = T - L (a limiter at T). +/// 3. Ballistics: g[n] = k * g[n-1] + (1 - k) * G, with k = attack coefficient while the reduction grows (G < g[n-1]) and +/// release coefficient otherwise; coefficient = exp(-1 / (sampleRate * time)) (one-pole smoothing, time constant in seconds). +/// 4. Output = x[n - lookAhead] * 10^(g[n] / 20): the gain computed from the current input is applied to the input delayed by +/// , so the reduction starts before the peak reaches the output. +/// +/// Properties may be set before (as in an object initializer); Initialize recomputes every +/// coefficient for the actual sample rate and reallocates the delay lines. Not thread-safe. +/// // ReSharper disable once InconsistentNaming internal class RMSCompressor : IAudioFilter { @@ -13,11 +31,17 @@ internal class RMSCompressor : IAudioFilter private double _kneeDb = 5; // in dB private double _rmsWindowTime = 0.030; // in seconds private double _lookAheadTime; // in seconds + + /// Look-ahead delay in samples per channel: round(sampleRate * ); 0 disables the delay line. private int _lookAheadSamples; + /// Sample rate in Hz; 44100 until sets the actual rate. private int _sampleRate = 44100; + + /// Interleaved channel count; 2 until sets the actual count. private int _channels = 2; + // One-pole smoothing coefficients (0 ~ 1) of attack, release and the mean-square detector: exp(-1 / (sampleRate * time)). private double _attackCoef; private double _releaseCoef; private double _rmsCoef; @@ -28,6 +52,10 @@ internal class RMSCompressor : IAudioFilter private double[][] _delayBuffers = Array.Empty(); // circular delay buffers private int[] _delayWriteIndices = Array.Empty(); + /// + /// Look-ahead delay in seconds, clamped to 0 ~ 1. Setting it reallocates (and clears) the delay lines. + /// Also reported as . + /// public double LookAheadTime { get => _lookAheadTime; @@ -38,20 +66,24 @@ internal class RMSCompressor : IAudioFilter } } + /// Output delay in seconds: . public double Latency => _lookAheadTime; + /// Threshold in dB relative to full scale (RMS level of a constant ±1.0 signal = 0 dB). public double ThresholdDb { get => _thresholdDb; set => _thresholdDb = value; } + /// Compression ratio (input dB above threshold : output dB above threshold), at least 1. Infinity limits at the threshold. public double Ratio { get => _ratio; set => _ratio = Math.Max(1, value); } + /// Attack time constant in seconds, at least 0.0001 (smaller values, including 0, are raised to it). public double AttackTime { get => _attackTime; @@ -62,6 +94,7 @@ internal class RMSCompressor : IAudioFilter } } + /// Release time constant in seconds, at least 0.0001. public double ReleaseTime { get => _releaseTime; @@ -72,12 +105,14 @@ internal class RMSCompressor : IAudioFilter } } + /// Total knee width in dB, centered on the threshold, at least 0 (0 = hard knee). public double KneeDb { get => _kneeDb; set => _kneeDb = Math.Max(0, value); } + /// Time constant of the mean-square detector in seconds, at least 0.0001. public double RmsWindowTime { get => _rmsWindowTime; @@ -88,6 +123,7 @@ internal class RMSCompressor : IAudioFilter } } + /// public void Initialize(int sampleRate, int channels) { _sampleRate = sampleRate; @@ -110,6 +146,7 @@ internal class RMSCompressor : IAudioFilter ReallocateDelayBuffers(); } + /// Allocates one zeroed circular buffer of samples per channel. private void ReallocateDelayBuffers() { if (_channels <= 0) return; @@ -139,6 +176,10 @@ internal class RMSCompressor : IAudioFilter _rmsCoef = Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); } + /// + /// Processes one sample of (see the class summary). A channel outside 0 ~ channels - 1 is + /// returned unchanged and does not touch any state. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public double ProcessSample(double sample, int channel) { @@ -162,10 +203,12 @@ internal class RMSCompressor : IAudioFilter { if (rmsDb > _thresholdDb + _kneeDb * 0.5) { + // Above the knee: full ratio. gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); } else if (rmsDb >= _thresholdDb - _kneeDb * 0.5) { + // Inside the knee: quadratic transition from 0 dB at T - W/2 to the full-ratio line at T + W/2. double diff = rmsDb - _thresholdDb + _kneeDb * 0.5; gStatic = (1 / _ratio - 1) * diff * diff / (2 * _kneeDb); } @@ -194,6 +237,7 @@ internal class RMSCompressor : IAudioFilter if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0) { + // Circular delay line: read the sample written _lookAheadSamples calls ago, then overwrite it with the current one. var buffer = _delayBuffers[channel]; int writeIdx = _delayWriteIndices[channel]; double delayedSample = buffer[writeIdx]; @@ -205,6 +249,7 @@ internal class RMSCompressor : IAudioFilter return sample * linearGain; } + /// public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) @@ -214,6 +259,7 @@ internal class RMSCompressor : IAudioFilter } } + /// Clears the detector, the gain envelope and the delay lines (the delay outputs zeros for the next look-ahead samples). public void Reset() { Array.Clear(_rmsState, 0, _rmsState.Length); diff --git a/Util/Audio/Mixer/Voice.cs b/Util/Audio/Mixer/Voice.cs index 123bf8b..460d5d0 100644 --- a/Util/Audio/Mixer/Voice.cs +++ b/Util/Audio/Mixer/Voice.cs @@ -6,11 +6,11 @@ using ADOFAIRenderer.Util.Audio.Clip; namespace ADOFAIRenderer.Util.Audio.Mixer; /// -/// One playing sound of a (the song, or one effect sound), created by . +/// One playing sound of a (the song, or one sound effect), created by . /// /// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start, /// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order. -/// Render thread: applies each change at sample round(time * OutputRate). +/// Render thread: applies each change at sample floor(time * OutputRate) (). /// /// Playback model (output samples n, clip positions in frames of the source at the output sample rate): /// - The voice is silent before its start sample and from its end sample on. @@ -21,20 +21,37 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// internal sealed class Voice { + /// Kind of a queued ; the meaning of depends on it. internal enum ChangeKind { + /// Value: playback rate, at least 0. Pitch, + + /// Value: linear gain. Volume, + + /// Value: 1 = loop on, 0 = loop off. Loop, + + /// Value: clip position in seconds, at least 0. Position, + + /// Value: end time in seconds relative to the render start. EndTime, + + /// Value: unused (0). Ends playback at the sample the change is applied at. Stop } + /// One timed change of a voice, queued by the main Unity thread and applied by the render thread. internal readonly struct Change { public readonly ChangeKind Kind; + + /// Output sample at which the change takes effect ( of the change time). public readonly long Sample; + + /// Argument of the change (see ). public readonly double Value; public Change(ChangeKind kind, long sample, double value) @@ -45,6 +62,7 @@ internal sealed class Voice } } + /// Mixer that owns this voice; the Set methods queue their changes on it. private readonly VoiceMixer _mixer; // ── Render thread state ── @@ -58,11 +76,22 @@ internal sealed class Voice /// Output sample at which playback stops (exclusive); long.MaxValue when not set. internal long EndSample = long.MaxValue; + /// Output sample of the current anchor (see the class summary). internal long AnchorSample; + + /// Clip position at , in frames at the output sample rate (may be fractional). internal double AnchorPosition; + + /// Clip frames advanced per output sample (1 = original speed), at least 0. internal double Pitch; + + /// Linear gain applied to the clip samples. internal double Volume; + + /// True while the clip repeats at . internal bool Loop; + + /// Set by the render thread when playback has ended; the voice is then released and removed from the mixer. internal bool Finished; /// @@ -74,7 +103,10 @@ internal sealed class Voice /// Loop length in frames at the output sample rate (source duration * output rate). internal double Period; + /// Read position registered on so the decoder thread keeps the chunks ahead loaded; null without a usable source. internal AudioCursor? Cursor; + + /// Chunk of currently held for reading (invalid when none). Render thread only. internal ChunkView View; internal Voice(VoiceMixer mixer, AudioClipSource? source, string name) @@ -128,6 +160,9 @@ internal sealed class Voice /// internal void Apply(in Change change, long sample, int outputRate) { + // Anchor changes made before the start take effect at the start sample (playback starts from the new state). + // EndTime uses the change's own value; Stop ends at the sample it is applied at, so a Stop before the start + // ends the voice before it plays. var at = Math.Max(sample, StartSample); switch (change.Kind) { @@ -155,6 +190,7 @@ internal sealed class Voice } } + /// Moves the anchor to , keeping the (wrapped) position there, before a pitch or loop change. private void Reanchor(long sample) { AnchorPosition = WrappedPositionAt(sample); diff --git a/Util/Audio/Mixer/VoiceMixer.cs b/Util/Audio/Mixer/VoiceMixer.cs index b522d4b..5e8796e 100644 --- a/Util/Audio/Mixer/VoiceMixer.cs +++ b/Util/Audio/Mixer/VoiceMixer.cs @@ -9,7 +9,7 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// Mixes every into interleaved double samples at the output sample rate and channel count. /// /// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed -/// to and the Voice Set methods). A time t maps to sample round(t * OutputRate). +/// to and the Voice Set methods). A time t maps to sample floor(t * OutputRate) (). /// /// Threads: /// - Main thread: and the Voice Set methods queue commands under a lock. Commands must be queued in @@ -25,16 +25,23 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// internal sealed class VoiceMixer : IDisposable { + /// Kind of a queued main-thread command. private enum CommandKind { + /// Adds a new voice to the mixed set (created by ). Play, + + /// Appends a to the voice's pending queue. Change } + /// Command queued by the main Unity thread and taken by the render thread at the start of . private readonly struct Command { public readonly CommandKind Kind; public readonly Voice Voice; + + /// The change for ; default for . public readonly Voice.Change Change; public Command(CommandKind kind, Voice voice, Voice.Change change) @@ -48,6 +55,8 @@ internal sealed class VoiceMixer : IDisposable private readonly AudioClipStore _store; private readonly CancellationToken _cancellationToken; + // Double-buffered command lists: the main Unity thread appends to _commands under _commandLock; the render thread swaps the + // two lists under the lock and processes _takenCommands outside it, then clears it for the next swap. private readonly object _commandLock = new(); private List _commands = new(); private List _takenCommands = new(); @@ -68,8 +77,10 @@ internal sealed class VoiceMixer : IDisposable _cancellationToken = cancellationToken; } + /// Output sample rate in Hz (the store's ). public int OutputRate { get; } + /// Interleaved channel count of the mixed output (1 or more). public int OutputChannels { get; } /// Voices currently mixed (started or waiting to start). Render thread only. @@ -113,6 +124,10 @@ internal sealed class VoiceMixer : IDisposable return voice; } + /// + /// Queues a change of that takes effect at sample (). + /// Called by the Voice Set methods on the main Unity thread. + /// internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value) { var change = new Voice.Change(kind, ToSample(time, OutputRate), value); @@ -138,10 +153,13 @@ internal sealed class VoiceMixer : IDisposable TakeCommands(); + // Iterated backwards so finished voices can be removed in place. for (var i = _voices.Count - 1; i >= 0; i--) { var voice = _voices[i]; var sample = firstSample; + // Split the block into segments at the samples of pending changes: each segment is mixed with constant + // voice state, and a change is applied when the segment reaches its sample (or at once when its sample is past). while (!voice.Finished) { var segmentEnd = endSample; @@ -162,6 +180,8 @@ internal sealed class VoiceMixer : IDisposable if (sample >= endSample) break; } + // The end sample is inside or before this block: nothing more will be mixed. Changes still pending for this voice + // (after the end) are dropped with it. if (!voice.Finished && endSample >= voice.EndSample) voice.Finished = true; if (voice.Finished) @@ -172,6 +192,10 @@ internal sealed class VoiceMixer : IDisposable } } + /// + /// Takes every queued command: Play adds the voice (with a cursor at frame 0 when its source is usable), Change appends to + /// the voice's pending queue (dropped for finished voices). Render thread only. + /// private void TakeCommands() { lock (_commandLock) @@ -203,6 +227,7 @@ internal sealed class VoiceMixer : IDisposable var end = Math.Min(to, voice.EndSample); if (start >= end) return; + // A voice whose source is missing or failed is silent; it finishes at its first segment inside [start, end). var source = voice.Source; if (source == null || source.IsFailed) { @@ -222,6 +247,7 @@ internal sealed class VoiceMixer : IDisposable var volume = voice.Volume; var offset = (int)(start - from) * channels; + // Move the cursor to the segment start so the decoder thread keeps the chunks ahead of it loaded. UpdateCursor(voice, (long)voice.WrappedPositionAt(start)); for (var n = start; n < end; n++, offset += channels) @@ -234,6 +260,8 @@ internal sealed class VoiceMixer : IDisposable voice.Finished = true; return; } + // Looping: a wrapped position is below Period, and Period (Duration * rate) is at most TotalFrames + // (TotalFrames rounds up), so this is a guard that reads the last frame. position = totalFrames - 1; } @@ -243,6 +271,7 @@ internal sealed class VoiceMixer : IDisposable if (!EnsureView(voice, index)) return; var view = voice.View; + // Integer position (pitch 1 with an integer anchor): copy the stored sample. if (fraction == 0.0) { for (var c = 0; c < channels; c++) @@ -250,6 +279,7 @@ internal sealed class VoiceMixer : IDisposable continue; } + // Linear interpolation; the next frame may lie in the following chunk, at the loop start or past the end. var nextIndex = index + 1; var nextInView = view.Contains(nextIndex); for (var c = 0; c < channels; c++) @@ -303,6 +333,7 @@ internal sealed class VoiceMixer : IDisposable return false; } + /// Moves the voice's cursor to and copies the voice's loop flag (writes only on change). private static void UpdateCursor(Voice voice, long frame) { var cursor = voice.Cursor; @@ -311,6 +342,7 @@ internal sealed class VoiceMixer : IDisposable if (cursor.Position != frame) cursor.Position = frame; } + /// Releases the voice's held chunk and cursor and drops its pending changes. Render thread only. private static void ReleaseVoice(Voice voice) { var source = voice.Source; diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 3f5ef28..d8af3b2 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -11,7 +11,7 @@ using UnityEngine; namespace ADOFAIRenderer.Util.Audio; /// -/// Audio of one render: the song and effect sounds as s of a , clips from an +/// Audio of one render: the song and sound effects as s of a , clips from an /// , followed by the filter chain (RMS compressor → look-ahead limiter → hard clip → -1 dB gain). /// /// Time: seconds relative to the render start (RenderManager.dspTimeBase); output sample n is at n / sample rate. @@ -21,15 +21,15 @@ namespace ADOFAIRenderer.Util.Audio; /// Threads: /// - Main thread: , , , , /// source/voice mapping, and after each frame (all commands with an earlier time have been queued). -/// - Render thread: waits until the main thread has committed the end of the samples it mixes, -/// so every command takes effect at its own sample. ends the waiting when the main thread stops. +/// - Render thread: waits until the main Unity thread has committed the end of the samples it mixes, +/// so every command takes effect at its own sample. ends the waiting when the main Unity thread stops. /// internal sealed class OfflineAudioMixer : IDisposable { - /// Gain applied to every clip. + /// Linear gain applied to every clip (song and effects), before the filter chain (0.5 ≈ -6 dB headroom). private const double ClipVolume = 0.5; - /// Additional gain of effect sounds (registered with ). + /// Additional gain of sound effects (registered with ). private const double EffectVolume = 2.0 / 3.0; /// @@ -41,21 +41,48 @@ internal sealed class OfflineAudioMixer : IDisposable /// Longest wait for a commit before checking for cancellation again, in milliseconds. private const int CommitWaitMs = 50; + /// Output channel count (interleaved). private readonly int _channels; + + /// Output sample rate in Hz. private readonly int _sampleRate; + private readonly AudioClipStore _store; private readonly VoiceMixer _mixer; + + /// Canceled by /; ends commit waits and chunk waits on the render thread. private readonly CancellationTokenSource _cancellation = new(); + + /// Filter chain applied in list order to every mixed block. private readonly List _filters = new(); + + /// Sum of the filter latencies in output samples: round(sum of * sample rate). private readonly int _latencySamples; // Main thread. + + /// + /// Mapping of game AudioSources to the state of the sound effect last started on them. Weak keys: a destroyed source + /// does not keep its state alive. Replaced by a new table to drop every mapping at once. + /// private ConditionalWeakTable _sourceStates = new(); + + /// Effect voices started since the last (finished ones are pruned periodically). private readonly List _effectVoices = new(); + + /// Count of at which the next prune runs. private int _effectVoicePruneAt = EffectVoicePruneCount; + + /// Song source, or null when there is no song or it could not be loaded. private AudioClipSource? _bgmSource; + + /// audioLib key (AudioClip.name) of the song. private string _bgmKey = ""; + + /// Current song voice, created by the first and replaced when the start time changes. private Voice? _bgmVoice; + + // Values last sent to the song voice (start in seconds relative to the render start); NaN before the first update. private double _bgmStart = double.NaN; private double _bgmPitch = double.NaN; private double _bgmVolume = double.NaN; @@ -66,11 +93,26 @@ internal sealed class OfflineAudioMixer : IDisposable // Render thread (and Dispose, under _renderLock). private readonly object _renderLock = new(); + + /// Index of the next output sample writes (frames per channel since the render start). private long _nextOutputSample; + + /// True once the filters have been fed samples [0, latency). private bool _primed; + + /// Reused interleaved mix buffer, grown to the largest block requested. private double[] _mixBuffer = Array.Empty(); + private bool _disposed; + /// + /// Creates the clip store (and starts its decoder thread), the voice mixer and the filter chain. + /// Filter settings: RMS compressor (threshold -10 dB, ratio 2, attack/release 0.1 s, knee 10 dB, RMS window 0.02 s), + /// limiter (RMS compressor with threshold 0 dB, infinite ratio, attack 0 raised to the 0.0001 s minimum, release 0.05 s, + /// hard knee, RMS window 0.02 s, look-ahead 0.01 s), hard clip at ±1.0, then -1 dB gain. + /// + /// Output channel count (interleaved). + /// Output sample rate in Hz; every clip is resampled to it. public OfflineAudioMixer(int targetChannels, int targetSampleRate) { _channels = targetChannels; @@ -96,6 +138,7 @@ internal sealed class OfflineAudioMixer : IDisposable _filters.Add(new HardClip()); _filters.Add(new Gain { Level = -1 }); + // Total delay of the chain; only the limiter's look-ahead contributes (0.01 s). double latency = 0; foreach (var filter in _filters) { @@ -148,6 +191,8 @@ internal sealed class OfflineAudioMixer : IDisposable double pitch = conductor.song.pitch; var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0; var start = conductor.dspTimeSong + countdown - dspTimeBase; + // levelData.volume is a percentage (100 = unchanged; scrConductor sets song.volume = volume * 0.01 the same way). + // Without a scnGame instance the level volume is 1.0. var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0; var volume = levelVolume * ClipVolume; @@ -181,6 +226,11 @@ internal sealed class OfflineAudioMixer : IDisposable GetOrAddSource(clipName); } + /// + /// Source of the game clip (audioLib key): an existing source, or one opened through + /// with the clip from AudioManager.FindOrLoadAudioClip. Null for an empty name or an + /// unreadable clip (the key is then remembered as missing by the store). Main thread only. + /// private AudioClipSource? GetOrAddSource(string clipName) { if (string.IsNullOrEmpty(clipName)) return null; @@ -190,9 +240,17 @@ internal sealed class OfflineAudioMixer : IDisposable } /// - /// Starts an effect sound at . An of 0 or more makes it loop until then. + /// Starts a sound effect at . An of 0 or more makes it loop until then. /// (when given) is mapped to the returned state, which reports and applies later changes through the AudioSource. /// + /// Game AudioSource playing the sound, or null to map it later with . + /// audioLib key of the clip. A clip that cannot be loaded gives a silent voice. + /// Scheduled start, seconds relative to the render start (a start before starts at now). + /// Loop end, seconds relative to the render start; negative for a sound that plays once. + /// Game volume (linear); multiplied by the clip and effect gains. + /// Playback rate at the start. + /// Current render time in seconds. + /// State of the new sound, also used by the AudioSource patches. public SourceState RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) { var clipSource = GetOrAddSource(clipName); @@ -200,6 +258,7 @@ internal sealed class OfflineAudioMixer : IDisposable var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now); if (loop) voice.SetEndTime(endTime, now); + // Amortized pruning: the list is scanned only when it has doubled since the last prune. if (_effectVoices.Count >= _effectVoicePruneAt) { _effectVoices.RemoveAll(v => v.IsFinished); @@ -224,7 +283,7 @@ internal sealed class OfflineAudioMixer : IDisposable _sourceStates.Add(source, state); } - /// State of the effect sound last started for . The mapping stays after the sound ends or stops. + /// State of the sound effect last started for . The mapping stays after the sound ends or stops. public bool TryGetSourceState(AudioSource? source, out SourceState state) { state = null!; @@ -232,7 +291,7 @@ internal sealed class OfflineAudioMixer : IDisposable } /// - /// Stops every effect sound at , including ones scheduled to start later + /// Stops every sound effect at , including ones scheduled to start later /// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of /// AudioManager._Play and scrConductor.PlayWithEndTime). The song voice keeps playing. /// @@ -269,7 +328,10 @@ internal sealed class OfflineAudioMixer : IDisposable } } - /// Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException. + /// + /// Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException. + /// May be called from any thread, more than once. + /// public void Cancel() { _cancellation.Cancel(); @@ -280,7 +342,10 @@ internal sealed class OfflineAudioMixer : IDisposable /// /// Writes the next output frames (interleaved float, continuing from the previous call). + /// Blocks until the main Unity thread has committed the voice samples it needs (, ). /// + /// Destination; must hold at least samplesNeeded * channels floats. Values beyond that are untouched. + /// Frames (samples per channel) to write. /// Chunk loading failed fatally (out of memory or the decoder thread stopped). /// The render was canceled (). public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded) @@ -289,6 +354,8 @@ internal sealed class OfflineAudioMixer : IDisposable { if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer)); + // First call: run voice samples [0, latency) through the filters and discard the output, which is the + // zero-filled delay lines. Afterwards output sample n is the filtered voice sample n. if (!_primed) { _primed = true; @@ -310,7 +377,7 @@ internal sealed class OfflineAudioMixer : IDisposable } } - /// Mixes voices at samples [first, first + frames) into the reused double buffer, after the main thread committed them. + /// Mixes voices at samples [first, first + frames) into the reused double buffer, after the main Unity thread committed them. private Span MixBlock(long first, int frames) { WaitForCommit(first + frames); @@ -323,6 +390,11 @@ internal sealed class OfflineAudioMixer : IDisposable return block; } + /// + /// Waits until the committed sample reaches (exclusive end of the block to mix), checking for + /// cancellation every ms. + /// + /// The render was canceled. private void WaitForCommit(long endSample) { lock (_commitLock) diff --git a/Util/Audio/SourceState.cs b/Util/Audio/SourceState.cs index 161c3ad..a52e67e 100644 --- a/Util/Audio/SourceState.cs +++ b/Util/Audio/SourceState.cs @@ -21,15 +21,33 @@ namespace ADOFAIRenderer.Util.Audio; /// internal sealed class SourceState { + /// Start of playback, seconds relative to the render start. private readonly double _start; + + /// Render time of the last position or pitch change (at least ), seconds. private double _anchorTime; + + /// Unwrapped clip position at , seconds into the clip. private double _anchorPosition; + + /// Last finite pitch set (reported as is; negative values play as 0). private double _pitch; + private readonly bool _loop; + + /// Scheduled end time, seconds relative to the render start; infinity when not set. private double _endTime = double.PositiveInfinity; + + /// True after ; playback does not resume. private bool _stopped; + + /// timeSamples reported while not playing: the last value set through time/timeSamples, 0 after Stop. private int _offsetSamples; + + /// Clip length in seconds. private double _length; + + /// Clip sample rate in Hz (at least 1), used to convert between time and timeSamples. private int _frequency; /// Voice that plays the sound. @@ -51,6 +69,7 @@ internal sealed class SourceState _frequency = Math.Max(1, frequency); } + /// Voice of the that plays the sound; changes made while playing are queued on it. public Voice Voice { get; } /// Takes the clip length and sample rate from the clip of the game AudioSource. @@ -69,26 +88,39 @@ internal sealed class SourceState Voice.SetEndTime(endTime, now); } + /// AudioSource.isPlaying at : not stopped and before both the scheduled end and the natural end. public bool IsPlaying(double now) => !_stopped && now < Math.Min(_endTime, NaturalEnd()); + /// AudioSource.time at : clip position in seconds while playing, otherwise 0. public float GetTime(double now) => IsPlaying(now) ? (float)PositionAt(now) : 0f; + /// AudioSource.timeSamples at : floor(position * frequency) while playing, otherwise the stored offset. public int GetTimeSamples(double now) => IsPlaying(now) ? (int)Math.Floor(PositionAt(now) * _frequency) : _offsetSamples; + /// AudioSource.pitch: the last finite value set. public float GetPitch() => (float)_pitch; + /// + /// AudioSource.time setter: stores floor(seconds * frequency) as the offset and, while playing, moves the position to + /// (negative values become 0) from (or from the start when not started yet). + /// public void SetTime(float seconds, double now) { _offsetSamples = (int)Math.Floor((double)seconds * _frequency); Seek(seconds, now); } + /// AudioSource.timeSamples setter: as with samples / frequency seconds. public void SetTimeSamples(int samples, double now) { _offsetSamples = samples; Seek((double)samples / _frequency, now); } + /// + /// AudioSource.pitch setter: ignores NaN and infinity. While playing, the position up to is kept + /// (new anchor) and the voice changes rate at ; otherwise only the reported value changes. + /// public void SetPitch(float pitch, double now) { if (float.IsNaN(pitch) || float.IsInfinity(pitch)) return; @@ -98,6 +130,7 @@ internal sealed class SourceState if (playing) Voice.SetPitch(pitch, now); } + /// AudioSource.Stop(): stops the voice at when playing, marks the state stopped and clears the offset. public void Stop(double now) { if (IsPlaying(now)) Voice.Stop(now); @@ -105,8 +138,13 @@ internal sealed class SourceState _offsetSamples = 0; } + /// Rate used for playback: with negative values treated as 0 (as in ). private double PlaybackPitch => Math.Max(0.0, _pitch); + /// + /// While playing, sets the clip position to (at least 0) from max(now, start) and queues the same + /// position change on the voice. + /// private void Seek(double seconds, double now) { if (!IsPlaying(now)) return; @@ -116,6 +154,7 @@ internal sealed class SourceState Voice.SetPosition(position, now); } + /// Moves the anchor to max(now, start), keeping the current position, before a pitch change. private void Reanchor(double now) { var time = Math.Max(now, _start); diff --git a/Util/BundleLoader.cs b/Util/BundleLoader.cs index 668e65c..3b02bfb 100644 --- a/Util/BundleLoader.cs +++ b/Util/BundleLoader.cs @@ -6,9 +6,16 @@ using UnityEngine; namespace ADOFAIRenderer.Util; +/// +/// Loads the Unity asset bundles (fonts, shaders) embedded in the mod DLL as manifest resources. One bundle set is +/// embedded per platform (Bundle/Win64, Bundle/OSX, Bundle/Linux). +/// Main thread (AssetBundle loading). +/// internal static class BundleLoader { + /// Resource folder of the running platform: "Win64" on Windows, "OSX" on macOS, "Linux" otherwise. private static readonly string BundlePlatform; + /// The mod assembly that contains the bundle resources. private static readonly Assembly? ExecutingAssembly; static BundleLoader() @@ -22,6 +29,16 @@ internal static class BundleLoader ExecutingAssembly = Assembly.GetExecutingAssembly(); } + /// + /// Loads the bundle stored as manifest resource "ADOFAIRenderer.Bundle.<platform>.<resourcePath>" with + /// AssetBundle.LoadFromStream. The resource stream is disposed when this method returns. The caller owns the returned + /// bundle. + /// Reference: https://docs.unity3d.com/ScriptReference/AssetBundle.LoadFromStream.html + /// + /// Bundle name inside the platform folder, for example "fonts" or "shaders". + /// The loaded bundle, or null when Unity fails to load it. + /// The resource is not embedded in the DLL. + /// The executing assembly is not available. public static AssetBundle GetBundle(string resourcePath) { if (ExecutingAssembly != null) diff --git a/Util/FFmpeg/FFmpegNativeDecoder.cs b/Util/FFmpeg/FFmpegNativeDecoder.cs index 0e67974..126fadb 100644 --- a/Util/FFmpeg/FFmpegNativeDecoder.cs +++ b/Util/FFmpeg/FFmpegNativeDecoder.cs @@ -10,9 +10,19 @@ namespace ADOFAIRenderer.Util.FFmpeg; /// /// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke. -/// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and +/// Supports both GPU hardware-accelerated decoding (device types of : D3D11VA, DXVA2, +/// CUDA, D3D12VA on Windows; VAAPI, CUDA or VideoToolbox on other platforms; others the decoder reports) and /// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback. /// +/// Used by VideoBGPlayer to decode the background video of a level. Only the first video stream is decoded; +/// audio and other streams are discarded by the demuxer. +/// +/// Threading: every public member takes _lock, so calls from different threads are serialized. +/// Width, Height, FPS, Duration and TotalFrames are set once in the constructor and read without the lock. +/// +/// Timestamps returned by the TryRead* methods are the presentation times of the stream in seconds +/// (PTS × stream time base), so a stream with a non-zero start time returns timestamps starting at that time. +/// /// ── Architecture & Lifecycle Flow ────────────────────────────────────────────── /// /// 1. Construction & Demuxing: @@ -42,6 +52,7 @@ namespace ADOFAIRenderer.Util.FFmpeg; /// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip). /// /// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame): +/// (Short forward distances skip a) and b) and only decode forward; see SeekToTime.) /// a) av_seek_frame: Fast backward keyframe seek. /// b) avcodec_flush_buffers: Clear internal decoder state. /// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead; @@ -72,7 +83,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// private record struct HwConfigCandidate { + /// Hardware device type to create with av_hwdevice_ctx_create (AVCodecHWConfig.device_type). public AVHWDeviceType DeviceType; + + /// Hardware surface pixel format the decoder outputs with this device (AVCodecHWConfig.pix_fmt, e.g. AV_PIX_FMT_D3D11). public AVPixelFormat PixelFormat; } @@ -80,9 +94,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// Preferred hardware device types in priority order. /// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config), /// sorted by this list; configurations not in the list follow in reported order. - /// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Other platforms: VAAPI or VideoToolbox. + /// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Linux: CUDA (NVIDIA), then VAAPI. macOS: VideoToolbox. + /// A device type that cannot be created on the machine (no driver, wrong platform) fails creation and is skipped. /// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by - /// avcodec_find_decoder report DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan in the bundled FFmpeg 8 build; + /// avcodec_find_decoder report hwaccel device types such as DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan (depending on the build); /// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name. /// private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[] @@ -116,6 +131,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.). + /// This class owns one reference; the codec context holds a second one (hw_device_ctx). + /// Both are released in Dispose (or when a hardware candidate fails). Null in software decoding. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html /// private AVBufferRef* _hwDeviceCtx = null; @@ -137,6 +154,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Frame structure holding converted pixel data produced by sws_scale (e.g., RGBA or fallback YUV). + /// Owns no pixel memory: during one conversion its data[]/linesize[] point to the caller's output buffers, + /// and they are cleared before and after (ClearConvertedFramePointers). /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pConvertedFrame = null; @@ -147,6 +166,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pCachedFrame = null; + + /// + /// Whether holds a frame that the next TryRead* call returns + /// before any new frame is decoded. Set by DecodeToTarget, cleared when the frame is consumed or dropped. + /// private bool _hasCachedFrame; /// @@ -172,7 +196,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// private AVCodecContext_get_format? _getFormatDelegate; - /// Whether hardware-accelerated decoding is currently active. + /// + /// Whether hardware-accelerated decoding is currently active. + /// Set when a hardware candidate is configured; cleared when it fails, or by GetFormat when the stream + /// cannot be decoded on the hardware surface format (the decoder then outputs software frames). + /// private bool _isHwDecoding; /// Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV). @@ -183,6 +211,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12). + /// Assumed NV12 when hardware decoding is configured, then updated from every transferred frame + /// (e.g. P010 for 10-bit streams). AV_PIX_FMT_NONE in software decoding. /// private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; @@ -190,11 +220,28 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable // Playback, Stream & Scaler Synchronization Fields // ────────────────────────────────────────────────────────────── + /// Index of the decoded video stream in the format context. -1 until a video stream is found. private int _videoStreamIndex = -1; + + /// + /// Index of the frame most recently returned (or skipped while seeking): round(timestamp × FPS) for frames with a timestamp, + /// the previous value + 1 for frames without one. 0 after opening. + /// private long _currentFrameIndex; + + /// Time base of the video stream (seconds per PTS unit). Used to convert PTS to seconds and seek targets to PTS. private AVRational _timeBase; + + /// Whether Open completed. The read and seek methods do nothing until then and after Dispose. private bool _isOpened; + + /// Whether Dispose has run. Prevents duplicate cleanup. private bool _disposed; + + /// + /// Whether the end of the input has been reached and the decoder was put into draining mode + /// (avcodec_send_packet with null). Reset by a keyframe seek (after avcodec_flush_buffers). + /// private bool _isDraining; /// Number of video packets skipped because the decoder rejected them as invalid data. @@ -203,12 +250,21 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// Number of skipped packets for which a warning is logged. private const int MaxLoggedSkippedPackets = 5; + /// Lock object serializing all public members (decoder state and native resources). private readonly object _lock = new(); // Fields to detect SwsContext updates + // Parameters the current _pSwsContext was created with; a frame with different values recreates the context. + /// Source pixel format of the current scaler context. private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; + + /// Destination pixel format of the current scaler context (RGBA, NV12 or YUV420P). private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; + + /// Source width of the current scaler context in pixels (the destination is always Width × Height). private int _srcWidth; + + /// Source height of the current scaler context in pixels. private int _srcHeight; /// YUV→RGB matrix id (SWS_CS_*) applied to the context. 0 when no matrix is set (non-RGBA output). @@ -221,22 +277,31 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable // Public Properties // ────────────────────────────────────────────────────────────── - /// Video width in pixels. + /// + /// Video width in pixels (codec context width after opening, or the stream parameters). + /// Every output of the TryRead* methods has this width; frames of another size are scaled to it. + /// public int Width { get; private set; } - /// Video height in pixels. + /// Video height in pixels. Every output of the TryRead* methods has this height. public int Height { get; private set; } - /// Frames per second. + /// + /// Frames per second: r_frame_rate of the stream, else avg_frame_rate, else 30. + /// For variable frame rate streams both values are approximations. + /// public double FPS { get; private set; } - /// Total duration in seconds. + /// Total duration in seconds (stream duration, else container duration, else 0). public double Duration { get; private set; } - /// Estimated total number of frames. + /// + /// Estimated total number of frames: the frame count of the stream header (nb_frames) when present, + /// otherwise round(duration × FPS); 0 when the duration is unknown. + /// public long TotalFrames { get; private set; } - /// Index of the currently read frame. + /// Index of the currently read frame (the frame most recently returned or skipped; see _currentFrameIndex). public long CurrentFrameIndex { get @@ -248,7 +313,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } } - /// Current playback time in seconds. + /// Current playback time in seconds: CurrentFrameIndex / FPS (0 when FPS is not positive). public double CurrentTime { get @@ -266,6 +331,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// in system RAM (typically ) rather than the GPU surface format /// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines. /// In software mode, returns the codec context's pixel format (e.g., YUV420P). + /// Before the first hardware frame is transferred, NV12 is returned as the expected download format; + /// the actual format (e.g. P010 for 10-bit video) is known after the first transfer. + /// The TryRead* methods convert any decoded format to their output format, so this value only selects the fastest path. /// public AVPixelFormat PixelFormat { @@ -287,6 +355,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]). + /// Taken from the codec context, else from the stream parameters; AVCOL_RANGE_UNSPECIFIED when neither carries a tag. + /// This property reads the color_range tags only; it does not inspect the pixel format (YUVJ* formats are full range). /// public AVColorRange ColorRange { @@ -394,10 +464,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Initializes a new instance of and opens the specified video file. - /// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back - /// to multi-threaded software decoding if unavailable. + /// Automatically attempts hardware acceleration (device types reported by the decoder, in the order of + /// ), falling back to multi-threaded software decoding if unavailable. /// /// Absolute or relative file path to the video file. + /// + /// The file cannot be opened, has no video stream, or its codec has no decoder or cannot be opened. + /// All native resources allocated so far are released before the exception propagates. + /// public FFmpegNativeDecoder(string videoPath) { Open(videoPath); @@ -425,10 +499,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } // 2. Retrieve stream information + // Reads (and buffers) the start of the file to fill in codec parameters, frame rates and durations. if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0) throw new Exception("Failed to retrieve stream info."); - // 3. Find primary video stream + // 3. Find primary video stream (the first stream whose codec type is video) _videoStreamIndex = -1; for (int i = 0; i < _pFormatContext->nb_streams; i++) { @@ -456,6 +531,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _timeBase = stream->time_base; // 4. Calculate FPS + // r_frame_rate: lowest frame rate that represents all timestamps accurately (guessed by libavformat). + // avg_frame_rate: average frame rate. Both are approximations for variable frame rate streams. + // Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->r_frame_rate); @@ -470,6 +548,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } // 5. Find decoder by codec ID + // avcodec_find_decoder returns the default decoder registered for the ID (e.g. libdav1d for AV1 + // when the build includes it, which reports no hardware configuration). var codecId = stream->codecpar->codec_id; var decoder = ffmpeg.avcodec_find_decoder(codecId); if (decoder == null) @@ -480,6 +560,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable if (_pCodecContext == null) throw new Exception("Failed to allocate codec context."); + // Copies the stream parameters (size, pixel format, extradata, color tags, ...) into the codec context. if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters."); @@ -492,6 +573,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; // Multi-threading settings for improved software decoding performance (auto thread count) + // thread_type = FF_THREAD_FRAME selects frame threading only (several frames decoded in parallel, + // adds one frame of delay per thread); slice threading is not enabled. + // TryInitializeHwDecoder leaves a fresh codec context when every hardware candidate fails. // Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html _pCodecContext->thread_count = 0; _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME; @@ -505,6 +589,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height; // 8. Calculate total frames and duration + // nb_frames is the frame count stored in the container (0 when unknown); + // otherwise the count is estimated from the stream or container duration and FPS. if (stream->nb_frames > 0) { TotalFrames = stream->nb_frames; @@ -531,6 +617,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable : _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0; // 9. Allocate frame and packet buffers + // Structures only; pixel buffers are attached by the decoder, av_hwframe_transfer_data or av_frame_move_ref. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html _pDecodedFrame = ffmpeg.av_frame_alloc(); _pHwTransferFrame = ffmpeg.av_frame_alloc(); @@ -552,8 +639,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Searches for supported hardware device configurations via avcodec_get_hw_config, - /// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA), + /// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA, VAAPI, ...), /// registers the get_format callback, and attempts to open the hardware decoder. + /// + /// Candidates are tried in the order of , then the remaining reported order. + /// When a candidate fails to open, its codec context and device are freed and a fresh codec context + /// (with the stream parameters) is allocated for the next candidate or for the software fallback. + /// On success, , and describe the device + /// and the codec context is open. /// /// Target decoder. /// Source video stream. @@ -562,6 +655,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable { // ReSharper disable once InconsistentNaming // Collect all supported hardware configs from decoder + // AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX (0x01): the decoder supports the hardware format through hw_device_ctx, + // which has to be set on the codec context before avcodec_open2. Only these configurations are used here. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/codec.h const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01; var candidates = new List(); for (int i = 0; ; i++) @@ -616,6 +712,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV) { + // QSV: first with an explicit child device (D3D11VA on Windows, VAAPI elsewhere), + // then without options (FFmpeg selects the child device type). var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); string childDeviceName = isWindows ? "d3d11va" : "vaapi"; AVDictionary* opts = null; @@ -631,6 +729,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } else { + // Default device of the type (device = null: the first / default adapter chosen by FFmpeg). err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0); } @@ -646,6 +745,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } // Assign hardware context and format selection callback + // _hwDeviceCtx takes over the reference created above; the codec context receives its own reference. _hwDeviceCtx = hwDeviceCtx; _hwPixFmt = candidate.PixelFormat; _hwDeviceType = candidate.DeviceType; @@ -663,6 +763,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable continue; } + // The delegate is kept in a field so that it is not collected while native code can still call it + // (FFmpeg.AutoGen marshals it to a function pointer). With thread_count = 1 it is called on the calling + // thread inside avcodec_send_packet / avcodec_receive_frame, which run under _lock. _getFormatDelegate = GetFormat; _pCodecContext->get_format = _getFormatDelegate; @@ -681,7 +784,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable return true; } - // Failed to open codec with this hardware configuration, clean up and try next + // Failed to open codec with this hardware configuration, clean up and try next. + // Freeing the codec context releases its hw_device_ctx reference; _hwDeviceCtx holds the last one. Melon.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate..."); if (_pCodecContext != null) @@ -717,8 +821,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// Callback invoked by libavcodec during decoding to negotiate the output pixel format. /// Confirms the hardware surface format () if offered by the decoder, /// or falls back to software pixel formats via avcodec_default_get_format. + /// Called at the start of decoding and again when stream parameters change (e.g. resolution). + /// The fallback clears ; the decoder then outputs frames in system memory + /// with the thread count set for hardware decoding (1). /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// + /// Codec context negotiating the format. + /// Formats the decoder can output for the stream, terminated by AV_PIX_FMT_NONE (hardware formats first). + /// The selected pixel format. private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts) { for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++) @@ -744,9 +854,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// then sends it to the decoder via avcodec_send_packet. /// A packet that the decoder rejects as invalid data (AVERROR_INVALIDDATA, e.g. a damaged packet) /// is skipped and the next packet is read. Any other decoder error is thrown. + /// Packets of other streams are discarded (they are normally not returned, see AVDISCARD_ALL in Open). + /// Called only after avcodec_receive_frame returned EAGAIN, so avcodec_send_packet accepts the packet. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// - /// True if a video packet was sent; false on EOF. + /// True if a video packet was sent; false on EOF or on a read error (both end the stream). private bool ReadAndSendPacket() { while (true) @@ -757,6 +869,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable if (_pPacket->stream_index == _videoStreamIndex) { + // avcodec_send_packet takes its own reference to (or copy of) the packet data, + // so the packet is unreferenced right away for the next av_read_frame. ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket); ffmpeg.av_packet_unref(_pPacket); @@ -785,9 +899,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV), /// it is automatically downloaded to system RAM via av_hwframe_transfer_data into _pHwTransferFrame, /// preserving presentation timestamps via av_frame_copy_props. + /// + /// Decode loop: avcodec_receive_frame → on EAGAIN read and send the next packet → retry. + /// When the input ends, a null packet switches the decoder to draining mode; the remaining delayed frames + /// are returned, then avcodec_receive_frame returns AVERROR_EOF. + /// The returned frame ( or ) holds its data until + /// the caller unreferences it (av_frame_unref) or the next frame is received. /// /// The software frame containing pixel data in system RAM. /// True if a frame was successfully decoded and available; false on EOF or draining. + /// Decoding, packet sending or the GPU-to-host transfer fails. private bool TryReceiveFrame(out AVFrame* outFrame) { outFrame = null; @@ -800,6 +921,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable // Check if the received frame is on a hardware surface if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) { + // av_hwframe_transfer_data with an empty destination frame allocates a system memory buffer + // in the first format the device supports for download (NV12 for 8-bit 4:2:0 on the common APIs). + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html (av_hwframe_transfer_data) ffmpeg.av_frame_unref(_pHwTransferFrame); int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0); if (transferRet < 0) @@ -809,6 +933,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } // Copy presentation timestamp and metadata properties + // (pts, best_effort_timestamp, color tags; av_hwframe_transfer_data copies pixel data only). + // The hardware surface is then released back to the decoder's pool. ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); @@ -822,6 +948,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable return true; } + // EAGAIN: the decoder needs more input. Feed the next video packet, or start draining at the end of the input. + // The return value of the flush call (null packet) is not checked; a failure shows up as an error or EOF + // from the next avcodec_receive_frame. if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return false; @@ -832,6 +961,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable ffmpeg.avcodec_send_packet(_pCodecContext, null); } } + // EOF: every delayed frame has been returned after draining. else if (ret == ffmpeg.AVERROR_EOF) { return false; @@ -886,8 +1016,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// Lazily initializes or updates the when source/destination formats, dimensions, /// or (for RGBA output) the colorspace / range of the source frame change. /// For RGBA output the YUV→RGB matrix and the source range are taken from . + /// For NV12/YUV420P output no matrix is set; libswscale converts between the formats with its default range handling. + /// The destination size is always × ; SWS_BILINEAR filters scaling and chroma resampling. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// + /// sws_getContext cannot create a context for the conversion. /// Pixel format of the source frame. /// Width of the source frame in pixels. /// Height of the source frame in pixels. @@ -933,6 +1066,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height})."); } + // A new context uses the BT.601 matrix and the range implied by the source pixel format (full for YUVJ*, + // limited otherwise); set the frame's matrix and range for YUV → RGBA. if (swsColorspace != 0) { ApplySwsColorspace(swsColorspace, fullRange); @@ -952,13 +1087,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable int* currentTable; int srcRange, dstRange, brightness, contrast, saturation; - // Returns a negative value when the context cannot change its colorspace details (e.g. YUV to YUV conversion). + // A negative return value means the details cannot be read; the context then keeps its default matrix and range. if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, ¤tInvTable, &srcRange, ¤tTable, &dstRange, &brightness, &contrast, &saturation) < 0) { return; } + // inv_table: coefficients of the source YUV space (used for YUV → RGB); table: coefficients of the destination. + // A failure of sws_setColorspaceDetails leaves the defaults of the context and is not reported. var coefficients = ffmpeg.sws_getCoefficients(swsColorspace); var invTable = new int_array4(); var table = new int_array4(); @@ -975,6 +1112,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state. /// Used by seeking to inspect candidate frames without falsely incrementing frame indices. + /// Uses best_effort_timestamp (libavcodec's estimate from PTS/DTS), else pts, converted with the stream time base. + /// A frame without either timestamp gets / FPS. /// private double GetFrameTimestamp(AVFrame* frame) { @@ -994,6 +1133,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter. + /// With a timestamp, = round(timestamp × FPS); without one, it is incremented by 1. /// private double GetTimestampAndUpdateIndex(AVFrame* frame) { @@ -1016,6 +1156,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Resets all data plane pointers and strides in to zero. /// Prevents stale or dangling pointers from previous conversions being retained across frames. + /// The frame never owns buffers (buf[] stays empty), so av_frame_free in Dispose does not touch caller memory. + /// 8 = AV_NUM_DATA_POINTERS, the size of AVFrame.data / linesize. /// private void ClearConvertedFramePointers() { @@ -1034,12 +1176,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies it in 32-bit RGBA format using libswscale. /// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P). + /// The YUV→RGB matrix and range follow the frame's tags (see InitOrUpdateSwsContext). Frames of another size + /// are scaled to Width × Height. A frame cached by a seek is returned first. /// - /// Pointer to destination memory storing RGBA data. + /// Pointer to destination memory storing RGBA data (at least outStride × Height bytes). /// Byte stride per row of destination memory (Width * 4). /// Whether to vertically flip the output to match Unity texture coordinates. /// Timestamp of the output frame in seconds. - /// True if a frame was successfully retrieved; false on EOF. + /// True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer. + /// Decoding, transfer or scaler creation fails. public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) { timestamp = 0; @@ -1063,6 +1208,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable timestamp = GetTimestampAndUpdateIndex(frame); InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame); + // Destination: the caller's buffer. vflip uses a negative stride starting at the last row, + // so sws_scale writes the rows bottom-up (Unity textures have their origin at the bottom-left). ClearConvertedFramePointers(); if (vflip) { @@ -1082,6 +1229,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _pConvertedFrame->data, _pConvertedFrame->linesize ); + // Release the decoded (or cached) frame's buffers; the output is now in the caller's buffer. ClearConvertedFramePointers(); ffmpeg.av_frame_unref(frame); return true; @@ -1090,6 +1238,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer. + /// Same as the byte* overload. /// public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp) { @@ -1098,7 +1247,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies it in RGBA format to a Unity NativeArray. + /// The rows are tightly packed (stride Width × 4). /// + /// The array is not created or is smaller than Width × Height × 4 bytes. public bool TryReadFrame(NativeArray outBuffer, bool vflip, out double timestamp) { if (!outBuffer.IsCreated) @@ -1119,16 +1270,19 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format. - /// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed, - /// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing. + /// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed: + /// the planes are copied row by row into the caller's buffers (e.g. texture data for Compute Shader processing). /// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale. + /// The Y/UV values keep the range of the source (see ); no YUV→RGB conversion happens here. + /// A frame cached by a seek is returned first. /// - /// Pointer to destination buffer for Y plane (Width * Height bytes). + /// Pointer to destination buffer for Y plane (yStride × Height bytes; Width bytes per row are written). /// Row stride in bytes for Y plane. - /// Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes). + /// Pointer to destination buffer for interleaved UV plane (uvStride × ceil(Height / 2) bytes; ceil(Width / 2) × 2 bytes per row are written). /// Row stride in bytes for UV plane. /// Timestamp of the output frame in seconds. - /// True if a frame was successfully retrieved; false on EOF. + /// True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer. + /// Decoding, transfer or scaler creation fails. // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( byte* yBuffer, int yStride, @@ -1155,6 +1309,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable timestamp = GetTimestampAndUpdateIndex(frame); + // Conversion path: any other format (e.g. YUV420P from software decoding, P010 from 10-bit hardware decoding) + // or a frame size different from Width × Height. sws_scale writes directly into the caller's planes. if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12); @@ -1175,12 +1331,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } else { + // Direct copy path: NV12 frame of the output size. + // The UV plane has ceil(Height / 2) rows of ceil(Width / 2) U/V byte pairs. int w = Width; int h = Height; int uvH = (h + 1) / 2; int uvRowBytes = ((w + 1) / 2) * 2; // Copy Y plane (row by row to safely skip linesize GPU padding) + // Buffer.MemoryCopy(source, destination, destinationSizeInBytes, bytesToCopy): the destination size + // argument is the stride, so a stride smaller than the row width throws instead of overrunning. for (int y = 0; y < h; y++) { Buffer.MemoryCopy( @@ -1208,6 +1368,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers. + /// Same as the byte* overload. /// // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( @@ -1224,7 +1385,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays. + /// The planes are tightly packed: Y stride Width, UV stride ceil(Width / 2) × 2 with ceil(Height / 2) rows. /// + /// An array is not created or is smaller than its plane. // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( NativeArray yBuffer, @@ -1261,7 +1424,18 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// Reads the next video frame and copies its planar Y, U, and V channel data separately. /// For native YUV420P sources, color conversion is bypassed and memory is copied directly. /// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale. + /// YUVJ420P (full range YUV420P) is copied as well; the values keep the source range (see ). + /// A frame cached by a seek is returned first. /// + /// Destination of the Y plane (yStride × Height bytes; Width bytes per row are written). + /// Row stride in bytes of the Y plane. + /// Destination of the U plane (uStride × ceil(Height / 2) bytes; ceil(Width / 2) bytes per row). + /// Row stride in bytes of the U plane. + /// Destination of the V plane (same size as U). + /// Row stride in bytes of the V plane. + /// Timestamp of the output frame in seconds. + /// True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer. + /// Decoding, transfer or scaler creation fails. // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( byte* yBuffer, int yStride, @@ -1289,6 +1463,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable timestamp = GetTimestampAndUpdateIndex(frame); + // Conversion path: any format other than YUV420P/YUVJ420P, or a frame size different from Width × Height. if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P); @@ -1311,6 +1486,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } else { + // Direct copy path: each plane row by row (source rows can be padded, linesize ≥ row width). int w = Width; int h = Height; int uvW = (w + 1) / 2; @@ -1354,6 +1530,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers. + /// Same as the byte* overload. /// // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( @@ -1372,7 +1549,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays. + /// The planes are tightly packed: Y stride Width, U/V stride ceil(Width / 2) with ceil(Height / 2) rows. /// + /// An array is not created or is smaller than its plane. // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( NativeArray yBuffer, @@ -1419,8 +1598,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// for the next TryRead* call. /// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred. /// When the end of the stream is reached before the target, no frame is cached. + /// Every discarded frame updates ; the cached target frame updates it when it is read. /// /// Target time in seconds. + /// Decoding or the GPU-to-host transfer of the target frame fails. private void DecodeToTarget(double timeSeconds) { while (true) @@ -1451,6 +1632,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } else { + // System memory frame: move the buffer references into the cache without copying + // (_pDecodedFrame is left blank for the next receive). ffmpeg.av_frame_unref(_pCachedFrame); ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame); } @@ -1465,6 +1648,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable continue; } + // Same input feeding and draining as TryReceiveFrame. if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return; @@ -1496,8 +1680,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read). /// A backward keyframe seek is followed by a forward decode up to the target. /// The target frame is transferred and preserved in ; when it is already there, nothing is decoded. + /// Negative times are treated as 0. The time is in the timeline of the stream timestamps (a stream that starts at + /// a non-zero time has no frame before its start; the first frame is returned for earlier targets). + /// When the target is past the last frame, no frame is cached and the next TryRead* call returns false. /// /// Target time in seconds to seek to. + /// Seeking (including the fallback seek to the start) or decoding fails. public void SeekToTime(double timeSeconds) { lock (_lock) @@ -1546,6 +1734,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _hasCachedFrame = false; } + // Keyframe seek. With a stream index, the timestamp is in the stream time base. + // AVSEEK_FLAG_BACKWARD: seek to the nearest keyframe at or before the target, so that decoding forward + // from there reaches the target frame. + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_seek_frame) long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase)); int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) @@ -1559,6 +1751,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } // Flush decoder internal buffers + // Drops the frames buffered from before the seek and ends draining mode, so new packets are accepted again. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html ffmpeg.avcodec_flush_buffers(_pCodecContext); _isDraining = false; @@ -1569,9 +1762,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } /// - /// Seeks to a specific frame index. + /// Seeks to a specific frame index: with frameIndex / FPS seconds. /// - /// Target frame index. + /// Target frame index (0-based, in units of 1 / FPS). public void SeekToFrame(long frameIndex) { SeekToTime(frameIndex / FPS); @@ -1582,7 +1775,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable // ────────────────────────────────────────────────────────────── /// - /// Translates an FFmpeg integer error code to a readable string using av_strerror. + /// Translates an FFmpeg integer error code to a readable string using av_strerror + /// (1024-byte stack buffer; "Unknown error (Code: n)" when there is no description). /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html /// private string GetFFmpegErrorString(int errnum) @@ -1598,6 +1792,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures. + /// Safe to call more than once; also called by the constructor on failure and by the finalizer. + /// Order: scaler → frames → packet → codec context (releases its hw_device_ctx reference) + /// → this object's device reference → input file and format context. + /// After Dispose, the read and seek methods return without doing anything. /// public void Dispose() { @@ -1681,6 +1879,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } } + /// + /// Finalizer. Releases the native resources when the decoder was not disposed (runs on the GC finalizer thread). + /// ~FFmpegNativeDecoder() { Dispose(); diff --git a/Util/FFmpeg/FFmpegNativeEncoder.cs b/Util/FFmpeg/FFmpegNativeEncoder.cs index adff6ce..f669afd 100644 --- a/Util/FFmpeg/FFmpegNativeEncoder.cs +++ b/Util/FFmpeg/FFmpegNativeEncoder.cs @@ -32,6 +32,18 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// 5. Dispose(): Release all FFmpeg resources (context, frames, scaler, I/O) /// + /// ── Threading ──────────────────────────────────────────────────────── + /// Every public method takes _lock, so calls from different threads are serialized. + /// In the renderer, the main Unity thread constructs and starts the encoder, and the render thread + /// (RenderManager.RenderThread) sends the video frames and audio samples, calls End() and Dispose(). + /// Dispose() may also be called from another thread to release an encoder whose render was stopped. + /// After End() or Dispose(), the Write*/Send* methods return without doing anything. + /// + /// ── Timestamps ─────────────────────────────────────────────────────── + /// Video PTS = number of frames submitted before the frame (codec time base 1 / FPS). + /// Audio PTS = number of samples per channel submitted before the frame (codec time base 1 / sample rate). + /// Both start at 0, so the two streams are aligned by the count of submitted frames and samples. + /// /// ── Reference Documentation ───────────────────────────────────────── /// • FFmpeg Encoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html /// • Send/Receive Encoding API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html @@ -45,15 +57,22 @@ namespace ADOFAIRenderer.Util.FFmpeg /// public struct AudioSettings { - /// Sample rate of input audio (Hz). e.g., 44100, 48000. + /// + /// Sample rate of input audio (Hz). e.g., 44100, 48000. If 0 or less, 48000 is used. + /// The encoder runs at the same rate (no sample rate conversion), so the rate has to be one the audio codec supports. + /// public int SampleRate; - /// Number of input audio channels. e.g., 1 (mono), 2 (stereo). + /// + /// Number of input audio channels. e.g., 1 (mono), 2 (stereo). If 0 or less, 2 is used for the channel layout. + /// The channel layout is the FFmpeg default layout for the count (av_channel_layout_default). + /// public int Channels; /// - /// Sample format of input audio. Default is AV_SAMPLE_FMT_FLT (32-bit float). + /// Sample format of input audio (interleaved). Default is AV_SAMPLE_FMT_FLT (32-bit float). /// Unity's AudioClip.GetData() returns float[], so FLT is suitable. + /// The value 0 (AV_SAMPLE_FMT_U8) is treated as "not set" and selects FLT. /// public AVSampleFormat InputSampleFormat; @@ -75,13 +94,19 @@ namespace ADOFAIRenderer.Util.FFmpeg /// public struct EncoderSettings { - /// Width resolution of output video (pixels). + /// + /// Width resolution of output video (pixels). Also the width of every input frame (no scaling is performed). + /// The 4:2:0 input size formulas (GetRequiredInputLength) assume an even value. + /// public int Width; - /// Height resolution of output video (pixels). + /// Height resolution of output video (pixels). Also the height of every input frame; even values assumed as for Width. public int Height; - /// Frames per second (e.g., 60.0, 29.97, etc.). + /// + /// Frames per second (e.g., 60.0, 29.97, etc.). Converted to a rational number (RationalUtil.Double2MaxPrecisionInt) + /// for the codec framerate and time base; every submitted frame advances the PTS by one frame interval. + /// public double FPS; /// Video codec type to use (H264, H265, AV1, VP9, VVC). @@ -90,19 +115,28 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Bitrate control mode (CQP, CBR, VBR). public RateControlMode Mode; - /// Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless) + /// + /// Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless) + /// Clamped to the range of the selected encoder (see ApplyQualitySettings); a negative value selects 18. + /// public int QualityValue; /// Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg. public int TargetBitrate; - /// Maximum bitrate in VBR mode (in kbps). Determines peak quality limit. + /// + /// Maximum bitrate in VBR mode (in kbps). Determines peak quality limit. + /// Used only when larger than TargetBitrate; otherwise the maximum is 1.5 × TargetBitrate. + /// public int MaxBitrate; - /// GOP (Group of Pictures) size (in seconds). Represents keyframe interval. + /// + /// GOP (Group of Pictures) size (in seconds). Represents keyframe interval. + /// Converted to frames as round(FPS × GopSize), at least 1. + /// public double GopSize; - /// Whether to force the use of software CPU encoder. + /// Whether to force the use of software CPU encoder (hardware encoders are not tried). public bool ForceSoftware; /// @@ -132,8 +166,9 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Video stream within the output file. - /// A single container (format context) can contain multiple streams (video, audio, etc.), - /// but here only one video stream is used. + /// A single container (format context) can contain multiple streams (video, audio, etc.); + /// this encoder writes one video stream and, when audio is enabled, one audio stream (). + /// Owned by and freed with it. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html /// private AVStream* _pVideoStream = null; @@ -150,8 +185,9 @@ namespace ADOFAIRenderer.Util.FFmpeg private AVFrame* _pSrcFrame = null; /// - /// Frame converted to the pixel format required by the encoder (YUV420P or NV12). - /// Actual pixel data memory is allocated via av_frame_get_buffer(). + /// Frame converted to the pixel format required by the encoder (: NV12, YUV420P or YUV420P10LE). + /// Actual pixel data memory is allocated via av_frame_get_buffer() (reference-counted buffers owned by the frame). + /// Written by the CPU conversion (sws_scale) or by the copy of a pre-converted frame, then submitted to the encoder. /// private AVFrame* _pDstFrame = null; @@ -180,8 +216,9 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Whether hw_frames_ctx mode (GPU frame pool) is active in the QSV encoder. - /// If true, NV12 → QSV hardware surface upload is required every frame. - /// If false, passes NV12 frames in system memory directly to the encoder. + /// If true, NV12 → QSV hardware surface upload is required every frame (SubmitDstFrame). + /// If false, the frame in system memory () is passed directly to the encoder; + /// this is the case for every non-QSV encoder and for QSV in system memory input mode. /// private bool _isQsvEncoder; @@ -189,7 +226,8 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Pixel format of the frames passed to the encoder (output of the color conversion). /// - Hardware encoders (NVENC, AMF, QSV): NV12 (Y plane + UV interleaved plane) /// - libvvenc: YUV420P10LE (10-bit, 3 planes) - /// - Other software encoders: YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling) + /// - Other encoders (software encoders, VideoToolbox): YUV420P (3 planes Y/U/V, 4:2:0 chroma subsampling) + /// Set when the selected encoder opens; read by FrameReader (through SelectedPixelFormat) to choose the GPU conversion. /// /// Reference - Major pixel format relationships: /// RGBA (Unity source, 4-channel 8-bit) @@ -207,7 +245,11 @@ namespace ADOFAIRenderer.Util.FFmpeg /// private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG; - /// Pixel format of the finally selected and running encoder. + /// + /// Pixel format of the finally selected and running encoder (the format of the frames the encoder receives). + /// A pre-converted input passed with isColorConverted = true has to be in this format (NV12 or YUV420P). + /// Valid after Start() returns; AV_PIX_FMT_YUV420P before. + /// public AVPixelFormat SelectedPixelFormat => _dstPixFmt; /// @@ -219,8 +261,14 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// GPU frame structure used in QSV hardware acceleration mode. /// Cached as a member variable to prevent allocation/deallocation overhead for each frame. + /// Allocated only in GPU frame pool input mode; holds a pool surface only while one frame is uploaded and sent. /// private AVFrame* _pHwFrame = null; + + /// + /// Whether the conversion path (GPU pre-converted or CPU RGBA) has been logged. + /// The path is logged once, on the first frame after Start(). + /// private bool _loggedConversionMode; // ────────────────────────────────────────────────────────────── @@ -233,7 +281,7 @@ namespace ADOFAIRenderer.Util.FFmpeg /// private AVCodecContext* _pAudioCodecContext = null; - /// Audio stream within the output file. + /// Audio stream within the output file. Owned by and freed with it. private AVStream* _pAudioStream = null; /// Packet structure to temporarily hold encoded audio packet data. @@ -242,6 +290,7 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Audio resampling context (SwrContext). /// Converts input PCM format (e.g., float interleaved) → encoder required format (e.g., float planar). + /// Input and output use the same sample rate and channel layout, so only the sample format and layout in memory change. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html /// private SwrContext* _pSwrContext = null; @@ -252,10 +301,13 @@ namespace ADOFAIRenderer.Util.FFmpeg /// private AVAudioFifo* _pAudioFifo = null; - /// Frame structure to pass to the audio encoder. + /// + /// Frame structure to pass to the audio encoder. Reused for every audio frame; + /// holds frame_size samples per channel in the encoder sample format. + /// private AVFrame* _pAudioFrame = null; - /// Accumulated index of encoded audio samples. Used as the PTS value. + /// Accumulated index of encoded audio samples (per channel). Used as the PTS value (time base 1 / sample rate). private long _audioSampleIndex; /// Whether audio encoding is enabled. @@ -267,6 +319,10 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Encoding settings passed at construction time (immutable). private readonly EncoderSettings _settings; + /// + /// Stores the encoding settings. No FFmpeg resource is allocated and the output file is not created until Start(). + /// + /// Video (and optional audio) encoding settings. Copied (struct). public FFmpegNativeEncoder(EncoderSettings settings) { _settings = settings; @@ -285,10 +341,11 @@ namespace ADOFAIRenderer.Util.FFmpeg /// (per codec: NVENC → QSV → AMF → VideoToolbox → software, for the encoders the codec has) /// With ForceSoftware, only the software encoders are returned. /// - /// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based) - /// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework) - /// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK) - /// - lib*: CPU-based software encoders (always available, but slower) + /// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based) + /// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework) + /// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK / oneVPL) + /// - videotoolbox: Hardware encoder of Apple platforms (macOS) + /// - lib*: CPU-based software encoders (available when the FFmpeg build includes the library, but slower) /// /// Iterates through this list in Start() to use the first workable encoder. /// @@ -318,7 +375,8 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Converts an FFmpeg error code (negative integer) to a human-readable string. /// e.g., -1 → "Operation not permitted" /// - /// Calls av_strerror() internally to write the message to a stack-allocated buffer. + /// Calls av_strerror() internally to write the message to a stack-allocated buffer (1024 bytes, NUL-terminated). + /// Returns "Unknown error (Code: n)" when av_strerror has no description for the code. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html /// private string GetFFmpegErrorString(int errnum) @@ -348,8 +406,18 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 6. Initialize frame buffers (AVFrame) and colorspace converter (SwsContext) /// /// On failure, calls Dispose() to clean up partially initialized resources before rethrowing the exception. + /// The encoder object cannot be started again after a failure (it is disposed); a new instance is required. + /// A file created by avio_open before the failure is left on disk (the caller deletes incomplete output). + /// Does nothing when the encoder is already started or disposed. /// - /// Output video file path (e.g., "C:\output\video.mp4") + /// + /// Output video file path (e.g., "C:/output/video.mp4"). The extension selects the container (mp4, mkv, webm, ...). + /// + /// + /// No container matches the extension, no candidate encoder can be opened, the file cannot be opened, + /// or the header cannot be written. Native library load failures (NotSupportedException) are wrapped with a hint + /// about the FFmpeg DLLs in the UserLibs folder. + /// public void Start(string outputPath) { lock (_lock) @@ -370,8 +438,8 @@ namespace ADOFAIRenderer.Util.FFmpeg if (_pFormatContext == null) throw new Exception("Failed to create output format context."); // ── Step 2: Add video stream ── - // Creates a single video track (stream) inside the container. - // Additional audio streams can be created here if needed. + // Creates a single video track (stream) inside the container (stream index 0). + // The audio stream, when enabled, is added after the encoder is opened (Step 4.5). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html _pVideoStream = ffmpeg.avformat_new_stream(_pFormatContext, null); if (_pVideoStream == null) throw new Exception("Failed to create video stream."); @@ -379,6 +447,9 @@ namespace ADOFAIRenderer.Util.FFmpeg // ── Step 3: Select and open encoder ── // Iterates through the priority list returned by GetEncoderCandidates() // to find the first available encoder. + // Each attempt allocates a new codec context. An attempt that fails frees its codec context + // (and its hardware device/frames references) before the next one, so after the loop + // _pCodecContext is either the opened encoder or null. var candidates = GetEncoderCandidates(); var openedSuccessfully = false; @@ -400,6 +471,9 @@ namespace ADOFAIRenderer.Util.FFmpeg _pCodecContext = ffmpeg.avcodec_alloc_context3(pCodec); if (_pCodecContext == null) continue; + // FF_COMPLIANCE_EXPERIMENTAL: allows avcodec_open2 to open encoders that libavcodec marks + // as experimental (AV_CODEC_CAP_EXPERIMENTAL); at a stricter level avcodec_open2 returns AVERROR_EXPERIMENTAL. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/avcodec.c (avcodec_open2) _pCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL; // ── Basic video parameters configuration ── @@ -419,15 +493,19 @@ namespace ADOFAIRenderer.Util.FFmpeg : AVColorRange.AVCOL_RANGE_JPEG; _pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709; // thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count. - // For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count, - // so setting 0 (auto) is required for parallel encoding utilizing all cores. + // For libvvenc, the FFmpeg wrapper passes a positive thread_count to VVenC as its thread count + // (m_numThreads); with 0 the wrapper leaves the VVenC default thread setting in place. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c _pCodecContext->thread_count = 0; // ── Pixel format configuration ── // Video encoders generally expect input in YUV420P or NV12 format. // - // • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE). - // • NV12: Y plane + UV interleaved plane. Used by the hardware encoders (NVENC, AMF, QSV). + // • YUV420P: 3 independent planes Y/U/V. Used by the software encoders (libvvenc: YUV420P10LE) + // and by VideoToolbox. + // • NV12: Y plane + UV interleaved plane. Used by the hardware encoders NVENC, AMF and QSV. + // selectedPixFmt is the format of the frames this class produces (_dstPixFmt once the encoder opens); + // _pCodecContext->pix_fmt is the format the encoder is told to accept. // • QSV: the converted frames are NV12 (selectedPixFmt). The codec context pix_fmt set here is // AV_PIX_FMT_QSV and is replaced in the QSV device setup below according to the input mode: // system memory input mode: pix_fmt = NV12, the encoder receives the NV12 frames directly. @@ -448,7 +526,10 @@ namespace ADOFAIRenderer.Util.FFmpeg } else if (encoder == VideoEncoder.LibVvenc) { - // libvvenc: Only 10-bit YUV420P10LE supported (yuv420p verified unsupported in this build) + // libvvenc: Only 10-bit YUV420P10LE supported (the wrapper's pix_fmts list is AV_PIX_FMT_YUV420P10 only). + // Pre-converted GPU input is NV12/YUV420P only, so libvvenc receives its frames through the + // CPU conversion (sws_scale) path; WritePreConvertedFrame rejects this format. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P10LE; _pCodecContext->pix_fmt = selectedPixFmt; } @@ -483,7 +564,12 @@ namespace ADOFAIRenderer.Util.FFmpeg var calculatedGop = Math.Max(1, (int)Math.Round(_settings.FPS * _settings.GopSize)); if (encoder == VideoEncoder.LibVvenc) { - // libvvenc (VVenC) requires IntraPeriod (gop_size) to be a multiple of its GOPSize (typically 32). + // gop_size for libvvenc: calculatedGop rounded to the nearest multiple of 32 (an exact half, e.g. 48, rounds down), at least 32. + // VVenC places intra frames at multiples of its GOP size (32 in its default GOP structure). + // The FFmpeg 8.0 libvvenc wrapper reads gop_size only to select all-intra coding (gop_size == 1); + // for any other value the intra period comes from the wrapper's "period" option + // (seconds, default 1), so this value does not change the keyframe interval of the output. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (m_IntraPeriodSec) _pCodecContext->gop_size = Math.Max(32, ((calculatedGop + 15) / 32) * 32); } else @@ -498,7 +584,8 @@ namespace ADOFAIRenderer.Util.FFmpeg // // Configuration per encoder: // SVT-AV1: svtav1-params "pred-struct=1" (low delay prediction structure). - // Setting max_b_frames on AVCodecContext makes SVT-AV1 report a "bad parameter" error. + // The FFmpeg libsvtav1 wrapper does not read max_b_frames; the prediction structure is set + // through svtav1-params, which the wrapper passes to svt_av1_enc_parse_parameter. // libvvenc: max_b_frames = -1 (encoder default GOP structure). // Others: max_b_frames = 0 (no B-frames, frames are encoded in presentation order). // @@ -506,14 +593,17 @@ namespace ADOFAIRenderer.Util.FFmpeg // // Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md // (PredStructure: 0 = all intra, 1 = low delay, 2 = random access, default 2) + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libsvtav1.c (config_enc_params) if (encoder == VideoEncoder.LibSvtAv1) { ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0); } else if (encoder == VideoEncoder.LibVvenc) { - // libvvenc relies on hierarchical GOP structures; forcing max_b_frames = 0 causes initialization failure. - // We explicitly set max_b_frames to -1 to allow libvvenc to use its default GOP reference distance. + // libvvenc uses the hierarchical GOP structure of VVenC (reordered frames). + // max_b_frames = -1 leaves the B-frame count to the encoder. The FFmpeg 8.0 wrapper does not read + // max_b_frames, so the GOP structure is the VVenC default in either case. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c _pCodecContext->max_b_frames = -1; } else @@ -533,16 +623,18 @@ namespace ADOFAIRenderer.Util.FFmpeg } // ── Apply quality/bitrate settings ── + // Writes bit_rate / rc_* fields or the encoder's private quality option. + // Private options are set on priv_data before avcodec_open2, which reads them. ApplyQualitySettings(encoder); // ── QSV Encoder Hardware Device Initialization ── // QSV (Quick Sync Video) uses fixed-function encoding hardware on Intel GPUs. // Using QSV in FFmpeg requires creating a hardware device context. // - // 3 strategies are attempted sequentially below: - // Strategy 1) Create QSV device directly with child_device_type=d3d11va option - // → FFmpeg internally creates D3D11 device as child and connects to QSV - // Strategy 2) Create D3D11VA device first, then derive QSV device + // 3 strategies are attempted sequentially below (child device: D3D11VA on Windows, VAAPI elsewhere): + // Strategy 1) Create QSV device directly with child_device_type=d3d11va (or vaapi) option + // → FFmpeg internally creates the child device and connects it to QSV + // Strategy 2) Create the D3D11VA (or VAAPI) device first, then derive QSV device // → Effective when Strategy 1 fails on some drivers // Strategy 3) Create QSV device directly (legacy method) // → Fallback on older drivers/systems @@ -555,6 +647,8 @@ namespace ADOFAIRenderer.Util.FFmpeg // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html if (encoder.Kind == EncoderKind.QSV) { + // hwDeviceCtx: local reference to the QSV device, released below after the codec context + // (or the frames context) has taken its own reference. AVBufferRef* hwDeviceCtx = null; var qsvReady = false; int err; @@ -597,11 +691,14 @@ namespace ADOFAIRenderer.Util.FFmpeg qsvReady = true; Melon.Logger.Msg($"{encoder.Name}: Successfully created device by deriving {childDeviceType} → QSV"); } + // The derived QSV device holds its own reference to the child device, + // so the local reference is released whether or not the derivation succeeded. ffmpeg.av_buffer_unref(&childDeviceCtx); } } // ── Strategy 3: Direct QSV creation (legacy) ── + // No options: FFmpeg selects the child device type of the platform by itself. if (!qsvReady) { err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0); @@ -734,6 +831,7 @@ namespace ADOFAIRenderer.Util.FFmpeg if (encoder.Kind == EncoderKind.NVENC) { // NVENC preset range: p1 (fastest) ~ p7 (highest quality). p4 = medium (default) + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/nvenc_h264.c ("preset") ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "p4", 0); } else if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265) @@ -744,23 +842,29 @@ namespace ADOFAIRenderer.Util.FFmpeg } else if (encoder == VideoEncoder.LibSvtAv1) { - // SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality) + // SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality), library default 8. + // Negative presets are for debugging; the FFmpeg option default -2 keeps the library default. + // Reference: https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Parameters.md (EncoderMode, --preset) ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "10", 0); } else if (encoder == VideoEncoder.LibAomAv1) { - // libaom cpu-used: 0 (highest quality) ~ 8 (highest speed) + // libaom cpu-used: 0 (highest quality) ~ 8 (highest speed), FFmpeg option default 1 + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libaomenc.c ("cpu-used", "usage") ffmpeg.av_opt_set(_pCodecContext->priv_data, "cpu-used", "6", 0); - // "realtime" usage: Single pass, low latency encoding mode + // "realtime" usage (AOM_USAGE_REALTIME, value 1): Single pass, low latency encoding mode. + // The other values are "good" (default) and "allintra". ffmpeg.av_opt_set(_pCodecContext->priv_data, "usage", "realtime", 0); } else if (encoder == VideoEncoder.LibVvenc) { // libvvenc preset: slower, slow, medium, fast, faster. Default is medium, change to faster to gain performance + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c ("preset") ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "faster", 0); } - // Other encoders (AMF, QSV, etc.) use default presets. + // Other encoders (AMF, QSV, VideoToolbox, libvpx-vp9) use their default presets / speed settings. // Setting unsupported options can cause errors, so skip. + // The av_opt_set results above are not checked: an option the encoder does not accept leaves its default. // ── Open encoder ── // avcodec_open2: Initializes encoder based on all parameters configured above. @@ -780,6 +884,8 @@ namespace ADOFAIRenderer.Util.FFmpeg } // If open fails, free codec context and QSV frame state, then try next candidate. + // avcodec_free_context also releases the hw_device_ctx / hw_frames_ctx references of the context, + // which releases the QSV device and frame pool of this attempt. var inputMode = encoder.Kind == EncoderKind.QSV ? (useFramePool ? " (GPU frame pool input mode)" : " (system memory input mode)") : ""; Melon.Logger.Warning($"Failed to initialize encoder {encoder.Name}{inputMode}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate."); var pCodecCtx = _pCodecContext; @@ -810,7 +916,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // Stream time base: unit of the timestamps stored in the container. // Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks. // When it is left unset, the muxer uses 1/90000, in which the interval of rates such as - // 59.94 fps (1501.5 ticks), 23.976 fps (3753.75 ticks) or 165 fps (545.45 ticks) is not an integer, + // 59.94 FPS (1501.5 ticks), 23.976 FPS (3753.75 ticks) or 165 FPS (545.45 ticks) is not an integer, // and the frame durations alternate between two values. // The muxer may still adjust the value in avformat_write_header (e.g. mp4 scales it to at least 10000); // packet timestamps are rescaled to the final value in SendFrameAndWritePackets. @@ -819,6 +925,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // ── Step 4.5 (Optional): Initialize audio stream ── // If EncoderSettings.Audio is set, add audio stream. + // Must run before avformat_write_header: streams cannot be added after the header is written. if (_settings.Audio.HasValue) { InitializeAudioStream(_settings.Audio.Value); @@ -835,8 +942,9 @@ namespace ADOFAIRenderer.Util.FFmpeg throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})"); } - // Write file header (ftyp, moov, container metadata). + // Write file header (container metadata; for mp4 the ftyp box, the moov index is written by av_write_trailer). // Packets can be written with av_interleaved_write_frame after this point. + // The muxer may change the stream time bases here; packets are rescaled to the final values when written. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null); if (headerResult < 0) @@ -864,6 +972,8 @@ namespace ADOFAIRenderer.Util.FFmpeg // The chroma planes are subsampled to half width and height (4:2:0) with the same filter, // so each chroma sample takes one source pixel instead of averaging the 2x2 block. // This scaler is used only by the CPU conversion path (WriteFrame with RGBA input). + // It is created for every encoder, also when all frames arrive pre-converted. + // For libvvenc the output is YUV420P10LE (8-bit RGBA values scaled to 10 bits). // // Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html _pSwsContext = ffmpeg.sws_getContext( @@ -874,6 +984,7 @@ namespace ADOFAIRenderer.Util.FFmpeg if (_pSwsContext == null) throw new Exception("Failed to initialize software scaler."); + // BT.709 matrix and the encoder color range for the RGBA → YUV conversion (see ConfigureSwsColorspace). ConfigureSwsColorspace(); _isInitialized = true; @@ -881,6 +992,8 @@ namespace ADOFAIRenderer.Util.FFmpeg } catch (NotSupportedException ex) { + // FFmpeg.AutoGen throws NotSupportedException when a native function cannot be bound + // (DLL missing, wrong version or corrupted). Dispose(); throw new Exception("Failed to load FFmpeg native library (DLL). Please check if FFmpeg DLL files (avcodec, avformat, etc.) in UserLibs folder are missing or corrupted, delete the DLL files in UserLibs, and restart the game.", ex); } @@ -904,10 +1017,15 @@ namespace ADOFAIRenderer.Util.FFmpeg /// ── Bitrate Control Mode Description ── /// /// • CBR (Constant Bitrate): Maintains a constant bitrate. Suitable for streaming. - /// → bit_rate = rc_max_rate = rc_min_rate (all identical) + /// → bit_rate = rc_max_rate = rc_min_rate (all identical), rc_buffer_size = 1 second of the bitrate + /// → libx264/libx265 additionally receive HRD/CBR options. Other encoders select their rate control mode + /// from these generic fields according to their own FFmpeg wrapper. /// /// • VBR (Variable Bitrate): Bitrate varies according to scene complexity. - /// → bit_rate = target value, rc_max_rate = maximum allowed value + /// → bit_rate = target value, rc_max_rate = maximum allowed value, rc_buffer_size = 2 × rc_max_rate + /// + /// libvvenc receives bit_rate only in CBR/VBR (see the libvvenc notes in the method body). + /// Bitrates in the settings are in kbps; the codec context fields are in bits per second (rc_buffer_size in bits). /// /// • CQP (Constant Quantization Parameter): Uses a fixed quantization value. /// → Lower value = higher quality/larger size, Higher value = lower quality/smaller size @@ -919,8 +1037,13 @@ namespace ADOFAIRenderer.Util.FFmpeg /// QSV: AVCodecContext.global_quality (Intel ICQ mode) /// Software: "crf" (Constant Rate Factor, smarter quality control) /// + /// VideoToolbox encoders take the software branch ("crf") in CQP mode; when the encoder has no such option, + /// TrySetOption logs that it was not applied and the encoder keeps its default rate control. + /// /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// + /// Encoder whose codec context (, allocated, not yet opened) is configured. + /// The rate control mode is not CBR, VBR or CQP. private void ApplyQualitySettings(VideoEncoder encoder) { switch (_settings.Mode) @@ -929,12 +1052,15 @@ namespace ADOFAIRenderer.Util.FFmpeg // kbps → bps conversion (FFmpeg bit_rate field is in bps) _pCodecContext->bit_rate = _settings.TargetBitrate * 1000L; - // libvvenc does not support rc_min_rate/rc_max_rate/rc_buffer_size, - // so set bit_rate only and delegate the rest to internal rate control. + // libvvenc: only bit_rate is set; VVenC's internal rate control handles the rest. + // The FFmpeg 8.0 libvvenc wrapper reads bit_rate and, with VVenC 1.8.0 or later, rc_max_rate + // (m_RCMaxBitrate); it does not read rc_min_rate or rc_buffer_size. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_rc_mode) if (encoder != VideoEncoder.LibVvenc) { // In CBR, set target/max/min bitrates all equal. - // rc_buffer_size determines the bitrate buffering size. + // rc_buffer_size determines the bitrate buffering size (VBV/HRD buffer, in bits): + // TargetBitrate × 1000 bits = one second of data at the target bitrate. _pCodecContext->rc_max_rate = _pCodecContext->rc_min_rate = _settings.TargetBitrate * 1000L; _pCodecContext->rc_buffer_size = _settings.TargetBitrate * 1000; @@ -961,7 +1087,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // kbps → bps conversion _pCodecContext->bit_rate = _settings.TargetBitrate * 1000L; - // Skip libvvenc as it does not support rc_max_rate/rc_buffer_size. + // libvvenc: only bit_rate is set (see the CBR case for the fields its wrapper reads). if (encoder != VideoEncoder.LibVvenc) { // If MaxBitrate is larger than TargetBitrate, use as is; @@ -970,7 +1096,8 @@ namespace ADOFAIRenderer.Util.FFmpeg ? _settings.MaxBitrate * 1000L : _settings.TargetBitrate * 1500L; _pCodecContext->rc_max_rate = maxRate; - // Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes. + // Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes + // (in bits: two seconds of data at the maximum bitrate). _pCodecContext->rc_buffer_size = (int)(maxRate * 2); } break; @@ -980,6 +1107,8 @@ namespace ADOFAIRenderer.Util.FFmpeg // encoder's option definition (AVOption min/max); a declared minimum of -1 means "auto". // NVENC: "qp" H.264/HEVC 0~51, AV1 0~255 // (libavcodec/nvenc_h264.c, nvenc_hevc.c: "qp" -1~51 / libavcodec/nvenc_av1.c: "qp" -1~255) + // The bounds of every encoder except QSV are read at run time (GetQualityRange); + // the values listed here are those of the FFmpeg sources. // AMF: "qp_i" H.264/HEVC 0~51, AV1 0~255 // (libavcodec/amfenc_h264.c, amfenc_hevc.c: "qp_i" -1~51 / libavcodec/amfenc_av1.c: "qp_i" -1~255) // QSV: AVCodecContext.global_quality 1~51 @@ -991,7 +1120,8 @@ namespace ADOFAIRenderer.Util.FFmpeg if (encoder.Kind == EncoderKind.QSV) { // QSV runs in ICQ (Intelligent Constant Quality) mode when global_quality > 0 and rc_max_rate is 0; - // global_quality 0 selects the encoder's default CQP. The ICQ quality is clamped to 1~51 by FFmpeg: + // the lower bound 1 keeps global_quality positive, so ICQ is selected (qsvenc.c select_rc_mode checks + // this condition before the bitrate-based modes). The ICQ quality is clamped to 1~51 by FFmpeg: // libavcodec/qsvenc.c: q->param.mfx.ICQQuality = av_clip(avctx->global_quality, 1, 51); // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/qsvenc.c qpMin = 1; @@ -1002,8 +1132,9 @@ namespace ADOFAIRenderer.Util.FFmpeg else if (encoder == VideoEncoder.LibVvenc) (qpMin, qpMax) = GetQualityRange("qp"); else (qpMin, qpMax) = GetQualityRange("crf"); // Software encoder (libx264, libx265, libvpx-vp9, etc.) - // Default value 18: Quality close to "visually lossless" in H.264. + // Default value 18 (used when QualityValue is negative): Quality close to "visually lossless" in H.264. // Provides similar level of quality in other codecs as well. + // The value is clamped to [qpMin, qpMax] of the encoder; a clamped value is logged as a warning. const int defaultQuality = 18; var requestedQp = _settings.QualityValue; int qpValue; @@ -1043,6 +1174,7 @@ namespace ADOFAIRenderer.Util.FFmpeg } else { + // Software encoders (libx264, libx265, libvpx-vp9, libsvtav1, libaom-av1) and VideoToolbox. // CRF (Constant Rate Factor): Smarter quality control than QP. // Automatically adjusts bitrate according to scene complexity to maintain consistent visual quality. // Range: 0 (lossless) ~ 51 (lowest quality) for x264/x265, 0~63 for VP9/AV1. Typically between 18 and 28. @@ -1057,7 +1189,11 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Sets a private option (priv_data) of the encoder. /// Returns true when FFmpeg accepts the option. Otherwise logs a warning with the FFmpeg error and returns false. + /// Must be called before avcodec_open2. The value string is parsed by av_opt_set according to the option type. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__opt__set__funcs.html /// + /// Option name as declared by the encoder (e.g. "crf", "qp", "x265-params"). + /// Option value as text. // ReSharper disable once UnusedMethodReturnValue.Local private bool TrySetOption(string name, string value) { @@ -1087,7 +1223,10 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Field order of AVQSVFramesContext: mfxFrameSurface1* surfaces, int nb_surfaces, int frame_type, mfxFrameInfo* info /// → frame_type is at offset sizeof(pointer) + sizeof(int) (12 bytes on 64-bit). /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavutil/hwcontext_qsv.h + /// Must be called before av_hwframe_ctx_init. Does nothing when hwctx is null. /// + /// QSV frames context allocated by av_hwframe_ctx_alloc on a QSV device. + /// Combination of MFX_MEMTYPE_* flags, or 0 to let FFmpeg choose. private static void SetQsvFrameType(AVHWFramesContext* framesCtx, int frameType) { if (framesCtx->hwctx == null) return; @@ -1142,14 +1281,21 @@ namespace ADOFAIRenderer.Util.FFmpeg /// → Highly efficient as no extra memory copy or separate flip operation is required /// /// stride = Width × 4 (RGBA has 4 bytes per pixel) + /// + /// The buffer is read synchronously by sws_scale inside this call and is not referenced after it returns. + /// Does nothing when the encoder is not started, already ended or disposed. /// - /// Starting address of Unity RGBA pixel data + /// + /// Starting address of Unity RGBA pixel data: Width × Height pixels, rows tightly packed (Width × 4 bytes), + /// bottom row first. The length is not checked here; the public Send* methods validate it. + /// public void WriteFrameFromBuffer(byte* buffer) { lock (_lock) { if (_disposed || !_isInitialized) return; + // _pSrcFrame does not own memory: its data[0]/linesize[0] point into the caller's buffer for this call only. var stride = _settings.Width * 4; // RGBA = 4 bytes per pixel _pSrcFrame->data[0] = buffer + stride * (_settings.Height - 1); // Starting address of last row _pSrcFrame->linesize[0] = -stride; // Vertical flip with negative stride @@ -1165,7 +1311,10 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Encodes a frame by obtaining a read-only pointer from Unity NativeArray<byte>. /// NativeArray is managed by Unity's native memory allocator, /// allowing processing of large amounts of data without GC (Garbage Collector) pressure. + /// The array holds RGBA pixels in the layout of . /// + /// RGBA frame data, at least Width × Height × 4 bytes. + /// The array is smaller than one RGBA frame. public void WriteFrameFromNativeBuffer(NativeArray buffer) { lock (_lock) @@ -1182,7 +1331,9 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV) /// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V) /// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it). + /// The 4:2:0 formulas use integer halves of Width and Height (exact for even dimensions). /// + /// True for pre-converted NV12/YUV420P input, false for RGBA input. private long GetRequiredInputLength(bool isColorConverted) { long w = _settings.Width, h = _settings.Height; @@ -1209,8 +1360,20 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Encodes a frame from IntPtr (native pointer). - /// Can be used from external native plugins or custom capture sources. + /// Used by the renderer for frames read back into native memory (AsyncGPUReadback), + /// and usable from external native plugins or custom capture sources. + /// The memory is read synchronously within this call; the caller may reuse or free it after the call returns. + /// Does nothing when the encoder is not started, already ended or disposed. /// + /// Start of the frame data. + /// Number of readable bytes at ; must be at least one frame (GetRequiredInputLength). + /// + /// True: the data is already in (NV12 or YUV420P, planes tightly packed, top row first). + /// False: the data is RGBA, bottom row first (flipped while converting). + /// + /// is smaller than one frame. + /// Pre-converted input while the encoder format is not NV12 or YUV420P. + /// The encoder or the muxer reports an error. public void SendDataFromNativeBuffer(IntPtr pointer, int length, bool isColorConverted) { lock (_lock) @@ -1234,7 +1397,13 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Encodes a frame from a managed byte[]. /// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer. + /// The array is not referenced after the call returns (the renderer returns it to its ArrayPool). + /// Data layout and exceptions as for . /// + /// Frame data. May be larger than one frame (e.g. a pooled array). + /// Number of valid bytes at the start of . + /// True for pre-converted NV12/YUV420P data, false for RGBA. + /// exceeds the array or is smaller than one frame. public void SendDataFromBuffer(byte[] buffer, int length, bool isColorConverted) { lock (_lock) @@ -1345,7 +1514,15 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data. + /// + /// Input layout (tightly packed, top row first, planes back to back): + /// NV12: Y (Width × Height) | UV interleaved (Width × Height/2) + /// YUV420P: Y (Width × Height) | U (Width/2 × Height/2) | V (Width/2 × Height/2) + /// Each plane is copied into , whose rows can be padded (linesize ≥ row width). + /// A plane is copied with one memory copy when the destination stride equals the row width, otherwise row by row. /// + /// Start of the frame data; the length was validated by the caller. + /// The encoder pixel format is neither NV12 nor YUV420P (e.g. YUV420P10LE). private void WritePreConvertedFrame(byte* buffer) { PrepareDstFrameForWrite(); @@ -1375,6 +1552,7 @@ namespace ADOFAIRenderer.Util.FFmpeg } // Copy UV Plane (positioned after Y plane size) + // Each UV row holds Width/2 U/V pairs = Width bytes; there are Height/2 rows. var pSrcUV = buffer + width * height; var pDstUV = _pDstFrame->data[1]; var dstStrideUV = _pDstFrame->linesize[1]; @@ -1413,7 +1591,7 @@ namespace ADOFAIRenderer.Util.FFmpeg } } - // Copy U Plane + // Copy U Plane (Width/2 × Height/2 bytes, positioned after the Y plane) var pSrcU = buffer + width * height; var pDstU = _pDstFrame->data[1]; var dstStrideU = _pDstFrame->linesize[1]; @@ -1432,7 +1610,7 @@ namespace ADOFAIRenderer.Util.FFmpeg } } - // Copy V Plane + // Copy V Plane (same size as U, positioned after the U plane) var pSrcV = pSrcU + chromaWidth * chromaHeight; var pDstV = _pDstFrame->data[2]; var dstStrideV = _pDstFrame->linesize[2]; @@ -1465,8 +1643,10 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 1. PrepareDstFrameForWrite: Makes the destination frame buffer writable /// (Allocates a new buffer if encoder is still referencing previous frame's buffer) /// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion - /// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface - /// 4. Pass to encoder after setting PTS + /// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface (SubmitDstFrame) + /// 4. Pass to encoder after setting PTS (SubmitDstFrame) + /// + /// The source is , set up by WriteFrameFromBuffer. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale) /// @@ -1493,6 +1673,14 @@ namespace ADOFAIRenderer.Util.FFmpeg SubmitDstFrame(); } + /// + /// Passes the filled destination frame () to the encoder with the next video PTS + /// and writes the packets the encoder returns. + /// GPU frame pool input mode (QSV, _isQsvEncoder): the frame is uploaded to a pool surface (_pHwFrame), which is sent. + /// Every other encoder / input mode: _pDstFrame itself is sent. + /// _frameIndex is incremented when the frame's PTS is assigned, right before SendFrameAndWritePackets. + /// + /// Surface allocation, upload, encoding or packet writing fails. private void SubmitDstFrame() { if (_isQsvEncoder) @@ -1500,6 +1688,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // ── QSV hw_frames_ctx mode: System memory → GPU surface upload ── // // av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool. + // Fails (e.g. ENOMEM) when every surface of the pool is still held by the encoder (QsvFramePoolSize). // av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface. // // The frame is copied from system memory to the GPU surface on every call; @@ -1526,6 +1715,7 @@ namespace ADOFAIRenderer.Util.FFmpeg finally { // Release reference immediately after use so it can be returned to pool. + // The encoder keeps its own reference to the surface until the frame is encoded. ffmpeg.av_frame_unref(_pHwFrame); } } @@ -1534,6 +1724,8 @@ namespace ADOFAIRenderer.Util.FFmpeg // ── General Encoders (Software or hw_device_ctx mode) ── // Pass software frame directly to encoder. // In hw_device_ctx mode, encoder handles GPU upload internally. + // avcodec_send_frame takes a new reference to (or a copy of) the frame data; _pDstFrame keeps its own + // reference, and PrepareDstFrameForWrite allocates a new buffer if the old one is still shared. _pDstFrame->pts = _frameIndex++; SendFrameAndWritePackets(_pDstFrame); } @@ -1563,9 +1755,14 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// ── Packet Timestamp Handling ── /// av_packet_rescale_ts: Converts PTS/DTS from codec's time_base to stream's time_base. - /// Mandatory because codec and container may use different time_bases. + /// Mandatory because codec and container may use different time_bases + /// (Start sets the stream time base to the codec time base, but the muxer can change it in avformat_write_header). /// e.g., codec time_base=1/60, stream time_base=1/90000 → PTS 150 → PTS 225000 /// + /// Errors: avcodec_send_frame errors (including AVERROR_EOF when called again after the flush) and packet + /// receive/write errors are thrown as exceptions. avcodec_send_frame does not return EAGAIN here, + /// because every call drains all available packets before returning. + /// /// Reference: /// Send/Receive API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// Packet: https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html @@ -1611,6 +1808,9 @@ namespace ADOFAIRenderer.Util.FFmpeg // av_interleaved_write_frame: Write packet to container. // "Interleaved" means packets from multiple streams (video+audio) are interleaved // and written in time order. This function is used even when only video is present. + // It takes ownership of the packet data (the packet is left blank), and it buffers packets + // until packets of the other streams with later timestamps arrive. + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html (av_interleaved_write_frame) response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket); if (response < 0) throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})"); @@ -1647,6 +1847,8 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Throws an exception when a flush or the trailer fails (e.g. the disk is full), because the file is then incomplete. /// End() runs once: after it returns or throws, further calls return immediately. + /// End() does not release resources and does not close the file handle; Dispose() must still be called. + /// Order: video flush → audio FIFO remainder and audio flush → trailer. /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html /// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavformat/mux.c (av_write_trailer) @@ -1688,12 +1890,16 @@ namespace ADOFAIRenderer.Util.FFmpeg // ================================================================ /// - /// Actual resource cleanup logic. Cleanup order is important: + /// Releases every native resource. Safe to call more than once and after a failed Start(); also called by the finalizer. + /// Cleanup order: /// /// 1. SwsContext (scaler) → independent, can be freed first - /// 2. AVFrame (src/dst) → freed before codec as codec may reference them - /// 3. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together - /// 4. AVFormatContext (format/IO) → includes closing file handle + /// 2. AVFrame (src/dst/hw) and AVPacket → the structures and their buffer references. + /// Buffers are reference-counted: a buffer the encoder still references stays alive until the codec context is freed. + /// 3. Video AVCodecContext (codec) → also releases its hw_frames_ctx/hw_device_ctx references + /// (the QSV frame pool and device are destroyed when their last reference goes away) + /// 4. Audio resources (DisposeAudioResources: resampler, FIFO, frame, packet, audio codec context) + /// 5. AVFormatContext (format/IO) → closes the file handle (avio_closep), then frees the context and its streams /// /// Dispose does not finish the file (no encoder flush, no trailer). A file is complete only after End() returned. /// Without End(), the output file is incomplete and is left to the caller (the renderer deletes it). @@ -1788,7 +1994,7 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Finalizer (destructor). Executed when collected by GC without Dispose() being called. - /// Safety net to prevent native resource leaks. + /// Safety net to prevent native resource leaks. Runs on the GC finalizer thread; the output file is not finished. /// ⚠️ Finalizer execution timing is not guaranteed, so always manage resources /// using a using statement or explicit Dispose() call. /// @@ -1809,8 +2015,14 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 5. Initialize FIFO buffer (arbitrary input size → collect in frame_size units) /// 6. Allocate audio frame and packet structures /// + /// Called by Start() after the video encoder is opened and before the header is written. + /// Defaults for unset values: codec AAC, 48000 Hz, 2 channels, 256000 bps, input format FLT. + /// The encoder sample rate equals the input rate; a rate the codec does not support makes avcodec_open2 fail. + /// Throws on any failure; Start() then disposes the encoder. + /// /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html /// + /// Audio settings of . private void InitializeAudioStream(AudioSettings audioSettings) { // ── 1. Determine and search audio codec ── @@ -1846,10 +2058,11 @@ namespace ADOFAIRenderer.Util.FFmpeg // Configure channel layout (FFmpeg 7.x+: use AVChannelLayout) ffmpeg.av_channel_layout_default(&_pAudioCodecContext->ch_layout, channels); - // time_base: Audio typically uses 1/sample_rate. + // time_base: Audio typically uses 1/sample_rate, so one PTS tick is one sample per channel. _pAudioCodecContext->time_base = new AVRational { num = 1, den = sampleRate }; - // Global header flag + // Global header flag (same rule as the video encoder): codec configuration (e.g. AAC AudioSpecificConfig) + // goes into extradata for containers that store it in the header. if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0) { _pAudioCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER; @@ -1870,6 +2083,7 @@ namespace ADOFAIRenderer.Util.FFmpeg if (ffmpeg.avcodec_parameters_from_context(_pAudioStream->codecpar, _pAudioCodecContext) < 0) throw new Exception("Failed to copy audio codec parameters to stream."); + // Stream time base hint; the muxer may change it in avformat_write_header (packets are rescaled when written). _pAudioStream->time_base = _pAudioCodecContext->time_base; // ── 4. Initialize resampler (SwrContext) ── @@ -1879,6 +2093,7 @@ namespace ADOFAIRenderer.Util.FFmpeg : AVSampleFormat.AV_SAMPLE_FMT_FLT; // Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes). + // Uses audioSettings.Channels as given (not the defaulted channel count used for the layouts below). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample) _audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels; @@ -1889,6 +2104,9 @@ namespace ADOFAIRenderer.Util.FFmpeg AVChannelLayout inLayout; ffmpeg.av_channel_layout_default(&inLayout, channels); + // Output: encoder layout/format/rate. Input: same layout and rate, input sample format. + // Only the sample format (and interleaved → planar layout) is converted; no resampling takes place. + // Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_alloc_set_opts2) ret = ffmpeg.swr_alloc_set_opts2( ppSwr, &_pAudioCodecContext->ch_layout, encoderSampleFmt, sampleRate, @@ -1904,6 +2122,7 @@ namespace ADOFAIRenderer.Util.FFmpeg throw new Exception($"Failed to initialize audio resampler: {GetFFmpegErrorString(ret)} (Code: {ret})"); // ── 5. Initialize FIFO buffer ── + // Holds samples in the encoder format. Initial capacity 1 sample; av_audio_fifo_write grows it as needed. _pAudioFifo = ffmpeg.av_audio_fifo_alloc(encoderSampleFmt, channels, 1); if (_pAudioFifo == null) throw new Exception("Failed to allocate audio FIFO buffer."); @@ -1916,6 +2135,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // Determine frame size: Some codecs (e.g. AAC) have a fixed frame_size. // Use default 1024 if frame_size is 0 (e.g. PCM family). + // frame_size is known after avcodec_open2 (e.g. AAC 1024, Opus 960 at 48 kHz). var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; _pAudioFrame->nb_samples = frameSize; _pAudioFrame->format = (int)encoderSampleFmt; @@ -1937,9 +2157,16 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Internally executes resampling → FIFO buffering → frame_size unit encoding pipeline. /// /// Thread safety: Safe to call from multiple threads as it uses _lock internally. + /// Does nothing when audio is disabled, or the encoder is not started, already ended or disposed. + /// Samples that do not fill a whole encoder frame stay in the FIFO until the next call or End(). /// - /// PCM audio sample data. Must be in interleaved format (LRLRLR...). + /// + /// PCM audio sample data in , interleaved (LRLRLR...). + /// May be larger than needed (e.g. a pooled array); only the first numSamples × channels values are read. + /// /// Number of samples per channel (= total float count / channel count). + /// is negative or exceeds the array. + /// Format conversion, FIFO, encoding or packet writing fails. public void WriteAudioSamples(float[] samples, int numSamples) { lock (_lock) @@ -1957,8 +2184,11 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Encodes raw PCM audio sample data from pointer into the audio stream. + /// Same behavior as the float[] overload, without the length check: the caller guarantees that + /// numSamples × channels interleaved samples in the input format are readable at . + /// The memory is read within this call only. /// - /// Pointer to PCM audio sample data. + /// Pointer to PCM audio sample data (interleaved, input sample format). /// Number of samples per channel. public void WriteAudioSamples(IntPtr samples, int numSamples) { @@ -1969,18 +2199,6 @@ namespace ADOFAIRenderer.Util.FFmpeg } } - /// - /// Internal implementation of audio sample encoding. - /// - /// Processing flow: - /// 1. Resample input PCM → encoder format via swr_convert - /// 2. Add resampled data to FIFO buffer - /// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder - /// - /// Reference: - /// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html - /// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html - /// /// /// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it. /// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer. @@ -1994,6 +2212,25 @@ namespace ADOFAIRenderer.Util.FFmpeg throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})"); } + /// + /// Internal implementation of audio sample encoding. + /// + /// Processing flow: + /// 1. Resample input PCM → encoder format via swr_convert + /// (same sample rate: one output sample per input sample, only the format and planar layout change) + /// 2. Add resampled data to FIFO buffer + /// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder + /// (each frame gets PTS = _audioSampleIndex, which then advances by frame_size) + /// + /// A temporary frame for the converted samples is allocated and freed on every call. + /// When that frame structure cannot be allocated, the call returns without encoding the samples. + /// + /// Reference: + /// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html + /// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html + /// + /// Interleaved input samples (single plane). + /// Number of samples per channel. private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples) { // ── 1. Resampling: Input format → Encoder format ── @@ -2013,6 +2250,9 @@ namespace ADOFAIRenderer.Util.FFmpeg throw new Exception($"Failed to allocate resampling output frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})"); // swr_convert: Converts input data to format required by encoder. + // Arguments: output planes and capacity (samples per channel), input planes and sample count. + // &inputBuffer is a one-element plane array, because the input is interleaved. + // Returns the number of samples per channel written to the output, or a negative error code. var convertedSamples = ffmpeg.swr_convert( _pSwrContext, (byte**)&pConvertedFrame->data, numSamples, @@ -2032,6 +2272,7 @@ namespace ADOFAIRenderer.Util.FFmpeg var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize) { + // The encoder may still reference the buffer of the previous frame; get a writable one first. MakeAudioFrameWritable(); _pAudioFrame->nb_samples = frameSize; @@ -2053,7 +2294,10 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// Sends audio frame to encoder and writes result packets to file. /// Uses same Send/Receive pattern as video SendFrameAndWritePackets. + /// Packets are rescaled from the codec time base (1 / sample rate) to the audio stream time base. /// + /// Audio frame to encode, or null to flush the audio encoder. + /// Sending, receiving or writing fails. private void SendAudioFrameAndWritePackets(AVFrame* pFrame) { var response = ffmpeg.avcodec_send_frame(_pAudioCodecContext, pFrame); @@ -2128,8 +2372,8 @@ namespace ADOFAIRenderer.Util.FFmpeg } /// - /// Releases audio-related FFmpeg resources. - /// Called inside Dispose(). + /// Releases audio-related FFmpeg resources (resampler, FIFO, frame, packet, audio codec context) and disables audio. + /// Called inside Dispose(). The audio stream itself belongs to the format context and is freed with it. /// private void DisposeAudioResources() { diff --git a/Util/FFmpeg/VideoEncoder.cs b/Util/FFmpeg/VideoEncoder.cs index 724a4e0..dc709c6 100644 --- a/Util/FFmpeg/VideoEncoder.cs +++ b/Util/FFmpeg/VideoEncoder.cs @@ -6,52 +6,107 @@ using ADOFAIRenderer.Data; namespace ADOFAIRenderer.Util.FFmpeg; +/// +/// Family of a video encoder. Selects the encoder-specific configuration in +/// (input pixel format, hardware device setup, quality option names, speed preset). +/// [SuppressMessage("ReSharper", "InconsistentNaming")] internal enum EncoderKind { + /// CPU encoder library linked into FFmpeg (libx264, libx265, libvpx-vp9, libsvtav1, libaom-av1, libvvenc). Software, + + /// NVIDIA NVENC hardware encoder (*_nvenc). NVENC, + + /// AMD Advanced Media Framework hardware encoder (*_amf). AMF, + + /// Intel Quick Sync Video hardware encoder (*_qsv), driven through a QSV hardware device context. QSV, + + /// Apple VideoToolbox hardware encoder (*_videotoolbox, macOS). VideoToolbox } +/// +/// Known FFmpeg video encoder: FFmpeg encoder name, output codec, GPU vendor and encoder family. +/// +/// Instances exist only as the static fields below (the constructor is private); compare them with == or +/// , which compare case-insensitively. +/// lists every encoder, and its order is the fallback order used by +/// GPUUtil.GetPrioritizedEncoders after the encoders of the active GPU vendor. +/// +/// An entry does not mean that the encoder exists in the loaded FFmpeg build or works on the current machine: +/// FFmpegNativeEncoder.Start looks each one up with avcodec_find_encoder_by_name and opens it, and skips the +/// encoders that are missing or fail to open. +/// Reference: https://ffmpeg.org/ffmpeg-codecs.html (Video Encoders) +/// [SuppressMessage("ReSharper", "InconsistentNaming")] internal sealed class VideoEncoder : IEquatable { // ReSharper disable MemberCanBePrivate.Global - + + // Each field: FFmpeg encoder name, output codec, GPU vendor whose hardware runs it + // (Unknown for CPU encoders), encoder family. + // ── H.264 ── + /// NVIDIA NVENC hardware encoder for H.264 (FFmpeg encoder "h264_nvenc"). public static readonly VideoEncoder H264_NVENC = new("h264_nvenc", Codec.H264, GPUVendor.NVIDIA, EncoderKind.NVENC); + /// Intel Quick Sync Video hardware encoder for H.264 (FFmpeg encoder "h264_qsv"). public static readonly VideoEncoder H264_QSV = new("h264_qsv", Codec.H264, GPUVendor.Intel, EncoderKind.QSV); + /// AMD AMF hardware encoder for H.264 (FFmpeg encoder "h264_amf"). public static readonly VideoEncoder H264_AMF = new("h264_amf", Codec.H264, GPUVendor.AMD, EncoderKind.AMF); + /// Apple VideoToolbox hardware encoder for H.264 (FFmpeg encoder "h264_videotoolbox"). public static readonly VideoEncoder H264_VideoToolbox = new("h264_videotoolbox", Codec.H264, GPUVendor.Apple, EncoderKind.VideoToolbox); + /// CPU software encoder for H.264 (FFmpeg encoder "libx264"). public static readonly VideoEncoder Libx264 = new("libx264", Codec.H264, GPUVendor.Unknown, EncoderKind.Software); // ── H.265 (HEVC) ── + /// NVIDIA NVENC hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_nvenc"). public static readonly VideoEncoder HEVC_NVENC = new("hevc_nvenc", Codec.H265, GPUVendor.NVIDIA, EncoderKind.NVENC); + /// Intel Quick Sync Video hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_qsv"). public static readonly VideoEncoder HEVC_QSV = new("hevc_qsv", Codec.H265, GPUVendor.Intel, EncoderKind.QSV); + /// AMD AMF hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_amf"). public static readonly VideoEncoder HEVC_AMF = new("hevc_amf", Codec.H265, GPUVendor.AMD, EncoderKind.AMF); + /// Apple VideoToolbox hardware encoder for H.265 (HEVC) (FFmpeg encoder "hevc_videotoolbox"). public static readonly VideoEncoder HEVC_VideoToolbox = new("hevc_videotoolbox", Codec.H265, GPUVendor.Apple, EncoderKind.VideoToolbox); + /// CPU software encoder for H.265 (HEVC) (FFmpeg encoder "libx265"). public static readonly VideoEncoder Libx265 = new("libx265", Codec.H265, GPUVendor.Unknown, EncoderKind.Software); // ── VP9 ── + // libavcodec has no "vp9_nvenc" encoder (NVENC does not encode VP9), so avcodec_find_encoder_by_name returns null + // for this entry and FFmpegNativeEncoder.Start skips it with a "not found" warning. + /// NVIDIA NVENC entry for VP9 (FFmpeg encoder name "vp9_nvenc"); never found in libavcodec, see above. public static readonly VideoEncoder VP9_NVENC = new("vp9_nvenc", Codec.VP9, GPUVendor.NVIDIA, EncoderKind.NVENC); + /// Intel Quick Sync Video hardware encoder for VP9 (FFmpeg encoder "vp9_qsv"). public static readonly VideoEncoder VP9_QSV = new("vp9_qsv", Codec.VP9, GPUVendor.Intel, EncoderKind.QSV); + /// CPU software encoder for VP9 (FFmpeg encoder "libvpx-vp9"). public static readonly VideoEncoder LibvpxVP9 = new("libvpx-vp9", Codec.VP9, GPUVendor.Unknown, EncoderKind.Software); // ── AV1 ── + /// NVIDIA NVENC hardware encoder for AV1 (FFmpeg encoder "av1_nvenc"). public static readonly VideoEncoder AV1_NVENC = new("av1_nvenc", Codec.AV1, GPUVendor.NVIDIA, EncoderKind.NVENC); + /// Intel Quick Sync Video hardware encoder for AV1 (FFmpeg encoder "av1_qsv"). public static readonly VideoEncoder AV1_QSV = new("av1_qsv", Codec.AV1, GPUVendor.Intel, EncoderKind.QSV); + /// AMD AMF hardware encoder for AV1 (FFmpeg encoder "av1_amf"). public static readonly VideoEncoder AV1_AMF = new("av1_amf", Codec.AV1, GPUVendor.AMD, EncoderKind.AMF); + /// CPU software encoder for AV1 (FFmpeg encoder "libsvtav1"). public static readonly VideoEncoder LibSvtAv1 = new("libsvtav1", Codec.AV1, GPUVendor.Unknown, EncoderKind.Software); + /// CPU software encoder for AV1 (FFmpeg encoder "libaom-av1"). public static readonly VideoEncoder LibAomAv1 = new("libaom-av1", Codec.AV1, GPUVendor.Unknown, EncoderKind.Software); // ── VVC ── + /// CPU software encoder for H.266 (VVC) (FFmpeg encoder "libvvenc"). public static readonly VideoEncoder LibVvenc = new("libvvenc", Codec.VVC, GPUVendor.Unknown, EncoderKind.Software); // ReSharper restore MemberCanBePrivate.Global + /// + /// Every known encoder, grouped by codec. Within a codec the order is NVENC → QSV → AMF → VideoToolbox → software + /// (for the families the codec has); GPUUtil.GetPrioritizedEncoders keeps this order for the encoders that do not + /// belong to the active GPU vendor. + /// public static readonly IReadOnlyList All = new[] { H264_NVENC, H264_QSV, H264_AMF, H264_VideoToolbox, Libx264, @@ -61,15 +116,26 @@ internal sealed class VideoEncoder : IEquatable LibVvenc }; + /// Lookup table of by (case-insensitive), used by . private static readonly Dictionary ByName = All.ToDictionary(e => e.Name, StringComparer.OrdinalIgnoreCase); + /// FFmpeg encoder name passed to avcodec_find_encoder_by_name (e.g. "h264_nvenc", "libx264"). public string Name { get; } + + /// Video codec of the bitstream the encoder produces. public Codec Codec { get; } + + /// GPU vendor whose hardware runs the encoder. for CPU (software) encoders. public GPUVendor Vendor { get; } + + /// Encoder family (software or hardware API). public EncoderKind Kind { get; } + + /// True for every family except . public bool IsHardware => Kind != EncoderKind.Software; + /// Creates an entry. Used only by the static fields of this class. private VideoEncoder(string name, Codec codec, GPUVendor vendor, EncoderKind kind) { Name = name; @@ -78,36 +144,46 @@ internal sealed class VideoEncoder : IEquatable Kind = kind; } + /// + /// Returns the known encoder with the given FFmpeg name (case-insensitive), + /// or null when is null, empty or not in . + /// public static VideoEncoder? FromName(string name) { return string.IsNullOrEmpty(name) ? null : ByName.GetValueOrDefault(name); } + /// True when is the same entry or has the same (case-insensitive). public bool Equals(VideoEncoder other) { if (ReferenceEquals(null, other)) return false; return ReferenceEquals(this, other) || string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase); } + /// True when is a equal to this one (see ). public override bool Equals(object? obj) { return ReferenceEquals(this, obj) || obj is VideoEncoder other && Equals(other); } + /// Case-insensitive hash of , consistent with . public override int GetHashCode() { return StringComparer.OrdinalIgnoreCase.GetHashCode(Name); } + /// Value equality through ; two null operands are equal. public static bool operator ==(VideoEncoder left, VideoEncoder right) { return Equals(left, right); } + /// Negation of the == operator. public static bool operator !=(VideoEncoder left, VideoEncoder right) { return !Equals(left, right); } + /// Returns the FFmpeg encoder name (). public override string ToString() => Name; } \ No newline at end of file diff --git a/Util/FileUtil.cs b/Util/FileUtil.cs index cca31c6..8ac3a19 100644 --- a/Util/FileUtil.cs +++ b/Util/FileUtil.cs @@ -2,8 +2,15 @@ namespace ADOFAIRenderer.Util; +/// File system helpers. internal static class FileUtil { + /// + /// Creates and its missing parent directories when it does not exist. + /// Exceptions of Directory.CreateDirectory are not caught (for example ArgumentException for an empty path, + /// IOException or UnauthorizedAccessException when the directory cannot be created). + /// + /// Directory path, absolute or relative to the current directory. public static void CreateDirectory(string path) { if (!Directory.Exists(path)) Directory.CreateDirectory(path); diff --git a/Util/FrameReader.cs b/Util/FrameReader.cs index 50508d8..3aaa414 100644 --- a/Util/FrameReader.cs +++ b/Util/FrameReader.cs @@ -13,25 +13,61 @@ using UnityEngine.Rendering; namespace ADOFAIRenderer.Util { + /// + /// GPU capture of rendered frames for the encoder. For each requested frame it optionally converts the RGBA texture to + /// the encoder's YUV 4:2:0 layout with a compute shader (ColorConversion.compute from the mod's asset bundle, kernels + /// CSMain_NV12 / CSMain_YUV420P), then reads the result back asynchronously (AsyncGPUReadback.RequestIntoNativeArray) + /// into a persistent native buffer and adds it to RenderManager.ImageQueue under its video frame index. + /// When the shader or the kernel for the pixel format is unavailable, the RGBA texture is read back unchanged and the + /// encoder converts it (IsColorConverted = false). + /// + /// Each request uses a from a pool (16 created up front, more allocated when the pool + /// is empty). A state stays out of the pool from the request until the render thread has sent its frame + /// (RenderManager.SendVideoFrame, then ) or the frame was dropped. + /// + /// + /// Threads: requests and readback completions run on the main Unity thread; is also called by + /// the render thread. The pool and the disposed flag are guarded by _poolLock. One instance per render, created in + /// RenderManager.SetupPlay with the pixel format of the opened encoder. + /// + /// internal class FrameReader : IDisposable { + // Property IDs of the compute shader parameters: source texture, output buffer and frame size in pixels. private static readonly int Height = Shader.PropertyToID("Height"); private static readonly int Width = Shader.PropertyToID("Width"); private static readonly int DestinationBuffer = Shader.PropertyToID("DestinationBuffer"); private static readonly int SourceTexture = Shader.PropertyToID("SourceTexture"); + /// Number of readback states created by InitializePool. private const int InitialBufferSize = 16; + /// States that are free for a new request. Guarded by _poolLock. private readonly Stack _statePool = new(); + /// Every state created by this reader (free, in flight or holding a queued frame), for Dispose. Guarded by _poolLock. private readonly List _allocatedStates = new(); + /// Lock for _statePool, _allocatedStates, _isInitialized, _isDisposed and ReadbackRequestState.IsInFlight. private readonly object _poolLock = new(); + /// True after InitializePool; false again after Dispose (RecycleState then disposes returned states). private bool _isInitialized; + /// Set by Dispose. Completions and recycles that arrive afterwards dispose their state. private volatile bool _isDisposed; + /// Color conversion compute shader from the asset bundle (ModAssets.ColorConversionShader); null when not loaded. private readonly ComputeShader? _computeShader; + /// Kernel index for the target pixel format; -1 when the format has no kernel or the kernel was not found. private readonly int _kernelIndex = -1; + /// True when frames are converted to YUV on the GPU (shader and kernel available). private readonly bool _useGPUConversion; + /// Pixel format of the read-back data: the target format with GPU conversion, AV_PIX_FMT_RGBA otherwise. private readonly AVPixelFormat _expectedFormat; + /// + /// Selects the conversion kernel for and fills the state pool. + /// + /// + /// Pixel format of the encoder input (FFmpegNativeEncoder.SelectedPixelFormat). NV12 and YUV420P use GPU conversion; + /// any other format falls back to RGBA readback. + /// public FrameReader(AVPixelFormat targetFormat) { _computeShader = ModAssets.ColorConversionShader; @@ -57,6 +93,7 @@ namespace ADOFAIRenderer.Util InitializePool(); } + /// Creates InitialBufferSize states once. Their buffers are allocated on first use (ReadbackRequestState.Setup). private void InitializePool() { lock (_poolLock) @@ -83,6 +120,15 @@ namespace ADOFAIRenderer.Util return _useGPUConversion ? pixels * 3 / 2 : pixels * 4; } + /// + /// Requests the capture of as video frame . Does nothing + /// outside the Render stage. Main thread. + /// With GPU conversion, the compute kernel writes the YUV data of the texture into the state's ComputeBuffer and that + /// buffer is read back; otherwise the texture itself is read back as RGBA. The buffers are sized from the texture + /// dimensions of this call. The result arrives in OnReadbackComplete. + /// + /// Frame to capture (game frame or the output of the frame hooks). + /// Video frame index; the key of the frame in ImageQueue. public void RequestFrameCapture(Texture texture, long currentFrame) { if (!RenderManager.IsCapturingActive) return; @@ -113,6 +159,8 @@ namespace ADOFAIRenderer.Util _computeShader.SetInt(Width, texture.width); _computeShader.SetInt(Height, texture.height); + // Thread group counts cover the frame in tiles of 32 x 16 pixels (NV12) or 64 x 16 pixels (YUV420P), rounded up. + // The tile size is the area one thread group of the ColorConversion.compute kernels processes. var threadGroupsX = _expectedFormat == AVPixelFormat.AV_PIX_FMT_NV12 ? (texture.width + 31) / 32 : (texture.width + 63) / 64; @@ -120,6 +168,8 @@ namespace ADOFAIRenderer.Util _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1); + // The native buffer must not be disposed until the request completes; Dispose skips states that are in flight. + // Reference: https://docs.unity3d.com/ScriptReference/Rendering.AsyncGPUReadback.RequestIntoNativeArray.html AsyncGPUReadback.RequestIntoNativeArray( ref state.NativeBuffer, state.GPUBuffer, state.CachedCallback ); @@ -136,6 +186,17 @@ namespace ADOFAIRenderer.Util } } + /// + /// Completion of a readback request (through ReadbackRequestState.CachedCallback). + /// + /// Reader disposed: the state is disposed. + /// Render aborted: the state is returned to the pool and nothing is queued. + /// Readback error: a black frame (managed buffer) is queued under the same index, so the render thread does not + /// wait forever for the missing index, and the state is returned to the pool. + /// Success: the state's native buffer is queued without copying; the state stays out of the pool until the + /// render thread has encoded the frame. + /// + /// internal void OnReadbackComplete(ReadbackRequestState state, AsyncGPUReadbackRequest request) { var frame = state.FrameIndex; @@ -227,6 +288,10 @@ namespace ADOFAIRenderer.Util return frame; } + /// + /// Returns a state to the pool after its frame was sent or dropped, or disposes it when the reader is disposed. + /// Called by the render thread (after sending) and by the main Unity thread (abort, errors, queue cleanup). + /// internal void RecycleState(ReadbackRequestState state) { lock (_poolLock) @@ -242,6 +307,10 @@ namespace ADOFAIRenderer.Util } } + /// + /// Adds the frame to RenderManager.ImageQueue under (replacing a frame with the same index) + /// and wakes the render thread. + /// private static void PushToQueue(long frame, RenderManager.CapturedFrame capturedFrame) { if (!RenderManager.Instance) return; @@ -252,6 +321,11 @@ namespace ADOFAIRenderer.Util RenderManager.Instance.FrameAvailableEvent.Set(); } + /// + /// Disposes every state that is not in flight and clears the pool. States in flight are disposed when their + /// readback completes. States still referenced by frames in ImageQueue are disposed too, so callers clear the queue + /// first (RenderManager.StopFFmpeg / RestoreRenderEnvironment). Main thread. + /// public void Dispose() { lock (_poolLock) @@ -269,20 +343,38 @@ namespace ADOFAIRenderer.Util } } + /// + /// One reusable capture slot of : the persistent native buffer the readback writes into, the + /// ComputeBuffer the conversion kernel writes (GPU conversion only), and the metadata of the frame it currently holds. + /// The buffers are kept between frames and reallocated only when the frame size or conversion mode changes. + /// internal class ReadbackRequestState : IDisposable { + /// Reader that receives the completion and owns the pool. private readonly FrameReader _owner; + /// Video frame index of the current request; -1 while the state is free. public long FrameIndex { get; private set; } + /// + /// Persistent native buffer receiving the readback: Width x Height x 3 / 2 bytes of YUV 4:2:0 with GPU conversion, + /// Width x Height x 4 bytes of RGBA otherwise. Read by the render thread through CapturedFrame.NativePointer. + /// public NativeArray NativeBuffer; + /// True between the request and its completion. Written under FrameReader._poolLock. public bool IsInFlight { get; internal set; } + /// Raw ComputeBuffer (4-byte elements) the conversion kernel writes; null without GPU conversion. public ComputeBuffer? GPUBuffer; + /// True when the current frame is converted to YUV on the GPU. public bool IsGPUConversion { get; private set; } + /// Width of the current frame in pixels. public int Width { get; private set; } + /// Height of the current frame in pixels. public int Height { get; private set; } + /// Completion delegate created once per state, so a request does not allocate a new delegate. public readonly Action CachedCallback; + /// Creates a free state without buffers. public ReadbackRequestState(FrameReader owner) { _owner = owner; @@ -290,6 +382,17 @@ namespace ADOFAIRenderer.Util FrameIndex = -1; } + /// + /// Prepares the state for a request: stores the frame metadata and (re)allocates the buffers when their size differs. + /// The native buffer holds width x height x 3 / 2 bytes with GPU conversion, otherwise . + /// The ComputeBuffer holds the same number of bytes as 4-byte elements (byte count / 4, rounded down). + /// + /// Video frame index of the request. + /// Readback size in bytes for RGBA readback (width x height x 4). + /// Whether the kernel converts the frame to YUV. + /// Pixel format of the read-back data (not used by this method). + /// Frame width in pixels. + /// Frame height in pixels. public void Setup(long frameIndex, int bufferSize, bool isGPUConversion, AVPixelFormat pixelFormat, int width, int height) { FrameIndex = frameIndex; @@ -320,16 +423,19 @@ namespace ADOFAIRenderer.Util } } + /// Readback completion; forwards to FrameReader.OnReadbackComplete. private void OnComplete(AsyncGPUReadbackRequest request) { _owner.OnReadbackComplete(this, request); } + /// Marks the state as free (FrameIndex -1). The buffers are kept for the next request. public void Recycle() { FrameIndex = -1; } + /// Releases the native buffer and the ComputeBuffer. public void Dispose() { if (NativeBuffer.IsCreated) NativeBuffer.Dispose(); diff --git a/Util/GPUUtil.cs b/Util/GPUUtil.cs index 586dcba..bdf4221 100644 --- a/Util/GPUUtil.cs +++ b/Util/GPUUtil.cs @@ -10,27 +10,48 @@ using UnityEngine; namespace ADOFAIRenderer.Util; +/// +/// GPU vendor, used to try the hardware encoders of the active GPU first (VideoEncoder.Vendor). Unknown is +/// also the vendor of the software encoders. +/// [SuppressMessage("ReSharper", "InconsistentNaming")] internal enum GPUVendor { Unknown, NVIDIA, AMD, Intel, Apple } +/// +/// Detection of the active GPU vendor and the NVIDIA driver version, and the encoder try order derived from the vendor. +/// Results are cached for the game session. Main thread (SystemInfo is read on first use). +/// internal static class GPUUtil { + // Device name patterns (SystemInfo.graphicsDeviceName) of the GPU families recognized per vendor. A name that matches + // none of them gives GPUVendor.Unknown (Apple on macOS), which only affects the encoder try order. + // NVIDIA: "NVIDIA GeForce RTX/GTX " with optional SUPER / Ti / Ti SUPER, memory size, "Laptop GPU" and a + // "/PCIe/SSE2" suffix. private const string NvidiaPattern = @"^NVIDIA GeForce [RG]TX [0-9]+(?> SUPER| Ti(?> SUPER)?)?(?> [0-9]+GB)?(?> Laptop GPU)?(?>/PCIe/SSE2)?$"; + // Intel: "Intel(R) UHD/HD Graphics " or "Intel(R) Arc(TM) Graphics", with an optional + // parenthesized suffix. private const string IntelPattern = @"^Intel(?>\(R\))? (?>U?HD Graphics [0-9]+|Arc\(TM\) [A-C][0-9]+ Graphics)(?> \([^)]+\))?$"; + // AMD: "AMD Radeon RX " with an optional " XT". // ReSharper disable once InconsistentNaming private const string AMDPattern = @"^AMD Radeon RX [0-9]+(?> XT)?$"; + /// Vendor found by the first GetActiveGPUVendor call. private static GPUVendor _cachedVendor = GPUVendor.Unknown; + /// True once _cachedVendor is set. private static bool _hasCachedVendor; + /// + /// Vendor of the GPU Unity renders with: the device name is matched against the NVIDIA, Intel and AMD patterns in + /// that order; on macOS an unmatched name gives Apple; otherwise Unknown. Logs the device name once. + /// private static GPUVendor GetActiveGPUVendor() { if (_hasCachedVendor) return _cachedVendor; @@ -72,9 +93,21 @@ internal static class GPUUtil return _cachedVendor; } + /// Result of the first GetNvidiaDriverVersion call (null when not available). private static double? _cachedNvidiaVersion; + /// True once _cachedNvidiaVersion is set. private static bool _hasCachedNvidiaVersion; + /// + /// NVIDIA driver version in NVIDIA's numbering (for example 561.09), or null when the active GPU is not detected as + /// NVIDIA or no version can be found. The result is cached. + /// + /// The version is derived from a four-part Windows driver version a.b.c.d as ((c mod 10) x 10000 + d) / 100, + /// for example 31.0.15.6109 gives 561.09. It is read from SystemInfo.graphicsDeviceVersion when that string contains a + /// four-part number, otherwise (Windows only) from the DriverVersion value of the first display adapter entry whose + /// ProviderName contains "NVIDIA" under the display adapter class key of the registry. + /// + /// public static double? GetNvidiaDriverVersion() { if (_hasCachedNvidiaVersion) return _cachedNvidiaVersion; @@ -103,6 +136,9 @@ internal static class GPUUtil // 2. Fallback to Windows Registry (only on Windows NT platforms via RegistryUtil) if (System.Environment.OSVersion.Platform != System.PlatformID.Win32NT) return null; + // Device class key of display adapters (class GUID {4d36e968-e325-11ce-bfc1-08002be10318}); each numbered subkey is + // one installed adapter driver. + // Reference: https://learn.microsoft.com/en-us/windows-hardware/drivers/install/system-defined-device-setup-classes-available-to-vendors const string classKey = @"SYSTEM\CurrentControlSet\Control\Class\{4d36e968-e325-11ce-bfc1-08002be10318}"; var subkeys = RegistryUtil.GetHKLMSubKeyNames(classKey); if (subkeys == null) return null; @@ -130,6 +166,12 @@ internal static class GPUUtil return null; } + /// + /// Encoders for in the order they are tried: the encoders of the active GPU vendor first, + /// then all others in VideoEncoder.All order (hardware encoders of other vendors and the software encoders). + /// + /// Video codec. + /// When true, only the software encoders of the codec are returned. public static List GetPrioritizedEncoders(Codec codec, bool forceSoftware = false) { var encoders = VideoEncoder.All.Where(e => e.Codec == codec).ToList(); diff --git a/Util/OS/FolderDialogUtil.cs b/Util/OS/FolderDialogUtil.cs index ccae4fc..ef6587e 100644 --- a/Util/OS/FolderDialogUtil.cs +++ b/Util/OS/FolderDialogUtil.cs @@ -10,12 +10,49 @@ using MelonLoader; namespace ADOFAIRenderer.Util.OS; +/// +/// Native "select folder" dialog used to choose the render output directory. +/// +/// - Windows: the shell's IFileOpenDialog (COM, FOS_PICKFOLDERS) on a dedicated STA thread owned by the +/// game window; when it cannot be created or shown, the legacy SHBrowseForFolderW dialog. +/// +/// - macOS: osascript running AppleScript choose folder. +/// +/// - Linux: zenity --file-selection --directory; when zenity does not return an existing directory (not installed, +/// cancelled or failed), kdialog --getexistingdirectory. +/// +/// +/// The dialog runs off the main Unity thread so the game keeps rendering frames while it is open. Only one dialog can be open +/// at a time. Failures are reported to the callback as null (same as cancel); exceptions are logged as warnings, except a +/// zenity start failure, which silently moves on to kdialog. +/// +/// [SuppressMessage("ReSharper", "InconsistentNaming")] [SuppressMessage("ReSharper", "UnusedMember.Local")] internal static class FolderDialogUtil { + // 1 while a dialog is open, 0 otherwise. Set with Interlocked.CompareExchange in OpenFolderBrowser and reset by the + // dialog worker before it dispatches the result. private static int _isDialogOpen; + /// + /// Opens the folder selection dialog asynchronously and returns immediately. + /// + /// When a dialog is already open, logs a message and returns without opening another one; + /// is not called in that case. Otherwise the dialog runs on a background thread (Windows: new STA thread + /// "FolderDialog-Win"; macOS/Linux: thread pool task) and receives the result. + /// + /// + /// Callback thread: when the caller has a (the main Unity thread has one), the callback + /// is posted to it and runs on the caller's thread; otherwise it runs on the dialog thread. Exceptions thrown by the + /// callback are caught and logged. + /// + /// On platforms other than Windows, macOS and Linux the callback receives null. + /// + /// Folder shown first; ignored when empty or not an existing directory. + /// + /// Receives the absolute path of the selected existing directory, or null when the user cancelled or the dialog failed. + /// public static void OpenFolderBrowser(string initialDirectory, Action onSelected) { if (Interlocked.CompareExchange(ref _isDialogOpen, 1, 0) != 0) @@ -24,12 +61,18 @@ internal static class FolderDialogUtil return; } + // Captured on the calling thread so the result can be posted back to it. var syncContext = SynchronizationContext.Current; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + // Resolved on the calling thread: GetActiveWindow only returns a window attached to the calling thread's + // message queue. var ownerHandle = GetGameWindowHandle(); + // Both dialogs run on a single-threaded apartment (STA) thread with OLE initialized; SHBrowseForFolder with + // BIF_NEWDIALOGSTYLE fails in a multithreaded apartment. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/ns-shlobj_core-browseinfow var thread = new Thread(() => { string? selected = null; @@ -45,6 +88,8 @@ internal static class FolderDialogUtil } finally { + // OleInitialize returns S_OK or S_FALSE (already initialized) on success; each successful call is + // balanced by one OleUninitialize. if (oleHr >= 0) { try @@ -98,6 +143,13 @@ internal static class FolderDialogUtil } } + /// + /// Delivers the dialog result: posts to when it is not null, + /// otherwise invokes it on the current (dialog) thread. Callback exceptions are caught and logged. + /// + /// Synchronization context of the thread that opened the dialog, or null. + /// Result callback. + /// Selected directory path, or null. private static void DispatchResult(SynchronizationContext? syncContext, Action onSelected, string? selected) { if (syncContext != null) @@ -129,20 +181,32 @@ internal static class FolderDialogUtil #region Windows Implementation + // COM class ID of the shell's common file dialog (FileOpenDialog) and the interface IDs of IFileOpenDialog and + // IShellItem (shobjidl_core.h). private static readonly Guid CLSID_FileOpenDialog = new("DC1C5A9C-E88A-4DDE-A5A1-60F82A20AEF7"); private static readonly Guid IID_IFileOpenDialog = new("d57c7288-d4ad-4768-be02-9d969532d960"); private static readonly Guid IID_IShellItem = new("43826D1E-E718-42EE-BC55-A1E261C37BFE"); + // HRESULT_FROM_WIN32(ERROR_CANCELLED) = 0x800704C7: IModalWindow::Show result when the user cancels the dialog. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shobjidl_core/nf-shobjidl_core-imodalwindow-show private const int ERROR_CANCELLED_HR = unchecked((int)0x800704C7); + // Active window attached to the calling thread's message queue, or NULL. [DllImport("user32.dll")] private static extern IntPtr GetActiveWindow(); + // Foreground window of the desktop (any process), or NULL. [DllImport("user32.dll")] private static extern IntPtr GetForegroundWindow(); + // Thread ID that created the window; the process ID is returned through lpdwProcessId. [DllImport("user32.dll")] private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId); + /// + /// Finds the game window to own the dialog, in order: the calling thread's active window, the foreground window when it + /// belongs to this process, then . + /// + /// Window handle, or when none is found or a call fails (the dialog is then unowned). private static IntPtr GetGameWindowHandle() { try @@ -175,6 +239,13 @@ internal static class FolderDialogUtil return IntPtr.Zero; } + /// + /// Shows the IFileOpenDialog folder picker; when it throws (cannot be created, or Show fails with an error other than + /// cancel), logs a warning and shows the legacy SHBrowseForFolder dialog. Runs on the STA dialog thread. + /// + /// Folder shown first, or empty. + /// Owner window, or . + /// Selected existing directory, or null on cancel. private static string? OpenWindowsFolderBrowser(string initialDirectory, IntPtr ownerHandle) { try @@ -188,16 +259,31 @@ internal static class FolderDialogUtil } } + /// + /// Shows the shell common file dialog in folder mode. + /// + /// Options: FOS_PICKFOLDERS (choose folders), FOS_FORCEFILESYSTEM (file system items only) and + /// FOS_PATHMUSTEXIST. The initial folder is set with SetFolder and SetDefaultFolder when it exists. + /// When Show fails with an owner window (error other than cancel), it is retried once without an owner. + /// The result item is read as a file system path (SIGDN_FILESYSPATH); the returned string is freed with + /// and the COM objects are released. + /// + /// + /// Folder shown first, or empty. + /// Owner window, or . + /// Selected existing directory; null on cancel or when the result has no file system path. + /// The dialog cannot be created, or Show fails with an error other than cancel. private static string? OpenWindowsFolderBrowserModern(string initialDirectory, IntPtr ownerHandle) { - // 1. Try native CoCreateInstance first (works on Mono and .NET without class registration) + // 1. Create the dialog with ole32 CoCreateInstance (in-process server), marshalled directly to IFileOpenDialog. var clsid = CLSID_FileOpenDialog; var iid = IID_IFileOpenDialog; int createHr = CoCreateInstance(ref clsid, IntPtr.Zero, 1 /* CLSCTX_INPROC_SERVER */, ref iid, out var dialog); if (createHr != 0) { - // 2. Fallback to Activator via Type.GetTypeFromCLSID + // 2. On failure, create it through the runtime's COM interop (Type.GetTypeFromCLSID + Activator.CreateInstance). + // createHr keeps the CoCreateInstance error for the exception below. var comType = Type.GetTypeFromCLSID(CLSID_FileOpenDialog); if (comType != null) { @@ -223,6 +309,7 @@ internal static class FolderDialogUtil var normalized = Path.GetFullPath(initialDirectory).TrimEnd('\\', '/'); if (Directory.Exists(normalized)) { + // The dialog takes folders as IShellItem; SHCreateItemFromParsingName creates one from the path. var shellItemGuid = IID_IShellItem; int itemHr = SHCreateItemFromParsingName(normalized, IntPtr.Zero, ref shellItemGuid, out var initialItem); if (itemHr == 0) @@ -293,8 +380,19 @@ internal static class FolderDialogUtil } } + /// + /// Shows the legacy shell folder browser (SHBrowseForFolderW) with the resizable dialog style, an edit box, and + /// file system folders only. The initial folder is selected from the dialog callback (BFFM_INITIALIZED -> + /// BFFM_SETSELECTIONW with the path string). Cancel and failure both return null. Native buffers and the returned + /// item ID list are freed before returning. + /// + /// Folder selected first, or empty. + /// Owner window, or . + /// Selected existing directory, or null. private static string? OpenWindowsFolderBrowserLegacy(string initialDirectory, IntPtr ownerHandle) { + // Display name buffer: MAX_PATH (260) UTF-16 characters, the size SHBrowseForFolder assumes. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/ns-shlobj_core-browseinfow IntPtr pszDisplayName = Marshal.AllocHGlobal(260 * sizeof(char)); IntPtr initialDirPtr = IntPtr.Zero; BrowseCallbackProc? callback = null; @@ -318,6 +416,9 @@ internal static class FolderDialogUtil var normalized = Path.GetFullPath(initialDirectory).TrimEnd('\\', '/'); if (Directory.Exists(normalized)) { + // lpData = bi.lParam = the UTF-16 path; wParam 1 (TRUE) tells BFFM_SETSELECTION it is a string. + // The delegate is kept alive until the dialog returns (GC.KeepAlive in finally). + // Reference: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/bb762598(v=vs.85) initialDirPtr = Marshal.StringToHGlobalUni(normalized); callback = (hwnd, uMsg, _, lpData) => { @@ -340,12 +441,12 @@ internal static class FolderDialogUtil pidl = SHBrowseForFolderW(ref bi); if (pidl == IntPtr.Zero && ownerHandle != IntPtr.Zero) { - // If it returned null with ownerHandle, check if retry with IntPtr.Zero is appropriate - // But note: cancelling also returns Zero. We do not redundantly re-prompt if cancelled. + // SHBrowseForFolderW returns NULL both on cancel and on failure, so no retry without the owner is made. } if (pidl != IntPtr.Zero) { + // SHGetPathFromIDListW writes up to MAX_PATH (260) characters. var sb = new StringBuilder(260); if (SHGetPathFromIDListW(pidl, sb)) { @@ -376,6 +477,10 @@ internal static class FolderDialogUtil #region COM & Win32 Interop Definitions + /// + /// FILEOPENDIALOGOPTIONS flags passed to IFileDialog::SetOptions (subset; values from shobjidl_core.h). + /// Reference: https://learn.microsoft.com/en-us/windows/win32/api/shobjidl_core/ne-shobjidl_core-_fileopendialogoptions + /// [Flags] private enum FOS : uint { @@ -386,11 +491,20 @@ internal static class FolderDialogUtil FOS_FILEMUSTEXIST = 0x00001000, } + /// + /// Display name form requested from IShellItem::GetDisplayName; SIGDN_FILESYSPATH is the item's file system + /// path (the call fails for items without one). + /// Reference: https://learn.microsoft.com/en-us/windows/win32/api/shobjidl_core/ne-shobjidl_core-sigdn + /// private enum SIGDN : uint { SIGDN_FILESYSPATH = 0x80058000 } + /// + /// COM interop declaration of the shell IShellItem interface (vtable order of shobjidl_core.h). Only + /// GetDisplayName is called; it keeps its HRESULT return (PreserveSig). + /// [ComImport] [Guid("43826D1E-E718-42EE-BC55-A1E261C37BFE")] [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] @@ -404,6 +518,11 @@ internal static class FolderDialogUtil void Compare(IShellItem psi, uint hint, out int piOrder); } + /// + /// COM interop declaration of IFileOpenDialog: the methods of IModalWindow (Show), IFileDialog + /// and IFileOpenDialog in vtable order. Show keeps its HRESULT return (PreserveSig); the other methods + /// throw on a failure HRESULT. + /// [ComImport] [Guid("d57c7288-d4ad-4768-be02-9d969532d960")] [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] @@ -438,12 +557,15 @@ internal static class FolderDialogUtil void GetSelectedItems(out IntPtr ppsai); } + // Initializes COM as a single-threaded apartment plus OLE on the calling thread; returns an HRESULT (S_OK / S_FALSE on success). [DllImport("ole32.dll", ExactSpelling = true)] private static extern int OleInitialize(IntPtr pvReserved); + // Balances one successful OleInitialize on the same thread. [DllImport("ole32.dll", ExactSpelling = true)] private static extern void OleUninitialize(); + // Creates a COM object of class rclsid and returns interface riid; declared here with the IFileOpenDialog return type. [DllImport("ole32.dll", ExactSpelling = true)] private static extern int CoCreateInstance( [In] ref Guid rclsid, @@ -452,6 +574,7 @@ internal static class FolderDialogUtil [In] ref Guid riid, [Out, MarshalAs(UnmanagedType.Interface)] out IFileOpenDialog ppv); + // Creates an IShellItem from a parsing name (file system path); returns an HRESULT (0 = S_OK). [DllImport("shell32.dll", CharSet = CharSet.Unicode, PreserveSig = true)] private static extern int SHCreateItemFromParsingName( [MarshalAs(UnmanagedType.LPWStr)] string pszPath, @@ -459,14 +582,22 @@ internal static class FolderDialogUtil [In] ref Guid riid, [Out, MarshalAs(UnmanagedType.Interface)] out IShellItem ppv); + // BROWSEINFO.ulFlags: file system directories only, resizable new-style dialog, edit box for typing a name. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/ns-shlobj_core-browseinfow private const uint BIF_RETURNONLYFSDIRS = 0x0001; private const uint BIF_NEWDIALOGSTYLE = 0x0040; private const uint BIF_EDITBOX = 0x0010; + // Browse dialog messages (shlobj_core.h): BFFM_INITIALIZED is sent to the callback when the dialog has finished + // initializing; BFFM_SETSELECTIONW (WM_USER + 103, Unicode) is sent to the dialog to select a folder. The reference + // describes the messages; the numeric values are the shlobj_core.h definitions. + // Reference:https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/bb762598(v=vs.85) private const uint BFFM_INITIALIZED = 1; private const uint BFFM_SETSELECTIONW = 0x0400 + 103; + // BFFCALLBACK: called by the browse dialog for dialog events; returns 0 except for BFFM_VALIDATEFAILED. private delegate int BrowseCallbackProc(IntPtr hwnd, uint uMsg, IntPtr lParam, IntPtr lpData); + // Unicode BROWSEINFOW structure passed to SHBrowseForFolderW (field order and types of shlobj_core.h). [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] private struct BROWSEINFO { @@ -480,13 +611,16 @@ internal static class FolderDialogUtil public int iImage; } + // Shows the modal browse dialog; returns the selected item's ID list (freed with CoTaskMemFree), or NULL on cancel or failure. [DllImport("shell32.dll", CharSet = CharSet.Unicode)] private static extern IntPtr SHBrowseForFolderW(ref BROWSEINFO lpbi); + // Converts an item ID list to a file system path; false when the item is not part of the file system. [DllImport("shell32.dll", CharSet = CharSet.Unicode)] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool SHGetPathFromIDListW(IntPtr pidl, StringBuilder pszPath); + // Sends a window message and waits for it to be processed (used from the browse dialog callback). [DllImport("user32.dll", CharSet = CharSet.Unicode)] private static extern IntPtr SendMessage(IntPtr hWnd, uint Msg, IntPtr wParam, IntPtr lParam); @@ -496,10 +630,21 @@ internal static class FolderDialogUtil #region macOS Implementation + /// + /// Runs osascript -e with the AppleScript POSIX path of (choose folder with prompt ... [default location ...]) + /// and blocks until the process exits. The initial directory is passed as a POSIX file in an AppleScript string literal + /// (backslashes and double quotes escaped). On cancel osascript prints nothing to standard output. + /// + /// Folder shown first; used only when it is an existing directory. + /// + /// The trimmed standard output when it is an existing directory; null on cancel, when the process cannot be started, or + /// on an exception (logged as a warning). + /// private static string? OpenMacFolderBrowser(string initialDirectory) { try { + // POSIX path of (...) converts the chosen folder alias to a slash-separated path. string script; if (!string.IsNullOrEmpty(initialDirectory) && Directory.Exists(initialDirectory)) { @@ -541,6 +686,17 @@ internal static class FolderDialogUtil #region Linux Implementation + /// + /// Runs zenity --file-selection --directory --title=... [--filename=initialDirectory] and blocks until it exits. + /// When zenity cannot be started, exits with a non-zero code (cancel or error) or prints a path that is not an existing + /// directory, runs kdialog --getexistingdirectory [initialDirectory] --title ... instead. Both commands print the + /// selected path to standard output. + /// + /// Folder shown first; used only when it is an existing directory. + /// + /// The selected existing directory; null when both dialogs return nothing, when kdialog cannot be started, or on an + /// exception (logged as a warning). + /// private static string? OpenLinuxFolderBrowser(string initialDirectory) { try diff --git a/Util/OS/OSNotification.cs b/Util/OS/OSNotification.cs index c755050..2f2f248 100644 --- a/Util/OS/OSNotification.cs +++ b/Util/OS/OSNotification.cs @@ -13,26 +13,36 @@ namespace ADOFAIRenderer.Util.OS; /// (WinRT ToastNotificationManager, script passed with -EncodedCommand). /// With a folder path, the script also registers the adofai-renderer:// URL protocol under /// HKCU\Software\Classes\adofai-renderer (open command: explorer.exe "<folder>") -/// so the toast button opens the folder; the key is overwritten on each notification and stays registered. +/// so the toast button opens the folder; the key is overwritten on each notification and stays registered, so every +/// toast opens the folder of the most recent notification. /// When PowerShell fails, a Win32 Shell_NotifyIcon balloon is shown via P/Invoke. +/// Runs on a dedicated background thread ("OSNotification-Win"). /// /// -/// - macOS: Uses osascript with AppleScript's display notification. +/// - macOS: Uses osascript with AppleScript's display notification (no folder action), started from the +/// calling thread without waiting. /// /// -/// - Linux: Uses notify-send (libnotify) with xdg-open for folder actions. +/// - Linux: Uses notify-send (libnotify) and, when it cannot be started, gdbus calling +/// org.freedesktop.Notifications.Notify. With a folder path, notify-send --action runs on a thread pool task +/// and xdg-open opens the folder when the action is chosen. /// +/// Other platforms: nothing is shown. Failures are logged as warnings; no exception reaches the caller. /// internal static class OSNotification { /// - /// Sends a native desktop notification. - /// If is provided, Windows and Linux will attempt to open - /// the folder when the notification is clicked/dismissed. + /// Sends a native desktop notification and returns without waiting for it to be shown (see the class summary for the + /// thread used on each platform). + /// If is provided, Windows and Linux offer to open the folder from the notification. + /// Exceptions thrown while starting the platform implementation are caught and logged as warnings. /// /// The notification title. /// The notification body text. - /// Optional folder path to open on click (Windows balloon click / Linux action). + /// + /// Optional folder path to open from the notification (Windows toast click or button / Windows balloon click / Linux action); + /// ignored on macOS. + /// public static void Show(string title, string message, string? folderPath = null) { try @@ -65,6 +75,13 @@ internal static class OSNotification /// private static class WindowsNotification { + /// + /// Starts a background thread ("OSNotification-Win") that tries the toast and shows the balloon when the toast fails. + /// Returns immediately. Exceptions on the thread are caught and logged. + /// + /// Notification title. + /// Notification body text. + /// Folder to open from the notification, or null. public static void Send(string title, string message, string? folderPath) { var thread = new Thread(() => @@ -95,11 +112,23 @@ internal static class OSNotification /// With a folder path, registers the adofai-renderer:// protocol in HKCU first (see class summary). /// Waits up to 5 seconds for the process. Returns true when it exits with code 0, /// false when it exits with another code, does not exit within 5 seconds, or fails to start. + /// + /// The toast XML uses the ToastGeneric template with the title and message as two text elements; with a folder + /// path it also sets launch and an "Open Render Folder" action to the adofai-renderer:// URI with protocol + /// activation. The notifier is created with the application ID "ADOFAI Renderer". The script is passed as Base64 of its + /// UTF-16LE bytes (-EncodedCommand), so no command line quoting is needed. + /// /// + /// Notification title (XML-escaped into the toast). + /// Notification body text (XML-escaped into the toast). + /// Folder opened by the protocol handler, or null for a toast without action. + /// True when PowerShell exited with code 0 within 5 seconds; false otherwise (the stderr text or exception is logged). private static bool TrySendToast(string title, string message, string? folderPath) { try { + // SecurityElement.Escape replaces < > & " ' with XML entities, so the text also cannot end the + // single-quoted PowerShell string that holds the XML. string xmlTitle = System.Security.SecurityElement.Escape(title) ?? title; string xmlMessage = System.Security.SecurityElement.Escape(message) ?? message; @@ -108,8 +137,13 @@ internal static class OSNotification if (!string.IsNullOrEmpty(folderPath)) { + // A single quote is written as two single quotes inside a single-quoted PowerShell string. var escapedFolder = folderPath.Replace("'", "''"); + // Script: register HKCU\Software\Classes\adofai-renderer as a URL protocol ("URL Protocol" value, default + // value "URL:adofai-renderer Protocol") whose shell\open\command runs explorer.exe on the folder, then load + // the WinRT types and show the toast. + toastXml = "" + "" + @@ -148,6 +182,7 @@ internal static class OSNotification "[Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier('ADOFAI Renderer').Show([Windows.UI.Notifications.ToastNotification]::new($xml))"; } + // -EncodedCommand takes Base64 of the UTF-16LE script text. byte[] bytes = System.Text.Encoding.Unicode.GetBytes(psScript); string base64 = Convert.ToBase64String(bytes); @@ -166,7 +201,8 @@ internal static class OSNotification process.Start(); process.WaitForExit(5000); - // If PowerShell exited with error, treat as failure + // ExitCode throws InvalidOperationException while the process is still running (timeout), which the catch + // below reports as a failure. If PowerShell exited with error, treat as failure if (process.ExitCode != 0) { string stderr = process.StandardError.ReadToEnd().Trim(); @@ -189,9 +225,14 @@ internal static class OSNotification #region Balloon via Shell_NotifyIcon // ReSharper disable InconsistentNaming + // Shell_NotifyIcon messages: add / delete the notification area icon. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-shell_notifyiconw private const int NIM_ADD = 0x00; private const int NIM_DELETE = 0x02; + // NOTIFYICONDATAW.uFlags: uCallbackMessage, hIcon, szTip and the balloon fields (szInfo, szInfoTitle, dwInfoFlags) + // are valid. NIIF_INFO: information icon in the balloon. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shellapi/ns-shellapi-notifyicondataw private const int NIF_MESSAGE = 0x01; private const int NIF_ICON = 0x02; private const int NIF_TIP = 0x04; @@ -199,18 +240,25 @@ internal static class OSNotification private const int NIIF_INFO = 0x01; + // WM_USER (winuser.h) starts the private window message range; WM_TRAYICON is the application-defined callback + // message of the icon. NIN_BALLOONUSERCLICK (shellapi.h, WM_USER + 5) arrives as lParam of the callback message when + // the user clicks the balloon. private const int WM_USER = 0x0400; private const int WM_TRAYICON = WM_USER + 1; private const int NIN_BALLOONUSERCLICK = 0x0405; + // Window style of the hidden message window (winuser.h), WM_CLOSE message ID (winuser.h), and the application-defined + // icon ID (any value unique for the window). private const int WS_OVERLAPPED = 0x00000000; private const uint WM_CLOSE = 0x0010; private const int TRAY_ICON_UID = 29471; // ReSharper restore InconsistentNaming + // Adds, modifies or deletes a notification area icon; returns false on failure. [DllImport("shell32.dll", CharSet = CharSet.Unicode)] private static extern bool Shell_NotifyIconW(int dwMessage, ref NotifyIconDataW lpData); + // Creates the window that owns the icon and receives its callback messages; returns NULL on failure. [DllImport("user32.dll", CharSet = CharSet.Unicode)] private static extern IntPtr CreateWindowExW( int dwExStyle, string lpClassName, string lpWindowName, @@ -221,9 +269,12 @@ internal static class OSNotification [return: MarshalAs(UnmanagedType.Bool)] private static extern bool DestroyWindow(IntPtr hWnd); + // Loads a predefined system icon when hInstance is NULL. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-loadiconw [DllImport("user32.dll")] private static extern IntPtr LoadIconW(IntPtr hInstance, IntPtr lpIconName); + // Message loop functions of the calling thread: GetMessageW blocks until a message arrives and returns false for WM_QUIT. [DllImport("user32.dll")] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool GetMessageW(out Msg lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax); @@ -234,10 +285,13 @@ internal static class OSNotification [DllImport("user32.dll")] private static extern IntPtr DispatchMessageW(ref Msg lpMsg); + // Places a message in the window's thread queue and returns without waiting (callable from any thread). [DllImport("user32.dll")] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool PostMessageW(IntPtr hWnd, uint msg, IntPtr wParam, IntPtr lParam); + // NOTIFYICONDATAW up to dwInfoFlags (without guidItem and hBalloonIcon); cbSize is the size of this layout. + // String fields hold at most SizeConst characters including the terminating null. [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] private struct NotifyIconDataW { @@ -259,6 +313,7 @@ internal static class OSNotification public int dwInfoFlags; } + // Win32 MSG structure filled by GetMessageW. [StructLayout(LayoutKind.Sequential)] private struct Msg { @@ -271,8 +326,22 @@ internal static class OSNotification public int pt_y; } + /// + /// Shows a notification area balloon through Shell_NotifyIconW, owned by a hidden message-only window. + /// + /// Without a folder path the icon stays for 500 ms and is then removed. With a folder path the method runs a message + /// loop on the calling thread for up to 12 seconds (a timer posts WM_CLOSE); a balloon click + /// (NIN_BALLOONUSERCLICK) opens the folder through the shell (UseShellExecute) and ends the loop. + /// The icon is deleted and the window destroyed before returning. Errors are logged as warnings. + /// + /// + /// Balloon title; truncated to 63 characters. + /// Balloon text; truncated to 255 characters. + /// Folder opened on balloon click, or null. private static void SendBalloon(string title, string message, string? folderPath) { + // szInfoTitle holds 64 and szInfo 256 characters including the terminating null. + // Reference: https://learn.microsoft.com/en-us/windows/win32/api/shellapi/ns-shellapi-notifyicondataw if (title.Length > 63) title = title.Substring(0, 63); if (message.Length > 255) message = message.Substring(0, 255); @@ -281,6 +350,7 @@ internal static class OSNotification try { + // HWND_MESSAGE ((HWND)-3, winuser.h) as parent creates a message-only window of the "Static" class. var hwndMessage = new IntPtr(-3); hWnd = CreateWindowExW( 0, "Static", "ADOFAIRendererNotify", @@ -293,6 +363,7 @@ internal static class OSNotification return; } + // 32512 = IDI_APPLICATION (winuser.h), the default application icon. IntPtr hIcon = LoadIconW(IntPtr.Zero, (IntPtr)32512); nid.cbSize = Marshal.SizeOf(typeof(NotifyIconDataW)); @@ -305,6 +376,7 @@ internal static class OSNotification nid.szInfo = message; nid.szInfoTitle = title; nid.dwInfoFlags = NIIF_INFO; + // uTimeout in milliseconds; deprecated as of Windows Vista (display time follows the system settings). nid.uTimeoutOrVersion = 4; Shell_NotifyIconW(NIM_ADD, ref nid); @@ -327,6 +399,7 @@ internal static class OSNotification if (msg.message == WM_TRAYICON) { + // No NIM_SETVERSION call: lParam carries the notification event (NIN_*) directly. int notifyMsg = (int)(long)msg.lParam; if (notifyMsg == NIN_BALLOONUSERCLICK) { @@ -388,6 +461,12 @@ internal static class OSNotification /// private static class MacNotification { + /// + /// Starts osascript -e "display notification ... with title ..." without waiting. Exceptions from starting the + /// process (for example osascript missing) propagate to . + /// + /// Notification title. + /// Notification body text. public static void Send(string title, string message) { // AppleScript string escaping: backslash and double-quote @@ -412,6 +491,9 @@ internal static class OSNotification RunProcess(startInfo); } + /// Escapes a value for use inside a double-quoted AppleScript string literal. + /// Raw text. + /// The text with each backslash and double quote preceded by a backslash. private static string EscapeAppleScript(string value) { // In AppleScript string literals, escape backslash and double-quote @@ -431,6 +513,13 @@ internal static class OSNotification /// private static class LinuxNotification { + /// + /// Shows the notification with an "Open Render Folder" action when is set (thread pool + /// task), otherwise a plain notification started from the calling thread. + /// + /// Notification summary. + /// Notification body. + /// Folder opened by the action, or null. public static void Send(string title, string message, string? folderPath) { if (!string.IsNullOrEmpty(folderPath)) @@ -443,6 +532,9 @@ internal static class OSNotification } } + /// Starts notify-send; when it cannot be started, starts gdbus. Neither process is waited for. + /// Notification summary. + /// Notification body. private static void ShowSimple(string title, string message) { // Try notify-send first @@ -453,6 +545,15 @@ internal static class OSNotification TryGdbus(title, message); } + /// + /// On a thread pool task, runs notify-send -a "ADOFAI Renderer" --action=open_folder=Open Render Folder and waits + /// for it to exit. notify-send prints the key of the chosen action ("open_folder") to standard output; in that case + /// xdg-open <folderPath> is started. When an exception occurs (for example notify-send missing), it is logged + /// and shows the notification without the action. + /// + /// Notification summary. + /// Notification body. + /// Folder passed to xdg-open. private static void ShowWithAction(string title, string message, string folderPath) { // Run on a background thread because notify-send with --action blocks @@ -505,6 +606,10 @@ internal static class OSNotification }); } + /// Starts notify-send -a "ADOFAI Renderer" <title> <message> without waiting. + /// Notification summary. + /// Notification body. + /// True when the process was started (its exit code is not checked); false when starting it threw. private static bool TryNotifySend(string title, string message) { try @@ -530,6 +635,13 @@ internal static class OSNotification } } + /// + /// Starts gdbus call --session to org.freedesktop.Notifications.Notify (arguments app_name, replaces_id, + /// app_icon, summary, body, actions, hints, expire_timeout) without waiting. Start failures are logged as warnings. + /// Reference: https://specifications.freedesktop.org/notification/latest/protocol.html + /// + /// Notification summary. + /// Notification body. private static void TryGdbus(string title, string message) { try diff --git a/Util/OS/RegistryUtil.cs b/Util/OS/RegistryUtil.cs index 91ecfc6..11467cf 100644 --- a/Util/OS/RegistryUtil.cs +++ b/Util/OS/RegistryUtil.cs @@ -2,16 +2,37 @@ using System; namespace ADOFAIRenderer.Util.OS; +/// +/// Read-only access to HKEY_LOCAL_MACHINE (64-bit registry view) through reflection. +/// +/// The mod targets netstandard2.1, which does not expose Microsoft.Win32.RegistryKey; the game runtime (Mono) +/// provides it in its core library. The types and methods are looked up once in the static constructor and invoked through +/// reflection. +/// +/// +/// Intended for Windows. On other platforms the result depends on the runtime's registry support; when the types or methods +/// are not found, or a call throws, every method returns null. +/// All failures (missing key, access denied, reflection errors) are caught and reported as null; nothing is logged. +/// Stateless and safe to call from any thread; each call opens and closes its own keys. +/// +/// internal static class RegistryUtil { + // Reflected Microsoft.Win32 types; null when the runtime does not provide them. private static readonly Type? RegistryKeyType; private static readonly Type? RegistryHiveType; private static readonly Type? RegistryViewType; + // RegistryKey.OpenBaseKey(RegistryHive, RegistryView) (static), OpenSubKey(string) (read-only open), + // GetSubKeyNames() and GetValue(string). private static readonly System.Reflection.MethodInfo? OpenBaseKeyMethod; private static readonly System.Reflection.MethodInfo? OpenSubKeyMethod; private static readonly System.Reflection.MethodInfo? GetSubKeyNamesMethod; private static readonly System.Reflection.MethodInfo? GetValueMethod; + /// + /// Resolves the registry types by name (, then the core library that defines + /// ) and their methods. Leaves the fields null on any failure. + /// static RegistryUtil() { try @@ -42,6 +63,8 @@ internal static class RegistryUtil } } + /// Closes a reflected RegistryKey (Dispose, or Close when it is not IDisposable). Ignores null and errors. + /// RegistryKey instance returned by a reflected call, or null. private static void CloseKey(object? key) { if (key == null || RegistryKeyType == null) return; @@ -63,6 +86,11 @@ internal static class RegistryUtil } } + /// + /// Returns the names of the direct subkeys of HKEY_LOCAL_MACHINE\<parentKeyPath> in the 64-bit registry view. + /// + /// Key path relative to HKEY_LOCAL_MACHINE, with backslash separators. + /// Subkey names, or null when the key does not exist, cannot be opened, or the registry is unavailable. // ReSharper disable once InconsistentNaming public static string[]? GetHKLMSubKeyNames(string parentKeyPath) { @@ -71,6 +99,10 @@ internal static class RegistryUtil try { + // RegistryHive.LocalMachine = 0x80000002 (-2147483646), RegistryView.Registry64 = 256. + // On a 32-bit OS a Registry64 request returns the 32-bit view. + // Reference: https://learn.microsoft.com/en-us/dotnet/api/microsoft.win32.registryhive + // Reference: https://learn.microsoft.com/en-us/dotnet/api/microsoft.win32.registryview object hiveLocalMachine = Enum.ToObject(RegistryHiveType!, 0x80000002); object viewRegistry64 = Enum.ToObject(RegistryViewType!, 256); @@ -95,6 +127,15 @@ internal static class RegistryUtil } } + /// + /// Reads a string value of HKEY_LOCAL_MACHINE\<keyPath> in the 64-bit registry view. + /// + /// Key path relative to HKEY_LOCAL_MACHINE, with backslash separators. + /// Value name; an empty string reads the key's default value. + /// + /// The value when it exists and is a string (REG_SZ / REG_EXPAND_SZ); null when the key or value does not exist, the value + /// has another type, or the registry is unavailable. + /// // ReSharper disable once InconsistentNaming public static string? GetHKLMValue(string keyPath, string valueName) { @@ -103,6 +144,7 @@ internal static class RegistryUtil try { + // Same hive and view constants as GetHKLMSubKeyNames. object hiveLocalMachine = Enum.ToObject(RegistryHiveType!, 0x80000002); object viewRegistry64 = Enum.ToObject(RegistryViewType!, 256); diff --git a/Util/ParticleUtil.cs b/Util/ParticleUtil.cs index 9c16058..021f63e 100644 --- a/Util/ParticleUtil.cs +++ b/Util/ParticleUtil.cs @@ -4,10 +4,22 @@ using UnityEngine; namespace ADOFAIRenderer.Util { + /// Particle system helpers. internal static class ParticleUtil { + /// + /// Renderer found for each particle system. Weak keys: an entry is dropped when its ParticleSystem object is collected. + /// Main thread. + /// private static readonly ConditionalWeakTable Cache = new(); + /// + /// Returns the ParticleSystemRenderer that belongs to . When a GameObject has several particle + /// systems and renderers, the renderer with the same component index as the particle system is used; otherwise the + /// first renderer of the GameObject. The result is cached per particle system. Main thread. + /// + /// Particle system; null or destroyed returns null. + /// The renderer, or null when the GameObject has none. public static ParticleSystemRenderer? GetRenderer(this ParticleSystem ps) { if (!ps) return null; diff --git a/Util/RationalUtil.cs b/Util/RationalUtil.cs index dcfba11..09081da 100644 --- a/Util/RationalUtil.cs +++ b/Util/RationalUtil.cs @@ -2,8 +2,19 @@ using System; namespace ADOFAIRenderer.Util { + /// Conversion of floating-point rates (frame rates) to integer fractions. internal static class RationalUtil { + /// + /// Best rational approximation Numerator / Denominator of by continued fraction expansion: + /// the last convergent whose numerator fits in int and whose denominator does not exceed + /// . Exact for values with a short expansion (for example 60 gives 60/1). The sign is + /// carried by the numerator; 0 gives 0/1. A magnitude above int.MaxValue has no valid convergent and gives 1/0. + /// Thread-safe (pure function). + /// + /// Value to convert, for example a frame rate in frames per second. + /// Largest allowed denominator (default 100000000). + /// Numerator and denominator (positive for magnitudes up to int.MaxValue). public static (int Numerator, int Denominator) Double2MaxPrecisionInt(double value, int maxDenominator = 100000000) { diff --git a/Util/VideoBGPlayer.cs b/Util/VideoBGPlayer.cs index b9ac9d1..e860172 100644 --- a/Util/VideoBGPlayer.cs +++ b/Util/VideoBGPlayer.cs @@ -9,9 +9,25 @@ using FFmpeg.AutoGen; namespace ADOFAIRenderer.Util; +/// +/// Plays a custom level's background video with FFmpeg (FFmpegNativeDecoder) in place of Unity's VideoPlayer while a +/// render is active, so the video frame shown always matches the render clock. Attached to the scnGame.videoBG object +/// by scnGamePatch.ReplaceVideo, which disables the native VideoPlayer; removed again by that patch when no render is +/// active. +/// +/// Every LateUpdate it computes the video time from the conductor (the same formula scrVfxPlus uses to start and +/// position the native video), decodes frames up to that time and draws the newest one either into the VideoPlayer's +/// target RenderTexture or onto a quad at the far plane of the background camera. +/// Decoded YUV frames are converted to RGBA on the GPU (ColorConversion.compute, kernels CSMain_NV12ToRGBA / +/// CSMain_YUV420PToRGBA); other pixel formats, or a missing shader, use the decoder's RGBA output (CPU conversion). +/// Main thread only. +/// +/// // ReSharper disable once InconsistentNaming internal class VideoBGPlayer : MonoBehaviour { + // Property IDs of the compute shader parameters: input planes, output texture, frame size in pixels, + // ColorRange (1 = full range, 0 = limited range) and ColorMatrix (FFmpegNativeDecoder.ColorMatrix). private static readonly int YTex = Shader.PropertyToID("_YTex"); private static readonly int UVTex = Shader.PropertyToID("_UVTex"); private static readonly int Result = Shader.PropertyToID("_Result"); @@ -22,6 +38,10 @@ internal class VideoBGPlayer : MonoBehaviour private static readonly int ColorRange = Shader.PropertyToID("ColorRange"); private static readonly int ColorMatrix = Shader.PropertyToID("ColorMatrix"); + /// + /// Conversion path of the current video: None = CPU RGBA texture, YUV420P = separate Y, U, V planes, + /// NV12 = Y plane and interleaved UV plane. + /// [SuppressMessage("ReSharper", "InconsistentNaming")] private enum GPUMode { @@ -30,6 +50,7 @@ internal class VideoBGPlayer : MonoBehaviour NV12 } + /// Decoder of the current video; null when not initialized or after CleanUp. private FFmpegNativeDecoder? _decoder; // CPU fallback rendering @@ -47,16 +68,34 @@ internal class VideoBGPlayer : MonoBehaviour private RenderTexture? _renderTexture; private int _kernelIndex = -1; + /// Presentation time in seconds of the frame currently shown (video time, 0 = start of the file). private double _currentFrameTimestamp; + /// Video time in seconds from which the next frame is decoded (current timestamp + 1 / FPS). private double _nextFrameTimestamp; + /// Conversion path selected by Init. private GPUMode _gpuMode = GPUMode.None; + /// Target RenderTexture of the native VideoPlayer; null when the video is shown on the background quad. private RenderTexture? _targetTexture; + /// Whether the level loops the video (miscSettings "loopVideo"). private bool _isLooping; + /// True after a successful Init; set to false by CleanUp or a playback error (playback then stops). private bool _isInitialized; + /// Background quad created by SetupBackgroundQuad when there is no target texture; hidden until the first frame. private GameObject? _quad; + /// Material of the background quad; its main texture is the converted frame. private Material? _material; + /// + /// Releases any previous video and opens . Allocates the plane textures for the GPU path + /// of the decoder's pixel format (chroma planes are (width + 1) / 2 x (height + 1) / 2), or an RGBA texture for the + /// CPU path. Errors are logged; the player then stays uninitialized and shows nothing. + /// + /// Full path of the video file. + /// RenderTexture the native VideoPlayer renders into, or null to draw on a background quad. + /// Whether the video loops. + /// Camera the background quad is attached to when is null. + /// VideoPlayer object; its layer is given to the background quad. // ReSharper disable once InconsistentNaming public void Init(string videoPath, RenderTexture? targetTexture, bool loop, Camera? targetCamera, GameObject videoBGGo) { @@ -169,12 +208,19 @@ internal class VideoBGPlayer : MonoBehaviour } } + /// Sets clamp wrapping and bilinear filtering. private static void ConfigureTexture(Texture2D tex) { tex.wrapMode = TextureWrapMode.Clamp; tex.filterMode = FilterMode.Bilinear; } + /// + /// Main thread, every frame after the song has started. Computes the video time, seeks when it moved backwards or more + /// than 1 s ahead of the shown frame, decodes frames until the next frame lies after that time, and draws the last + /// decoded frame. Before the video starts (negative time) the target texture is cleared to black and the quad hidden. + /// When decoding fails at the end of the file, a looping video seeks to 0; otherwise the last frame stays. + /// private void LateUpdate() { if (!_isInitialized || _decoder == null) return; @@ -182,6 +228,9 @@ internal class VideoBGPlayer : MonoBehaviour var cond = scrConductor.instance; if (!cond || !cond.hasSongStarted) return; + // Song position in seconds of song time, as scrConductor computes songposition_minusi from dspTime with the input + // offset calibration_i = 0 (the render sets the input offset to 0), minus the separate countdown length. + // scrVfxPlus starts the native video at songposition_minusi - countdown + vidOffset (scrVfxPlus.cs). var pitch = cond.song.pitch; var separateCountdownOffset = cond.separateCountdownTime ? cond.crotchetAtStart * cond.adjustedCountdownTicks : 0.0; @@ -215,6 +264,7 @@ internal class VideoBGPlayer : MonoBehaviour } + // scrVfxPlus hides the tutorial (editor) background when the native video starts (scrVfxPlus.cs). if (ADOBase.customLevel) { ADOBase.customLevel.ShowTutorialBackground(false); @@ -222,6 +272,7 @@ internal class VideoBGPlayer : MonoBehaviour try { + // A seek is cheaper than decoding many frames when the time jumped forward by more than 1 s. if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0) { _decoder.SeekToTime(targetVideoTime); @@ -298,6 +349,10 @@ internal class VideoBGPlayer : MonoBehaviour } } + /// + /// Uploads the decoded planes, converts them to RGBA with the compute shader (or uploads the RGBA texture), and shows + /// the result: blitted into the target texture, or set as the quad's main texture (the quad is activated). + /// private void RenderToTarget() { if (_decoder == null) return; @@ -318,6 +373,7 @@ internal class VideoBGPlayer : MonoBehaviour _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0); _computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix); + // One thread group per 8 x 8 output pixels, rounded up (group size expected by the ColorConversion.compute kernels). int threadGroupsX = (w + 7) / 8; int threadGroupsY = (h + 7) / 8; _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1); @@ -353,6 +409,7 @@ internal class VideoBGPlayer : MonoBehaviour _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0); _computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix); + // One thread group per 8 x 8 output pixels, rounded up (group size expected by the ColorConversion.compute kernels). int threadGroupsX = (w + 7) / 8; int threadGroupsY = (h + 7) / 8; _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1); @@ -394,11 +451,16 @@ internal class VideoBGPlayer : MonoBehaviour } } + /// Releases the decoder and GPU resources when the component is destroyed. private void OnDestroy() { CleanUp(); } + /// + /// Disposes the decoder and destroys the textures, the RenderTexture, the background quad and its material. + /// The player is uninitialized afterwards. Called by Init, on errors and on destroy. + /// public void CleanUp() { _isInitialized = false; @@ -446,6 +508,12 @@ internal class VideoBGPlayer : MonoBehaviour } } + /// + /// Creates a quad as a child of , 0.5 units in front of its far clip plane, scaled to + /// fill the view (orthographic size or field of view), on the layer of . Its shader is + /// taken from scrCamera's camera quad material when available, otherwise the first of Sprites/Default, Unlit/Texture + /// and Standard that exists. The quad starts inactive. + /// // ReSharper disable once InconsistentNaming private void SetupBackgroundQuad(Camera targetCamera, GameObject videoBGGo) {