diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 9d9c2f1..ba0d6db 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -18,6 +18,7 @@ namespace ADOFAIRenderer.Patch.Engine; [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.FindOrLoadAudioClipExternal))] public static class AudioManagerFindOrLoadAudioClipExternalPatch { + /// Records in when its audioLib key is not loaded yet. Runs the original always. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static void Prefix(AudioManager __instance, string path) @@ -28,9 +29,18 @@ public static class AudioManagerFindOrLoadAudioClipExternalPatch } } +/// +/// AudioManager._Play (private; AudioManager.Play forwards to it): registers each scheduled effect sound (hit sounds, PlaySound +/// events, hold sounds without an end time) as an effect voice of the offline mixer and maps the returned AudioSource to it. +/// Active while capturing (RenderManager.IsCapturingActive) with RenderAudio on. The original always runs: it creates the +/// AudioSource the game keeps using (its PlayScheduled call is blocked during rendering by another patch), and the game sets +/// time/pitch/SetScheduledEndTime on it afterwards, which the patches forward to the mapped state. +/// The game volume is passed through; the start pitch is 1 because _Play sets pitch = 1 before PlayScheduled. +/// [HarmonyPatch(typeof(AudioManager), "_Play")] public static class AudioManagerPlayPatch { + /// Opens the clip's source so its first chunk starts loading. Returns true (the original runs). [HarmonyPrefix] public static bool Prefix(string snd, double time, AudioMixerGroup group, float volume, int priority) { @@ -42,6 +52,10 @@ public static class AudioManagerPlayPatch return true; // Let the original method execute to prevent NullReferenceException } + /// + /// Registers the sound at (absolute DSP time, converted to render time) as a non-looping effect voice, + /// maps to it and queues the source's game object for deactivation at the end of the frame. + /// [HarmonyPostfix] // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __result, string snd, double time, AudioMixerGroup group, float volume, int priority) @@ -63,6 +77,10 @@ public static class AudioManagerPlayPatch /// /// scrConductor.PlayWithEndTime: registers a looping effect voice until the end time. The AudioSource it creates with /// AudioManager.MakeSource is mapped to that voice (). +/// Active while capturing (RenderManager.IsCapturingActive) with RenderAudio on; the original always runs. The game calls it for +/// hold sounds with an end time above 0 (scrConductor hold sound scheduling); it sets loop = true, pitch = 1, PlayScheduled(time) and +/// SetScheduledEndTime(endTime) on the new source. pitch and SetScheduledEndTime reach the mapped state through the +/// patches; PlayScheduled is blocked during rendering by another patch. /// [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] public static class PlayWithEndTimePatch @@ -70,9 +88,14 @@ public static class PlayWithEndTimePatch /// State registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. internal static SourceState? PendingState; + /// Clears when PlayWithEndTime returns or throws, so it cannot map a later MakeSource call. [HarmonyFinalizer] public static void Finalizer() => PendingState = null; + /// + /// Opens the clip's source and registers a looping voice from to + /// (absolute DSP times, converted to render time); the state waits in . Returns true (the original runs). + /// [HarmonyPrefix] public static bool Prefix(string snd, double time, double endTime, float volume, int priority) { @@ -96,10 +119,18 @@ public static class PlayWithEndTimePatch } } -/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (). +/// +/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (). +/// Active only while a PlayWithEndTime call registered a state (capturing with RenderAudio on); every other MakeSource call +/// (including the one inside AudioManager._Play) is left unchanged. +/// [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))] public static class AudioManagerMakeSourcePatch { + /// + /// Takes the pending state, maps to it and queues the source's game object for deactivation at + /// the end of the frame. + /// [HarmonyPostfix] // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __result) @@ -173,6 +204,7 @@ internal static class EffectSource [HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)] public static class AudioSourceIsPlayingPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref bool __result) @@ -187,6 +219,7 @@ public static class AudioSourceIsPlayingPatch [HarmonyPatch(typeof(AudioSource), "time", MethodType.Getter)] public static class AudioSourceTimeGetterPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref float __result) @@ -204,6 +237,7 @@ public static class AudioSourceTimeGetterPatch [HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)] public static class AudioSourceTimePatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, float value) @@ -214,10 +248,14 @@ public static class AudioSourceTimePatch } } -/// AudioSource.timeSamples getter (): clip position in samples of the clip. +/// +/// AudioSource.timeSamples getter (): clip position in samples of the clip while playing; while not playing, +/// the last value set through time or timeSamples (0 after Stop()). +/// [HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Getter)] public static class AudioSourceTimeSamplesGetterPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref int __result) @@ -232,6 +270,7 @@ public static class AudioSourceTimeSamplesGetterPatch [HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Setter)] public static class AudioSourceTimeSamplesPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, int value) @@ -246,6 +285,7 @@ public static class AudioSourceTimeSamplesPatch [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Getter)] public static class AudioSourcePitchGetterPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, ref float __result) @@ -260,6 +300,7 @@ public static class AudioSourcePitchGetterPatch [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] public static class AudioSourcePitchPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, float value) @@ -274,6 +315,7 @@ public static class AudioSourcePitchPatch [HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))] public static class AudioSourceSetScheduledEndTimePatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance, double time) @@ -292,6 +334,7 @@ public static class AudioSourceSetScheduledEndTimePatch [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])] public static class AudioSourceStopPatch { + /// When finds a mapped state, uses it and returns false (Unity is skipped); otherwise returns true. [HarmonyPrefix] // ReSharper disable once InconsistentNaming public static bool Prefix(AudioSource __instance) @@ -306,11 +349,13 @@ public static class AudioSourceStopPatch /// AudioManager.StopAllSounds destroys every source in liveSources (all AudioManager._Play and scrConductor.PlayWithEndTime sources), /// so every effect voice stops at the current render time, including ones scheduled to start later. /// Callers include scrConductor.PlayHitTimes (from its second call), ScrubMusicToTile/Time, KillAllSoundsCoroutine, -/// scrController.FailAction and scnGame.Play. +/// scrController.FailAction, scrController.DebugUpdate, scnGame.Play and PauseMenu.Choose. +/// Active while capturing (RenderManager.IsCapturingActive) with RenderAudio on; the original always runs (it destroys the sources). /// [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.StopAllSounds))] public static class AudioManagerStopAllSoundsPatch { + /// Stops every effect voice while capturing with RenderAudio on. The original always runs (no return value). [HarmonyPrefix] public static void Prefix() { diff --git a/Util/Audio/AudioSpectrumGenerator.cs b/Util/Audio/AudioSpectrumGenerator.cs index 58d1c4c..669761c 100644 --- a/Util/Audio/AudioSpectrumGenerator.cs +++ b/Util/Audio/AudioSpectrumGenerator.cs @@ -28,14 +28,31 @@ internal static class AudioSpectrumGenerator /// private const double MonoGain = 0.70710678118654752; + /// Store created by this class when the mixer store cannot be shared (disposed by ); otherwise null. private static AudioClipStore? _ownStore; + + /// Store the song source belongs to: the mixer store or . private static AudioClipStore? _store; + + /// Song source at the output sample rate, or null when not initialized or the song could not be loaded. private static AudioClipSource? _source; + + /// Cursor on moved to each analysis window, so the decoder thread loads the chunks there. private static AudioCursor? _cursor; + + /// Chunk currently held for (invalid when none). private static ChunkView _view; + + /// True after a chunk loading error was logged (logged once per render). private static bool _loadErrorLogged; + /// RenderManager.frameCount of the spectra in ; -1 when empty. private static long _cachedFrame = -1; + + /// + /// Spectra computed in the current render frame, by spectrum length and source channel. Several game objects read the + /// spectrum every frame; within one frame the render time does not change, so they receive the same values. + /// private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new(); /// Registers the song clip. Call after the is created (null when audio is not rendered). @@ -93,6 +110,17 @@ internal static class AudioSpectrumGenerator CachedSpectrums.Clear(); } + /// + /// Fills with the magnitude spectrum of the song at the current render time + /// (called by the AudioSource.GetSpectrumData patch while capturing with ProcessAudio on, for any AudioSource). + /// Bin k covers frequency k * outputRate / (2 * samples.Length) Hz in output time; values are 0 ~ 1. + /// Zeros are written when the song is not available, the conductor has not started the song, or the length is not a power of two. + /// is not read: the Blackman-Harris window is always used (every GetSpectrumData call in the game + /// passes FFTWindow.BlackmanHarris). Main thread only. + /// + /// Destination; its length is the spectrum length (power of two). + /// Requested channel (0 = left, 1 = right), clamped to the song's channels; ignored for mono songs. + /// Requested window (not used). public static void GetSpectrumData(float[] samples, int channel, FFTWindow window) { var n = samples.Length; @@ -127,6 +155,8 @@ internal static class AudioSpectrumGenerator var now = render.dspTimeBase + render.AudioTime; var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown; + // Window of 2n output samples centered on the current time: output sample i reads source frame first + i * pitch, + // so the window starts n output samples (n * pitch source frames) before the song position. var fftSize = 2 * n; var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0); var first = songSeconds * _store.OutputRate - n * pitch; @@ -134,6 +164,7 @@ internal static class AudioSpectrumGenerator var windowSamples = new float[fftSize]; try { + // Keep the cursor at the window start so the decoder thread loads (and keeps) the chunks the window reads. _cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1); for (var i = 0; i < fftSize; i++) { @@ -178,6 +209,10 @@ internal static class AudioSpectrumGenerator return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel]; } + /// + /// Magnitude spectrum of (length n, a power of two) after the Blackman-Harris window: + /// writes min(n / 2, spectrum.Length) bins, bin k = min(|X[k]| * 2.5 / n, 1). + /// private static void CalculateFFT(float[] samples, float[] spectrum) { var n = samples.Length; @@ -199,10 +234,10 @@ internal static class AudioSpectrumGenerator im[i] = 0f; } - // Cooley-Tukey Radix-2 FFT + // Cooley-Tukey Radix-2 FFT (iterative, in place, decimation in time); m = log2(n) stages. var m = (int)Math.Round(Math.Log(n, 2)); - // Bit-reversal + // Bit-reversal: reorder the input so element i moves to the index with the bits of i reversed (j tracks that index). var j = 0; for (var i = 0; i < n - 1; i++) { @@ -225,7 +260,8 @@ internal static class AudioSpectrumGenerator j += k; } - // FFT stages + // FFT stages: stage s combines pairs of transforms of length le/2 into length le = 2^s with butterflies. + // (ur, ui) is the twiddle factor exp(-i * pi * sub / le2), advanced by multiplying with (wr, wi) = exp(-i * pi / le2). for (var stage = 1; stage <= m; stage++) { var le = 1 << stage; diff --git a/Util/Audio/AudioSyncCalculator.cs b/Util/Audio/AudioSyncCalculator.cs index 21e7bd4..a4a1626 100644 --- a/Util/Audio/AudioSyncCalculator.cs +++ b/Util/Audio/AudioSyncCalculator.cs @@ -2,29 +2,50 @@ using System; namespace ADOFAIRenderer.Util.Audio; +/// +/// Splits the audio sample stream into per-video-frame sample counts so that audio and video stay aligned without drift. +/// +/// The frame rate is converted to an exact fraction Numerator / Denominator (). +/// Frame k receives floor((k + 1) * sampleRate * Denominator / Numerator) - floor(k * sampleRate * Denominator / Numerator) samples: +/// the remainder of each division is carried into the next frame, so the sum over any number of frames equals the exact +/// sample count of that duration rounded down (for example 44100 Hz at 60 fps gives 735 every frame; at 144 fps three frames of 306 +/// are followed by one of 307, 306.25 on average). +/// Used by the render thread only (RenderManager.WriteAudioForFrame, once per frame sent to the encoder). +/// internal class AudioSyncCalculator { + /// Audio sample rate in Hz (samples per channel per second). private readonly long _sampleRate; + + /// Video frame rate as a fraction (frames per second = Numerator / Denominator). private readonly (long Numerator, long Denominator) _fpsRational; + /// Total samples handed out since construction or (kept for debugging, not read). // ReSharper disable once NotAccessedField.Local private long _currentSample; + + /// Remainder of the last division, in units of 1 / Numerator samples (0 ~ Numerator - 1). private long _remainder; + /// Audio sample rate in Hz. + /// Video frame rate in frames per second (the encoded frame rate). public AudioSyncCalculator(long sampleRate, double fps) { _sampleRate = sampleRate; _fpsRational = RationalUtil.Double2MaxPrecisionInt(fps); } + /// Restarts the sequence at frame 0 (the remainder is cleared). public void Reset() { _currentSample = 0; _remainder = 0; } + /// Returns the sample count (frames per channel) of the next video frame and advances to the following frame. public int GetNextFrameSamples() { + // samplesNeeded = floor((sampleRate * Denominator + remainder) / Numerator); the new remainder carries the fraction. var numerator = _sampleRate * _fpsRational.Denominator + _remainder; var samplesNeeded = numerator / _fpsRational.Numerator; diff --git a/Util/Audio/Clip/AudioClipPaths.cs b/Util/Audio/Clip/AudioClipPaths.cs index 1900d18..701d311 100644 --- a/Util/Audio/Clip/AudioClipPaths.cs +++ b/Util/Audio/Clip/AudioClipPaths.cs @@ -20,6 +20,7 @@ internal static class AudioClipPaths /// Suffix of the game's audioLib keys for external clips. public const string ExternalSuffix = "*external"; + /// File path by audioLib key; also the lock object. Entries are kept for the whole game session. private static readonly Dictionary Paths = new(); /// audioLib key of an external clip loaded from . diff --git a/Util/Audio/Clip/AudioClipResolver.cs b/Util/Audio/Clip/AudioClipResolver.cs index d62ee06..bcc7010 100644 --- a/Util/Audio/Clip/AudioClipResolver.cs +++ b/Util/Audio/Clip/AudioClipResolver.cs @@ -16,6 +16,10 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// internal static class AudioClipResolver { + /// + /// Returns the source registered under in , opening it on first use + /// (file first, then the Unity clip; see the class summary). A key that could not be opened is remembered as null. + /// /// Store of the current render. /// audioLib key of the clip (for example "song.ogg*external" or "sndKick"). /// Loaded Unity clip for the key, used when the clip has no file; may be null when the key has a file. @@ -38,6 +42,9 @@ internal static class AudioClipResolver Melon.Logger.Warning($"[AudioClipResolver] FFmpeg could not open '{path}' for '{key}', using the Unity clip: {e.Message}"); } + // The reader opened here becomes the first decode stream; additional streams reopen the file with OpenFile. + // When no source results (store disposed, or the source could not be created), the reader is disposed here + // (readers allow Dispose more than once). if (fileReader != null) { var source = store.GetOrAdd(key, () => fileReader, () => OpenFile(path)); diff --git a/Util/Audio/Clip/AudioClipSource.cs b/Util/Audio/Clip/AudioClipSource.cs index 7f856f2..401bd3e 100644 --- a/Util/Audio/Clip/AudioClipSource.cs +++ b/Util/Audio/Clip/AudioClipSource.cs @@ -13,9 +13,13 @@ namespace ADOFAIRenderer.Util.Audio.Clip; internal sealed class AudioCursor { private readonly AudioClipSource _source; + + /// Backing value of , accessed with Interlocked (64-bit reads are not atomic on every platform). private long _position; + private volatile bool _loop; + /// Created by . internal AudioCursor(AudioClipSource source, long position, bool loop) { _source = source; @@ -39,7 +43,10 @@ internal sealed class AudioCursor } } - /// When true, the chunk after the last chunk is chunk 0. + /// + /// When true, the chunk after the last chunk is chunk 0 (the window wraps to the clip start), and a position past the end + /// wraps into the clip. Setting it wakes the decoder thread. + /// public bool Loop { get => _loop; @@ -70,8 +77,10 @@ internal readonly struct ChunkView /// Channel count of the stored samples (same as the source clip). public readonly int Channels; + /// Index of the chunk in its source (used by ). public readonly int ChunkIndex; + /// Created by ; default(ChunkView) is the invalid view. internal ChunkView(float[] data, long startFrame, int frameCount, int channels, int chunkIndex) { Data = data; @@ -81,10 +90,13 @@ internal readonly struct ChunkView ChunkIndex = chunkIndex; } + /// True for a view that holds a chunk; false for default(ChunkView). public bool IsValid => Data != null; + /// Source frame after the last valid frame (exclusive end). public long EndFrame => StartFrame + FrameCount; + /// True when source frame is one of the valid frames of this chunk. public bool Contains(long frame) => frame >= StartFrame && frame < StartFrame + FrameCount; /// @@ -171,26 +183,45 @@ internal sealed class AudioClipSource internal int DecodeStreamLimit { get; set; } = MaxDecodeStreams; private readonly AudioClipStore _store; + + /// Reader passed to the constructor (the first decode stream); disposed again in . private readonly IPCMReader _firstReader; + + /// Opens another reader for an additional decode stream; null when only the first reader can be used. private readonly Func? _openReader; // Guarded by _sync: chunk table, hold counts, cursors, demand set, loaded set. private readonly object _sync = new(); + + /// Sample array of each chunk, null while not loaded. private readonly float[]?[] _chunks; + + /// Number of consumers holding each chunk (TryAcquire minus Release); a held chunk is not freed. private readonly int[] _holdCounts; + private readonly List _cursors = new(); + + /// Chunks a consumer asked for while they were not loaded; an index is removed when the chunk is acquired. private readonly HashSet _demanded = new(); + + /// Indices of the loaded chunks (non-null entries of ). private readonly HashSet _loaded = new(); + private int _residentLoadedCount; // Resident: chunks [0, _residentLoadedCount) are loaded // Guarded by _loadLock: decode streams. private readonly object _loadLock = new(); private readonly List _streams = new(); + + /// Incremented on every load; stored in for least-recently-used selection. private long _streamUseCounter; + + /// True after opening an additional reader failed; no further readers are opened for this source. private bool _openFailed; private volatile bool _isFailed; + /// Key the source is registered under in the store (the game's audioLib key / AudioClip.name). public string Key { get; } /// Channel count of the stored samples (same as the clip). @@ -205,12 +236,19 @@ internal sealed class AudioClipSource /// Clip length in seconds: clip frames / clip rate. public double Duration { get; } + /// Frames per chunk: / (rounded down). public int FramesPerChunk { get; } + /// Number of chunks: ceil( / ); 0 for an empty clip. public int ChunkCount { get; } + /// + /// for forward-only readers and clips of at most one chunk, otherwise + /// . Fixed at construction. + /// public ChunkPolicy Policy { get; } + /// True when is . public bool IsResident => Policy == ChunkPolicy.Resident; /// True after loading failed. A failed source returns no chunks. @@ -279,6 +317,7 @@ internal sealed class AudioClipSource return cursor; } + /// Unregisters ; its window chunks become freeable on the decoder thread's next pass. public void RemoveCursor(AudioCursor cursor) { lock (_sync) @@ -288,6 +327,7 @@ internal sealed class AudioClipSource _store.RequestService(); } + /// Wakes the decoder thread after a cursor changed chunk or loop flag (resident sources ignore cursors). internal void OnCursorMoved() { if (!IsResident) _store.RequestService(); @@ -325,6 +365,7 @@ internal sealed class AudioClipSource return false; } + /// Ends one hold of the chunk of taken by . An invalid view is ignored. public void Release(in ChunkView view) { if (!view.IsValid) return; @@ -335,6 +376,7 @@ internal sealed class AudioClipSource } } + /// View of chunk ; the last chunk's frame count stops at . private ChunkView CreateView(int index, float[] data) { var start = (long)index * FramesPerChunk; @@ -406,6 +448,8 @@ internal sealed class AudioClipSource return true; } + // For each cursor, the first missing chunk of its window; the lowest slot over all cursors wins + // (ties go to the earlier cursor). foreach (var cursor in _cursors) { if (!TryGetWindowStart(cursor, out var first, out var loop)) continue; @@ -441,6 +485,10 @@ internal sealed class AudioClipSource return false; } + /// + /// First chunk of the cursor's window: the chunk of its position (negative positions count as 0; positions past the end + /// wrap when looping). False when a non-looping cursor is past the end (no window). Caller holds _sync. + /// private bool TryGetWindowStart(AudioCursor cursor, out int first, out bool loop) { var position = cursor.Position; @@ -456,6 +504,7 @@ internal sealed class AudioClipSource return true; } + /// Chunk index of window slot after ; wraps when looping, false past the end otherwise. private bool TryWindowIndex(int first, int slot, bool loop, out int index) { index = first + slot; @@ -493,6 +542,7 @@ internal sealed class AudioClipSource stream.LastUse = ++_streamUseCounter; stream.Fill(start, frames, data); } + // Out of memory fails the whole render (store fatal error); any other error makes only this source silent. catch (OutOfMemoryException e) { ReturnArray(data); @@ -568,12 +618,14 @@ internal sealed class AudioClipSource return oldest!; } + /// Marks the source failed (no more chunks are returned or loaded); a resident source also closes its readers now. Caller holds _loadLock. private void Fail() { _isFailed = true; if (IsResident) DisposeStreams(); } + /// Disposes every decode stream (reader and resampler) and clears the list. Caller holds _loadLock. private void DisposeStreams() { foreach (var stream in _streams) @@ -581,6 +633,7 @@ internal sealed class AudioClipSource _streams.Clear(); } + /// Gives a chunk array back to the store: exact arrays (resident) are uncounted, pooled arrays (windowed) are returned to the pool. private void ReturnArray(float[]? data) { if (data == null) return; @@ -609,5 +662,6 @@ internal sealed class AudioClipSource } } + /// ceil(a / b) for a ≥ 0, b > 0. private static long CeilDiv(long a, long b) => (a + b - 1) / b; } diff --git a/Util/Audio/Clip/AudioClipStore.cs b/Util/Audio/Clip/AudioClipStore.cs index 631c41e..c94f0db 100644 --- a/Util/Audio/Clip/AudioClipStore.cs +++ b/Util/Audio/Clip/AudioClipStore.cs @@ -29,18 +29,34 @@ internal sealed class AudioClipStore : IDisposable /// Longest wait for the decoder thread to stop in , in milliseconds. private const int DecoderStopTimeoutMs = 3000; + /// Sources by key; a null value marks a key whose reader could not be created. Also the lock of the source tables. private readonly Dictionary _sources = new(); + + /// Non-null sources in creation order, scanned by the decoder thread. Guarded by the lock. private readonly List _sourceList = new(); + + /// Free windowed chunk arrays ( floats each), guarded by . private readonly Stack _chunkPool = new(); + private readonly object _poolLock = new(); + + /// Monitor the decoder thread pulses after each load (or failure); consumers in wait on it. private readonly object _signal = new(); + + /// Wakes the decoder thread when a cursor, demand or source changes (). private readonly AutoResetEvent _serviceRequested = new(false); private Thread? _decoderThread; private volatile bool _stopRequested; + + /// Bytes of allocated sample arrays (see ), updated with Interlocked. private long _allocatedBytes; + private long _peakBytes; + + /// First fatal error (out of memory, or an exception that stopped the decoder thread); null while none. private volatile Exception? _fatalError; + private volatile bool _disposed; /// Sample rate every source is resampled to, in Hz. @@ -52,9 +68,12 @@ internal sealed class AudioClipStore : IDisposable /// Highest value of since creation. public long PeakBytes => Interlocked.Read(ref _peakBytes); - /// Out-of-memory error raised while loading, or null. + /// + /// Out-of-memory error raised while opening or loading a clip, or the exception that stopped the decoder thread; null while none. + /// public Exception? FatalError => _fatalError; + /// Sample rate in Hz every source is resampled to (positive). Call to begin decoding. public AudioClipStore(int outputRate) { if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate)); @@ -75,6 +94,11 @@ internal sealed class AudioClipStore : IDisposable _decoderThread.Start(); } + /// + /// Decoder thread body: loads one chunk per iteration, or sleeps until (at most + /// ms) when nothing is missing. An unexpected exception ends the thread and is kept as the + /// fatal error, so waiting consumers fail instead of waiting forever. + /// private void DecoderLoop() { try @@ -124,6 +148,7 @@ internal sealed class AudioClipStore : IDisposable return true; } + /// Wakes the decoder thread to re-evaluate what to free and load. Any thread. internal void RequestService() => _serviceRequested.Set(); // ── Sources ── @@ -169,6 +194,8 @@ internal sealed class AudioClipStore : IDisposable Melon.Logger.Error($"[AudioClipStore] Failed to open clip '{key}': {e}"); } + // The reader is created outside the lock (it may decode or copy a whole clip). If another caller registered the + // same key meanwhile, its entry wins and this source is discarded. lock (_sources) { if (_disposed || _sources.TryGetValue(key, out var raced)) @@ -188,6 +215,10 @@ internal sealed class AudioClipStore : IDisposable return source; } + /// + /// True when has been registered; is then the source, or null when its reader + /// could not be created. Any thread. + /// public bool TryGet(string key, out AudioClipSource? source) { lock (_sources) @@ -214,6 +245,8 @@ internal sealed class AudioClipStore : IDisposable if (source.IsFailed || _disposed) return default; if (source.TryAcquire(frame, out var view)) return view; + // Retry under the signal lock so a load completed between the first try and the wait is not missed; + // the timed wait also rechecks cancellation and failure states periodically. lock (_signal) { if (source.TryAcquire(frame, out view)) return view; @@ -224,6 +257,7 @@ internal sealed class AudioClipStore : IDisposable } } + /// Throws wrapping when one is set. private void ThrowIfFatal() { var error = _fatalError; @@ -231,6 +265,7 @@ internal sealed class AudioClipStore : IDisposable throw new InvalidOperationException("Audio clip loading failed: " + error.Message, error); } + /// Wakes every consumer waiting in (after a load, a failure, or on dispose). internal void NotifyChunkLoaded() { lock (_signal) @@ -239,12 +274,14 @@ internal sealed class AudioClipStore : IDisposable } } + /// Logs a load error of one source (the source is already marked failed and plays silence) and wakes waiting consumers. internal void ReportSourceFailure(AudioClipSource source, Exception error) { Melon.Logger.Error($"[AudioClipStore] Failed to load clip '{source.Key}', it will be silent: {error}"); NotifyChunkLoaded(); } + /// Logs an out-of-memory error, keeps it as (first one wins) and wakes waiting consumers. internal void ReportFatal(AudioClipSource source, OutOfMemoryException error) { Melon.Logger.Error($"[AudioClipStore] Out of memory while loading clip '{source.Key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {error}"); @@ -254,6 +291,7 @@ internal sealed class AudioClipStore : IDisposable // ── Arrays ── + /// Takes a pooled array of floats, or allocates (and counts) a new one. Contents are undefined. internal float[] RentChunkArray() { lock (_poolLock) @@ -265,6 +303,7 @@ internal sealed class AudioClipStore : IDisposable return data; } + /// Puts a rented array back into the pool (it stays counted); after dispose it is dropped and uncounted. internal void ReturnChunkArray(float[] data) { if (_disposed) @@ -278,6 +317,7 @@ internal sealed class AudioClipStore : IDisposable } } + /// Allocates a counted array of exactly floats (resident chunks, copied Unity clip blocks). internal float[] AllocateExactArray(int floats) { var data = new float[floats]; @@ -285,8 +325,10 @@ internal sealed class AudioClipStore : IDisposable return data; } + /// Uncounts an array from ; the array itself is left to the garbage collector. internal void FreeExactArray(float[] data) => AddAllocated(-data.LongLength * sizeof(float)); + /// Adds (negative to subtract) to the allocation counter and raises the peak lock-free. private void AddAllocated(long bytes) { var current = Interlocked.Add(ref _allocatedBytes, bytes); @@ -297,6 +339,7 @@ internal sealed class AudioClipStore : IDisposable } } + /// Logs the source count and the current and peak sample memory in MiB. public void LogMemoryUsage() { int sourceCount; diff --git a/Util/Audio/Clip/DecodeStream.cs b/Util/Audio/Clip/DecodeStream.cs index 9ff5bf8..a962fc1 100644 --- a/Util/Audio/Clip/DecodeStream.cs +++ b/Util/Audio/Clip/DecodeStream.cs @@ -18,18 +18,34 @@ internal sealed class DecodeStream : IDisposable /// Input frames read from the reader per resampler call. private const int ReadBlockFrames = 4096; + /// Resampler from the reader's rate to the output rate; null when the rates are equal (samples are copied). private readonly PCMResampler? _resampler; + private readonly int _channels; private readonly int _outputRate; + + /// Input block of frames, interleaved. private readonly float[] _readBuffer; + + /// Scratch output for frames discarded after a restart; allocated on first use. private float[]? _discardBuffer; + + /// True once the reader returned no more input and the resampler is being drained with Flush. private bool _flushing; + + /// True once the input and the resampler tail are exhausted; the stream must restart before producing more. private bool _ended; + /// Input frames per alignment step: input rate / gcd (1 without resampling). private readonly long _inputStep; + + /// Output frames per alignment step: output rate / gcd (1 without resampling). private readonly long _outputStep; + + /// Input frames fed before the requested position on a restart, so the filter history matches a continuous run. private readonly long _prerollInputFrames; + /// Reader of this stream (owned; disposed with the stream). public IPCMReader Reader { get; } /// Output frame the stream continues at, or -1 when the stream is not positioned. @@ -41,6 +57,8 @@ internal sealed class DecodeStream : IDisposable /// Number of times the stream started anew (seek + resampler reset). public int RestartCount { get; private set; } + /// Reader of the clip; ownership passes to the stream. + /// Output sample rate in Hz. public DecodeStream(IPCMReader reader, int outputRate) { Reader = reader; @@ -58,6 +76,7 @@ internal sealed class DecodeStream : IDisposable // History fed before the first kept output frame when a stream starts mid-clip. // The filter spans about filter_size input frames per output frame at ratios up to 1:1 // and filter_size * (input rate / output rate) input frames when downsampling. + // The preroll is four times that span plus one alignment step (the restart position is rounded down to a step). var ratio = Math.Max(1L, (reader.SampleRate + outputRate - 1L) / outputRate); _prerollInputFrames = 4L * _resampler.FilterSize * ratio + _inputStep; } @@ -77,6 +96,7 @@ internal sealed class DecodeStream : IDisposable var channels = _channels; var filled = 0; + // Same rate: output frame = input frame; seek only when the reader is not already at the start. if (_resampler == null) { if (Reader.Position != start) @@ -115,6 +135,7 @@ internal sealed class DecodeStream : IDisposable fixed (float* pIn = _readBuffer) fixed (float* pOut = data) { + // Produce and drop the output frames between the aligned restart position and the requested start. while (discard > 0 && !_ended) { _discardBuffer ??= new float[ReadBlockFrames * channels]; @@ -167,12 +188,14 @@ internal sealed class DecodeStream : IDisposable return 0; } + /// Releases the resampler and the reader. public void Dispose() { _resampler?.Dispose(); Reader.Dispose(); } + /// Greatest common divisor of two positive integers (Euclid). private static int GCD(int a, int b) { while (b != 0) (a, b) = (b, a % b); diff --git a/Util/Audio/Clip/FFmpegPCMReader.cs b/Util/Audio/Clip/FFmpegPCMReader.cs index 3690b6a..504b7de 100644 --- a/Util/Audio/Clip/FFmpegPCMReader.cs +++ b/Util/Audio/Clip/FFmpegPCMReader.cs @@ -31,12 +31,20 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader /// private const double SeekMarginSeconds = 0.25; + // FFmpeg objects owned by this reader, freed by Dispose (null after). private AVFormatContext* _format; private AVCodecContext* _codec; + + /// Sample format converter: decoder format to interleaved float, same rate and layout. private SwrContext* _swr; + private AVPacket* _packet; private AVFrame* _frame; + + /// Index of the decoded audio stream in the container; every other stream is discarded by the demuxer. private readonly int _streamIndex; + + /// Time base of the stream's timestamps (seconds per tick). private readonly AVRational _timeBase; private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary) @@ -49,14 +57,27 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader private long _nextDecodedFrame; // Frame index of the next decoded frame when counting sequentially private bool _syncToTimestamp; // True until the first frame after open/seek sets _nextDecodedFrame from its timestamp + // _flushSent: the null packet (drain request) was sent at the end of the file. + // _endOfStream: the decoder returned EOF, or nothing more after the drain request. private bool _flushSent; private bool _endOfStream; + /// File path the reader was opened with. public string Path { get; } + + /// public int Channels { get; } + + /// public int SampleRate { get; } + + /// public long TotalFrames { get; } + + /// False: the reader seeks within the file. public bool IsForwardOnly => false; + + /// public long Position { get; private set; } /// True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary). @@ -65,6 +86,12 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader /// Number of demuxer seeks performed (for diagnostics). public int SeekCount { get; private set; } + /// + /// Opens , selects the best audio stream and opens its decoder. For MP3 files the whole file is + /// scanned once to count frames (see the class summary). + /// + /// The file cannot be opened, has no decodable audio stream, has no duration + /// (non-MP3), or an FFmpeg setup call fails. Everything allocated so far is freed. public FFmpegPCMReader(string path) { Path = path; @@ -181,6 +208,11 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}"); } + /// + /// Frame count from the stream duration, or from the container duration (AV_TIME_BASE units, microseconds) when the stream + /// has none. + /// + /// Neither duration is known. private long EstimateTotalFrames(AVStream* stream) { if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0) @@ -190,6 +222,13 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader throw new InvalidOperationException($"Audio file '{Path}' has no duration."); } + /// + /// Moves to (0 ~ ). A target within the pending block or up to the seek margin + /// after it is reached by decoding forward; otherwise the demuxer seeks backward (AVSEEK_FLAG_BACKWARD) to target - margin + /// (or to the stream start if that fails) and the decoder is flushed. Decoded samples before the target are discarded by Read. + /// + /// is outside 0 ~ TotalFrames. + /// The demuxer cannot seek. public void Seek(long frame) { if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); @@ -205,6 +244,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader return; } + // Target timestamp in stream ticks; the lead (silent) frames have no packets, so they are subtracted first. var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) @@ -222,6 +262,12 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader Position = frame; } + /// + /// + /// Frames before the first decoded frame (MP3 lead silence) and gaps between decoded timestamps are returned as zeros. + /// Returns fewer frames than requested when decoding ends before . + /// + /// Reading, decoding or sample conversion fails. public int Read(float[] buffer, int frameCount) { var limit = (int)Math.Min(frameCount, TotalFrames - Position); @@ -264,6 +310,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader } /// Decodes the next audio frame into the pending buffer. Returns false at the end of the stream. + /// + /// Send/receive loop: receive a frame; on EAGAIN read and send the next packet of the stream; at the end of the file send a + /// null packet to drain the decoder, then receive until EOF. + /// private bool DecodeNextFrame() { while (!_endOfStream) @@ -315,6 +365,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader return false; } + /// + /// Converts the decoded frame to interleaved float into the pending buffer and assigns its frame position: from its timestamp + /// for the first frame after open/seek (the current position when it has none), sequential afterwards. + /// private void StoreFrame() { var samples = _frame->nb_samples; @@ -347,6 +401,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader _nextDecodedFrame += samples; } + /// Frees every FFmpeg object and closes the file. Safe to call more than once and on a partly constructed reader. public void Dispose() { if (_frame != null) @@ -381,6 +436,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader } } + /// FFmpeg error text for (av_strerror) followed by the numeric code. private static string ErrorString(int error) { var buffer = stackalloc byte[1024]; diff --git a/Util/Audio/Clip/MP3HeaderFrame.cs b/Util/Audio/Clip/MP3HeaderFrame.cs index fe91fb8..deae053 100644 --- a/Util/Audio/Clip/MP3HeaderFrame.cs +++ b/Util/Audio/Clip/MP3HeaderFrame.cs @@ -37,8 +37,13 @@ internal static class MP3HeaderFrame return SilentFrames(file); } + /// + /// for a seekable stream positioned anywhere (the stream is read from offset 0). + /// Only the first valid frame header within after the ID3v2 tags is examined. + /// public static int SilentFrames(Stream stream) { + // Skip consecutive ID3v2 tags: 10-byte header + syncsafe size (+ 10-byte footer when flagged). var tag = new byte[10]; long offset = 0; while (true) @@ -65,6 +70,14 @@ internal static class MP3HeaderFrame return 0; } + /// + /// Parses the 4-byte frame header at . False for a missing sync or a reserved version, layer, bitrate + /// index (15) or sample rate index (3). + /// + /// Bytes to parse. + /// Offset of the first header byte in . + /// Samples per channel in the frame (384, 576 or 1152). + /// Offset of a Xing/Info tag from the header start: 4 + CRC (2) + Layer III side information (0 for Layers I/II). private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset) { samplesPerFrame = 0; @@ -91,6 +104,7 @@ internal static class MP3HeaderFrame return true; } + /// True when the ASCII string is at within the first bytes. private static bool HasTag(byte[] data, int length, int at, string tag) { if (at < 0 || at + tag.Length > length) return false; @@ -99,6 +113,7 @@ internal static class MP3HeaderFrame return true; } + /// Reads until bytes or the end of the stream; returns the bytes read. private static int ReadFully(Stream stream, byte[] buffer, int count) { var total = 0; diff --git a/Util/Audio/Clip/MP3MonoLayoutReader.cs b/Util/Audio/Clip/MP3MonoLayoutReader.cs index b1e35ab..c1c0c8e 100644 --- a/Util/Audio/Clip/MP3MonoLayoutReader.cs +++ b/Util/Audio/Clip/MP3MonoLayoutReader.cs @@ -16,6 +16,8 @@ namespace ADOFAIRenderer.Util.Audio.Clip; internal sealed class MP3MonoLayoutReader : IPCMReader { private readonly IPCMReader _inner; + + /// Reused buffer for inner frames, grown to the largest request. private float[] _innerBuffer = Array.Empty(); /// Mono reader of the MP3 file with the game's MP3 timing (). Owned by this reader. @@ -26,10 +28,19 @@ internal sealed class MP3MonoLayoutReader : IPCMReader Position = inner.Position * 2; } + /// Always 1. public int Channels => 1; + + /// The inner reader's sample rate (the game plays the doubled data at this rate). public int SampleRate => _inner.SampleRate; + + /// Twice the inner frame count. public long TotalFrames => _inner.TotalFrames * 2; + + /// public bool IsForwardOnly => _inner.IsForwardOnly; + + /// public long Position { get; private set; } /// Moves to ; the inner reader moves to the inner frame of the next even position. @@ -39,12 +50,14 @@ internal sealed class MP3MonoLayoutReader : IPCMReader Position = frame; } + /// public int Read(float[] buffer, int frameCount) { var limit = (int)Math.Min(frameCount, TotalFrames - Position); var written = 0; while (written < limit) { + // Odd position: the zero slot after an inner sample (also the first frame after a seek to an odd frame). if ((Position & 1) == 1) { buffer[written++] = 0f; @@ -52,6 +65,7 @@ internal sealed class MP3MonoLayoutReader : IPCMReader continue; } + // Even position: read enough inner frames for the remaining output, then interleave each with a zero. var want = (limit - written + 1) / 2; if (_innerBuffer.Length < want) _innerBuffer = new float[want]; var got = _inner.Read(_innerBuffer, want); @@ -69,5 +83,6 @@ internal sealed class MP3MonoLayoutReader : IPCMReader return written; } + /// Disposes the inner reader. public void Dispose() => _inner.Dispose(); } diff --git a/Util/Audio/Clip/MemoryPCMReader.cs b/Util/Audio/Clip/MemoryPCMReader.cs index 4d62840..16b42a0 100644 --- a/Util/Audio/Clip/MemoryPCMReader.cs +++ b/Util/Audio/Clip/MemoryPCMReader.cs @@ -11,19 +11,38 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// internal sealed class MemoryPCMReader : IPCMReader { + /// Block arrays; an entry is set to null when the block is released. private readonly float[]?[] _blocks; + + /// Called with each released block (for example to uncount its memory in the store); may be null. private readonly Action? _onBlockReleased; + private int _releasedCount; // Blocks [0, _releasedCount) are released + /// public int Channels { get; } + + /// public int SampleRate { get; } + + /// public long TotalFrames { get; } + + /// True: blocks are released as they are read. public bool IsForwardOnly => true; + + /// public long Position { get; private set; } /// Frames per block (all blocks except the last). public int FramesPerBlock { get; } + /// Interleaved blocks in frame order; the reader takes ownership of the arrays (the list is copied). + /// Frames in every block except possibly the last (positive). + /// Channel count (positive). + /// Sample rate in Hz. + /// Frame count; the blocks must hold at least this many frames. + /// Called with each block when it is released; may be null. public MemoryPCMReader(IReadOnlyList blocks, int framesPerBlock, int channels, int sampleRate, long totalFrames, Action? onBlockReleased) { @@ -40,6 +59,8 @@ internal sealed class MemoryPCMReader : IPCMReader TotalFrames = totalFrames; } + /// + /// lies in a block that was already released. public void Seek(long frame) { if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); @@ -48,6 +69,8 @@ internal sealed class MemoryPCMReader : IPCMReader Position = frame; } + /// + /// Blocks whose end the position has passed are released after each copy. public int Read(float[] buffer, int frameCount) { var total = (int)Math.Min(frameCount, TotalFrames - Position); @@ -68,6 +91,7 @@ internal sealed class MemoryPCMReader : IPCMReader return written; } + /// Releases every block with an index below that is not released yet, in order. private void ReleaseBlocksBefore(int blockIndex) { while (_releasedCount < blockIndex && _releasedCount < _blocks.Length) @@ -79,5 +103,6 @@ internal sealed class MemoryPCMReader : IPCMReader } } + /// Releases every remaining block. Safe to call more than once. public void Dispose() => ReleaseBlocksBefore(_blocks.Length); } diff --git a/Util/Audio/Clip/PCMResampler.cs b/Util/Audio/Clip/PCMResampler.cs index c9d4ef0..31040c6 100644 --- a/Util/Audio/Clip/PCMResampler.cs +++ b/Util/Audio/Clip/PCMResampler.cs @@ -11,16 +11,27 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// internal sealed unsafe class PCMResampler : IDisposable { + /// Resampler context owned by this instance; null after . private SwrContext* _swr; + + /// True after until . private bool _flushed; + /// Channel count of input and output (interleaved). public int Channels { get; } + + /// Input sample rate in Hz. public int InputRate { get; } + + /// Output sample rate in Hz. public int OutputRate { get; } /// Value of the SwrContext "filter_size" option (filter length per phase in taps at a 1:1 ratio). public int FilterSize { get; } + /// Creates and initializes a float-to-float resampler with the default channel layout for . + /// A count or rate is not positive. + /// libswresample cannot allocate or initialize the context. public PCMResampler(int channels, int inputRate, int outputRate) { if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); @@ -45,6 +56,8 @@ internal sealed unsafe class PCMResampler : IDisposable throw new InvalidOperationException($"swr_alloc_set_opts2 failed ({ret})."); _swr = swr; + // Falls back to 32, the libswresample default of "filter_size", when the option cannot be read. + // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libswresample/options.c (filter_size) long filterSize; FilterSize = ffmpeg.av_opt_get_int(_swr, "filter_size", 0, &filterSize) >= 0 ? (int)filterSize : 32; @@ -96,6 +109,7 @@ internal sealed unsafe class PCMResampler : IDisposable return ret; } + /// Frees the context. Safe to call more than once. public void Dispose() { if (_swr == null) return; diff --git a/Util/Audio/Clip/UnityClipLoader.cs b/Util/Audio/Clip/UnityClipLoader.cs index c9acbe5..ab9dcc6 100644 --- a/Util/Audio/Clip/UnityClipLoader.cs +++ b/Util/Audio/Clip/UnityClipLoader.cs @@ -25,6 +25,9 @@ internal static class UnityClipLoader if (clip.loadType == AudioClipLoadType.Streaming) throw new NotSupportedException($"AudioClip '{clip.name}' is a streaming clip; GetData cannot read it."); + // Clips with preloadAudioData off start unloaded; LoadAudioData loads the data synchronously, or asynchronously for + // clips with loadInBackground (GetData below then fails while the data is still loading). + // Reference: https://docs.unity3d.com/ScriptReference/AudioClip.LoadAudioData.html if (clip.loadState == AudioDataLoadState.Unloaded) clip.LoadAudioData(); diff --git a/Util/Audio/Filter/Gain.cs b/Util/Audio/Filter/Gain.cs index bafa79c..62384d7 100644 --- a/Util/Audio/Filter/Gain.cs +++ b/Util/Audio/Filter/Gain.cs @@ -2,22 +2,31 @@ using System; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Constant gain: every sample is multiplied by 10^( / 20). Stateless and without latency. +/// internal class Gain : IAudioFilter { + /// Gain in dB (0 = unchanged, -1 ≈ ×0.891, -6 ≈ ×0.501). Read on every sample, so changes apply immediately. public double Level = 0; + /// public void Initialize(int sampleRate, int channels) { } + /// Returns × 10^( / 20); is not used. public double ProcessSample(double sample, int channel) => sample * Math.Pow(10, Level / 20); + /// public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); } + /// public void Reset() { } + /// public double Latency => 0.0; } \ No newline at end of file diff --git a/Util/Audio/Filter/HardClip.cs b/Util/Audio/Filter/HardClip.cs index 2e20f6c..8a361da 100644 --- a/Util/Audio/Filter/HardClip.cs +++ b/Util/Audio/Filter/HardClip.cs @@ -3,15 +3,22 @@ using System.Runtime.CompilerServices; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Hard clipper: limits every sample to [-, +]. Stateless and without latency. +/// In the chain of it catches the peaks the RMS-based limiter lets through before the final gain. +/// internal class HardClip : IAudioFilter { + /// Clip level as a linear amplitude (1.0 = full scale). // ReSharper disable once MemberCanBePrivate.Global // ReSharper disable once FieldCanBeMadeReadOnly.Global // ReSharper disable once ConvertToConstant.Global public double Threshold = 1.0; + /// public void Initialize(int sampleRate, int channels) { } + /// Returns clamped to ±; is not used. [MethodImpl(MethodImplOptions.AggressiveInlining)] public double ProcessSample(double sample, int channel) { @@ -20,13 +27,16 @@ internal class HardClip : IAudioFilter return sample; } + /// public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); } + /// public void Reset() { } + /// public double Latency => 0.0; } \ No newline at end of file diff --git a/Util/Audio/Filter/IAudioFilter.cs b/Util/Audio/Filter/IAudioFilter.cs index ebf9a00..26460bf 100644 --- a/Util/Audio/Filter/IAudioFilter.cs +++ b/Util/Audio/Filter/IAudioFilter.cs @@ -2,14 +2,34 @@ using System; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Stage of the output filter chain of . Samples are doubles at nominal full scale ±1.0. +/// A filter keeps its own state between calls, so consecutive blocks form one continuous signal. +/// Used by one thread at a time (the render thread during rendering). +/// internal interface IAudioFilter { + /// + /// Sets the sample rate (Hz) and interleaved channel count of the signal and allocates per-channel state. + /// Called once before the first sample; the state is cleared as by . + /// void Initialize(int sampleRate, int channels); + /// Processes one sample of channel (0 ~ channels - 1) and returns the output sample. double ProcessSample(double sample, int channel); + + /// + /// Processes in place. The buffer holds interleaved frames starting at channel 0 + /// (length = frames * channels of ). + /// void ProcessBlock(Span buffer); + /// Clears the internal state (envelopes, delay lines) without changing the settings. void Reset(); + /// + /// Delay of the output relative to the input, in seconds (0 for filters without look-ahead). + /// sums the latencies of the chain and compensates them. + /// double Latency { get; } } \ No newline at end of file diff --git a/Util/Audio/Filter/RMSCompressor.cs b/Util/Audio/Filter/RMSCompressor.cs index ea07d27..3e82f85 100644 --- a/Util/Audio/Filter/RMSCompressor.cs +++ b/Util/Audio/Filter/RMSCompressor.cs @@ -3,6 +3,24 @@ using System.Runtime.CompilerServices; namespace ADOFAIRenderer.Util.Audio.Filter; +/// +/// Feed-forward RMS compressor with soft knee, attack/release ballistics in the dB domain and optional look-ahead. +/// Each channel is detected and gain-reduced independently (no stereo link). +/// +/// Per sample of channel c: +/// 1. Mean square: ms[n] = r * ms[n-1] + (1 - r) * x[n]², r = exp(-1 / (sampleRate * )); +/// level L = 10 * log10(ms + 1e-12) dB (the 1e-12 floor keeps silence finite, about -120 dB). +/// 2. Static curve (gain reduction G ≤ 0 dB, T = , R = , W = ): +/// G = (1/R - 1) * (L - T) above T + W/2; G = (1/R - 1) * (L - T + W/2)² / (2W) inside the knee; 0 below T - W/2. +/// With W = 0 the knee is hard (G = (1/R - 1) * (L - T) above T). R = infinity gives G = T - L (a limiter at T). +/// 3. Ballistics: g[n] = k * g[n-1] + (1 - k) * G, with k = attack coefficient while the reduction grows (G < g[n-1]) and +/// release coefficient otherwise; coefficient = exp(-1 / (sampleRate * time)) (one-pole smoothing, time constant in seconds). +/// 4. Output = x[n - lookAhead] * 10^(g[n] / 20): the gain computed from the current input is applied to the input delayed by +/// , so the reduction starts before the peak reaches the output. +/// +/// Properties may be set before (as in an object initializer); Initialize recomputes every +/// coefficient for the actual sample rate and reallocates the delay lines. Not thread-safe. +/// // ReSharper disable once InconsistentNaming internal class RMSCompressor : IAudioFilter { @@ -13,11 +31,17 @@ internal class RMSCompressor : IAudioFilter private double _kneeDb = 5; // in dB private double _rmsWindowTime = 0.030; // in seconds private double _lookAheadTime; // in seconds + + /// Look-ahead delay in samples per channel: round(sampleRate * ); 0 disables the delay line. private int _lookAheadSamples; + /// Sample rate in Hz; 44100 until sets the actual rate. private int _sampleRate = 44100; + + /// Interleaved channel count; 2 until sets the actual count. private int _channels = 2; + // One-pole smoothing coefficients (0 ~ 1) of attack, release and the mean-square detector: exp(-1 / (sampleRate * time)). private double _attackCoef; private double _releaseCoef; private double _rmsCoef; @@ -28,6 +52,10 @@ internal class RMSCompressor : IAudioFilter private double[][] _delayBuffers = Array.Empty(); // circular delay buffers private int[] _delayWriteIndices = Array.Empty(); + /// + /// Look-ahead delay in seconds, clamped to 0 ~ 1. Setting it reallocates (and clears) the delay lines. + /// Also reported as . + /// public double LookAheadTime { get => _lookAheadTime; @@ -38,20 +66,24 @@ internal class RMSCompressor : IAudioFilter } } + /// Output delay in seconds: . public double Latency => _lookAheadTime; + /// Threshold in dB relative to full scale (RMS level of a constant ±1.0 signal = 0 dB). public double ThresholdDb { get => _thresholdDb; set => _thresholdDb = value; } + /// Compression ratio (input dB above threshold : output dB above threshold), at least 1. Infinity limits at the threshold. public double Ratio { get => _ratio; set => _ratio = Math.Max(1, value); } + /// Attack time constant in seconds, at least 0.0001 (smaller values, including 0, are raised to it). public double AttackTime { get => _attackTime; @@ -62,6 +94,7 @@ internal class RMSCompressor : IAudioFilter } } + /// Release time constant in seconds, at least 0.0001. public double ReleaseTime { get => _releaseTime; @@ -72,12 +105,14 @@ internal class RMSCompressor : IAudioFilter } } + /// Total knee width in dB, centered on the threshold, at least 0 (0 = hard knee). public double KneeDb { get => _kneeDb; set => _kneeDb = Math.Max(0, value); } + /// Time constant of the mean-square detector in seconds, at least 0.0001. public double RmsWindowTime { get => _rmsWindowTime; @@ -88,6 +123,7 @@ internal class RMSCompressor : IAudioFilter } } + /// public void Initialize(int sampleRate, int channels) { _sampleRate = sampleRate; @@ -110,6 +146,7 @@ internal class RMSCompressor : IAudioFilter ReallocateDelayBuffers(); } + /// Allocates one zeroed circular buffer of samples per channel. private void ReallocateDelayBuffers() { if (_channels <= 0) return; @@ -139,6 +176,10 @@ internal class RMSCompressor : IAudioFilter _rmsCoef = Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); } + /// + /// Processes one sample of (see the class summary). A channel outside 0 ~ channels - 1 is + /// returned unchanged and does not touch any state. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public double ProcessSample(double sample, int channel) { @@ -162,10 +203,12 @@ internal class RMSCompressor : IAudioFilter { if (rmsDb > _thresholdDb + _kneeDb * 0.5) { + // Above the knee: full ratio. gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); } else if (rmsDb >= _thresholdDb - _kneeDb * 0.5) { + // Inside the knee: quadratic transition from 0 dB at T - W/2 to the full-ratio line at T + W/2. double diff = rmsDb - _thresholdDb + _kneeDb * 0.5; gStatic = (1 / _ratio - 1) * diff * diff / (2 * _kneeDb); } @@ -194,6 +237,7 @@ internal class RMSCompressor : IAudioFilter if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0) { + // Circular delay line: read the sample written _lookAheadSamples calls ago, then overwrite it with the current one. var buffer = _delayBuffers[channel]; int writeIdx = _delayWriteIndices[channel]; double delayedSample = buffer[writeIdx]; @@ -205,6 +249,7 @@ internal class RMSCompressor : IAudioFilter return sample * linearGain; } + /// public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) @@ -214,6 +259,7 @@ internal class RMSCompressor : IAudioFilter } } + /// Clears the detector, the gain envelope and the delay lines (the delay outputs zeros for the next look-ahead samples). public void Reset() { Array.Clear(_rmsState, 0, _rmsState.Length); diff --git a/Util/Audio/Mixer/Voice.cs b/Util/Audio/Mixer/Voice.cs index 123bf8b..faf6a9f 100644 --- a/Util/Audio/Mixer/Voice.cs +++ b/Util/Audio/Mixer/Voice.cs @@ -10,7 +10,7 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// /// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start, /// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order. -/// Render thread: applies each change at sample round(time * OutputRate). +/// Render thread: applies each change at sample floor(time * OutputRate) (). /// /// Playback model (output samples n, clip positions in frames of the source at the output sample rate): /// - The voice is silent before its start sample and from its end sample on. @@ -21,20 +21,37 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// internal sealed class Voice { + /// Kind of a queued ; the meaning of depends on it. internal enum ChangeKind { + /// Value: playback rate, at least 0. Pitch, + + /// Value: linear gain. Volume, + + /// Value: 1 = loop on, 0 = loop off. Loop, + + /// Value: clip position in seconds, at least 0. Position, + + /// Value: end time in seconds relative to the render start. EndTime, + + /// Value: unused (0). Ends playback at the sample the change is applied at. Stop } + /// One timed change of a voice, queued by the main thread and applied by the render thread. internal readonly struct Change { public readonly ChangeKind Kind; + + /// Output sample at which the change takes effect ( of the change time). public readonly long Sample; + + /// Argument of the change (see ). public readonly double Value; public Change(ChangeKind kind, long sample, double value) @@ -45,6 +62,7 @@ internal sealed class Voice } } + /// Mixer that owns this voice; the Set methods queue their changes on it. private readonly VoiceMixer _mixer; // ── Render thread state ── @@ -58,11 +76,22 @@ internal sealed class Voice /// Output sample at which playback stops (exclusive); long.MaxValue when not set. internal long EndSample = long.MaxValue; + /// Output sample of the current anchor (see the class summary). internal long AnchorSample; + + /// Clip position at , in frames at the output sample rate (may be fractional). internal double AnchorPosition; + + /// Clip frames advanced per output sample (1 = original speed), at least 0. internal double Pitch; + + /// Linear gain applied to the clip samples. internal double Volume; + + /// True while the clip repeats at . internal bool Loop; + + /// Set by the render thread when playback has ended; the voice is then released and removed from the mixer. internal bool Finished; /// @@ -74,7 +103,10 @@ internal sealed class Voice /// Loop length in frames at the output sample rate (source duration * output rate). internal double Period; + /// Read position registered on so the decoder thread keeps the chunks ahead loaded; null without a usable source. internal AudioCursor? Cursor; + + /// Chunk of currently held for reading (invalid when none). Render thread only. internal ChunkView View; internal Voice(VoiceMixer mixer, AudioClipSource? source, string name) @@ -128,6 +160,9 @@ internal sealed class Voice /// internal void Apply(in Change change, long sample, int outputRate) { + // Anchor changes made before the start take effect at the start sample (playback starts from the new state). + // EndTime uses the change's own value; Stop ends at the sample it is applied at, so a Stop before the start + // ends the voice before it plays. var at = Math.Max(sample, StartSample); switch (change.Kind) { @@ -155,6 +190,7 @@ internal sealed class Voice } } + /// Moves the anchor to , keeping the (wrapped) position there, before a pitch or loop change. private void Reanchor(long sample) { AnchorPosition = WrappedPositionAt(sample); diff --git a/Util/Audio/Mixer/VoiceMixer.cs b/Util/Audio/Mixer/VoiceMixer.cs index b522d4b..871ef69 100644 --- a/Util/Audio/Mixer/VoiceMixer.cs +++ b/Util/Audio/Mixer/VoiceMixer.cs @@ -9,7 +9,7 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// Mixes every into interleaved double samples at the output sample rate and channel count. /// /// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed -/// to and the Voice Set methods). A time t maps to sample round(t * OutputRate). +/// to and the Voice Set methods). A time t maps to sample floor(t * OutputRate) (). /// /// Threads: /// - Main thread: and the Voice Set methods queue commands under a lock. Commands must be queued in @@ -25,16 +25,23 @@ namespace ADOFAIRenderer.Util.Audio.Mixer; /// internal sealed class VoiceMixer : IDisposable { + /// Kind of a queued main-thread command. private enum CommandKind { + /// Adds a new voice to the mixed set (created by ). Play, + + /// Appends a to the voice's pending queue. Change } + /// Command queued by the main thread and taken by the render thread at the start of . private readonly struct Command { public readonly CommandKind Kind; public readonly Voice Voice; + + /// The change for ; default for . public readonly Voice.Change Change; public Command(CommandKind kind, Voice voice, Voice.Change change) @@ -48,6 +55,8 @@ internal sealed class VoiceMixer : IDisposable private readonly AudioClipStore _store; private readonly CancellationToken _cancellationToken; + // Double-buffered command lists: the main thread appends to _commands under _commandLock; the render thread swaps the + // two lists under the lock and processes _takenCommands outside it, then clears it for the next swap. private readonly object _commandLock = new(); private List _commands = new(); private List _takenCommands = new(); @@ -68,8 +77,10 @@ internal sealed class VoiceMixer : IDisposable _cancellationToken = cancellationToken; } + /// Output sample rate in Hz (the store's ). public int OutputRate { get; } + /// Interleaved channel count of the mixed output (1 or more). public int OutputChannels { get; } /// Voices currently mixed (started or waiting to start). Render thread only. @@ -113,6 +124,10 @@ internal sealed class VoiceMixer : IDisposable return voice; } + /// + /// Queues a change of that takes effect at sample (). + /// Called by the Voice Set methods on the main thread. + /// internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value) { var change = new Voice.Change(kind, ToSample(time, OutputRate), value); @@ -138,10 +153,13 @@ internal sealed class VoiceMixer : IDisposable TakeCommands(); + // Iterated backwards so finished voices can be removed in place. for (var i = _voices.Count - 1; i >= 0; i--) { var voice = _voices[i]; var sample = firstSample; + // Split the block into segments at the samples of pending changes: each segment is mixed with constant + // voice state, and a change is applied when the segment reaches its sample (or at once when its sample is past). while (!voice.Finished) { var segmentEnd = endSample; @@ -162,6 +180,8 @@ internal sealed class VoiceMixer : IDisposable if (sample >= endSample) break; } + // The end sample is inside or before this block: nothing more will be mixed. Changes still pending for this voice + // (after the end) are dropped with it. if (!voice.Finished && endSample >= voice.EndSample) voice.Finished = true; if (voice.Finished) @@ -172,6 +192,10 @@ internal sealed class VoiceMixer : IDisposable } } + /// + /// Takes every queued command: Play adds the voice (with a cursor at frame 0 when its source is usable), Change appends to + /// the voice's pending queue (dropped for finished voices). Render thread only. + /// private void TakeCommands() { lock (_commandLock) @@ -203,6 +227,7 @@ internal sealed class VoiceMixer : IDisposable var end = Math.Min(to, voice.EndSample); if (start >= end) return; + // A voice whose source is missing or failed is silent; it finishes at its first segment inside [start, end). var source = voice.Source; if (source == null || source.IsFailed) { @@ -222,6 +247,7 @@ internal sealed class VoiceMixer : IDisposable var volume = voice.Volume; var offset = (int)(start - from) * channels; + // Move the cursor to the segment start so the decoder thread keeps the chunks ahead of it loaded. UpdateCursor(voice, (long)voice.WrappedPositionAt(start)); for (var n = start; n < end; n++, offset += channels) @@ -234,6 +260,8 @@ internal sealed class VoiceMixer : IDisposable voice.Finished = true; return; } + // Looping: a wrapped position is below Period, and Period (Duration * rate) is at most TotalFrames + // (TotalFrames rounds up), so this is a guard that reads the last frame. position = totalFrames - 1; } @@ -243,6 +271,7 @@ internal sealed class VoiceMixer : IDisposable if (!EnsureView(voice, index)) return; var view = voice.View; + // Integer position (pitch 1 with an integer anchor): copy the stored sample. if (fraction == 0.0) { for (var c = 0; c < channels; c++) @@ -250,6 +279,7 @@ internal sealed class VoiceMixer : IDisposable continue; } + // Linear interpolation; the next frame may lie in the following chunk, at the loop start or past the end. var nextIndex = index + 1; var nextInView = view.Contains(nextIndex); for (var c = 0; c < channels; c++) @@ -303,6 +333,7 @@ internal sealed class VoiceMixer : IDisposable return false; } + /// Moves the voice's cursor to and copies the voice's loop flag (writes only on change). private static void UpdateCursor(Voice voice, long frame) { var cursor = voice.Cursor; @@ -311,6 +342,7 @@ internal sealed class VoiceMixer : IDisposable if (cursor.Position != frame) cursor.Position = frame; } + /// Releases the voice's held chunk and cursor and drops its pending changes. Render thread only. private static void ReleaseVoice(Voice voice) { var source = voice.Source; diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 3f5ef28..c32191f 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -26,7 +26,7 @@ namespace ADOFAIRenderer.Util.Audio; /// internal sealed class OfflineAudioMixer : IDisposable { - /// Gain applied to every clip. + /// Linear gain applied to every clip (song and effects), before the filter chain (0.5 ≈ -6 dB headroom). private const double ClipVolume = 0.5; /// Additional gain of effect sounds (registered with ). @@ -41,21 +41,48 @@ internal sealed class OfflineAudioMixer : IDisposable /// Longest wait for a commit before checking for cancellation again, in milliseconds. private const int CommitWaitMs = 50; + /// Output channel count (interleaved). private readonly int _channels; + + /// Output sample rate in Hz. private readonly int _sampleRate; + private readonly AudioClipStore _store; private readonly VoiceMixer _mixer; + + /// Canceled by /; ends commit waits and chunk waits on the render thread. private readonly CancellationTokenSource _cancellation = new(); + + /// Filter chain applied in list order to every mixed block. private readonly List _filters = new(); + + /// Sum of the filter latencies in output samples: round(sum of * sample rate). private readonly int _latencySamples; // Main thread. + + /// + /// Mapping of game AudioSources to the state of the effect sound last started on them. Weak keys: a destroyed source + /// does not keep its state alive. Replaced by a new table to drop every mapping at once. + /// private ConditionalWeakTable _sourceStates = new(); + + /// Effect voices started since the last (finished ones are pruned periodically). private readonly List _effectVoices = new(); + + /// Count of at which the next prune runs. private int _effectVoicePruneAt = EffectVoicePruneCount; + + /// Song source, or null when there is no song or it could not be loaded. private AudioClipSource? _bgmSource; + + /// audioLib key (AudioClip.name) of the song. private string _bgmKey = ""; + + /// Current song voice, created by the first and replaced when the start time changes. private Voice? _bgmVoice; + + // Values last sent to the song voice (start in seconds relative to the render start); NaN before the first update. private double _bgmStart = double.NaN; private double _bgmPitch = double.NaN; private double _bgmVolume = double.NaN; @@ -66,11 +93,26 @@ internal sealed class OfflineAudioMixer : IDisposable // Render thread (and Dispose, under _renderLock). private readonly object _renderLock = new(); + + /// Index of the next output sample writes (frames per channel since the render start). private long _nextOutputSample; + + /// True once the filters have been fed samples [0, latency). private bool _primed; + + /// Reused interleaved mix buffer, grown to the largest block requested. private double[] _mixBuffer = Array.Empty(); + private bool _disposed; + /// + /// Creates the clip store (and starts its decoder thread), the voice mixer and the filter chain. + /// Filter settings: RMS compressor (threshold -10 dB, ratio 2, attack/release 0.1 s, knee 10 dB, RMS window 0.02 s), + /// limiter (RMS compressor with threshold 0 dB, infinite ratio, attack 0 raised to the 0.0001 s minimum, release 0.05 s, + /// hard knee, RMS window 0.02 s, look-ahead 0.01 s), hard clip at ±1.0, then -1 dB gain. + /// + /// Output channel count (interleaved). + /// Output sample rate in Hz; every clip is resampled to it. public OfflineAudioMixer(int targetChannels, int targetSampleRate) { _channels = targetChannels; @@ -96,6 +138,7 @@ internal sealed class OfflineAudioMixer : IDisposable _filters.Add(new HardClip()); _filters.Add(new Gain { Level = -1 }); + // Total delay of the chain; only the limiter's look-ahead contributes (0.01 s). double latency = 0; foreach (var filter in _filters) { @@ -148,6 +191,8 @@ internal sealed class OfflineAudioMixer : IDisposable double pitch = conductor.song.pitch; var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0; var start = conductor.dspTimeSong + countdown - dspTimeBase; + // levelData.volume is a percentage (100 = unchanged; scrConductor sets song.volume = volume * 0.01 the same way). + // Without a scnGame instance the level volume is 1.0. var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0; var volume = levelVolume * ClipVolume; @@ -181,6 +226,11 @@ internal sealed class OfflineAudioMixer : IDisposable GetOrAddSource(clipName); } + /// + /// Source of the game clip (audioLib key): an existing source, or one opened through + /// with the clip from AudioManager.FindOrLoadAudioClip. Null for an empty name or an + /// unreadable clip (the key is then remembered as missing by the store). Main thread only. + /// private AudioClipSource? GetOrAddSource(string clipName) { if (string.IsNullOrEmpty(clipName)) return null; @@ -193,6 +243,14 @@ internal sealed class OfflineAudioMixer : IDisposable /// Starts an effect sound at . An of 0 or more makes it loop until then. /// (when given) is mapped to the returned state, which reports and applies later changes through the AudioSource. /// + /// Game AudioSource playing the sound, or null to map it later with . + /// audioLib key of the clip. A clip that cannot be loaded gives a silent voice. + /// Scheduled start, seconds relative to the render start (a start before starts at now). + /// Loop end, seconds relative to the render start; negative for a sound that plays once. + /// Game volume (linear); multiplied by the clip and effect gains. + /// Playback rate at the start. + /// Current render time in seconds. + /// State of the new sound, also used by the AudioSource patches. public SourceState RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) { var clipSource = GetOrAddSource(clipName); @@ -200,6 +258,7 @@ internal sealed class OfflineAudioMixer : IDisposable var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now); if (loop) voice.SetEndTime(endTime, now); + // Amortized pruning: the list is scanned only when it has doubled since the last prune. if (_effectVoices.Count >= _effectVoicePruneAt) { _effectVoices.RemoveAll(v => v.IsFinished); @@ -269,7 +328,10 @@ internal sealed class OfflineAudioMixer : IDisposable } } - /// Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException. + /// + /// Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException. + /// May be called from any thread, more than once. + /// public void Cancel() { _cancellation.Cancel(); @@ -280,7 +342,10 @@ internal sealed class OfflineAudioMixer : IDisposable /// /// Writes the next output frames (interleaved float, continuing from the previous call). + /// Blocks until the main thread has committed the voice samples it needs (, ). /// + /// Destination; must hold at least samplesNeeded * channels floats. Values beyond that are untouched. + /// Frames (samples per channel) to write. /// Chunk loading failed fatally (out of memory or the decoder thread stopped). /// The render was canceled (). public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded) @@ -289,6 +354,8 @@ internal sealed class OfflineAudioMixer : IDisposable { if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer)); + // First call: run voice samples [0, latency) through the filters and discard the output, which is the + // zero-filled delay lines. Afterwards output sample n is the filtered voice sample n. if (!_primed) { _primed = true; @@ -323,6 +390,11 @@ internal sealed class OfflineAudioMixer : IDisposable return block; } + /// + /// Waits until the committed sample reaches (exclusive end of the block to mix), checking for + /// cancellation every ms. + /// + /// The render was canceled. private void WaitForCommit(long endSample) { lock (_commitLock) diff --git a/Util/Audio/SourceState.cs b/Util/Audio/SourceState.cs index 161c3ad..a52e67e 100644 --- a/Util/Audio/SourceState.cs +++ b/Util/Audio/SourceState.cs @@ -21,15 +21,33 @@ namespace ADOFAIRenderer.Util.Audio; /// internal sealed class SourceState { + /// Start of playback, seconds relative to the render start. private readonly double _start; + + /// Render time of the last position or pitch change (at least ), seconds. private double _anchorTime; + + /// Unwrapped clip position at , seconds into the clip. private double _anchorPosition; + + /// Last finite pitch set (reported as is; negative values play as 0). private double _pitch; + private readonly bool _loop; + + /// Scheduled end time, seconds relative to the render start; infinity when not set. private double _endTime = double.PositiveInfinity; + + /// True after ; playback does not resume. private bool _stopped; + + /// timeSamples reported while not playing: the last value set through time/timeSamples, 0 after Stop. private int _offsetSamples; + + /// Clip length in seconds. private double _length; + + /// Clip sample rate in Hz (at least 1), used to convert between time and timeSamples. private int _frequency; /// Voice that plays the sound. @@ -51,6 +69,7 @@ internal sealed class SourceState _frequency = Math.Max(1, frequency); } + /// Voice of the that plays the sound; changes made while playing are queued on it. public Voice Voice { get; } /// Takes the clip length and sample rate from the clip of the game AudioSource. @@ -69,26 +88,39 @@ internal sealed class SourceState Voice.SetEndTime(endTime, now); } + /// AudioSource.isPlaying at : not stopped and before both the scheduled end and the natural end. public bool IsPlaying(double now) => !_stopped && now < Math.Min(_endTime, NaturalEnd()); + /// AudioSource.time at : clip position in seconds while playing, otherwise 0. public float GetTime(double now) => IsPlaying(now) ? (float)PositionAt(now) : 0f; + /// AudioSource.timeSamples at : floor(position * frequency) while playing, otherwise the stored offset. public int GetTimeSamples(double now) => IsPlaying(now) ? (int)Math.Floor(PositionAt(now) * _frequency) : _offsetSamples; + /// AudioSource.pitch: the last finite value set. public float GetPitch() => (float)_pitch; + /// + /// AudioSource.time setter: stores floor(seconds * frequency) as the offset and, while playing, moves the position to + /// (negative values become 0) from (or from the start when not started yet). + /// public void SetTime(float seconds, double now) { _offsetSamples = (int)Math.Floor((double)seconds * _frequency); Seek(seconds, now); } + /// AudioSource.timeSamples setter: as with samples / frequency seconds. public void SetTimeSamples(int samples, double now) { _offsetSamples = samples; Seek((double)samples / _frequency, now); } + /// + /// AudioSource.pitch setter: ignores NaN and infinity. While playing, the position up to is kept + /// (new anchor) and the voice changes rate at ; otherwise only the reported value changes. + /// public void SetPitch(float pitch, double now) { if (float.IsNaN(pitch) || float.IsInfinity(pitch)) return; @@ -98,6 +130,7 @@ internal sealed class SourceState if (playing) Voice.SetPitch(pitch, now); } + /// AudioSource.Stop(): stops the voice at when playing, marks the state stopped and clears the offset. public void Stop(double now) { if (IsPlaying(now)) Voice.Stop(now); @@ -105,8 +138,13 @@ internal sealed class SourceState _offsetSamples = 0; } + /// Rate used for playback: with negative values treated as 0 (as in ). private double PlaybackPitch => Math.Max(0.0, _pitch); + /// + /// While playing, sets the clip position to (at least 0) from max(now, start) and queues the same + /// position change on the voice. + /// private void Seek(double seconds, double now) { if (!IsPlaying(now)) return; @@ -116,6 +154,7 @@ internal sealed class SourceState Voice.SetPosition(position, now); } + /// Moves the anchor to max(now, start), keeping the current position, before a pitch change. private void Reanchor(double now) { var time = Math.Max(now, _start);