From 0a12c12ff584f65540c66b3de47349f98095586251aa3f91016eaeeda0b35e9b Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 00:30:02 +0900 Subject: [PATCH 01/14] Add Chunked Audio Clip Store --- Util/Audio/Clip/AudioClipSource.cs | 608 +++++++++++++++++++++++++++++ Util/Audio/Clip/AudioClipStore.cs | 337 ++++++++++++++++ Util/Audio/Clip/DecodeStream.cs | 181 +++++++++ Util/Audio/Clip/IPCMReader.cs | 42 ++ Util/Audio/Clip/MemoryPCMReader.cs | 83 ++++ Util/Audio/Clip/PCMResampler.cs | 106 +++++ Util/Audio/Clip/UnityClipLoader.cs | 55 +++ 7 files changed, 1412 insertions(+) create mode 100644 Util/Audio/Clip/AudioClipSource.cs create mode 100644 Util/Audio/Clip/AudioClipStore.cs create mode 100644 Util/Audio/Clip/DecodeStream.cs create mode 100644 Util/Audio/Clip/IPCMReader.cs create mode 100644 Util/Audio/Clip/MemoryPCMReader.cs create mode 100644 Util/Audio/Clip/PCMResampler.cs create mode 100644 Util/Audio/Clip/UnityClipLoader.cs diff --git a/Util/Audio/Clip/AudioClipSource.cs b/Util/Audio/Clip/AudioClipSource.cs new file mode 100644 index 0000000..9cb11ae --- /dev/null +++ b/Util/Audio/Clip/AudioClipSource.cs @@ -0,0 +1,608 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using MelonLoader; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Read position registered on an by one consumer (a playing voice or the spectrum generator). +/// For a windowed source, the chunk containing and the following chunks +/// ( chunks in total) are kept loaded. +/// +internal sealed class AudioCursor +{ + private readonly AudioClipSource _source; + private long _position; + private volatile bool _loop; + + internal AudioCursor(AudioClipSource source, long position, bool loop) + { + _source = source; + _position = position; + _loop = loop; + } + + /// + /// Frame in the source at the output sample rate (0 ~ TotalFrames - 1). + /// Written by the consumer that reads the source; read by the decoder thread. + /// Moving into another chunk wakes the decoder thread. + /// + public long Position + { + get => Interlocked.Read(ref _position); + set + { + var previous = Interlocked.Exchange(ref _position, value); + if (previous / _source.FramesPerChunk != value / _source.FramesPerChunk) + _source.OnCursorMoved(); + } + } + + /// When true, the chunk after the last chunk is chunk 0. + public bool Loop + { + get => _loop; + set + { + _loop = value; + _source.OnCursorMoved(); + } + } +} + +/// +/// One loaded chunk of an , held by a consumer between +/// and . +/// The chunk is not freed while held. +/// +internal readonly struct ChunkView +{ + /// Interleaved samples. Frame f of the source is at (f - StartFrame) * Channels. + public readonly float[] Data; + + /// Source frame of the first frame in . + public readonly long StartFrame; + + /// Valid frames in this chunk. Frames past the end of the source are not included. + public readonly int FrameCount; + + /// Channel count of the stored samples (same as the source clip). + public readonly int Channels; + + public readonly int ChunkIndex; + + internal ChunkView(float[] data, long startFrame, int frameCount, int channels, int chunkIndex) + { + Data = data; + StartFrame = startFrame; + FrameCount = frameCount; + Channels = channels; + ChunkIndex = chunkIndex; + } + + public bool IsValid => Data != null; + + public long EndFrame => StartFrame + FrameCount; + + public bool Contains(long frame) => frame >= StartFrame && frame < StartFrame + FrameCount; + + /// + /// Sample of mapped to output channel of . + /// Mapping: mono output averages all source channels; a mono source is copied to every output channel; + /// otherwise output channel c takes source channel c, and output channels beyond the source take source channel 0. + /// must be inside this chunk. + /// + public float GetSample(long frame, int outChannel, int outChannels) + { + var baseIndex = (int)(frame - StartFrame) * Channels; + if (Channels == 1) + return Data[baseIndex]; + + if (outChannels == 1) + { + var sum = 0f; + for (var c = 0; c < Channels; c++) + sum += Data[baseIndex + c]; + return sum / Channels; + } + + return Data[baseIndex + (outChannel < Channels ? outChannel : 0)]; + } +} + +/// +/// How an keeps its chunks. +/// +internal enum ChunkPolicy +{ + /// + /// Only the chunks required by registered cursors (current chunk and the next SlotCount - 1 chunks per cursor), + /// chunks requested by a waiting consumer, and chunks currently held are kept. Other chunks are freed and reloaded on demand. + /// Used for readers that can seek (files decoded with FFmpeg) when the clip is longer than one chunk. + /// + Windowed, + + /// + /// All chunks are loaded once, in order from chunk 0, starting when the source is created, and kept until the store is disposed. + /// Used for forward-only readers (clip data copied from Unity, released as it is consumed) and for clips of one chunk or less. + /// + Resident +} + +/// +/// Decoded audio of one clip, resampled to the output sample rate and stored in fixed-size chunks. +/// Channel count stays as in the clip. +/// +/// Chunks: chunk k covers output frames [k * FramesPerChunk, (k + 1) * FramesPerChunk), where +/// = / Channels (rounded down; a frame never spans two chunks). +/// Windowed chunks use pooled arrays of ChunkFloats floats; resident chunks use arrays of their exact length. +/// See for which chunks are kept. +/// +/// Threads: cursors and TryAcquire/Release may be used from any thread. +/// Loading and freeing run only on the store's decoder thread (), through +/// , and . +/// +internal sealed class AudioClipSource +{ + /// Floats per chunk array: 2^22 floats = 16 MiB. + public const int ChunkFloats = 1 << 22; + + /// Chunks kept per cursor of a windowed source: the chunk containing the cursor and the next SlotCount - 1 chunks. + public const int SlotCount = 3; + + /// Load priority of a chunk a consumer is waiting for (lowest value is loaded first). + internal const int PriorityDemanded = 0; + + /// Load priority of slot s of a cursor window is PriorityWindow + s. + internal const int PriorityWindow = 1; + + /// Load priority of resident chunks nobody is waiting for. + internal const int PriorityResident = PriorityWindow + SlotCount; + + /// + /// Most decode streams (reader + resampler) a windowed source opens. A load continues the stream positioned at the chunk start, + /// otherwise opens another stream up to this count, otherwise restarts the least recently used stream. + /// Each stream holds one open reader (file handle and decoder state, measured 0.3 ~ 2.7 MiB for ogg/flac/mp3 with the clip harness). + /// + public const int MaxDecodeStreams = 4; + + /// Decode stream limit of this source ( unless changed for tests). + internal int DecodeStreamLimit { get; set; } = MaxDecodeStreams; + + private readonly AudioClipStore _store; + private readonly IPCMReader _firstReader; + private readonly Func? _openReader; + + // Guarded by _sync: chunk table, hold counts, cursors, demand set, loaded set. + private readonly object _sync = new(); + private readonly float[]?[] _chunks; + private readonly int[] _holdCounts; + private readonly List _cursors = new(); + private readonly HashSet _demanded = new(); + private readonly HashSet _loaded = new(); + private int _residentLoadedCount; // Resident: chunks [0, _residentLoadedCount) are loaded + + // Guarded by _loadLock: decode streams. + private readonly object _loadLock = new(); + private readonly List _streams = new(); + private long _streamUseCounter; + private bool _openFailed; + + private volatile bool _isFailed; + + public string Key { get; } + + /// Channel count of the stored samples (same as the clip). + public int Channels { get; } + + /// Sample rate of the stored samples in Hz (the output sample rate). + public int Frequency { get; } + + /// Frame count at : ceil(clip frames * Frequency / clip rate). + public long TotalFrames { get; } + + /// Clip length in seconds: clip frames / clip rate. + public double Duration { get; } + + public int FramesPerChunk { get; } + + public int ChunkCount { get; } + + public ChunkPolicy Policy { get; } + + public bool IsResident => Policy == ChunkPolicy.Resident; + + /// True after loading failed. A failed source returns no chunks. + public bool IsFailed => _isFailed; + + /// True when every chunk of a resident source is loaded. + public bool IsFullyLoaded + { + get + { + lock (_sync) return _loaded.Count == ChunkCount; + } + } + + /// Number of chunk loads performed (for diagnostics). + public int LoadCount { get; private set; } + + /// Decode streams opened so far (for diagnostics). + public int StreamCount + { + get + { + lock (_loadLock) return _streams.Count; + } + } + + /// Store that owns this source. + /// Clip name or path the source is registered under. + /// First reader; also defines the clip format. + /// Opens another reader of the same clip for an additional decode stream, or null when not possible. + /// Output sample rate in Hz. + internal AudioClipSource(AudioClipStore store, string key, IPCMReader reader, Func? openReader, int outputRate) + { + if (reader.Channels <= 0) throw new ArgumentException("Reader channel count must be positive.", nameof(reader)); + if (reader.SampleRate <= 0) throw new ArgumentException("Reader sample rate must be positive.", nameof(reader)); + if (reader.TotalFrames < 0) throw new ArgumentException("Reader frame count must not be negative.", nameof(reader)); + + _store = store; + _firstReader = reader; + _openReader = openReader; + Key = key; + Channels = reader.Channels; + Frequency = outputRate; + Duration = (double)reader.TotalFrames / reader.SampleRate; + TotalFrames = CeilDiv(reader.TotalFrames * outputRate, reader.SampleRate); + FramesPerChunk = ChunkFloats / Channels; + ChunkCount = (int)CeilDiv(TotalFrames, FramesPerChunk); + Policy = reader.IsForwardOnly || ChunkCount <= 1 ? ChunkPolicy.Resident : ChunkPolicy.Windowed; + + _chunks = new float[ChunkCount][]; + _holdCounts = new int[ChunkCount]; + _streams.Add(new DecodeStream(reader, outputRate)); + } + + // ── Cursors ── + + /// Registers a read position. The decoder thread loads the chunks it requires. + public AudioCursor AddCursor(long position, bool loop) + { + var cursor = new AudioCursor(this, position, loop); + lock (_sync) + { + _cursors.Add(cursor); + } + _store.RequestService(); + return cursor; + } + + public void RemoveCursor(AudioCursor cursor) + { + lock (_sync) + { + _cursors.Remove(cursor); + } + _store.RequestService(); + } + + internal void OnCursorMoved() + { + if (!IsResident) _store.RequestService(); + } + + // ── Consumer access ── + + /// Chunk index containing (0 ~ TotalFrames - 1). + public int ChunkIndexOf(long frame) => (int)(frame / FramesPerChunk); + + /// + /// Holds the chunk containing when it is loaded. + /// When it is not loaded, records a load request (kept until the chunk is acquired), wakes the decoder thread and returns false. + /// Every successful call must be paired with . + /// + public bool TryAcquire(long frame, out ChunkView view) + { + view = default; + if (_isFailed || frame < 0 || frame >= TotalFrames) return false; + + var index = ChunkIndexOf(frame); + lock (_sync) + { + var data = _chunks[index]; + if (data != null) + { + _demanded.Remove(index); + _holdCounts[index]++; + view = CreateView(index, data); + return true; + } + _demanded.Add(index); + } + _store.RequestService(); + return false; + } + + public void Release(in ChunkView view) + { + if (!view.IsValid) return; + lock (_sync) + { + if (_holdCounts[view.ChunkIndex] > 0) + _holdCounts[view.ChunkIndex]--; + } + } + + private ChunkView CreateView(int index, float[] data) + { + var start = (long)index * FramesPerChunk; + var count = (int)Math.Min(FramesPerChunk, TotalFrames - start); + return new ChunkView(data, start, count, Channels, index); + } + + // ── Decoder thread ── + + /// + /// Windowed: frees loaded chunks that are not required (see ) and not held. + /// Resident: does nothing. Decoder thread only. + /// + internal void FreeUnused() + { + if (IsResident) return; + + List? freed = null; + lock (_sync) + { + if (_loaded.Count == 0) return; + List? remove = null; + foreach (var index in _loaded) + { + if (_holdCounts[index] > 0 || IsRequired(index)) continue; + (remove ??= new List()).Add(index); + } + if (remove == null) return; + + foreach (var index in remove) + { + (freed ??= new List()).Add(_chunks[index]!); + _chunks[index] = null; + _loaded.Remove(index); + } + } + + foreach (var data in freed!) + _store.ReturnChunkArray(data); + } + + /// + /// Next chunk to load and its priority (lower loads first), or false when nothing is missing. + /// Windowed: demanded chunks, then each cursor's window by slot. + /// Resident: the next chunk in order; priority is while a consumer is waiting on this source. + /// Decoder thread only. + /// + internal bool TryGetNextLoad(out int index, out int priority) + { + index = -1; + priority = int.MaxValue; + if (_isFailed) return false; + + lock (_sync) + { + if (IsResident) + { + if (_residentLoadedCount >= ChunkCount) return false; + index = _residentLoadedCount; + priority = _demanded.Count > 0 ? PriorityDemanded : PriorityResident; + return true; + } + + foreach (var demanded in _demanded) + { + if (_chunks[demanded] != null) continue; + index = demanded; + priority = PriorityDemanded; + return true; + } + + foreach (var cursor in _cursors) + { + if (!TryGetWindowStart(cursor, out var first, out var loop)) continue; + for (var slot = 0; slot < SlotCount; slot++) + { + if (!TryWindowIndex(first, slot, loop, out var candidate)) break; + if (_chunks[candidate] != null) continue; + if (PriorityWindow + slot < priority) + { + index = candidate; + priority = PriorityWindow + slot; + } + break; + } + } + return index >= 0; + } + } + + /// True when a demand or a cursor window covers . Caller holds _sync. + private bool IsRequired(int index) + { + if (_demanded.Contains(index)) return true; + foreach (var cursor in _cursors) + { + if (!TryGetWindowStart(cursor, out var first, out var loop)) continue; + for (var slot = 0; slot < SlotCount; slot++) + { + if (!TryWindowIndex(first, slot, loop, out var candidate)) break; + if (candidate == index) return true; + } + } + return false; + } + + private bool TryGetWindowStart(AudioCursor cursor, out int first, out bool loop) + { + var position = cursor.Position; + loop = cursor.Loop; + first = 0; + if (position < 0) position = 0; + if (position >= TotalFrames) + { + if (!loop || TotalFrames == 0) return false; + position %= TotalFrames; + } + first = ChunkIndexOf(position); + return true; + } + + private bool TryWindowIndex(int first, int slot, bool loop, out int index) + { + index = first + slot; + if (index < ChunkCount) return true; + if (!loop) return false; + index %= ChunkCount; + return true; + } + + /// + /// Loads chunk if it is not loaded. Resident sources load only the next chunk in order. + /// Decoder thread only. + /// + /// True when this call loaded the chunk. + internal bool LoadChunk(int index) + { + if (_isFailed || index < 0 || index >= ChunkCount) return false; + + lock (_loadLock) + { + lock (_sync) + { + if (_chunks[index] != null) return false; + if (IsResident && index != _residentLoadedCount) + throw new InvalidOperationException($"Resident clip '{Key}' loads chunks in order: expected {_residentLoadedCount}, got {index}."); + } + + float[]? data = null; + try + { + var start = (long)index * FramesPerChunk; + var frames = (int)Math.Min(FramesPerChunk, TotalFrames - start); + data = IsResident ? _store.AllocateExactArray(frames * Channels) : _store.RentChunkArray(); + var stream = SelectStream(start); + stream.LastUse = ++_streamUseCounter; + stream.Fill(start, frames, data); + } + catch (OutOfMemoryException e) + { + ReturnArray(data); + Fail(); + _store.ReportFatal(this, e); + return false; + } + catch (Exception e) + { + ReturnArray(data); + Fail(); + _store.ReportSourceFailure(this, e); + return false; + } + + lock (_sync) + { + _chunks[index] = data; + _loaded.Add(index); + if (IsResident) _residentLoadedCount++; + } + LoadCount++; + + // A resident source never reads again after its last chunk; the reader releases its data now. + if (IsResident && _residentLoadedCount == ChunkCount) + DisposeStreams(); + } + + _store.NotifyChunkLoaded(); + return true; + } + + /// + /// Decode stream for a load starting at output frame : + /// the stream positioned at ; otherwise a new stream while fewer than + /// are open (windowed sources only); otherwise the least recently used stream. Caller holds _loadLock. + /// + private DecodeStream SelectStream(long start) + { + DecodeStream? oldest = null; + foreach (var stream in _streams) + { + if (stream.NextOutFrame == start) return stream; + if (oldest == null || stream.LastUse < oldest.LastUse) oldest = stream; + } + + if (!IsResident && !_openFailed && _openReader != null && _streams.Count < DecodeStreamLimit) + { + IPCMReader? reader = null; + try + { + reader = _openReader(); + if (reader != null) + { + var stream = new DecodeStream(reader, Frequency); + _streams.Add(stream); + return stream; + } + } + catch (Exception e) + { + reader?.Dispose(); + Melon.Logger.Warning($"[AudioClipSource] Could not open another decode stream for '{Key}', reusing open streams: {e.Message}"); + } + _openFailed = true; + } + + return oldest!; + } + + private void Fail() + { + _isFailed = true; + if (IsResident) DisposeStreams(); + } + + private void DisposeStreams() + { + foreach (var stream in _streams) + stream.Dispose(); + _streams.Clear(); + } + + private void ReturnArray(float[]? data) + { + if (data == null) return; + if (IsResident) _store.FreeExactArray(data); + else _store.ReturnChunkArray(data); + } + + /// Releases all chunk arrays and readers. Called by the store after the decoder thread stopped. + internal void DisposeInternal() + { + lock (_loadLock) + { + lock (_sync) + { + foreach (var index in _loaded) + { + ReturnArray(_chunks[index]); + _chunks[index] = null; + } + _loaded.Clear(); + _cursors.Clear(); + _demanded.Clear(); + } + DisposeStreams(); + _firstReader.Dispose(); + } + } + + private static long CeilDiv(long a, long b) => (a + b - 1) / b; +} diff --git a/Util/Audio/Clip/AudioClipStore.cs b/Util/Audio/Clip/AudioClipStore.cs new file mode 100644 index 0000000..7c24eba --- /dev/null +++ b/Util/Audio/Clip/AudioClipStore.cs @@ -0,0 +1,337 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using MelonLoader; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Clip sources of one render, keyed by clip name or path, with a shared pool of chunk arrays and one decoder thread. +/// +/// Threads: +/// - Decoder thread (): the only thread that decodes, resamples, loads and frees chunks. +/// Each step loads one chunk, the one with the lowest priority value over all sources +/// (a chunk a consumer waits for, then cursor windows by slot, then resident chunks in order). +/// - Consumers (render thread, main thread for the spectrum): waits for the decoder thread and never loads. +/// - Main thread: creates readers (for Unity clips this copies the clip data). +/// +/// Memory has no limit; current and peak usage are tracked and logged by . +/// An while loading is kept in and rethrown by . +/// +internal sealed class AudioClipStore : IDisposable +{ + /// Longest wait of the decoder thread between checks when nothing signals it, in milliseconds. + private const int DecoderIdleWaitMs = 50; + + /// Longest wait of a consumer between checks when no chunk is loaded, in milliseconds. + private const int ConsumerWaitMs = 50; + + private readonly Dictionary _sources = new(); + private readonly List _sourceList = new(); + private readonly Stack _chunkPool = new(); + private readonly object _poolLock = new(); + private readonly object _signal = new(); + private readonly AutoResetEvent _serviceRequested = new(false); + + private Thread? _decoderThread; + private volatile bool _stopRequested; + private long _allocatedBytes; + private long _peakBytes; + private volatile Exception? _fatalError; + private volatile bool _disposed; + + /// Sample rate every source is resampled to, in Hz. + public int OutputRate { get; } + + /// Bytes of sample arrays currently allocated (loaded chunks, pooled chunk arrays and copied Unity clip data). + public long AllocatedBytes => Interlocked.Read(ref _allocatedBytes); + + /// Highest value of since creation. + public long PeakBytes => Interlocked.Read(ref _peakBytes); + + /// Out-of-memory error raised while loading, or null. + public Exception? FatalError => _fatalError; + + public AudioClipStore(int outputRate) + { + if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate)); + OutputRate = outputRate; + } + + // ── Decoder thread ── + + /// Starts the decoder thread. Called once. + public void Start() + { + if (_decoderThread != null) throw new InvalidOperationException("Decoder thread already started."); + _decoderThread = new Thread(DecoderLoop) + { + Name = "Audio Clip Decoder", + IsBackground = true + }; + _decoderThread.Start(); + } + + private void DecoderLoop() + { + try + { + while (!_stopRequested) + { + if (!LoadNext()) + _serviceRequested.WaitOne(DecoderIdleWaitMs); + } + } + catch (Exception e) + { + Melon.Logger.Error($"[AudioClipStore] Decoder thread stopped: {e}"); + _fatalError ??= e; + NotifyChunkLoaded(); + } + } + + /// + /// Frees unused windowed chunks, then loads the single chunk with the lowest priority value over all sources. + /// Runs on the decoder thread (tests may call it directly while the thread is not started). + /// + /// True when a chunk was loaded or a load was attempted. + internal bool LoadNext() + { + AudioClipSource[] sources; + lock (_sources) + { + if (_disposed) return false; + sources = _sourceList.ToArray(); + } + + AudioClipSource? best = null; + var bestIndex = -1; + var bestPriority = int.MaxValue; + foreach (var source in sources) + { + source.FreeUnused(); + if (!source.TryGetNextLoad(out var index, out var priority) || priority >= bestPriority) continue; + best = source; + bestIndex = index; + bestPriority = priority; + } + + if (best == null) return false; + best.LoadChunk(bestIndex); + return true; + } + + internal void RequestService() => _serviceRequested.Set(); + + // ── Sources ── + + /// + /// Returns the source registered under , creating it with on first use. + /// A key whose reader could not be created is remembered as null and not retried. + /// For readers that can seek, is kept and called again on the decoder thread + /// to open additional decode streams (see ). + /// + public AudioClipSource? GetOrAdd(string key, Func createReader) + { + lock (_sources) + { + if (_disposed) return null; + if (_sources.TryGetValue(key, out var existing)) return existing; + } + + AudioClipSource? source = null; + try + { + var reader = createReader(); + if (reader != null) + { + try + { + source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : createReader, OutputRate); + } + catch + { + reader.Dispose(); + throw; + } + } + } + catch (OutOfMemoryException e) + { + Melon.Logger.Error($"[AudioClipStore] Out of memory while opening clip '{key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {e}"); + _fatalError ??= e; + } + catch (Exception e) + { + Melon.Logger.Error($"[AudioClipStore] Failed to open clip '{key}': {e}"); + } + + lock (_sources) + { + if (_disposed || _sources.TryGetValue(key, out var raced)) + { + source?.DisposeInternal(); + return _disposed ? null : _sources[key]; + } + _sources[key] = source; + if (source != null) _sourceList.Add(source); + } + + if (source != null) + { + Melon.Logger.Msg($"[AudioClipStore] Opened clip '{key}': {source.Channels}ch, {source.Duration:F3}s, {source.TotalFrames} frames at {OutputRate}Hz, {source.ChunkCount} chunk(s), {source.Policy}"); + RequestService(); + } + return source; + } + + public bool TryGet(string key, out AudioClipSource? source) + { + lock (_sources) + { + return _sources.TryGetValue(key, out source); + } + } + + // ── Consumers ── + + /// + /// Holds the chunk containing , waiting for the decoder thread until it is loaded. + /// Returns an invalid view when the source failed, the frame is outside the source, or the store was disposed. + /// + /// Loading ran out of memory or the decoder thread stopped. + /// was canceled while waiting. + public ChunkView Acquire(AudioClipSource source, long frame, CancellationToken cancellationToken = default) + { + if (frame < 0 || frame >= source.TotalFrames) return default; + + while (true) + { + ThrowIfFatal(); + if (source.IsFailed || _disposed) return default; + if (source.TryAcquire(frame, out var view)) return view; + + lock (_signal) + { + if (source.TryAcquire(frame, out view)) return view; + if (source.IsFailed || _fatalError != null || _disposed) continue; + Monitor.Wait(_signal, ConsumerWaitMs); + } + cancellationToken.ThrowIfCancellationRequested(); + } + } + + private void ThrowIfFatal() + { + var error = _fatalError; + if (error != null) + throw new InvalidOperationException("Audio clip loading failed: " + error.Message, error); + } + + internal void NotifyChunkLoaded() + { + lock (_signal) + { + Monitor.PulseAll(_signal); + } + } + + internal void ReportSourceFailure(AudioClipSource source, Exception error) + { + Melon.Logger.Error($"[AudioClipStore] Failed to load clip '{source.Key}', it will be silent: {error}"); + NotifyChunkLoaded(); + } + + internal void ReportFatal(AudioClipSource source, OutOfMemoryException error) + { + Melon.Logger.Error($"[AudioClipStore] Out of memory while loading clip '{source.Key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {error}"); + _fatalError ??= error; + NotifyChunkLoaded(); + } + + // ── Arrays ── + + internal float[] RentChunkArray() + { + lock (_poolLock) + { + if (_chunkPool.Count > 0) return _chunkPool.Pop(); + } + var data = new float[AudioClipSource.ChunkFloats]; + AddAllocated(data.LongLength * sizeof(float)); + return data; + } + + internal void ReturnChunkArray(float[] data) + { + if (_disposed) + { + AddAllocated(-data.LongLength * sizeof(float)); + return; + } + lock (_poolLock) + { + _chunkPool.Push(data); + } + } + + internal float[] AllocateExactArray(int floats) + { + var data = new float[floats]; + AddAllocated(data.LongLength * sizeof(float)); + return data; + } + + internal void FreeExactArray(float[] data) => AddAllocated(-data.LongLength * sizeof(float)); + + private void AddAllocated(long bytes) + { + var current = Interlocked.Add(ref _allocatedBytes, bytes); + long peak; + while (current > (peak = Interlocked.Read(ref _peakBytes))) + { + if (Interlocked.CompareExchange(ref _peakBytes, current, peak) == peak) break; + } + } + + public void LogMemoryUsage() + { + int sourceCount; + lock (_sources) sourceCount = _sourceList.Count; + Melon.Logger.Msg($"[AudioClipStore] Clips: {sourceCount}, sample memory now {AllocatedBytes / 1048576.0:F1} MiB, peak {PeakBytes / 1048576.0:F1} MiB"); + } + + /// Stops the decoder thread, then releases every source and pooled array. + public void Dispose() + { + AudioClipSource[] sources; + lock (_sources) + { + if (_disposed) return; + _disposed = true; + sources = _sourceList.ToArray(); + } + + _stopRequested = true; + _serviceRequested.Set(); + NotifyChunkLoaded(); + _decoderThread?.Join(); + + LogMemoryUsage(); + + lock (_sources) + { + _sourceList.Clear(); + _sources.Clear(); + } + + foreach (var source in sources) + source.DisposeInternal(); + + lock (_poolLock) + { + while (_chunkPool.Count > 0) + AddAllocated(-_chunkPool.Pop().LongLength * sizeof(float)); + } + } +} diff --git a/Util/Audio/Clip/DecodeStream.cs b/Util/Audio/Clip/DecodeStream.cs new file mode 100644 index 0000000..9ff5bf8 --- /dev/null +++ b/Util/Audio/Clip/DecodeStream.cs @@ -0,0 +1,181 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// One reader plus its resampler, producing output-rate frames of a clip as a continuous stream. +/// Consecutive calls whose ranges follow each other continue the stream (reader position and resampler state); +/// any other range starts a new stream near the requested position. +/// +/// Restart: the input position is moved back by a preroll and down to a multiple of in_rate / gcd(in_rate, out_rate), +/// where input frame k * (in_rate / gcd) maps exactly to output frame k * (out_rate / gcd); +/// the resampler is reset and output before the requested frame is discarded. +/// Measured with the clip harness: output is bit-identical to one continuous conversion of the whole clip. +/// Used by one thread at a time (the store's decoder thread). +/// +internal sealed class DecodeStream : IDisposable +{ + /// Input frames read from the reader per resampler call. + private const int ReadBlockFrames = 4096; + + private readonly PCMResampler? _resampler; + private readonly int _channels; + private readonly int _outputRate; + private readonly float[] _readBuffer; + private float[]? _discardBuffer; + private bool _flushing; + private bool _ended; + + private readonly long _inputStep; + private readonly long _outputStep; + private readonly long _prerollInputFrames; + + public IPCMReader Reader { get; } + + /// Output frame the stream continues at, or -1 when the stream is not positioned. + public long NextOutFrame { get; private set; } = -1; + + /// Value of the owner's use counter at the last (for least-recently-used selection). + public long LastUse { get; set; } + + /// Number of times the stream started anew (seek + resampler reset). + public int RestartCount { get; private set; } + + public DecodeStream(IPCMReader reader, int outputRate) + { + Reader = reader; + _channels = reader.Channels; + _outputRate = outputRate; + _readBuffer = new float[ReadBlockFrames * _channels]; + + if (reader.SampleRate != outputRate) + { + _resampler = new PCMResampler(_channels, reader.SampleRate, outputRate); + var gcd = GCD(reader.SampleRate, outputRate); + _inputStep = reader.SampleRate / gcd; + _outputStep = outputRate / gcd; + + // History fed before the first kept output frame when a stream starts mid-clip. + // The filter spans about filter_size input frames per output frame at ratios up to 1:1 + // and filter_size * (input rate / output rate) input frames when downsampling. + var ratio = Math.Max(1L, (reader.SampleRate + outputRate - 1L) / outputRate); + _prerollInputFrames = 4L * _resampler.FilterSize * ratio + _inputStep; + } + else + { + _inputStep = 1; + _outputStep = 1; + } + } + + /// + /// Writes output frames [start, start + frames) into and zeros the rest of the array. + /// Frames past the end of the decoded input are zero. + /// + public unsafe void Fill(long start, int frames, float[] data) + { + var channels = _channels; + var filled = 0; + + if (_resampler == null) + { + if (Reader.Position != start) + { + Reader.Seek(start); + RestartCount++; + } + while (filled < frames) + { + var request = Math.Min(ReadBlockFrames, frames - filled); + var read = Reader.Read(_readBuffer, request); + if (read <= 0) break; + Array.Copy(_readBuffer, 0, data, filled * channels, read * channels); + filled += read; + } + Array.Clear(data, filled * channels, data.Length - filled * channels); + NextOutFrame = start + frames; + return; + } + + long discard = 0; + if (NextOutFrame != start) + { + // Input frame of the requested output frame, moved back by the preroll and down to a multiple of _inputStep. + var inputFrame = start * Reader.SampleRate / _outputRate - _prerollInputFrames; + var step = inputFrame > 0 ? inputFrame / _inputStep : 0; + + _resampler.Reset(); + Reader.Seek(step * _inputStep); + _flushing = false; + _ended = false; + discard = start - step * _outputStep; + RestartCount++; + } + + fixed (float* pIn = _readBuffer) + fixed (float* pOut = data) + { + while (discard > 0 && !_ended) + { + _discardBuffer ??= new float[ReadBlockFrames * channels]; + fixed (float* pDiscard = _discardBuffer) + { + var capacity = (int)Math.Min(ReadBlockFrames, discard); + discard -= Pull(pIn, pDiscard, capacity); + } + } + + while (filled < frames && !_ended) + filled += Pull(pIn, pOut + (long)filled * channels, frames - filled); + } + + Array.Clear(data, filled * channels, data.Length - filled * channels); + NextOutFrame = _ended ? -1 : start + frames; + } + + /// + /// Writes 1 ~ resampled frames to , reading input as needed. + /// Returns 0 and sets _ended once the input and the resampler's flushed tail are exhausted. + /// + private unsafe int Pull(float* input, float* output, int capacity) + { + var resampler = _resampler!; + if (!_flushing) + { + while (true) + { + var produced = resampler.Convert(input, 0, output, capacity); + if (produced > 0) return produced; + + var read = Reader.Read(_readBuffer, ReadBlockFrames); + if (read <= 0) + { + _flushing = true; + break; + } + + produced = resampler.Convert(input, read, output, capacity); + if (produced > 0) return produced; + } + } + + // Input is exhausted: take the resampler's remaining tail until it returns 0. + var flushed = resampler.Flush(output, capacity); + if (flushed > 0) return flushed; + + _ended = true; + return 0; + } + + public void Dispose() + { + _resampler?.Dispose(); + Reader.Dispose(); + } + + private static int GCD(int a, int b) + { + while (b != 0) (a, b) = (b, a % b); + return a; + } +} diff --git a/Util/Audio/Clip/IPCMReader.cs b/Util/Audio/Clip/IPCMReader.cs new file mode 100644 index 0000000..6561218 --- /dev/null +++ b/Util/Audio/Clip/IPCMReader.cs @@ -0,0 +1,42 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Sequential reader of decoded PCM at the clip's own sample rate and channel count. +/// Samples are interleaved 32-bit float. Positions and counts are in frames (one frame = one sample per channel). +/// Used by one thread at a time (the store's decoder thread). may be called more than once. +/// +internal interface IPCMReader : IDisposable +{ + /// Channel count of the decoded audio (1 or more). + int Channels { get; } + + /// Sample rate of the decoded audio in Hz. + int SampleRate { get; } + + /// Total frame count of the clip at . + long TotalFrames { get; } + + /// + /// True when the data can be read only once, from frame 0 to the end (data is released as it is read). + /// The source then keeps every chunk (). + /// + bool IsForwardOnly { get; } + + /// Frame index that the next starts at. + long Position { get; } + + /// + /// Moves the read position to (0 ~ ). + /// A forward-only reader accepts only frames whose data has not been released. + /// + void Seek(long frame); + + /// + /// Reads up to frames into starting at index 0 and advances the position. + /// must hold at least frameCount * Channels floats. + /// + /// Frames read. 0 at the end of the clip. + int Read(float[] buffer, int frameCount); +} diff --git a/Util/Audio/Clip/MemoryPCMReader.cs b/Util/Audio/Clip/MemoryPCMReader.cs new file mode 100644 index 0000000..4d62840 --- /dev/null +++ b/Util/Audio/Clip/MemoryPCMReader.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Forward-only over PCM held in memory as a list of blocks. +/// Block b holds frames [b * FramesPerBlock, (b + 1) * FramesPerBlock) interleaved; the last block may be shorter. +/// Each block is released (passed to the release callback and dropped) as soon as the read position passes its end, +/// and all remaining blocks are released on . +/// +internal sealed class MemoryPCMReader : IPCMReader +{ + private readonly float[]?[] _blocks; + private readonly Action? _onBlockReleased; + private int _releasedCount; // Blocks [0, _releasedCount) are released + + public int Channels { get; } + public int SampleRate { get; } + public long TotalFrames { get; } + public bool IsForwardOnly => true; + public long Position { get; private set; } + + /// Frames per block (all blocks except the last). + public int FramesPerBlock { get; } + + public MemoryPCMReader(IReadOnlyList blocks, int framesPerBlock, int channels, int sampleRate, long totalFrames, + Action? onBlockReleased) + { + if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); + if (framesPerBlock <= 0) throw new ArgumentOutOfRangeException(nameof(framesPerBlock)); + if (totalFrames > (long)blocks.Count * framesPerBlock) throw new ArgumentException("Blocks hold fewer frames than totalFrames.", nameof(blocks)); + + _blocks = new float[blocks.Count][]; + for (var i = 0; i < blocks.Count; i++) _blocks[i] = blocks[i]; + _onBlockReleased = onBlockReleased; + FramesPerBlock = framesPerBlock; + Channels = channels; + SampleRate = sampleRate; + TotalFrames = totalFrames; + } + + public void Seek(long frame) + { + if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); + if (frame < (long)_releasedCount * FramesPerBlock) + throw new InvalidOperationException($"Frame {frame} was already released by the forward-only reader."); + Position = frame; + } + + public int Read(float[] buffer, int frameCount) + { + var total = (int)Math.Min(frameCount, TotalFrames - Position); + if (total <= 0) return 0; + + var written = 0; + while (written < total) + { + var blockIndex = (int)(Position / FramesPerBlock); + var block = _blocks[blockIndex] ?? throw new InvalidOperationException($"Block {blockIndex} was already released."); + var offset = (int)(Position - (long)blockIndex * FramesPerBlock); + var count = Math.Min(total - written, FramesPerBlock - offset); + Array.Copy(block, offset * Channels, buffer, written * Channels, count * Channels); + written += count; + Position += count; + ReleaseBlocksBefore((int)(Position / FramesPerBlock)); + } + return written; + } + + private void ReleaseBlocksBefore(int blockIndex) + { + while (_releasedCount < blockIndex && _releasedCount < _blocks.Length) + { + var block = _blocks[_releasedCount]; + _blocks[_releasedCount] = null; + _releasedCount++; + if (block != null) _onBlockReleased?.Invoke(block); + } + } + + public void Dispose() => ReleaseBlocksBefore(_blocks.Length); +} diff --git a/Util/Audio/Clip/PCMResampler.cs b/Util/Audio/Clip/PCMResampler.cs new file mode 100644 index 0000000..c9d4ef0 --- /dev/null +++ b/Util/Audio/Clip/PCMResampler.cs @@ -0,0 +1,106 @@ +using System; +using FFmpeg.AutoGen; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Sample rate converter for interleaved 32-bit float PCM (libswresample). +/// Input and output have the same channel count; only the sample rate changes. +/// One instance keeps filter state between calls, so consecutive calls form one continuous stream. +/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html +/// +internal sealed unsafe class PCMResampler : IDisposable +{ + private SwrContext* _swr; + private bool _flushed; + + public int Channels { get; } + public int InputRate { get; } + public int OutputRate { get; } + + /// Value of the SwrContext "filter_size" option (filter length per phase in taps at a 1:1 ratio). + public int FilterSize { get; } + + public PCMResampler(int channels, int inputRate, int outputRate) + { + if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); + if (inputRate <= 0) throw new ArgumentOutOfRangeException(nameof(inputRate)); + if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate)); + + Channels = channels; + InputRate = inputRate; + OutputRate = outputRate; + + AVChannelLayout layout; + ffmpeg.av_channel_layout_default(&layout, channels); + + SwrContext* swr = null; + var ret = ffmpeg.swr_alloc_set_opts2( + &swr, + &layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, outputRate, + &layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, inputRate, + 0, null); + ffmpeg.av_channel_layout_uninit(&layout); + if (ret < 0 || swr == null) + throw new InvalidOperationException($"swr_alloc_set_opts2 failed ({ret})."); + _swr = swr; + + long filterSize; + FilterSize = ffmpeg.av_opt_get_int(_swr, "filter_size", 0, &filterSize) >= 0 ? (int)filterSize : 32; + + Reset(); + } + + /// Discards all buffered input and filter history. The next call starts a new stream. + public void Reset() + { + // swr_init on an initialized context re-initializes it. + // Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_init, swr_close) + var ret = ffmpeg.swr_init(_swr); + if (ret < 0) + throw new InvalidOperationException($"swr_init failed ({ret})."); + _flushed = false; + } + + /// + /// Feeds frames and writes up to frames. + /// Input that does not fit in the output is buffered inside the context and returned by later calls. + /// may be 0 to take only buffered output. + /// + /// Frames written to . + public int Convert(float* input, int inputFrames, float* output, int outputCapacity) + { + if (_flushed) throw new InvalidOperationException("Resampler was flushed. Call Reset first."); + + // swr_convert treats a null input array as a flush request, so a non-null pointer is passed even for 0 frames. + float dummy = 0; + var inPlane = (byte*)(input != null ? input : &dummy); + var outPlane = (byte*)output; + var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, &inPlane, inputFrames); + if (ret < 0) + throw new InvalidOperationException($"swr_convert failed ({ret})."); + return ret; + } + + /// + /// Ends the input stream and writes up to remaining frames. + /// Call repeatedly until it returns 0. is not allowed afterwards until . + /// + public int Flush(float* output, int outputCapacity) + { + _flushed = true; + var outPlane = (byte*)output; + var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, null, 0); + if (ret < 0) + throw new InvalidOperationException($"swr_convert (flush) failed ({ret})."); + return ret; + } + + public void Dispose() + { + if (_swr == null) return; + var swr = _swr; + ffmpeg.swr_free(&swr); + _swr = null; + } +} diff --git a/Util/Audio/Clip/UnityClipLoader.cs b/Util/Audio/Clip/UnityClipLoader.cs new file mode 100644 index 0000000..c9acbe5 --- /dev/null +++ b/Util/Audio/Clip/UnityClipLoader.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using UnityEngine; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Copies a Unity into memory for clips without a file on disk +/// (built-in hit sounds from Resources, official/DLC level audio). +/// runs on the main thread only, so the whole clip is copied once on the main thread +/// and the decoder thread reads the copy through a . +/// Streaming clips () are rejected because GetData does not read them. +/// Reference: https://docs.unity3d.com/ScriptReference/AudioClip.GetData.html +/// +internal static class UnityClipLoader +{ + /// + /// Copies in blocks of floats or less + /// (FramesPerBlock = ChunkFloats / channels frames per GetData call). Main thread only. + /// Block memory is counted in and released block by block as the decoder thread reads it. + /// + public static MemoryPCMReader Load(AudioClip clip, AudioClipStore store) + { + if (!clip) throw new ArgumentNullException(nameof(clip)); + if (clip.loadType == AudioClipLoadType.Streaming) + throw new NotSupportedException($"AudioClip '{clip.name}' is a streaming clip; GetData cannot read it."); + + if (clip.loadState == AudioDataLoadState.Unloaded) + clip.LoadAudioData(); + + var channels = clip.channels; + var totalFrames = (long)clip.samples; + var framesPerBlock = AudioClipSource.ChunkFloats / channels; + var blocks = new List(); + try + { + for (long start = 0; start < totalFrames; start += framesPerBlock) + { + var frames = (int)Math.Min(framesPerBlock, totalFrames - start); + var block = store.AllocateExactArray(frames * channels); + blocks.Add(block); + // GetData fills the whole array; the offset is in frames (samples per channel). + if (!clip.GetData(block, (int)start)) + throw new InvalidOperationException($"AudioClip.GetData failed for '{clip.name}' at frame {start} (load state {clip.loadState})."); + } + } + catch + { + foreach (var block in blocks) store.FreeExactArray(block); + throw; + } + + return new MemoryPCMReader(blocks, framesPerBlock, channels, clip.frequency, totalFrames, store.FreeExactArray); + } +} From 82776fc572e83da2cb45e9917a1bc0051e529382bff6eab7e95a83b83a91f248 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 00:30:07 +0900 Subject: [PATCH 02/14] Add FFmpeg Audio File Reader --- Util/Audio/Clip/FFmpegPCMReader.cs | 327 +++++++++++++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 Util/Audio/Clip/FFmpegPCMReader.cs diff --git a/Util/Audio/Clip/FFmpegPCMReader.cs b/Util/Audio/Clip/FFmpegPCMReader.cs new file mode 100644 index 0000000..c9896a3 --- /dev/null +++ b/Util/Audio/Clip/FFmpegPCMReader.cs @@ -0,0 +1,327 @@ +using System; +using System.Runtime.InteropServices; +using FFmpeg.AutoGen; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// that decodes the best audio stream of a file with libavformat/libavcodec. +/// Output keeps the stream's sample rate and channel count; libswresample converts only the sample format to interleaved float. +/// +/// Frame positions: frame 0 is the first sample the decoder outputs for the stream's start time. +/// The position of the first frame decoded after opening or seeking is taken from its timestamp +/// (best_effort_timestamp - stream start_time, in samples); following frames are counted sequentially. +/// Seek moves the demuxer to before the target and discards decoded samples before the target. +/// +/// comes from the stream duration (or the container duration) and may differ from the decoded length; +/// reads stop at whichever ends first. +/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API) +/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame) +/// +internal sealed unsafe class FFmpegPCMReader : IPCMReader +{ + /// + /// Time the demuxer is moved before a seek target, in seconds. Decoders that depend on previous packets + /// (for example the MP3 bit reservoir) produce settled output after this margin; the samples in it are discarded. + /// + private const double SeekMarginSeconds = 0.25; + + private AVFormatContext* _format; + private AVCodecContext* _codec; + private SwrContext* _swr; + private AVPacket* _packet; + private AVFrame* _frame; + private readonly int _streamIndex; + private readonly AVRational _timeBase; + private readonly long _startPts; + + // Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels]. + private float[] _pending = Array.Empty(); + private int _pendingOffset; + private int _pendingCount; + private long _pendingStart; + + private long _nextDecodedFrame; // Frame index of the next decoded frame when counting sequentially + private bool _syncToTimestamp; // True until the first frame after open/seek sets _nextDecodedFrame from its timestamp + private bool _flushSent; + private bool _endOfStream; + + public string Path { get; } + public int Channels { get; } + public int SampleRate { get; } + public long TotalFrames { get; } + public bool IsForwardOnly => false; + public long Position { get; private set; } + + /// Number of demuxer seeks performed (for diagnostics). + public int SeekCount { get; private set; } + + public FFmpegPCMReader(string path) + { + Path = path; + try + { + AVFormatContext* format = null; + var ret = ffmpeg.avformat_open_input(&format, path, null, null); + if (ret < 0) throw new InvalidOperationException($"Failed to open audio file '{path}': {ErrorString(ret)}"); + _format = format; + + ret = ffmpeg.avformat_find_stream_info(_format, null); + if (ret < 0) throw new InvalidOperationException($"Failed to read stream info of '{path}': {ErrorString(ret)}"); + + AVCodec* decoder = null; + _streamIndex = ffmpeg.av_find_best_stream(_format, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &decoder, 0); + if (_streamIndex < 0 || decoder == null) throw new InvalidOperationException($"No decodable audio stream in '{path}': {ErrorString(_streamIndex)}"); + + for (var i = 0; i < (int)_format->nb_streams; i++) + { + if (i != _streamIndex) _format->streams[i]->discard = AVDiscard.AVDISCARD_ALL; + } + + var stream = _format->streams[_streamIndex]; + _timeBase = stream->time_base; + _startPts = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0; + + _codec = ffmpeg.avcodec_alloc_context3(decoder); + if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed."); + ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar); + if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}"); + _codec->pkt_timebase = _timeBase; + ret = ffmpeg.avcodec_open2(_codec, decoder, null); + if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}"); + + Channels = _codec->ch_layout.nb_channels; + SampleRate = _codec->sample_rate; + if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz."); + + TotalFrames = EstimateTotalFrames(stream); + + SwrContext* swr = null; + ret = ffmpeg.swr_alloc_set_opts2(&swr, + &_codec->ch_layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, SampleRate, + &_codec->ch_layout, _codec->sample_fmt, SampleRate, 0, null); + if (ret < 0 || swr == null) throw new InvalidOperationException($"swr_alloc_set_opts2 failed: {ErrorString(ret)}"); + _swr = swr; + ret = ffmpeg.swr_init(_swr); + if (ret < 0) throw new InvalidOperationException($"swr_init failed: {ErrorString(ret)}"); + + _packet = ffmpeg.av_packet_alloc(); + _frame = ffmpeg.av_frame_alloc(); + if (_packet == null || _frame == null) throw new InvalidOperationException("Failed to allocate packet/frame."); + + _syncToTimestamp = true; + } + catch + { + Dispose(); + throw; + } + } + + private long EstimateTotalFrames(AVStream* stream) + { + if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0) + return ffmpeg.av_rescale_q(stream->duration, _timeBase, new AVRational { num = 1, den = SampleRate }); + if (_format->duration != ffmpeg.AV_NOPTS_VALUE && _format->duration > 0) + return ffmpeg.av_rescale(_format->duration, SampleRate, ffmpeg.AV_TIME_BASE); + throw new InvalidOperationException($"Audio file '{Path}' has no duration."); + } + + public void Seek(long frame) + { + if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); + if (frame == Position) return; + + var marginFrames = (long)(SeekMarginSeconds * SampleRate); + + // Target inside the pending block or less than the seek margin after it: decode forward and discard. + if (!_syncToTimestamp && frame >= _pendingStart + _pendingOffset + && frame <= _pendingStart + _pendingOffset + _pendingCount + marginFrames) + { + Position = frame; + return; + } + + var targetTs = _startPts + ffmpeg.av_rescale_q(Math.Max(0, frame - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); + var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD); + if (ret < 0) + ret = ffmpeg.av_seek_frame(_format, _streamIndex, _startPts, ffmpeg.AVSEEK_FLAG_BACKWARD); + if (ret < 0) + throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}"); + + ffmpeg.avcodec_flush_buffers(_codec); + SeekCount++; + _pendingCount = 0; + _pendingOffset = 0; + _syncToTimestamp = true; + _flushSent = false; + _endOfStream = false; + Position = frame; + } + + public int Read(float[] buffer, int frameCount) + { + var limit = (int)Math.Min(frameCount, TotalFrames - Position); + var written = 0; + while (written < limit) + { + if (_pendingCount == 0 || Position >= _pendingStart + _pendingOffset + _pendingCount) + { + if (!DecodeNextFrame()) break; + continue; + } + + var pendingPosition = _pendingStart + _pendingOffset; + if (Position > pendingPosition) + { + // Discard decoded samples before the read position (after a seek). + var skip = (int)Math.Min(Position - pendingPosition, _pendingCount); + _pendingOffset += skip; + _pendingCount -= skip; + continue; + } + if (Position < pendingPosition) + { + // A gap in the decoded timestamps: fill with silence up to the next decoded sample. + var gap = (int)Math.Min(pendingPosition - Position, limit - written); + Array.Clear(buffer, written * Channels, gap * Channels); + written += gap; + Position += gap; + continue; + } + + var count = Math.Min(_pendingCount, limit - written); + Array.Copy(_pending, _pendingOffset * Channels, buffer, written * Channels, count * Channels); + _pendingOffset += count; + _pendingCount -= count; + written += count; + Position += count; + } + return written; + } + + /// Decodes the next audio frame into the pending buffer. Returns false at the end of the stream. + private bool DecodeNextFrame() + { + while (!_endOfStream) + { + var ret = ffmpeg.avcodec_receive_frame(_codec, _frame); + if (ret == 0) + { + StoreFrame(); + ffmpeg.av_frame_unref(_frame); + return true; + } + if (ret == ffmpeg.AVERROR_EOF) + { + _endOfStream = true; + break; + } + if (ret != ffmpeg.AVERROR(ffmpeg.EAGAIN)) + throw new InvalidOperationException($"Audio decoding failed in '{Path}': {ErrorString(ret)}"); + + if (_flushSent) + { + _endOfStream = true; + break; + } + + ret = ffmpeg.av_read_frame(_format, _packet); + if (ret < 0) + { + if (ret != ffmpeg.AVERROR_EOF) + throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}"); + ffmpeg.avcodec_send_packet(_codec, null); + _flushSent = true; + continue; + } + + try + { + if (_packet->stream_index != _streamIndex) continue; + ret = ffmpeg.avcodec_send_packet(_codec, _packet); + // A corrupt packet is skipped; decoding continues with the next one. + if (ret < 0 && ret != ffmpeg.AVERROR_INVALIDDATA && ret != ffmpeg.AVERROR(ffmpeg.EAGAIN)) + throw new InvalidOperationException($"Failed to send audio packet of '{Path}': {ErrorString(ret)}"); + } + finally + { + ffmpeg.av_packet_unref(_packet); + } + } + return false; + } + + private void StoreFrame() + { + var samples = _frame->nb_samples; + if (samples <= 0) return; + + if (_syncToTimestamp) + { + var pts = _frame->best_effort_timestamp; + _nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE + ? ffmpeg.av_rescale_q(pts - _startPts, _timeBase, new AVRational { num = 1, den = SampleRate }) + : Position; + _syncToTimestamp = false; + } + + if (_pending.Length < samples * Channels) + _pending = new float[samples * Channels]; + + fixed (float* pPending = _pending) + { + var outPlane = (byte*)pPending; + var converted = ffmpeg.swr_convert(_swr, &outPlane, samples, _frame->extended_data, samples); + if (converted < 0) + throw new InvalidOperationException($"Sample format conversion failed in '{Path}': {ErrorString(converted)}"); + samples = converted; + } + + _pendingStart = _nextDecodedFrame; + _pendingOffset = 0; + _pendingCount = samples; + _nextDecodedFrame += samples; + } + + public void Dispose() + { + if (_frame != null) + { + var frame = _frame; + ffmpeg.av_frame_free(&frame); + _frame = null; + } + if (_packet != null) + { + var packet = _packet; + ffmpeg.av_packet_free(&packet); + _packet = null; + } + if (_swr != null) + { + var swr = _swr; + ffmpeg.swr_free(&swr); + _swr = null; + } + if (_codec != null) + { + var codec = _codec; + ffmpeg.avcodec_free_context(&codec); + _codec = null; + } + if (_format != null) + { + var format = _format; + ffmpeg.avformat_close_input(&format); + _format = null; + } + } + + private static string ErrorString(int error) + { + var buffer = stackalloc byte[1024]; + ffmpeg.av_strerror(error, buffer, 1024); + return $"{Marshal.PtrToStringAnsi((IntPtr)buffer)} ({error})"; + } +} From 35606d6e581913bd770cf894a56c648bf1e88886a6a4aa28d72eb40f1b4000ec Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 01:59:53 +0900 Subject: [PATCH 03/14] Match Game MP3 Decoding Timing --- Util/Audio/Clip/FFmpegPCMReader.cs | 75 +++++++++++++++-- Util/Audio/Clip/MP3HeaderFrame.cs | 107 +++++++++++++++++++++++++ Util/Audio/Clip/MP3MonoLayoutReader.cs | 73 +++++++++++++++++ 3 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 Util/Audio/Clip/MP3HeaderFrame.cs create mode 100644 Util/Audio/Clip/MP3MonoLayoutReader.cs diff --git a/Util/Audio/Clip/FFmpegPCMReader.cs b/Util/Audio/Clip/FFmpegPCMReader.cs index c9896a3..3690b6a 100644 --- a/Util/Audio/Clip/FFmpegPCMReader.cs +++ b/Util/Audio/Clip/FFmpegPCMReader.cs @@ -15,6 +15,11 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// /// comes from the stream duration (or the container duration) and may differ from the decoded length; /// reads stop at whichever ends first. +/// +/// MP3: decoded with the game's timing (MP3Sharp, see ): encoder delay and padding are kept +/// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed +/// before it when the file starts with a Xing/Info/VBRI header frame. is the sum of all packet +/// durations plus that silence, counted once when the file is opened. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API) /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame) /// @@ -33,7 +38,8 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader private AVFrame* _frame; private readonly int _streamIndex; private readonly AVRational _timeBase; - private readonly long _startPts; + private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames + private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary) // Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels]. private float[] _pending = Array.Empty(); @@ -53,6 +59,9 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader public bool IsForwardOnly => false; public long Position { get; private set; } + /// True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary). + public bool IsMP3 { get; } + /// Number of demuxer seeks performed (for diagnostics). public int SeekCount { get; private set; } @@ -80,13 +89,20 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader var stream = _format->streams[_streamIndex]; _timeBase = stream->time_base; - _startPts = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0; + _ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0; _codec = ffmpeg.avcodec_alloc_context3(decoder); if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed."); ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar); if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}"); _codec->pkt_timebase = _timeBase; + IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3; + if (IsMP3) + { + // Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead). + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL) + _codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL; + } ret = ffmpeg.avcodec_open2(_codec, decoder, null); if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}"); @@ -94,7 +110,16 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader SampleRate = _codec->sample_rate; if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz."); - TotalFrames = EstimateTotalFrames(stream); + if (IsMP3) + { + _leadFrames = MP3HeaderFrame.SilentFrames(path); + ScanPackets(out _ptsOrigin, out var decodedFrames); + TotalFrames = _leadFrames + decodedFrames; + } + else + { + TotalFrames = EstimateTotalFrames(stream); + } SwrContext* swr = null; ret = ffmpeg.swr_alloc_set_opts2(&swr, @@ -118,6 +143,44 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader } } + /// + /// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed + /// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes + /// to a full frame when trimming is off. + /// + private void ScanPackets(out long firstPts, out long frames) + { + firstPts = ffmpeg.AV_NOPTS_VALUE; + long durationTicks = 0; + var packet = ffmpeg.av_packet_alloc(); + try + { + int ret; + while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0) + { + if (packet->stream_index == _streamIndex) + { + if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts; + durationTicks += packet->duration; + } + ffmpeg.av_packet_unref(packet); + } + if (ret != ffmpeg.AVERROR_EOF) + throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}"); + } + finally + { + ffmpeg.av_packet_free(&packet); + } + + if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0; + frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate }); + + var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD); + if (seek < 0) + throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}"); + } + private long EstimateTotalFrames(AVStream* stream) { if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0) @@ -142,10 +205,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader return; } - var targetTs = _startPts + ffmpeg.av_rescale_q(Math.Max(0, frame - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); + var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) - ret = ffmpeg.av_seek_frame(_format, _streamIndex, _startPts, ffmpeg.AVSEEK_FLAG_BACKWARD); + ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}"); @@ -261,7 +324,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader { var pts = _frame->best_effort_timestamp; _nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE - ? ffmpeg.av_rescale_q(pts - _startPts, _timeBase, new AVRational { num = 1, den = SampleRate }) + ? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate }) : Position; _syncToTimestamp = false; } diff --git a/Util/Audio/Clip/MP3HeaderFrame.cs b/Util/Audio/Clip/MP3HeaderFrame.cs new file mode 100644 index 0000000..c74e03b --- /dev/null +++ b/Util/Audio/Clip/MP3HeaderFrame.cs @@ -0,0 +1,107 @@ +using System; +using System.IO; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Detects the VBR/info header frame ("Xing", "Info" or "VBRI") at the start of an MP3 file. +/// +/// The game decodes MP3 files with MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.MP3Stream), which has no header-frame handling: +/// it decodes that frame as one frame of silence and keeps the encoder delay. FFmpeg's MP3 demuxer consumes the header frame +/// without output. Measured with the clip harness (LAME 192k CBR, LAME VBR, 44.1/48 kHz, with and without ID3v2): +/// MP3Sharp output is FFmpeg's untrimmed output preceded by one frame of silence when the header frame exists, +/// and identical in timing when it does not. +/// +/// Frame layout used here (ISO/IEC 11172-3 and 13818-3 frame header; Xing/Info tag after the Layer III side information, +/// VBRI tag 32 bytes after the frame header): +/// - Header: 11 sync bits, version (2 bits: 3 = MPEG-1, 2 = MPEG-2, 0 = MPEG-2.5), layer (2 bits: 1 = III, 2 = II, 3 = I), +/// protection bit (0 = 16-bit CRC follows), bitrate index, sample rate index, padding, private, channel mode (3 = mono). +/// - Layer III side information: MPEG-1 32 bytes (17 mono), MPEG-2/2.5 17 bytes (9 mono). +/// - Samples per frame: Layer I 384, Layer II 1152, Layer III 1152 (MPEG-1) or 576 (MPEG-2/2.5). +/// - ID3v2 tags before the first frame: "ID3", version (2 bytes), flags (bit 0x10 = 10-byte footer), size (4 bytes, 7 bits each). +/// +internal static class MP3HeaderFrame +{ + /// Bytes searched for the first frame sync after the ID3v2 tags. + private const int SyncSearchBytes = 64 * 1024; + + /// + /// Frames of silence MP3Sharp outputs for the header frame of : + /// the samples per frame of the first frame when it is a Xing/Info/VBRI header frame, otherwise 0. + /// + public static int SilentFrames(string path) + { + using var file = File.OpenRead(path); + return SilentFrames(file); + } + + public static int SilentFrames(Stream stream) + { + var tag = new byte[10]; + long offset = 0; + while (true) + { + stream.Position = offset; + if (ReadFully(stream, tag, 10) < 10 || tag[0] != 'I' || tag[1] != 'D' || tag[2] != '3') break; + var size = (tag[6] & 0x7F) << 21 | (tag[7] & 0x7F) << 14 | (tag[8] & 0x7F) << 7 | (tag[9] & 0x7F); + offset += 10 + size + ((tag[5] & 0x10) != 0 ? 10 : 0); + } + + stream.Position = offset; + var data = new byte[SyncSearchBytes]; + var length = ReadFully(stream, data, data.Length); + for (var i = 0; i + 4 <= length; i++) + { + if (!TryParseHeader(data, i, out var samplesPerFrame, out var tagOffset)) continue; + + // Xing/Info after the side information; VBRI at a fixed offset of 32 bytes after the 4-byte header. + if (HasTag(data, length, i + tagOffset, "Xing") || HasTag(data, length, i + tagOffset, "Info") + || HasTag(data, length, i + 4 + 32, "VBRI")) + return samplesPerFrame; + return 0; + } + return 0; + } + + private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset) + { + samplesPerFrame = 0; + tagOffset = 0; + if (data[i] != 0xFF || (data[i + 1] & 0xE0) != 0xE0) return false; + + var version = (data[i + 1] >> 3) & 0x3; + var layer = (data[i + 1] >> 1) & 0x3; + var hasCrc = (data[i + 1] & 0x1) == 0; + var bitrateIndex = (data[i + 2] >> 4) & 0xF; + var sampleRateIndex = (data[i + 2] >> 2) & 0x3; + var mono = ((data[i + 3] >> 6) & 0x3) == 3; + if (version == 1 || layer == 0 || bitrateIndex == 0xF || sampleRateIndex == 3) return false; + + var mpeg1 = version == 3; + samplesPerFrame = layer switch + { + 3 => 384, + 2 => 1152, + _ => mpeg1 ? 1152 : 576 + }; + var sideInfo = layer == 1 ? (mpeg1 ? (mono ? 17 : 32) : (mono ? 9 : 17)) : 0; + tagOffset = 4 + (hasCrc ? 2 : 0) + sideInfo; + return true; + } + + private static bool HasTag(byte[] data, int length, int at, string tag) + { + if (at < 0 || at + tag.Length > length) return false; + for (var k = 0; k < tag.Length; k++) + if (data[at + k] != tag[k]) return false; + return true; + } + + private static int ReadFully(Stream stream, byte[] buffer, int count) + { + var total = 0; + int read; + while (total < count && (read = stream.Read(buffer, total, count - total)) > 0) total += read; + return total; + } +} diff --git a/Util/Audio/Clip/MP3MonoLayoutReader.cs b/Util/Audio/Clip/MP3MonoLayoutReader.cs new file mode 100644 index 0000000..b1e35ab --- /dev/null +++ b/Util/Audio/Clip/MP3MonoLayoutReader.cs @@ -0,0 +1,73 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// that reproduces the game's output for mono MP3 files. +/// +/// MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.Buffer16BitStereo) writes every frame in stereo layout and fills only the +/// channel 0 slot for mono streams, while MP3Stream.ChannelCount reports 1. The game creates a 1-channel clip from that data, +/// so a mono MP3 plays as [s0, 0, s1, 0, ...]: twice as long, at half speed and one octave lower +/// (confirmed in game with TestMaps/mp3_mono, 2026-10-13). +/// +/// Output: 1 channel at the inner reader's sample rate. Frame 2k is inner frame k, frame 2k + 1 is 0. +/// = 2 * inner TotalFrames. +/// +internal sealed class MP3MonoLayoutReader : IPCMReader +{ + private readonly IPCMReader _inner; + private float[] _innerBuffer = Array.Empty(); + + /// Mono reader of the MP3 file with the game's MP3 timing (). Owned by this reader. + public MP3MonoLayoutReader(IPCMReader inner) + { + if (inner.Channels != 1) throw new ArgumentException("Inner reader must be mono.", nameof(inner)); + _inner = inner; + Position = inner.Position * 2; + } + + public int Channels => 1; + public int SampleRate => _inner.SampleRate; + public long TotalFrames => _inner.TotalFrames * 2; + public bool IsForwardOnly => _inner.IsForwardOnly; + public long Position { get; private set; } + + /// Moves to ; the inner reader moves to the inner frame of the next even position. + public void Seek(long frame) + { + _inner.Seek((frame + 1) / 2); + Position = frame; + } + + public int Read(float[] buffer, int frameCount) + { + var limit = (int)Math.Min(frameCount, TotalFrames - Position); + var written = 0; + while (written < limit) + { + if ((Position & 1) == 1) + { + buffer[written++] = 0f; + Position++; + continue; + } + + var want = (limit - written + 1) / 2; + if (_innerBuffer.Length < want) _innerBuffer = new float[want]; + var got = _inner.Read(_innerBuffer, want); + if (got == 0) break; + + for (var i = 0; i < got && written < limit; i++) + { + buffer[written++] = _innerBuffer[i]; + Position++; + if (written >= limit) break; + buffer[written++] = 0f; + Position++; + } + } + return written; + } + + public void Dispose() => _inner.Dispose(); +} From 7ea2d8fe3fdfc7b1b234b5764966456229d87a92b2bfb709d5d4d19329edee10 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 02:00:18 +0900 Subject: [PATCH 04/14] Resolve Audio Clips From Level Files --- Patch/Engine/AudioPatches.cs | 20 ++++++++++ Util/Audio/Clip/AudioClipPaths.cs | 46 +++++++++++++++++++++ Util/Audio/Clip/AudioClipResolver.cs | 60 ++++++++++++++++++++++++++++ Util/Audio/Clip/AudioClipSource.cs | 5 +++ Util/Audio/Clip/AudioClipStore.cs | 25 ++++++++---- 5 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 Util/Audio/Clip/AudioClipPaths.cs create mode 100644 Util/Audio/Clip/AudioClipResolver.cs diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index c79495c..5bcf75e 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -1,11 +1,31 @@ using System; using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util.Audio.Clip; using HarmonyLib; using UnityEngine; using UnityEngine.Audio; namespace ADOFAIRenderer.Patch.Engine; +/// +/// Records the file path of each external clip the game loads (always active, also outside rendering, +/// because levels load their song and PlaySound files before a render starts). +/// FindOrLoadAudioClipExternal returns the clip already in audioLib when its key exists, so the path is recorded +/// only when the key is missing and the file is about to be loaded. +/// +[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.FindOrLoadAudioClipExternal))] +public static class AudioManagerFindOrLoadAudioClipExternalPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static void Prefix(AudioManager __instance, string path) + { + if (string.IsNullOrEmpty(path)) return; + if (__instance.audioLib.ContainsKey(AudioClipPaths.KeyOf(path))) return; + AudioClipPaths.Record(path); + } +} + [HarmonyPatch(typeof(AudioManager), "_Play")] public static class AudioManagerPlayPatch { diff --git a/Util/Audio/Clip/AudioClipPaths.cs b/Util/Audio/Clip/AudioClipPaths.cs new file mode 100644 index 0000000..1900d18 --- /dev/null +++ b/Util/Audio/Clip/AudioClipPaths.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.IO; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// File path of every external audio clip the game loads, keyed by the game's audioLib key. +/// +/// The game loads external clips with AudioManager.FindOrLoadAudioClipExternal(path, ...) and stores them in audioLib under +/// Path.GetFileName(path) + "*external" (the same string becomes AudioClip.name). Recorded paths: +/// - Song: Path.Combine(Path.GetDirectoryName(levelPath), levelData.songFilename) (scnGame song loading). +/// - PlaySound event sound: Path.Combine(Path.GetDirectoryName(levelPath), hitsound) (scnGame.LoadSoundCo); +/// ffxPlaySound plays it as hitsound + "*external". +/// is called only when the game actually loads the file (the key is not in audioLib yet), +/// so the recorded path is the file of the clip audioLib holds under that key. +/// Thread-safe. +/// +internal static class AudioClipPaths +{ + /// Suffix of the game's audioLib keys for external clips. + public const string ExternalSuffix = "*external"; + + private static readonly Dictionary Paths = new(); + + /// audioLib key of an external clip loaded from . + public static string KeyOf(string path) => Path.GetFileName(path) + ExternalSuffix; + + /// Records under (path), replacing an earlier path of the same key. + public static void Record(string path) + { + if (string.IsNullOrEmpty(path)) return; + lock (Paths) + { + Paths[KeyOf(path)] = path; + } + } + + /// Recorded path for an audioLib key (or AudioClip.name) of an external clip. + public static bool TryGetPath(string key, out string path) + { + lock (Paths) + { + return Paths.TryGetValue(key, out path!); + } + } +} diff --git a/Util/Audio/Clip/AudioClipResolver.cs b/Util/Audio/Clip/AudioClipResolver.cs new file mode 100644 index 0000000..d62ee06 --- /dev/null +++ b/Util/Audio/Clip/AudioClipResolver.cs @@ -0,0 +1,60 @@ +using System; +using System.IO; +using MelonLoader; +using UnityEngine; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Opens the for a clip the game plays, keyed by the game's audioLib key (also AudioClip.name): +/// - Key recorded in and the file exists: decoded from the file with +/// (mono MP3 through ) +/// (song files and PlaySound files of custom levels). +/// - Otherwise, or when FFmpeg cannot open the file: the Unity clip is copied with +/// (built-in hit sounds from Resources, official/DLC level songs from Addressables). +/// Main thread only (the Unity copy calls AudioClip.GetData). +/// +internal static class AudioClipResolver +{ + /// Store of the current render. + /// audioLib key of the clip (for example "song.ogg*external" or "sndKick"). + /// Loaded Unity clip for the key, used when the clip has no file; may be null when the key has a file. + /// The source, or null when neither the file nor the clip can be read. + public static AudioClipSource? GetOrAdd(AudioClipStore store, string key, AudioClip? clip) + { + if (store.TryGet(key, out var existing)) return existing; + + if (AudioClipPaths.TryGetPath(key, out var path) && File.Exists(path)) + { + IPCMReader? fileReader = null; + try + { + fileReader = OpenFile(path); + if (fileReader is MP3MonoLayoutReader) + Melon.Logger.Warning($"[AudioClipResolver] Mono MP3 '{path}' is rendered like the game: half speed, one octave lower."); + } + catch (Exception e) + { + Melon.Logger.Warning($"[AudioClipResolver] FFmpeg could not open '{path}' for '{key}', using the Unity clip: {e.Message}"); + } + + if (fileReader != null) + { + var source = store.GetOrAdd(key, () => fileReader, () => OpenFile(path)); + if (source == null) fileReader.Dispose(); + return source; + } + } + + if (!clip) return store.GetOrAdd(key, () => null); + return store.GetOrAdd(key, () => UnityClipLoader.Load(clip!, store)); + } + + /// Opens with FFmpeg; mono MP3 files are wrapped in (game layout). + private static IPCMReader OpenFile(string path) + { + var reader = new FFmpegPCMReader(path); + if (!reader.IsMP3 || reader.Channels != 1) return reader; + return new MP3MonoLayoutReader(reader); + } +} diff --git a/Util/Audio/Clip/AudioClipSource.cs b/Util/Audio/Clip/AudioClipSource.cs index 9cb11ae..7f856f2 100644 --- a/Util/Audio/Clip/AudioClipSource.cs +++ b/Util/Audio/Clip/AudioClipSource.cs @@ -545,6 +545,11 @@ internal sealed class AudioClipSource try { reader = _openReader(); + if (reader is { IsForwardOnly: true }) + { + reader.Dispose(); + reader = null; + } if (reader != null) { var stream = new DecodeStream(reader, Frequency); diff --git a/Util/Audio/Clip/AudioClipStore.cs b/Util/Audio/Clip/AudioClipStore.cs index 7c24eba..631c41e 100644 --- a/Util/Audio/Clip/AudioClipStore.cs +++ b/Util/Audio/Clip/AudioClipStore.cs @@ -26,6 +26,9 @@ internal sealed class AudioClipStore : IDisposable /// Longest wait of a consumer between checks when no chunk is loaded, in milliseconds. private const int ConsumerWaitMs = 50; + /// Longest wait for the decoder thread to stop in , in milliseconds. + private const int DecoderStopTimeoutMs = 3000; + private readonly Dictionary _sources = new(); private readonly List _sourceList = new(); private readonly Stack _chunkPool = new(); @@ -128,10 +131,10 @@ internal sealed class AudioClipStore : IDisposable /// /// Returns the source registered under , creating it with on first use. /// A key whose reader could not be created is remembered as null and not retried. - /// For readers that can seek, is kept and called again on the decoder thread - /// to open additional decode streams (see ). + /// For readers that can seek, additional decode streams (see ) are opened on the + /// decoder thread with , or with when it is null. /// - public AudioClipSource? GetOrAdd(string key, Func createReader) + public AudioClipSource? GetOrAdd(string key, Func createReader, Func? openAdditionalReader = null) { lock (_sources) { @@ -147,7 +150,7 @@ internal sealed class AudioClipStore : IDisposable { try { - source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : createReader, OutputRate); + source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : openAdditionalReader ?? createReader, OutputRate); } catch { @@ -315,7 +318,12 @@ internal sealed class AudioClipStore : IDisposable _stopRequested = true; _serviceRequested.Set(); NotifyChunkLoaded(); - _decoderThread?.Join(); + + // A decoder thread blocked inside a read (for example a file on an unresponsive drive) is left running: + // it is a background thread, and its sources keep their readers so the blocked call does not touch freed FFmpeg contexts. + var decoderStopped = _decoderThread == null || _decoderThread.Join(DecoderStopTimeoutMs); + if (!decoderStopped) + Melon.Logger.Warning($"[AudioClipStore] Decoder thread did not stop within {DecoderStopTimeoutMs} ms; its readers are left open."); LogMemoryUsage(); @@ -325,8 +333,11 @@ internal sealed class AudioClipStore : IDisposable _sources.Clear(); } - foreach (var source in sources) - source.DisposeInternal(); + if (decoderStopped) + { + foreach (var source in sources) + source.DisposeInternal(); + } lock (_poolLock) { From a21d86d3a612896e44f5dfb72d1b6d62ec9bc607051ea98d97f4bcef32419258 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 09:43:07 +0900 Subject: [PATCH 05/14] Add Voice Mixer With Timed Changes --- Util/Audio/Mixer/Voice.cs | 156 +++++++++++++++ Util/Audio/Mixer/VoiceMixer.cs | 341 +++++++++++++++++++++++++++++++++ 2 files changed, 497 insertions(+) create mode 100644 Util/Audio/Mixer/Voice.cs create mode 100644 Util/Audio/Mixer/VoiceMixer.cs diff --git a/Util/Audio/Mixer/Voice.cs b/Util/Audio/Mixer/Voice.cs new file mode 100644 index 0000000..a5206ce --- /dev/null +++ b/Util/Audio/Mixer/Voice.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using ADOFAIRenderer.Util.Audio.Clip; + +namespace ADOFAIRenderer.Util.Audio.Mixer; + +/// +/// One playing sound of a (the song, or one effect sound), created by . +/// +/// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start, +/// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order. +/// Render thread: applies each change at sample round(time * OutputRate). +/// +/// Playback model (output samples n, clip positions in frames of the source at the output sample rate): +/// - The voice is silent before its start sample and from its end sample on. +/// - Clip position at sample n = anchor position + (n - anchor sample) * pitch. A position or pitch change sets a new anchor +/// at the sample where it takes effect, so a pitch change keeps the position continuous. +/// - Looping: the position wraps at the clip length ( * output rate, fractional). +/// Not looping: the voice ends when the position reaches . +/// +internal sealed class Voice +{ + internal enum ChangeKind + { + Pitch, + Volume, + Loop, + Position, + EndTime, + Stop + } + + internal readonly struct Change + { + public readonly ChangeKind Kind; + public readonly long Sample; + public readonly double Value; + + public Change(ChangeKind kind, long sample, double value) + { + Kind = kind; + Sample = sample; + Value = value; + } + } + + private readonly VoiceMixer _mixer; + + // ── Render thread state ── + + /// Changes not yet applied, in time order. Render thread only. + internal readonly Queue Pending = new(); + + /// First output sample of playback. + internal long StartSample; + + /// Output sample at which playback stops (exclusive); long.MaxValue when not set. + internal long EndSample = long.MaxValue; + + internal long AnchorSample; + internal double AnchorPosition; + internal double Pitch; + internal double Volume; + internal bool Loop; + internal bool Finished; + + /// Loop length in frames at the output sample rate (source duration * output rate). + internal double Period; + + internal AudioCursor? Cursor; + internal ChunkView View; + + internal Voice(VoiceMixer mixer, AudioClipSource? source, string name) + { + _mixer = mixer; + Source = source; + Name = name; + } + + /// Clip played by this voice; null plays silence (the clip could not be loaded). + public AudioClipSource? Source { get; } + + /// Name for logs (the game's clip key). + public string Name { get; } + + /// Sets the playback rate (1 = original speed). Negative values are treated as 0. + public void SetPitch(double pitch, double time) => _mixer.Enqueue(this, ChangeKind.Pitch, time, Math.Max(0.0, pitch)); + + /// Sets the linear gain applied to the clip samples. + public void SetVolume(double volume, double time) => _mixer.Enqueue(this, ChangeKind.Volume, time, volume); + + /// Turns looping on or off. Turning it off lets the current repetition play to the end of the clip. + public void SetLoop(bool loop, double time) => _mixer.Enqueue(this, ChangeKind.Loop, time, loop ? 1.0 : 0.0); + + /// + /// Moves playback to seconds into the clip. + /// Before the voice starts, this sets the position playback starts from. + /// + public void SetPosition(double clipSeconds, double time) => _mixer.Enqueue(this, ChangeKind.Position, time, Math.Max(0.0, clipSeconds)); + + /// Stops playback at (seconds relative to the render start). + public void SetEndTime(double endTime, double time) => _mixer.Enqueue(this, ChangeKind.EndTime, time, endTime); + + /// Stops playback at . + public void Stop(double time) => _mixer.Enqueue(this, ChangeKind.Stop, time, 0.0); + + /// Unwrapped clip position at output sample (sample at or after the anchor). + internal double PositionAt(long sample) => AnchorPosition + (sample - AnchorSample) * Pitch; + + /// Clip position at , wrapped into the loop when looping. + internal double WrappedPositionAt(long sample) + { + var position = PositionAt(sample); + if (Loop && Period > 0 && position >= Period) position %= Period; + return position; + } + + /// + /// Applies at output sample (its own sample, or a later one when the change + /// arrived after its sample was mixed). Before the start sample, changes replace the starting state. Render thread only. + /// + internal void Apply(in Change change, long sample, int outputRate) + { + var at = Math.Max(sample, StartSample); + switch (change.Kind) + { + case ChangeKind.Pitch: + Reanchor(at); + Pitch = change.Value; + break; + case ChangeKind.Volume: + Volume = change.Value; + break; + case ChangeKind.Loop: + Reanchor(at); + Loop = change.Value != 0.0; + break; + case ChangeKind.Position: + AnchorSample = at; + AnchorPosition = change.Value * outputRate; + break; + case ChangeKind.EndTime: + EndSample = VoiceMixer.ToSample(change.Value, outputRate); + break; + case ChangeKind.Stop: + EndSample = Math.Min(EndSample, sample); + break; + } + } + + private void Reanchor(long sample) + { + AnchorPosition = WrappedPositionAt(sample); + AnchorSample = sample; + } +} diff --git a/Util/Audio/Mixer/VoiceMixer.cs b/Util/Audio/Mixer/VoiceMixer.cs new file mode 100644 index 0000000..9be63e6 --- /dev/null +++ b/Util/Audio/Mixer/VoiceMixer.cs @@ -0,0 +1,341 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using ADOFAIRenderer.Util.Audio.Clip; + +namespace ADOFAIRenderer.Util.Audio.Mixer; + +/// +/// Mixes every into interleaved double samples at the output sample rate and channel count. +/// +/// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed +/// to and the Voice Set methods). A time t maps to sample round(t * OutputRate). +/// +/// Threads: +/// - Main thread: and the Voice Set methods queue commands under a lock. Commands must be queued in +/// non-decreasing time order per voice. +/// - Render thread: takes the queued commands and mixes a block of samples. A command takes effect at its sample; +/// a command whose sample is before the block being mixed takes effect at the start of that block. +/// Chunks are read through , which waits for the decoder thread. +/// +/// Sample reading: clip positions are in frames of the source at the output rate (sources are resampled when loaded). +/// An integer position copies the stored sample; a fractional position interpolates linearly between the frame and the next one +/// (the next frame of a looping voice after the last frame is frame 0; after the end of a non-looping clip it is 0). +/// Channel mapping follows . +/// +internal sealed class VoiceMixer : IDisposable +{ + private enum CommandKind + { + Play, + Change + } + + private readonly struct Command + { + public readonly CommandKind Kind; + public readonly Voice Voice; + public readonly Voice.Change Change; + + public Command(CommandKind kind, Voice voice, Voice.Change change) + { + Kind = kind; + Voice = voice; + Change = change; + } + } + + private readonly AudioClipStore _store; + private readonly CancellationToken _cancellationToken; + + private readonly object _commandLock = new(); + private List _commands = new(); + private List _takenCommands = new(); + + // Render thread only. + private readonly List _voices = new(); + private bool _disposed; + + /// Store the voices' sources belong to; its OutputRate is the mixer's sample rate. + /// Interleaved channel count of the mixed output. + /// Cancels waits for chunks in . + public VoiceMixer(AudioClipStore store, int outputChannels, CancellationToken cancellationToken = default) + { + if (outputChannels < 1) throw new ArgumentOutOfRangeException(nameof(outputChannels)); + _store = store; + OutputRate = store.OutputRate; + OutputChannels = outputChannels; + _cancellationToken = cancellationToken; + } + + public int OutputRate { get; } + + public int OutputChannels { get; } + + /// Voices currently mixed (started or waiting to start). Render thread only. + public int ActiveVoiceCount => _voices.Count; + + /// Output sample nearest to . + public static long ToSample(double seconds, int rate) => (long)Math.Round(seconds * rate, MidpointRounding.AwayFromZero); + + // ── Main thread ── + + /// + /// Schedules to start at . + /// A start time before (the current time) starts the voice at . + /// + /// Clip to play; null plays silence. + /// Name for logs. + /// Start of playback, seconds relative to the render start. + /// Linear gain. + /// Playback rate (1 = original speed); negative values are treated as 0. + /// Repeats the clip until the voice is stopped. + /// Current time, seconds relative to the render start. + public Voice Play(AudioClipSource? source, string name, double startTime, double volume, double pitch, bool loop, double time) + { + var voice = new Voice(this, source, name) + { + StartSample = Math.Max(ToSample(startTime, OutputRate), ToSample(time, OutputRate)), + Volume = volume, + Pitch = Math.Max(0.0, pitch), + Loop = loop, + Period = source != null ? source.Duration * OutputRate : 0.0 + }; + voice.AnchorSample = voice.StartSample; + + lock (_commandLock) + { + _commands.Add(new Command(CommandKind.Play, voice, default)); + } + return voice; + } + + internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value) + { + var change = new Voice.Change(kind, ToSample(time, OutputRate), value); + lock (_commandLock) + { + _commands.Add(new Command(CommandKind.Change, voice, change)); + } + } + + // ── Render thread ── + + /// + /// Adds the mix of output samples [, firstSample + frames) to + /// (interleaved, frames = destination.Length / OutputChannels). Blocks are expected in increasing sample order. + /// + /// Chunk loading failed fatally (see ). + /// The cancellation token was canceled while waiting for a chunk. + public void Mix(long firstSample, Span destination) + { + if (_disposed) throw new ObjectDisposedException(nameof(VoiceMixer)); + var frames = destination.Length / OutputChannels; + var endSample = firstSample + frames; + + TakeCommands(); + + for (var i = _voices.Count - 1; i >= 0; i--) + { + var voice = _voices[i]; + var sample = firstSample; + while (!voice.Finished) + { + var segmentEnd = endSample; + if (voice.Pending.Count > 0) + { + var next = voice.Pending.Peek(); + if (next.Sample <= sample) + { + voice.Apply(voice.Pending.Dequeue(), sample, OutputRate); + continue; + } + segmentEnd = Math.Min(segmentEnd, next.Sample); + } + + if (segmentEnd > sample) + MixSegment(voice, sample, segmentEnd, destination.Slice((int)(sample - firstSample) * OutputChannels)); + sample = segmentEnd; + if (sample >= endSample) break; + } + + if (!voice.Finished && endSample >= voice.EndSample) + voice.Finished = true; + if (voice.Finished) + { + ReleaseVoice(voice); + _voices.RemoveAt(i); + } + } + } + + private void TakeCommands() + { + lock (_commandLock) + { + (_commands, _takenCommands) = (_takenCommands, _commands); + } + + foreach (var command in _takenCommands) + { + var voice = command.Voice; + if (command.Kind == CommandKind.Play) + { + if (voice.Source is { IsFailed: false } source) + voice.Cursor = source.AddCursor(0, voice.Loop); + _voices.Add(voice); + } + else if (!voice.Finished) + { + voice.Pending.Enqueue(command.Change); + } + } + _takenCommands.Clear(); + } + + /// Mixes over samples [from, to) with its current state into (starting at sample from). + private void MixSegment(Voice voice, long from, long to, Span destination) + { + var start = Math.Max(from, voice.StartSample); + var end = Math.Min(to, voice.EndSample); + if (start >= end) return; + + var source = voice.Source; + if (source == null || source.IsFailed) + { + voice.Finished = true; + return; + } + + var totalFrames = source.TotalFrames; + if (voice.Volume == 0.0) + { + // Nothing to add; a non-looping voice still ends at the end of the clip. + if (!voice.Loop && voice.PositionAt(end - 1) >= totalFrames) voice.Finished = true; + return; + } + + var channels = OutputChannels; + var volume = voice.Volume; + var offset = (int)(start - from) * channels; + + UpdateCursor(voice, (long)voice.WrappedPositionAt(start)); + + for (var n = start; n < end; n++, offset += channels) + { + var position = voice.WrappedPositionAt(n); + if (position >= totalFrames) + { + if (!voice.Loop) + { + voice.Finished = true; + return; + } + position = totalFrames - 1; + } + + var index = (long)position; + var fraction = position - index; + + if (!EnsureView(voice, index)) return; + var view = voice.View; + + if (fraction == 0.0) + { + for (var c = 0; c < channels; c++) + destination[offset + c] += view.GetSample(index, c, channels) * volume; + continue; + } + + var nextIndex = index + 1; + var nextInView = view.Contains(nextIndex); + for (var c = 0; c < channels; c++) + { + double low = view.GetSample(index, c, channels); + double high = nextInView ? view.GetSample(nextIndex, c, channels) : ReadOutside(voice, nextIndex, c); + destination[offset + c] += (low + fraction * (high - low)) * volume; + } + } + } + + /// Sample of frame outside the voice's current view (next chunk, loop start, or past the end). + private double ReadOutside(Voice voice, long frame, int channel) + { + var source = voice.Source!; + if (frame >= source.TotalFrames) + { + if (!voice.Loop) return 0.0; + frame = 0; + } + + var view = _store.Acquire(source, frame, _cancellationToken); + if (!view.IsValid) return 0.0; + try + { + return view.GetSample(frame, channel, OutputChannels); + } + finally + { + source.Release(view); + } + } + + /// Makes voice.View the chunk containing . False when the source cannot provide it (voice finished). + private bool EnsureView(Voice voice, long frame) + { + if (voice.View.IsValid && voice.View.Contains(frame)) return true; + + var source = voice.Source!; + if (voice.View.IsValid) + { + source.Release(voice.View); + voice.View = default; + } + + UpdateCursor(voice, frame); + voice.View = _store.Acquire(source, frame, _cancellationToken); + if (voice.View.IsValid) return true; + + voice.Finished = true; + return false; + } + + private static void UpdateCursor(Voice voice, long frame) + { + var cursor = voice.Cursor; + if (cursor == null) return; + if (cursor.Loop != voice.Loop) cursor.Loop = voice.Loop; + if (cursor.Position != frame) cursor.Position = frame; + } + + private static void ReleaseVoice(Voice voice) + { + var source = voice.Source; + if (source == null) return; + if (voice.View.IsValid) + { + source.Release(voice.View); + voice.View = default; + } + if (voice.Cursor != null) + { + source.RemoveCursor(voice.Cursor); + voice.Cursor = null; + } + voice.Pending.Clear(); + } + + /// Releases every voice's chunk and cursor. Render thread (or after the render thread stopped). + public void Dispose() + { + if (_disposed) return; + _disposed = true; + foreach (var voice in _voices) + ReleaseVoice(voice); + _voices.Clear(); + lock (_commandLock) + { + _commands.Clear(); + } + } +} From 3d2ffe9dfe4653e1421c5ed1ba2f110047b451ca3f7eb946f6c886771cb0bb62 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 11:30:42 +0900 Subject: [PATCH 06/14] Fix MP3 Header Read While Game Streams Song --- Util/Audio/Clip/MP3HeaderFrame.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Util/Audio/Clip/MP3HeaderFrame.cs b/Util/Audio/Clip/MP3HeaderFrame.cs index c74e03b..fe91fb8 100644 --- a/Util/Audio/Clip/MP3HeaderFrame.cs +++ b/Util/Audio/Clip/MP3HeaderFrame.cs @@ -31,7 +31,9 @@ internal static class MP3HeaderFrame /// public static int SilentFrames(string path) { - using var file = File.OpenRead(path); + // The game keeps the song open while it streams it (MP3Sharp.MP3Stream: new FileStream(path, FileMode.Open), + // read/write access with read sharing), so the file is opened with read/write/delete sharing. + using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); return SilentFrames(file); } From cb0fd1bd96636273b9bf4c92522e64455464e1e0f5424c86b44724a19db7cef5 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 11:30:57 +0900 Subject: [PATCH 07/14] Switch Render Audio To Voice Mixer --- Core/RenderManager.cs | 26 +- Patch/Engine/AudioPatches.cs | 12 +- Util/Audio/AudioResampler.cs | 100 ----- Util/Audio/Filter/Gain.cs | 8 +- Util/Audio/Filter/HardClip.cs | 6 +- Util/Audio/Filter/IAudioFilter.cs | 4 +- Util/Audio/Filter/RMSCompressor.cs | 106 ++--- Util/Audio/OfflineAudioMixer.cs | 630 ++++++++++++----------------- 8 files changed, 351 insertions(+), 541 deletions(-) delete mode 100644 Util/Audio/AudioResampler.cs diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index 9afecb8..3b94580 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -146,6 +146,9 @@ internal class RenderManager : MonoBehaviour public int frameCountBase; public OfflineAudioMixer? AudioMixer { get; private set; } + + /// Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands. + public double AudioTime => frameCount / Configuration.RenderFPS; private AudioSyncCalculator? SyncCalculator { get; set; } private FrameReader? _frameReader; @@ -412,7 +415,16 @@ internal class RenderManager : MonoBehaviour // ReSharper disable once Unity.IncorrectMethodSignature public void OnPostFrame(RenderTexture? texture) { - if (Instance.stage < RenderStage.Render || !texture) return; + if (Instance.stage < RenderStage.Render) return; + + // Audio commands of this frame are queued: update the song voice and let the render thread mix up to the next frame. + if (AudioMixer != null) + { + AudioMixer.UpdateBGM(AudioTime, dspTimeBase); + AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS); + } + + if (!texture) return; // [Backpressure Throttling] isThrottling = false; @@ -682,7 +694,7 @@ internal class RenderManager : MonoBehaviour throw; } - var encoder = videoFFmpeg!; + var encoder = videoFFmpeg; RenderThread = new Thread(() => RunRenderThread(encoder)) { IsBackground = true, @@ -816,17 +828,13 @@ internal class RenderManager : MonoBehaviour { if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return; - var frameIdx = (int)(passCount - 1); - var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); - var frameTime = frameIdx / effectiveVideoFps; - var channels = Configuration.AudioConfig.AudioChannels; var samplesNeeded = syncCalculator.GetNextFrameSamples(); var audioBuffer = ArrayPool.Shared.Rent(samplesNeeded * channels); try { - AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase); + AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded); videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded); } finally @@ -865,6 +873,8 @@ internal class RenderManager : MonoBehaviour private void StopFFmpeg(bool success) { endRequested = true; + if (success) AudioMixer?.Complete(); + else AudioMixer?.Cancel(); if (!success) { MarkCancelled(); @@ -1011,7 +1021,7 @@ internal class RenderManager : MonoBehaviour _frameReader?.Dispose(); ResetFrameHookState(); - AudioMixer?.Reset(); AudioMixer = null; + AudioMixer?.Dispose(); AudioMixer = null; SyncCalculator = null; AudioSpectrumGenerator.Clear(); diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 5bcf75e..0483521 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -50,7 +50,7 @@ public static class AudioManagerPlayPatch double relativeStartTime = time - RenderManager.Instance.dspTimeBase; // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource - RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f); + RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f, RenderManager.Instance.AudioTime); } } } @@ -74,7 +74,7 @@ public static class PlayWithEndTimePatch double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; // Register looping sound event to the offline mixer (default pitch = 1.0f) - mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f); + mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); } } return true; // Let the original method execute @@ -91,9 +91,9 @@ public static class AudioSourcePitchPatch if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) { var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent)) + if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) { - soundEvent.Pitch = value; + voice.SetPitch(value, RenderManager.Instance.AudioTime); } } } @@ -109,9 +109,9 @@ public static class AudioSourceSetScheduledEndTimePatch if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) { var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent)) + if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) { - soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase; + voice.SetEndTime(time - RenderManager.Instance.dspTimeBase, RenderManager.Instance.AudioTime); } } } diff --git a/Util/Audio/AudioResampler.cs b/Util/Audio/AudioResampler.cs deleted file mode 100644 index 12a8ff0..0000000 --- a/Util/Audio/AudioResampler.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; - -namespace ADOFAIRenderer.Util.Audio; - -internal static class AudioResampler -{ - public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency) - { - // ── Safety Validation ── - if (inputSamples.Length == 0 || - inputChannels <= 0 || targetChannels <= 0 || - inputFrequency <= 0 || targetFrequency <= 0) - { - return Array.Empty(); - } - - // If format is identical, return original samples - if (inputFrequency == targetFrequency && inputChannels == targetChannels) - return inputSamples; - - int inputLength = inputSamples.Length / inputChannels; - int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency); - float[] outputSamples = new float[targetLength * targetChannels]; - - for (int i = 0; i < targetLength; i++) - { - double srcIndex = (double)i * inputFrequency / targetFrequency; - long idxLow = (long)Math.Floor(srcIndex); - long idxHigh = idxLow + 1; - float t = (float)(srcIndex - idxLow); - - if (idxHigh >= inputLength) idxHigh = inputLength - 1; - if (idxLow >= inputLength) idxLow = inputLength - 1; - - if (targetChannels == 1) // 1. Mono Output - { - float valLow, valHigh; - if (inputChannels == 1) // Mono -> Mono - { - valLow = inputSamples[idxLow]; - valHigh = inputSamples[idxHigh]; - } - else // Multi -> Mono (Downmix: Average all channels) - { - float sumLow = 0f; - float sumHigh = 0f; - for (int c = 0; c < inputChannels; c++) - { - sumLow += inputSamples[idxLow * inputChannels + c]; - sumHigh += inputSamples[idxHigh * inputChannels + c]; - } - valLow = sumLow / inputChannels; - valHigh = sumHigh / inputChannels; - } - outputSamples[i] = valLow + t * (valHigh - valLow); - } - else if (targetChannels == 2) // 2. Stereo Output - { - if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right) - { - float valLow = inputSamples[idxLow]; - float valHigh = inputSamples[idxHigh]; - float val = valLow + t * (valHigh - valLow); - outputSamples[i * 2] = val; - outputSamples[i * 2 + 1] = val; - } - else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels) - { - float valLowL = inputSamples[idxLow * inputChannels]; - float valHighL = inputSamples[idxHigh * inputChannels]; - outputSamples[i * 2] = valLowL + t * (valHighL - valLowL); - - float valLowR = inputSamples[idxLow * inputChannels + 1]; - float valHighR = inputSamples[idxHigh * inputChannels + 1]; - outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR); - } - } - else // 3. Multi-channel Output (e.g., 5.1 Surround) - { - for (int c = 0; c < targetChannels; c++) - { - float valLow, valHigh; - if (c < inputChannels) - { - valLow = inputSamples[idxLow * inputChannels + c]; - valHigh = inputSamples[idxHigh * inputChannels + c]; - } - else - { - valLow = inputSamples[idxLow * inputChannels]; - valHigh = inputSamples[idxHigh * inputChannels]; - } - outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow); - } - } - } - - return outputSamples; - } -} \ No newline at end of file diff --git a/Util/Audio/Filter/Gain.cs b/Util/Audio/Filter/Gain.cs index 3c33a22..bafa79c 100644 --- a/Util/Audio/Filter/Gain.cs +++ b/Util/Audio/Filter/Gain.cs @@ -4,14 +4,14 @@ namespace ADOFAIRenderer.Util.Audio.Filter; internal class Gain : IAudioFilter { - public float Level = 0f; + public double Level = 0; public void Initialize(int sampleRate, int channels) { } - public float ProcessSample(float sample, int channel) => - sample * (float)Math.Pow(10, Level / 20); + public double ProcessSample(double sample, int channel) => + sample * Math.Pow(10, Level / 20); - public void ProcessBlock(Span buffer) + public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); diff --git a/Util/Audio/Filter/HardClip.cs b/Util/Audio/Filter/HardClip.cs index adc58c4..2e20f6c 100644 --- a/Util/Audio/Filter/HardClip.cs +++ b/Util/Audio/Filter/HardClip.cs @@ -8,19 +8,19 @@ internal class HardClip : IAudioFilter // ReSharper disable once MemberCanBePrivate.Global // ReSharper disable once FieldCanBeMadeReadOnly.Global // ReSharper disable once ConvertToConstant.Global - public float Threshold = 1.0f; + public double Threshold = 1.0; public void Initialize(int sampleRate, int channels) { } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public float ProcessSample(float sample, int channel) + public double ProcessSample(double sample, int channel) { if (sample > Threshold) return Threshold; if (sample < -Threshold) return -Threshold; return sample; } - public void ProcessBlock(Span buffer) + public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); diff --git a/Util/Audio/Filter/IAudioFilter.cs b/Util/Audio/Filter/IAudioFilter.cs index 63c684b..ebf9a00 100644 --- a/Util/Audio/Filter/IAudioFilter.cs +++ b/Util/Audio/Filter/IAudioFilter.cs @@ -6,8 +6,8 @@ internal interface IAudioFilter { void Initialize(int sampleRate, int channels); - float ProcessSample(float sample, int channel); - void ProcessBlock(Span buffer); + double ProcessSample(double sample, int channel); + void ProcessBlock(Span buffer); void Reset(); diff --git a/Util/Audio/Filter/RMSCompressor.cs b/Util/Audio/Filter/RMSCompressor.cs index 571f126..ea07d27 100644 --- a/Util/Audio/Filter/RMSCompressor.cs +++ b/Util/Audio/Filter/RMSCompressor.cs @@ -6,84 +6,84 @@ namespace ADOFAIRenderer.Util.Audio.Filter; // ReSharper disable once InconsistentNaming internal class RMSCompressor : IAudioFilter { - private float _thresholdDb = -20f; - private float _ratio = 4f; - private float _attackTime = 0.010f; // in seconds - private float _releaseTime = 0.100f; // in seconds - private float _kneeDb = 5f; // in dB - private float _rmsWindowTime = 0.030f; // in seconds - private float _lookAheadTime; // in seconds + private double _thresholdDb = -20; + private double _ratio = 4; + private double _attackTime = 0.010; // in seconds + private double _releaseTime = 0.100; // in seconds + private double _kneeDb = 5; // in dB + private double _rmsWindowTime = 0.030; // in seconds + private double _lookAheadTime; // in seconds private int _lookAheadSamples; private int _sampleRate = 44100; private int _channels = 2; - private float _attackCoef; - private float _releaseCoef; - private float _rmsCoef; + private double _attackCoef; + private double _releaseCoef; + private double _rmsCoef; // Per-channel state - private float[] _rmsState = Array.Empty(); // running mean-square (MS) state - private float[] _gainState = Array.Empty(); // smoothed gain reduction in dB - private float[][] _delayBuffers = Array.Empty(); // circular delay buffers + private double[] _rmsState = Array.Empty(); // running mean-square (MS) state + private double[] _gainState = Array.Empty(); // smoothed gain reduction in dB + private double[][] _delayBuffers = Array.Empty(); // circular delay buffers private int[] _delayWriteIndices = Array.Empty(); - public float LookAheadTime + public double LookAheadTime { get => _lookAheadTime; set { - _lookAheadTime = Math.Max(0f, Math.Min(1.0f, value)); + _lookAheadTime = Math.Max(0, Math.Min(1.0, value)); UpdateLookAheadSettings(); } } public double Latency => _lookAheadTime; - public float ThresholdDb + public double ThresholdDb { get => _thresholdDb; set => _thresholdDb = value; } - public float Ratio + public double Ratio { get => _ratio; - set => _ratio = Math.Max(1f, value); + set => _ratio = Math.Max(1, value); } - public float AttackTime + public double AttackTime { get => _attackTime; set { - _attackTime = Math.Max(0.0001f, value); + _attackTime = Math.Max(0.0001, value); UpdateAttackCoef(); } } - public float ReleaseTime + public double ReleaseTime { get => _releaseTime; set { - _releaseTime = Math.Max(0.0001f, value); + _releaseTime = Math.Max(0.0001, value); UpdateReleaseCoef(); } } - public float KneeDb + public double KneeDb { get => _kneeDb; - set => _kneeDb = Math.Max(0f, value); + set => _kneeDb = Math.Max(0, value); } - public float RmsWindowTime + public double RmsWindowTime { get => _rmsWindowTime; set { - _rmsWindowTime = Math.Max(0.0001f, value); + _rmsWindowTime = Math.Max(0.0001, value); UpdateRmsCoef(); } } @@ -98,8 +98,8 @@ internal class RMSCompressor : IAudioFilter UpdateRmsCoef(); UpdateLookAheadSettings(); - _rmsState = new float[channels]; - _gainState = new float[channels]; + _rmsState = new double[channels]; + _gainState = new double[channels]; Reset(); } @@ -114,33 +114,33 @@ internal class RMSCompressor : IAudioFilter { if (_channels <= 0) return; - _delayBuffers = new float[_channels][]; + _delayBuffers = new double[_channels][]; _delayWriteIndices = new int[_channels]; for (int i = 0; i < _channels; i++) { - _delayBuffers[i] = _lookAheadSamples > 0 ? new float[_lookAheadSamples] : Array.Empty(); + _delayBuffers[i] = _lookAheadSamples > 0 ? new double[_lookAheadSamples] : Array.Empty(); _delayWriteIndices[i] = 0; } } private void UpdateAttackCoef() { - _attackCoef = (float)Math.Exp(-1.0 / (_sampleRate * _attackTime)); + _attackCoef = Math.Exp(-1.0 / (_sampleRate * _attackTime)); } private void UpdateReleaseCoef() { - _releaseCoef = (float)Math.Exp(-1.0 / (_sampleRate * _releaseTime)); + _releaseCoef = Math.Exp(-1.0 / (_sampleRate * _releaseTime)); } private void UpdateRmsCoef() { - _rmsCoef = (float)Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); + _rmsCoef = Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public float ProcessSample(float sample, int channel) + public double ProcessSample(double sample, int channel) { if (channel < 0 || channel >= _channels) { @@ -149,54 +149,54 @@ internal class RMSCompressor : IAudioFilter // Calculate running Mean-Square (MS) level // v_c[n] = alpha * v_c[n-1] + (1 - alpha) * x^2[n] - float sq = sample * sample; - _rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0f - _rmsCoef) * sq; + double sq = sample * sample; + _rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0 - _rmsCoef) * sq; - float rmsSq = Math.Max(0f, _rmsState[channel]); + double rmsSq = Math.Max(0, _rmsState[channel]); // Convert to dB. 10 * log10(rmsSq) is equivalent to 20 * log10(sqrt(rmsSq)) - float rmsDb = 10.0f * (float)Math.Log10(rmsSq + 1e-12f); + double rmsDb = 10.0 * Math.Log10(rmsSq + 1e-12); // Calculate static envelope curve (gain reduction in dB) - float gStatic = 0f; - if (_kneeDb > 0f) + double gStatic = 0; + if (_kneeDb > 0) { - if (rmsDb > _thresholdDb + _kneeDb * 0.5f) + if (rmsDb > _thresholdDb + _kneeDb * 0.5) { - gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb); + gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); } - else if (rmsDb >= _thresholdDb - _kneeDb * 0.5f) + else if (rmsDb >= _thresholdDb - _kneeDb * 0.5) { - float diff = rmsDb - _thresholdDb + _kneeDb * 0.5f; - gStatic = (1f / _ratio - 1f) * diff * diff / (2f * _kneeDb); + double diff = rmsDb - _thresholdDb + _kneeDb * 0.5; + gStatic = (1 / _ratio - 1) * diff * diff / (2 * _kneeDb); } } else { if (rmsDb > _thresholdDb) { - gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb); + gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); } } // Apply ballistics (attack and release) on the dB gain reduction - float lastGain = _gainState[channel]; - float targetGain = gStatic; + double lastGain = _gainState[channel]; + double targetGain = gStatic; // Note: Since targetGain and lastGain are <= 0 (gain reduction in dB), // targetGain < lastGain means target gain reduction is larger (more negative), // so we are in the attack phase. - float coef = targetGain < lastGain ? _attackCoef : _releaseCoef; - float currentGain = coef * lastGain + (1f - coef) * targetGain; + double coef = targetGain < lastGain ? _attackCoef : _releaseCoef; + double currentGain = coef * lastGain + (1 - coef) * targetGain; _gainState[channel] = currentGain; // Convert back to linear gain multiplier - float linearGain = (float)Math.Pow(10.0, currentGain / 20.0); + double linearGain = Math.Pow(10.0, currentGain / 20.0); if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0) { var buffer = _delayBuffers[channel]; int writeIdx = _delayWriteIndices[channel]; - float delayedSample = buffer[writeIdx]; + double delayedSample = buffer[writeIdx]; buffer[writeIdx] = sample; _delayWriteIndices[channel] = (writeIdx + 1) % _lookAheadSamples; return delayedSample * linearGain; @@ -205,7 +205,7 @@ internal class RMSCompressor : IAudioFilter return sample * linearGain; } - public void ProcessBlock(Span buffer) + public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) { diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 18190be..82fbf9a 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -1,416 +1,316 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Threading; +using ADOFAIRenderer.Util.Audio.Clip; using ADOFAIRenderer.Util.Audio.Filter; +using ADOFAIRenderer.Util.Audio.Mixer; using MelonLoader; using UnityEngine; namespace ADOFAIRenderer.Util.Audio; -internal class AudioClipCache +/// +/// Audio of one render: the song and effect sounds as s of a , clips from an +/// , followed by the filter chain (RMS compressor → look-ahead limiter → hard clip → -1 dB gain). +/// +/// Time: seconds relative to the render start (RenderManager.dspTimeBase); output sample n is at n / sample rate. +/// The limiter delays its output by its look-ahead (), so output sample n is mixed from +/// voices at sample n + latency, and the filters are primed with samples [0, latency) before the first block. +/// +/// Threads: +/// - Main thread: , , , , +/// source/voice mapping, and after each frame (all commands with an earlier time have been queued). +/// - Render thread: waits until the main thread has committed the end of the samples it mixes, +/// so every command takes effect at its own sample. ends the waiting when the main thread stops. +/// +internal sealed class OfflineAudioMixer : IDisposable { - private readonly Dictionary _cache = new(); + /// Gain applied to every clip. + private const double ClipVolume = 0.5; - public class CachedClipData - { - public float[] Samples = null!; // Pre-resampled to target channels and sample rate - public double Duration; - } + /// Additional gain of effect sounds (registered with ). + private const double EffectVolume = 2.0 / 3.0; - public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true) - { - if (string.IsNullOrEmpty(clipName)) return null; + /// Longest wait for a commit before checking for cancellation again, in milliseconds. + private const int CommitWaitMs = 50; - lock (_cache) - { - if (_cache.TryGetValue(clipName, out var cached)) - return cached; + private readonly int _channels; + private readonly int _sampleRate; + private readonly AudioClipStore _store; + private readonly VoiceMixer _mixer; + private readonly CancellationTokenSource _cancellation = new(); + private readonly List _filters = new(); + private readonly int _latencySamples; - if (!allowLoad) return null; + // Main thread. + private ConditionalWeakTable _sourceVoices = new(); + private AudioClipSource? _bgmSource; + private string _bgmKey = ""; + private Voice? _bgmVoice; + private double _bgmStart = double.NaN; + private double _bgmPitch = double.NaN; + private double _bgmVolume = double.NaN; - if (!AudioManager.Instance) return null; + // Shared: committed output sample (voices are final before it), guarded by _commitLock. + private readonly object _commitLock = new(); + private long _committedSample = long.MinValue; - var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName); - if (!clip) return null; - - try - { - var totalSamples = clip.samples; - var channels = clip.channels; - var rawSamples = new float[totalSamples * channels]; - clip.GetData(rawSamples, 0); - - // Auto-resample during cache generation - var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate); - - var data = new CachedClipData - { - Samples = resampled, - Duration = (double)resampled.Length / (targetChannels * targetSampleRate) - }; - - _cache[clipName] = data; - return data; - } - catch (Exception e) - { - MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}"); - return null; - } - } - } - - public void Clear() - { - lock (_cache) - { - _cache.Clear(); - } - } -} - -internal class OfflineAudioMixer -{ - public class SoundEvent - { - public string ClipName = null!; - public double StartTime; // In seconds relative to dspTimeBase - public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot) - public float Volume; - public float Pitch; - } - - private readonly List _activeEvents = new(); - private ConditionalWeakTable _sourceMappings = new(); - private readonly object _lock = new(); - - // Target Audio Settings - private readonly int _targetChannels; - private readonly int _targetSampleRate; - private const float ClipVolume = .5f; - - // Audio Filter Settings - private readonly LinkedList _filters = new(); - - // BGM Cached Samples (Pre-resampled) - private float[]? _bgmSamples; - private double _bgmDuration; - - // Clip Cache - private readonly AudioClipCache _clipCache = new(); - private bool _isPrimed; + // Render thread (and Dispose, under _renderLock). + private readonly object _renderLock = new(); + private long _nextOutputSample; + private bool _primed; + private double[] _mixBuffer = Array.Empty(); + private bool _disposed; public OfflineAudioMixer(int targetChannels, int targetSampleRate) { - _targetChannels = targetChannels; - _targetSampleRate = targetSampleRate; + _channels = targetChannels; + _sampleRate = targetSampleRate; + _store = new AudioClipStore(targetSampleRate); + _mixer = new VoiceMixer(_store, targetChannels, _cancellation.Token); var rmsCompressor = new RMSCompressor { ThresholdDb = -10, Ratio = 2, - AttackTime = 0.1f, ReleaseTime = 0.1f, - KneeDb = 10, RmsWindowTime = 0.02f + AttackTime = 0.1, ReleaseTime = 0.1, + KneeDb = 10, RmsWindowTime = 0.02 }; - rmsCompressor.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(rmsCompressor); - var limiter = new RMSCompressor { - ThresholdDb = 0, Ratio = float.PositiveInfinity, - AttackTime = 0f, ReleaseTime = 0.05f, - KneeDb = 0, RmsWindowTime = 0.02f, - LookAheadTime = 0.01f + ThresholdDb = 0, Ratio = double.PositiveInfinity, + AttackTime = 0, ReleaseTime = 0.05, + KneeDb = 0, RmsWindowTime = 0.02, + LookAheadTime = 0.01 }; - limiter.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(limiter); - - var hardClip = new HardClip(); - hardClip.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(hardClip); + _filters.Add(rmsCompressor); + _filters.Add(limiter); + _filters.Add(new HardClip()); + _filters.Add(new Gain { Level = -1 }); - var truePeakSafetyGain = new Gain { Level = -1f }; - truePeakSafetyGain.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(truePeakSafetyGain); + double latency = 0; + foreach (var filter in _filters) + { + filter.Initialize(_sampleRate, _channels); + latency += filter.Latency; + } + _latencySamples = (int)Math.Round(latency * _sampleRate); + + _store.Start(); } + // ── Main thread ── + + /// Registers the song clip. The song voice is created and kept in sync by . public void InitializeBGM(AudioClip? clip) { - lock (_lock) + if (!clip) { - if (!clip) - { - _bgmSamples = null; - _bgmDuration = 0; - return; - } - - try - { - int originalSamples = clip.samples; - int originalChannels = clip.channels; - float[] rawSamples = new float[originalSamples * originalChannels]; - clip.GetData(rawSamples, 0); - - // Pre-resample the BGM to the target format - _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate); - _bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate); - - Melon.Logger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}"); - } - catch (Exception e) - { - Melon.Logger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}"); - _bgmSamples = null; - _bgmDuration = 0; - } + _bgmSource = null; + return; } + + _bgmKey = clip!.name; + _bgmSource = AudioClipResolver.GetOrAdd(_store, _bgmKey, clip); + if (_bgmSource == null) + { + Melon.Logger.Error($"[OfflineAudioMixer] Song clip '{_bgmKey}' could not be loaded, the song will be silent."); + return; + } + Melon.Logger.Msg($"[OfflineAudioMixer] Song '{_bgmKey}': {_bgmSource.Channels} ch, {_bgmSource.Duration:F2} s, {_bgmSource.Policy}"); } - public void CacheClip(string clipName) + /// + /// Updates the song voice from the conductor (start time, pitch) and the level volume. + /// A changed start time restarts the song voice at position (now - start) * pitch; pitch and volume changes take effect at . + /// Start time: dspTimeSong + separate countdown (crotchetAtStart / pitch * adjustedCountdownTicks when separateCountdownTime), + /// the time scrConductor.StartMusicCo schedules the song at. + /// + /// Current time, seconds relative to the render start. + /// AudioSettings.dspTime at the render start. + public void UpdateBGM(double now, double dspTimeBase) { - _clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true); - } - - public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch) - { - lock (_lock) - { - var ev = new SoundEvent - { - ClipName = clipName, - StartTime = startTime, - EndTime = endTime, - Volume = volume, - Pitch = pitch - }; - _activeEvents.Add(ev); - - if (!source) return; - _sourceMappings.Remove(source); - _sourceMappings.Add(source, ev); - } - } - - public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch) - { - RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch); - } - - public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev) - { - if (source == null) - { - ev = null!; - return false; - } - lock (_lock) - { - return _sourceMappings.TryGetValue(source, out ev); - } - } - - public bool RemoveSourceMapping(AudioSource source) - { - lock (_lock) - { - return _sourceMappings.Remove(source); - } - } - - public void Reset() - { - lock (_lock) - { - _activeEvents.Clear(); - _sourceMappings = new ConditionalWeakTable(); - _bgmSamples = null; - _bgmDuration = 0; - _clipCache.Clear(); - _isPrimed = false; - foreach (var filter in _filters) - { - filter.Reset(); - } - } - } - - private void PrimeFilters(double totalLatency, double dspTimeBase) - { - int latencySamples = (int)Math.Round(totalLatency * _targetSampleRate); - if (latencySamples <= 0) return; - - // Rent temporary buffer to prime filters with initial audio - float[] tempBuffer = System.Buffers.ArrayPool.Shared.Rent(latencySamples * _targetChannels); - try - { - Array.Clear(tempBuffer, 0, latencySamples * _targetChannels); - - // Mix starting from -totalLatency to 0 - MixBGM(tempBuffer, latencySamples, -totalLatency, dspTimeBase, totalLatency); - MixEffectSounds(tempBuffer, latencySamples, -totalLatency, totalLatency); - - // Feed the buffer into filters to prime circular delay buffers - foreach (var filter in _filters) - { - filter.ProcessBlock(tempBuffer.AsSpan(0, latencySamples * _targetChannels)); - } - } - finally - { - System.Buffers.ArrayPool.Shared.Return(tempBuffer); - } - } - - public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) - { - // mixedBuffer size must be at least samplesNeeded * _targetChannels - Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels); - - lock (_lock) - { - double totalLatency = 0; - foreach (var filter in _filters) - { - totalLatency += filter.Latency; - } - - // Prime filters on first frame execution if latency exists - if (!_isPrimed && totalLatency > 0) - { - PrimeFilters(totalLatency, dspTimeBase); - _isPrimed = true; - } - - // 1. Mix BGM - MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase, totalLatency); - - // 2. Mix Hitsounds/Effect Sounds - MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime, totalLatency); - - foreach (var filter in _filters) - filter.ProcessBlock(mixedBuffer.AsSpan(0, samplesNeeded * _targetChannels)); - - for (var i = 0; i < samplesNeeded * _targetChannels; i++) - { - if (Math.Abs(mixedBuffer[i]) > 1) Melon.Logger.Msg("Warning Why Audio Sample Is Larger Than 1"); - } - } - } - - private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase, double totalLatency) - { - if (_bgmSamples == null || _bgmSamples.Length == 0) return; - + if (_bgmSource == null) return; var conductor = scrConductor.instance; if (!conductor) return; double pitch = conductor.song.pitch; - var dspTimeSong = conductor.dspTimeSong; - var separateCountdownOffset = conductor.separateCountdownTime - ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks - : 0.0; + var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0; + var start = conductor.dspTimeSong + countdown - dspTimeBase; + var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0; + var volume = levelVolume * ClipVolume; - // BGM Playback start time relative to dspTimeBase - var time1 = dspTimeSong + separateCountdownOffset; - var relativeTime1 = time1 - dspTimeBase; - var adjustedRelativeTime1 = relativeTime1 - totalLatency; - - var totalSamples = _bgmSamples.Length / _targetChannels; - - for (int s = 0; s < samplesNeeded; s++) + if (_bgmVoice == null || start != _bgmStart) { - double sampleTime = frameStartTime + (double)s / _targetSampleRate; - double clipTime = (sampleTime - adjustedRelativeTime1) * pitch; + // Song position at time t is (t - start) * pitch, also when the start is already past. + _bgmVoice?.Stop(now); + _bgmVoice = _mixer.Play(_bgmSource, _bgmKey, Math.Max(start, now), volume, pitch, false, now); + if (start < now) _bgmVoice.SetPosition((now - start) * pitch, now); + _bgmStart = start; + _bgmPitch = pitch; + _bgmVolume = volume; + return; + } - if (clipTime >= 0 && clipTime < _bgmDuration) - { - double sampleIdxDouble = clipTime * _targetSampleRate; - long idxLow = (long)Math.Floor(sampleIdxDouble); - long idxHigh = idxLow + 1; - float t = (float)(sampleIdxDouble - idxLow); - - if (idxHigh >= totalSamples) idxHigh = totalSamples - 1; - if (idxLow >= totalSamples) idxLow = totalSamples - 1; - - for (var c = 0; c < _targetChannels; c++) - { - float valLow = _bgmSamples[idxLow * _targetChannels + c]; - float valHigh = _bgmSamples[idxHigh * _targetChannels + c]; - mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * (scnGame.instance.levelData.volume * 0.01f) - * ClipVolume; - } - } + if (pitch != _bgmPitch) + { + _bgmVoice.SetPitch(pitch, now); + _bgmPitch = pitch; + } + if (volume != _bgmVolume) + { + _bgmVoice.SetVolume(volume, now); + _bgmVolume = volume; } } - private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double totalLatency) + /// Prepares the source of the game clip (audioLib key) so its first chunk starts loading. + public void CacheClip(string clipName) { - for (var i = _activeEvents.Count - 1; i >= 0; i--) + GetOrAddSource(clipName); + } + + private AudioClipSource? GetOrAddSource(string clipName) + { + if (string.IsNullOrEmpty(clipName)) return null; + if (_store.TryGet(clipName, out var existing)) return existing; + var clip = AudioManager.Instance ? AudioManager.Instance.FindOrLoadAudioClip(clipName) : null; + return AudioClipResolver.GetOrAdd(_store, clipName, clip); + } + + /// + /// Starts an effect sound at . An of 0 or more makes it loop until then. + /// (when given) is mapped to the voice for later changes through the AudioSource. + /// + public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) + { + var clipSource = GetOrAddSource(clipName); + var loop = endTime >= 0; + var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now); + if (loop) voice.SetEndTime(endTime, now); + + if (!source) return; + _sourceVoices.Remove(source!); + _sourceVoices.Add(source!, voice); + } + + /// Voice started for by . + public bool TryGetVoice(AudioSource? source, out Voice voice) + { + voice = null!; + return source != null && _sourceVoices.TryGetValue(source, out voice); + } + + public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source); + + /// + /// Marks every command with a time before as queued + /// (called after each frame with the time of the next frame). + /// + public void Commit(double time) + { + var sample = VoiceMixer.ToSample(time, _sampleRate); + lock (_commitLock) { - var ev = _activeEvents[i]; - var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false); - if (clipData == null) - { - _activeEvents.RemoveAt(i); - continue; - } - - var isSoundFinished = false; - var totalSamples = clipData.Samples.Length / _targetChannels; - double adjustedStartTime = ev.StartTime - totalLatency; - double adjustedEndTime = ev.EndTime >= 0 ? ev.EndTime - totalLatency : -1.0; - - for (var s = 0; s < samplesNeeded; s++) - { - var sampleTime = frameStartTime + (double)s / _targetSampleRate; - - // If it is a looping sound and has reached or exceeded its end time, stop playing. - if (adjustedEndTime >= 0 && sampleTime >= adjustedEndTime) - { - isSoundFinished = true; - break; - } - - var elapsed = sampleTime - adjustedStartTime; - if (elapsed < 0) continue; // Sound hasn't started yet - - var soundTime = elapsed * ev.Pitch; - - if (soundTime >= clipData.Duration) - { - if (ev.EndTime < 0) // One-shot - { - isSoundFinished = true; - break; - } - - // Looping - soundTime %= clipData.Duration; - } - - // Linear interpolation for pitch shifting in the mixer loop - double sampleIdxDouble = soundTime * _targetSampleRate; - long idxLow = (long)Math.Floor(sampleIdxDouble); - long idxHigh = idxLow + 1; - float t = (float)(sampleIdxDouble - idxLow); - - if (idxHigh >= totalSamples) idxHigh = totalSamples - 1; - if (idxLow >= totalSamples) idxLow = totalSamples - 1; - - for (var c = 0; c < _targetChannels; c++) - { - float valLow = clipData.Samples[idxLow * _targetChannels + c]; - float valHigh = clipData.Samples[idxHigh * _targetChannels + c]; - mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume - * ClipVolume * 2 / 3; - } - } - - // Clean up finished sound events (both one-shot and looping) - if (isSoundFinished) - { - _activeEvents.RemoveAt(i); - } + if (sample <= _committedSample) return; + _committedSample = sample; + Monitor.PulseAll(_commitLock); } } -} \ No newline at end of file + + /// No more commands will be queued (render ending): mixing no longer waits for commits. + public void Complete() + { + lock (_commitLock) + { + _committedSample = long.MaxValue; + Monitor.PulseAll(_commitLock); + } + } + + /// Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException. + public void Cancel() + { + _cancellation.Cancel(); + Complete(); + } + + // ── Render thread ── + + /// + /// Writes the next output frames (interleaved float, continuing from the previous call). + /// + /// Chunk loading failed fatally (out of memory or the decoder thread stopped). + /// The render was canceled (). + public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded) + { + lock (_renderLock) + { + if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer)); + + if (!_primed) + { + _primed = true; + if (_latencySamples > 0) + { + var prime = MixBlock(0, _latencySamples); + foreach (var filter in _filters) + filter.ProcessBlock(prime); + } + } + + var block = MixBlock(_nextOutputSample + _latencySamples, samplesNeeded); + foreach (var filter in _filters) + filter.ProcessBlock(block); + _nextOutputSample += samplesNeeded; + + for (var i = 0; i < block.Length; i++) + mixedBuffer[i] = (float)block[i]; + } + } + + /// Mixes voices at samples [first, first + frames) into the reused double buffer, after the main thread committed them. + private Span MixBlock(long first, int frames) + { + WaitForCommit(first + frames); + + var length = frames * _channels; + if (_mixBuffer.Length < length) _mixBuffer = new double[length]; + var block = _mixBuffer.AsSpan(0, length); + block.Clear(); + _mixer.Mix(first, block); + return block; + } + + private void WaitForCommit(long endSample) + { + lock (_commitLock) + { + while (_committedSample < endSample) + { + _cancellation.Token.ThrowIfCancellationRequested(); + Monitor.Wait(_commitLock, CommitWaitMs); + } + } + _cancellation.Token.ThrowIfCancellationRequested(); + } + + /// + /// Cancels waits, then releases the voices, the clip store (decoder thread) and the filter state. + /// Waits for a running to return. + /// + public void Dispose() + { + Cancel(); + lock (_renderLock) + { + if (_disposed) return; + _disposed = true; + _mixer.Dispose(); + _store.Dispose(); + foreach (var filter in _filters) + filter.Reset(); + } + _sourceVoices = new ConditionalWeakTable(); + } +} From 5055e4dfbb9b8691ec18d6ee8f7f679ebba4deb1e2c057174d2e32d5bd8ed76a Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 11:57:07 +0900 Subject: [PATCH 08/14] Apply AudioSource Playback Control To Voices --- Patch/Engine/AudioPatches.cs | 104 +++++++++++++++++++++++++++++++- Util/Audio/Mixer/Voice.cs | 7 +++ Util/Audio/OfflineAudioMixer.cs | 36 +++++++++-- 3 files changed, 141 insertions(+), 6 deletions(-) diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 0483521..0a518fa 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -1,6 +1,7 @@ using System; using ADOFAIRenderer.Core; using ADOFAIRenderer.Util.Audio.Clip; +using ADOFAIRenderer.Util.Audio.Mixer; using HarmonyLib; using UnityEngine; using UnityEngine.Audio; @@ -55,9 +56,19 @@ public static class AudioManagerPlayPatch } } +/// +/// scrConductor.PlayWithEndTime: registers a looping effect voice until the end time. The AudioSource it creates with +/// AudioManager.MakeSource is mapped to that voice (). +/// [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] public static class PlayWithEndTimePatch { + /// Voice registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. + internal static Voice? PendingVoice; + + [HarmonyFinalizer] + public static void Finalizer() => PendingVoice = null; + [HarmonyPrefix] public static bool Prefix(string snd, double time, double endTime, float volume, int priority) { @@ -74,13 +85,51 @@ public static class PlayWithEndTimePatch double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; // Register looping sound event to the offline mixer (default pitch = 1.0f) - mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); + PendingVoice = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); } } return true; // Let the original method execute } } +/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its voice (). +[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))] +public static class AudioManagerMakeSourcePatch +{ + [HarmonyPostfix] + // ReSharper disable once InconsistentNaming + public static void Postfix(AudioSource __result) + { + var voice = PlayWithEndTimePatch.PendingVoice; + if (voice == null || !__result) return; + PlayWithEndTimePatch.PendingVoice = null; + RenderManager.Instance.AudioMixer?.MapSource(__result, voice); + } +} + +/// +/// AudioSource.isPlaying getter: a source mapped to a voice reports true until the voice has finished, including before its +/// scheduled start, as a source started with PlayScheduled does. During rendering PlayScheduled is blocked, so without this +/// the game sees every effect source as stopped and AudioManager.Update returns it to the reusable pool on the next frame. +/// Voice completion is set by the render thread, which mixes behind the main thread, so the value turns false at or after +/// the render time the voice ends. +/// +[HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)] +public static class AudioSourceIsPlayingPatch +{ + [HarmonyPostfix] + // ReSharper disable once InconsistentNaming + public static void Postfix(AudioSource __instance, ref bool __result) + { + if (__result || !RenderManager.IsCapturingActive) return; + var mixer = RenderManager.Instance.AudioMixer; + if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) + { + __result = !voice.IsFinished; + } + } +} + [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] public static class AudioSourcePitchPatch { @@ -117,6 +166,32 @@ public static class AudioSourceSetScheduledEndTimePatch } } +/// +/// AudioSource.time setter: moves the playback position of the mapped voice (clip seconds) from the current render time. +/// Set before the scheduled start (ffxPlaySound sets it right after AudioManager.Play), it becomes the start position. +/// +[HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)] +public static class AudioSourceTimePatch +{ + [HarmonyPostfix] + // ReSharper disable once InconsistentNaming + public static void Postfix(AudioSource __instance, float value) + { + if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) + { + var mixer = RenderManager.Instance.AudioMixer; + if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) + { + voice.SetPosition(value, RenderManager.Instance.AudioTime); + } + } + } +} + +/// +/// AudioSource.Stop(): stops the mapped voice at the current render time and removes the mapping. +/// AudioManager.Update calls Stop() on sources it recycles, which are finished (). +/// [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])] public static class AudioSourceStopPatch { @@ -126,7 +201,32 @@ public static class AudioSourceStopPatch { if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) { - RenderManager.Instance.AudioMixer?.RemoveSourceMapping(__instance); + var mixer = RenderManager.Instance.AudioMixer; + if (mixer == null) return; + if (mixer.TryGetVoice(__instance, out var voice)) + { + voice.Stop(RenderManager.Instance.AudioTime); + } + mixer.RemoveSourceMapping(__instance); + } + } +} + +/// +/// AudioManager.StopAllSounds destroys every source in liveSources (all AudioManager._Play and scrConductor.PlayWithEndTime sources), +/// so every effect voice stops at the current render time, including ones scheduled to start later. +/// Callers include scrConductor.PlayHitTimes (from its second call), ScrubMusicToTile/Time, KillAllSoundsCoroutine, +/// scrController.FailAction and scnGame.Play. +/// +[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.StopAllSounds))] +public static class AudioManagerStopAllSoundsPatch +{ + [HarmonyPrefix] + public static void Prefix() + { + if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) + { + RenderManager.Instance.AudioMixer?.StopAllEffects(RenderManager.Instance.AudioTime); } } } \ No newline at end of file diff --git a/Util/Audio/Mixer/Voice.cs b/Util/Audio/Mixer/Voice.cs index a5206ce..123bf8b 100644 --- a/Util/Audio/Mixer/Voice.cs +++ b/Util/Audio/Mixer/Voice.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading; using ADOFAIRenderer.Util.Audio.Clip; namespace ADOFAIRenderer.Util.Audio.Mixer; @@ -64,6 +65,12 @@ internal sealed class Voice internal bool Loop; internal bool Finished; + /// + /// Playback has ended and the voice was released by the render thread. Readable from any thread; + /// the value can lag behind the render thread. + /// + public bool IsFinished => Volatile.Read(ref Finished); + /// Loop length in frames at the output sample rate (source duration * output rate). internal double Period; diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 82fbf9a..0fa528b 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -32,6 +32,9 @@ internal sealed class OfflineAudioMixer : IDisposable /// Additional gain of effect sounds (registered with ). private const double EffectVolume = 2.0 / 3.0; + /// Number of tracked effect voices at which finished ones are removed from the list used by . + private const int EffectVoicePruneCount = 1024; + /// Longest wait for a commit before checking for cancellation again, in milliseconds. private const int CommitWaitMs = 50; @@ -45,6 +48,7 @@ internal sealed class OfflineAudioMixer : IDisposable // Main thread. private ConditionalWeakTable _sourceVoices = new(); + private readonly List _effectVoices = new(); private AudioClipSource? _bgmSource; private string _bgmKey = ""; private Voice? _bgmVoice; @@ -182,16 +186,26 @@ internal sealed class OfflineAudioMixer : IDisposable /// Starts an effect sound at . An of 0 or more makes it loop until then. /// (when given) is mapped to the voice for later changes through the AudioSource. /// - public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) + public Voice RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) { var clipSource = GetOrAddSource(clipName); var loop = endTime >= 0; var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now); if (loop) voice.SetEndTime(endTime, now); - if (!source) return; - _sourceVoices.Remove(source!); - _sourceVoices.Add(source!, voice); + if (_effectVoices.Count >= EffectVoicePruneCount) + _effectVoices.RemoveAll(v => v.IsFinished); + _effectVoices.Add(voice); + + if (source) MapSource(source!, voice); + return voice; + } + + /// Maps to for later changes through the AudioSource (replaces an earlier mapping). + public void MapSource(AudioSource source, Voice voice) + { + _sourceVoices.Remove(source); + _sourceVoices.Add(source, voice); } /// Voice started for by . @@ -203,6 +217,19 @@ internal sealed class OfflineAudioMixer : IDisposable public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source); + /// + /// Stops every effect sound at , including ones scheduled to start later + /// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of + /// AudioManager._Play and scrConductor.PlayWithEndTime). The song voice keeps playing. + /// + public void StopAllEffects(double now) + { + foreach (var voice in _effectVoices) + voice.Stop(now); + _effectVoices.Clear(); + _sourceVoices = new ConditionalWeakTable(); + } + /// /// Marks every command with a time before as queued /// (called after each frame with the time of the next frame). @@ -312,5 +339,6 @@ internal sealed class OfflineAudioMixer : IDisposable filter.Reset(); } _sourceVoices = new ConditionalWeakTable(); + _effectVoices.Clear(); } } From d0f6fa9ecde4b7af131303bcf8a2914bfb2444a8a61bf59174327dbd75445d67 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 12:01:49 +0900 Subject: [PATCH 09/14] Fix Repeated Effect Voice List Scans --- Util/Audio/OfflineAudioMixer.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 0fa528b..17ee323 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -32,7 +32,10 @@ internal sealed class OfflineAudioMixer : IDisposable /// Additional gain of effect sounds (registered with ). private const double EffectVolume = 2.0 / 3.0; - /// Number of tracked effect voices at which finished ones are removed from the list used by . + /// + /// Smallest number of tracked effect voices at which finished ones are removed from the list used by . + /// After each removal the next threshold is twice the remaining count (at least this value). + /// private const int EffectVoicePruneCount = 1024; /// Longest wait for a commit before checking for cancellation again, in milliseconds. @@ -49,6 +52,7 @@ internal sealed class OfflineAudioMixer : IDisposable // Main thread. private ConditionalWeakTable _sourceVoices = new(); private readonly List _effectVoices = new(); + private int _effectVoicePruneAt = EffectVoicePruneCount; private AudioClipSource? _bgmSource; private string _bgmKey = ""; private Voice? _bgmVoice; @@ -193,8 +197,11 @@ internal sealed class OfflineAudioMixer : IDisposable var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now); if (loop) voice.SetEndTime(endTime, now); - if (_effectVoices.Count >= EffectVoicePruneCount) + if (_effectVoices.Count >= _effectVoicePruneAt) + { _effectVoices.RemoveAll(v => v.IsFinished); + _effectVoicePruneAt = Math.Max(EffectVoicePruneCount, _effectVoices.Count * 2); + } _effectVoices.Add(voice); if (source) MapSource(source!, voice); From 89f15eca1e215a2b2e3bf82e224911508e8269a828df690b892d57b6223c5113 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 15:20:04 +0900 Subject: [PATCH 10/14] Read Spectrum Samples From Clip Store --- Core/RenderManager.cs | 16 +- Util/Audio/AudioSpectrumGenerator.cs | 226 +++++++++++++++------------ Util/Audio/OfflineAudioMixer.cs | 3 + 3 files changed, 136 insertions(+), 109 deletions(-) diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index 3b94580..c53b3c1 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -592,19 +592,19 @@ internal class RenderManager : MonoBehaviour scrController.instance.levelWasSkipped = true; if (!scrConductor.instance || !scrConductor.instance.song) yield break; - if (Configuration.AdvancedConfig.ProcessAudio) - { - AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip); - } - if (Configuration.AudioConfig.RenderAudio) { AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate); AudioMixer.InitializeBGM(scrConductor.instance.song.clip); - + var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps); } + + if (Configuration.AdvancedConfig.ProcessAudio) + { + AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer); + } scrConductor.instance.song.clip = null; scrConductor.instance.song.time = -1000; @@ -1021,10 +1021,10 @@ internal class RenderManager : MonoBehaviour _frameReader?.Dispose(); ResetFrameHookState(); + AudioSpectrumGenerator.Clear(); AudioMixer?.Dispose(); AudioMixer = null; SyncCalculator = null; - - AudioSpectrumGenerator.Clear(); + RenderContext.Reset(); } diff --git a/Util/Audio/AudioSpectrumGenerator.cs b/Util/Audio/AudioSpectrumGenerator.cs index 306334a..d8567db 100644 --- a/Util/Audio/AudioSpectrumGenerator.cs +++ b/Util/Audio/AudioSpectrumGenerator.cs @@ -1,164 +1,187 @@ using System; using System.Collections.Generic; using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util.Audio.Clip; +using MelonLoader; using UnityEngine; namespace ADOFAIRenderer.Util.Audio; +/// +/// Replacement for AudioSource.GetSpectrumData during rendering: the spectrum of the song as the song AudioSource would output it. +/// +/// Signal: the song clip from an at the output sample rate (AudioSettings.outputSampleRate), +/// played at conductor.song.pitch (linear interpolation between frames) and scaled by conductor.song.volume. +/// Unity analyzes the output signal of the source: bin spacing follows the output rate, pitch moves the frequencies, +/// and volume scales the magnitudes (measured with a Unity 6000.3.10f1 test project). +/// Channel: stereo clips use the requested channel (0 = left, 1 = right); a mono clip is scaled by . +/// Window: twice the requested spectrum length in output samples, centered on the current render time. +/// +/// The song source is shared with the store when its output rate matches; otherwise a separate +/// store is created. A cursor at the analysis position keeps the needed chunks loaded. Main thread only. +/// internal static class AudioSpectrumGenerator { - // ReSharper disable MemberCanBePrivate.Global - public static float[]? AudioClipData { get; private set; } - public static int Channels { get; private set; } - public static int Frequency { get; private set; } - public static int TotalSamples { get; private set; } - // ReSharper restore MemberCanBePrivate.Global + /// + /// Gain of a mono clip relative to one channel of a stereo clip with the same samples. + /// Measured with a Unity 6000.3.10f1 test project: the spectrum of a mono clip is about 0.70 times that of each stereo channel (1/√2). + /// + private const double MonoGain = 0.70710678118654752; + + private static AudioClipStore? _ownStore; + private static AudioClipStore? _store; + private static AudioClipSource? _source; + private static AudioCursor? _cursor; + private static ChunkView _view; + private static bool _loadErrorLogged; private static long _cachedFrame = -1; - private static readonly Dictionary CachedSpectrums = new(); + private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new(); - public static void Initialize(AudioClip? clip) + /// Registers the song clip. Call after the is created (null when audio is not rendered). + public static void Initialize(AudioClip? clip, OfflineAudioMixer? mixer) { - if (!clip) - { - Clear(); - return; - } + Clear(); + if (!clip) return; try { - Channels = clip.channels; - Frequency = clip.frequency; - int samplesCount = clip.samples; - - float[] rawSamples = new float[samplesCount * Channels]; - clip.GetData(rawSamples, 0); - - if (Channels == 1) + var rate = AudioSettings.outputSampleRate; + if (mixer != null && mixer.Store.OutputRate == rate) { - AudioClipData = rawSamples; - TotalSamples = samplesCount; + _store = mixer.Store; } else { - AudioClipData = new float[samplesCount]; - TotalSamples = samplesCount; - for (int i = 0; i < samplesCount; i++) - { - float sum = 0f; - for (int c = 0; c < Channels; c++) - { - sum += rawSamples[i * Channels + c]; - } - AudioClipData[i] = sum / Channels; - } + _ownStore = new AudioClipStore(rate); + _ownStore.Start(); + _store = _ownStore; } - MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}"); + + _source = AudioClipResolver.GetOrAdd(_store, clip!.name, clip); + if (_source == null) + { + Melon.Logger.Error($"[AudioSpectrumGenerator] Song clip '{clip.name}' could not be loaded, the spectrum will be empty."); + return; + } + _cursor = _source.AddCursor(0, false); + Melon.Logger.Msg($"[AudioSpectrumGenerator] Song '{clip.name}': {_source.Channels} ch at {rate} Hz ({(_ownStore != null ? "own store" : "shared with the mixer")})"); } catch (Exception e) { - MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}"); + Melon.Logger.Error($"[AudioSpectrumGenerator] Failed to initialize: {e}"); Clear(); } } + /// Releases the chunk, the cursor and the own store. Call before the mixer store is disposed. public static void Clear() { - AudioClipData = null; - TotalSamples = 0; - Channels = 0; - Frequency = 0; - lock (CachedSpectrums) + if (_source != null) { - _cachedFrame = -1; - CachedSpectrums.Clear(); + _source.Release(_view); + if (_cursor != null) _source.RemoveCursor(_cursor); } + _view = default; + _cursor = null; + _source = null; + _store = null; + _ownStore?.Dispose(); + _ownStore = null; + _loadErrorLogged = false; + _cachedFrame = -1; + CachedSpectrums.Clear(); } public static void GetSpectrumData(float[] samples, int channel, FFTWindow window) { - if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song) - { - Array.Clear(samples, 0, samples.Length); - return; - } - - var conductor = scrConductor.instance; - var pitch = conductor.song.pitch; - var separateCountdownOffset = conductor.separateCountdownTime ? - conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0; - - // Calculate the actual scheduled start time of the song - var songScheduledStartTime = conductor.skipOffset ? - conductor.dspTimeSong + conductor.addoffset / pitch : - conductor.dspTimeSong + separateCountdownOffset / pitch; - - if (conductor.dspTime < songScheduledStartTime) - { - Array.Clear(samples, 0, samples.Length); - return; - } - - var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset; - var n = samples.Length; - if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2 + var conductor = scrConductor.instance; + if (_source == null || _store == null || !conductor || !conductor!.hasSongStarted || !conductor.song + || n == 0 || (n & (n - 1)) != 0) { - Array.Clear(samples, 0, samples.Length); + Array.Clear(samples, 0, n); return; } - var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1; - - lock (CachedSpectrums) + var render = RenderManager.Instance; + var currentFrame = render.frameCount; + if (currentFrame != _cachedFrame) { - if (currentFrame != _cachedFrame) - { - _cachedFrame = currentFrame; - CachedSpectrums.Clear(); - } - - if (CachedSpectrums.TryGetValue(n, out var cached)) - { - Array.Copy(cached, samples, n); - return; - } + _cachedFrame = currentFrame; + CachedSpectrums.Clear(); } + var sourceChannel = _source.Channels == 1 ? 0 : Math.Min(Math.Max(channel, 0), _source.Channels - 1); + if (CachedSpectrums.TryGetValue((n, sourceChannel), out var cached)) + { + Array.Copy(cached, samples, n); + return; + } + + // Song position (seconds of song time) at the current render time, as in scrConductor.StartMusicCo: + // the song starts at dspTimeSong + separate countdown (crotchetAtStart * adjustedCountdownTicks in song time). + double pitch = conductor.song.pitch; + double volume = conductor.song.volume; + var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0; + var now = render.dspTimeBase + render.AudioTime; + var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown; + var fftSize = 2 * n; - var frequency = Frequency; - var totalSamples = TotalSamples; - var currentSampleIndex = (long)(songTime * frequency); + var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0); + var first = songSeconds * _store.OutputRate - n * pitch; var windowSamples = new float[fftSize]; - for (var i = 0; i < fftSize; i++) + try { - var idx = currentSampleIndex - fftSize / 2 + i; - if (idx >= 0 && idx < totalSamples) + _cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1); + for (var i = 0; i < fftSize; i++) { - windowSamples[i] = AudioClipData[idx]; + var position = first + i * pitch; + var index = (long)Math.Floor(position); + var fraction = position - index; + var s0 = ReadSample(index, sourceChannel); + var s1 = fraction > 0 ? ReadSample(index + 1, sourceChannel) : 0.0; + windowSamples[i] = (float)((s0 + (s1 - s0) * fraction) * gain); } - else + } + catch (InvalidOperationException e) + { + if (!_loadErrorLogged) { - windowSamples[i] = 0f; + _loadErrorLogged = true; + Melon.Logger.Error($"[AudioSpectrumGenerator] Song samples are unavailable, the spectrum will be empty: {e.Message}"); } + Array.Clear(samples, 0, n); + return; } var spectrum = new float[n]; CalculateFFT(windowSamples, spectrum); + CachedSpectrums[(n, sourceChannel)] = spectrum; + Array.Copy(spectrum, samples, n); + } - lock (CachedSpectrums) + /// Sample of source channel at ; 0 outside the clip or when the source failed. + /// Chunk loading failed fatally (see ). + private static double ReadSample(long frame, int channel) + { + var source = _source!; + if (frame < 0 || frame >= source.TotalFrames) return 0.0; + if (!_view.IsValid || !_view.Contains(frame)) { - CachedSpectrums[n] = spectrum; - Array.Copy(spectrum, samples, n); + source.Release(_view); + _view = _store!.Acquire(source, frame); + if (!_view.IsValid) return 0.0; } + return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel]; } private static void CalculateFFT(float[] samples, float[] spectrum) { var n = samples.Length; - // Apply Blackman-Harris window + // Blackman-Harris window, periodic form: w(i) = a0 - a1 cos(2πi/n) + a2 cos(4πi/n) - a3 cos(6πi/n), i = 0 ~ n - 1 const float a0 = 0.35875f; const float a1 = 0.48829f; const float a2 = 0.14128f; @@ -169,7 +192,7 @@ internal static class AudioSpectrumGenerator for (var i = 0; i < n; i++) { - var angle = 2.0 * Math.PI * i / (n - 1); + var angle = 2.0 * Math.PI * i / n; var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle)); re[i] = samples[i] * w; im[i] = 0f; @@ -230,12 +253,13 @@ internal static class AudioSpectrumGenerator } } - // Normalize and fill spectrum - var factor = 2.0f / n; + // Magnitude scale 2.5 / n (n = FFT size, twice the spectrum length), clamped to 1.0: the scale of Unity GetSpectrumData. + // A sine of amplitude A centered on a bin gives A * a0 * 1.25 with the Blackman-Harris window. + var factor = 2.5f / n; var outLen = Math.Min(spectrum.Length, n / 2); for (var i = 0; i < outLen; i++) { - spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor; + spectrum[i] = Math.Min((float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor, 1f); } } -} \ No newline at end of file +} diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 17ee323..bee91d9 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -107,6 +107,9 @@ internal sealed class OfflineAudioMixer : IDisposable _store.Start(); } + /// Clip store of this render (output rate = mixer sample rate), also read by . + internal AudioClipStore Store => _store; + // ── Main thread ── /// Registers the song clip. The song voice is created and kept in sync by . From 455436e7445fe632f3b9c8881a0765b22efb011c6631f3f2edfc6f99b7f58196 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 15:37:16 +0900 Subject: [PATCH 11/14] Fix Audio Cleanup On Render Abort --- Core/RenderManager.cs | 39 +++++++++++++++++++--------- Util/Audio/AudioSpectrumGenerator.cs | 1 + 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index c53b3c1..da1900d 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -258,8 +258,10 @@ internal class RenderManager : MonoBehaviour if (stage != RenderStage.None) MarkFailed("the game exited or RenderManager was destroyed during rendering."); ResetRenderState(success: false); - + AudioSpectrumGenerator.Clear(); + // Cancels the render thread's audio waits, then releases the clip store (decoder thread, readers, chunks). + AudioMixer?.Dispose(); AudioMixer = null; ResetFrameHookState(); @@ -726,23 +728,35 @@ internal class RenderManager : MonoBehaviour endSuccessful = !IsRenderAborted; } + catch (OperationCanceledException) when (IsRenderAborted) + { + // Audio waits end with OperationCanceledException when the render is cancelled or has failed (OfflineAudioMixer.Cancel). + Melon.Logger.Msg("Render thread stopped: the render was aborted."); + DisposeEncoderAfterStop(videoFFmpeg); + } catch (Exception ex) { Melon.Logger.Error($"FFmpeg rendering thread error: {ex}"); MarkFailed("encoder thread error. See the log for details."); - try - { - videoFFmpeg.Dispose(); - } - catch (Exception disposeEx) - { - Melon.Logger.Error($"Encoder dispose after thread error also failed: {disposeEx.Message}"); - } - UnregisterEncoder(videoFFmpeg); + DisposeEncoderAfterStop(videoFFmpeg); } } + /// Releases the encoder after the render thread stopped early (abort or error) and removes it from the encoder list. + private void DisposeEncoderAfterStop(FFmpegNativeEncoder videoFFmpeg) + { + try + { + videoFFmpeg.Dispose(); + } + catch (Exception disposeEx) + { + Melon.Logger.Error($"Encoder dispose after the render thread stopped also failed: {disposeEx.Message}"); + } + UnregisterEncoder(videoFFmpeg); + } + /// /// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio /// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready. @@ -873,12 +887,13 @@ internal class RenderManager : MonoBehaviour private void StopFFmpeg(bool success) { endRequested = true; - if (success) AudioMixer?.Complete(); - else AudioMixer?.Cancel(); if (!success) { MarkCancelled(); } + // The abort state is set first: the render thread reads it when its audio wait is cancelled. + if (success) AudioMixer?.Complete(); + else AudioMixer?.Cancel(); FrameAvailableEvent.Set(); skipPressToStart = false; _renderSetup = false; diff --git a/Util/Audio/AudioSpectrumGenerator.cs b/Util/Audio/AudioSpectrumGenerator.cs index d8567db..58d1c4c 100644 --- a/Util/Audio/AudioSpectrumGenerator.cs +++ b/Util/Audio/AudioSpectrumGenerator.cs @@ -171,6 +171,7 @@ internal static class AudioSpectrumGenerator if (!_view.IsValid || !_view.Contains(frame)) { source.Release(_view); + _view = default; _view = _store!.Acquire(source, frame); if (!_view.IsValid) return 0.0; } From 5136127e39a77e43a4f9a197ac832a3bd6d4c5b63f21166fce09fb274f99881f Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 19:20:59 +0900 Subject: [PATCH 12/14] Round Voice Start Times Down To Output Samples --- Util/Audio/Mixer/VoiceMixer.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Util/Audio/Mixer/VoiceMixer.cs b/Util/Audio/Mixer/VoiceMixer.cs index 9be63e6..b522d4b 100644 --- a/Util/Audio/Mixer/VoiceMixer.cs +++ b/Util/Audio/Mixer/VoiceMixer.cs @@ -75,8 +75,11 @@ internal sealed class VoiceMixer : IDisposable /// Voices currently mixed (started or waiting to start). Render thread only. public int ActiveVoiceCount => _voices.Count; - /// Output sample nearest to . - public static long ToSample(double seconds, int rate) => (long)Math.Round(seconds * rate, MidpointRounding.AwayFromZero); + /// + /// Output sample containing (seconds × rate rounded down), the sample at which + /// AudioSource.PlayScheduled starts a sound scheduled between two output samples. + /// + public static long ToSample(double seconds, int rate) => (long)Math.Floor(seconds * rate); // ── Main thread ── From 69d098e04cc7330c0a220ca94113154bdb8a07bed8d29cbb4313f2711385086d Mon Sep 17 00:00:00 2001 From: copecone Date: Wed, 14 Oct 2026 21:55:13 +0900 Subject: [PATCH 13/14] Emulate AudioSource Playback State For Effect Sounds --- Patch/Engine/AudioPatches.cs | 207 ++++++++++++++++++++------------ Util/Audio/OfflineAudioMixer.cs | 38 +++--- Util/Audio/SourceState.cs | 140 +++++++++++++++++++++ 3 files changed, 290 insertions(+), 95 deletions(-) create mode 100644 Util/Audio/SourceState.cs diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 0a518fa..0202995 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -1,7 +1,7 @@ using System; using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util.Audio; using ADOFAIRenderer.Util.Audio.Clip; -using ADOFAIRenderer.Util.Audio.Mixer; using HarmonyLib; using UnityEngine; using UnityEngine.Audio; @@ -63,11 +63,11 @@ public static class AudioManagerPlayPatch [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] public static class PlayWithEndTimePatch { - /// Voice registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. - internal static Voice? PendingVoice; + /// State registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. + internal static SourceState? PendingState; [HarmonyFinalizer] - public static void Finalizer() => PendingVoice = null; + public static void Finalizer() => PendingState = null; [HarmonyPrefix] public static bool Prefix(string snd, double time, double endTime, float volume, int priority) @@ -85,14 +85,14 @@ public static class PlayWithEndTimePatch double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; // Register looping sound event to the offline mixer (default pitch = 1.0f) - PendingVoice = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); + PendingState = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); } } return true; // Let the original method execute } } -/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its voice (). +/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (). [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))] public static class AudioManagerMakeSourcePatch { @@ -100,115 +100,166 @@ public static class AudioManagerMakeSourcePatch // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __result) { - var voice = PlayWithEndTimePatch.PendingVoice; - if (voice == null || !__result) return; - PlayWithEndTimePatch.PendingVoice = null; - RenderManager.Instance.AudioMixer?.MapSource(__result, voice); + var state = PlayWithEndTimePatch.PendingState; + if (state == null || !__result) return; + PlayWithEndTimePatch.PendingState = null; + RenderManager.Instance.AudioMixer?.MapSource(__result, state); } } /// -/// AudioSource.isPlaying getter: a source mapped to a voice reports true until the voice has finished, including before its -/// scheduled start, as a source started with PlayScheduled does. During rendering PlayScheduled is blocked, so without this -/// the game sees every effect source as stopped and AudioManager.Update returns it to the reusable pool on the next frame. -/// Voice completion is set by the render thread, which mixes behind the main thread, so the value turns false at or after -/// the render time the voice ends. +/// AudioSource members of an effect source mapped to a during rendering: the patches below skip the +/// Unity implementation and read or change the state instead (isPlaying, time, timeSamples, pitch, SetScheduledEndTime, Stop). +/// Sources without a mapping (the song, menu sounds) use the Unity implementation. +/// +internal static class EffectSource +{ + /// State mapped to while capturing, with the current render time in . + public static bool TryGet(AudioSource source, out SourceState state, out double now) + { + state = null!; + now = 0; + if (!RenderManager.IsCapturingActive) return false; + var mixer = RenderManager.Instance.AudioMixer; + if (mixer == null || !mixer.TryGetSourceState(source, out state)) return false; + now = RenderManager.Instance.AudioTime; + return true; + } +} + +/// +/// AudioSource.isPlaying getter (): true from the scheduling call until Stop(), the scheduled end time or +/// the end of the clip, including before the scheduled start. AudioManager.Update returns a source to its reusable pool once this is false. /// [HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)] public static class AudioSourceIsPlayingPatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, ref bool __result) + public static bool Prefix(AudioSource __instance, ref bool __result) { - if (__result || !RenderManager.IsCapturingActive) return; - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - __result = !voice.IsFinished; - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + __result = state.IsPlaying(now); + return false; } } -[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] -public static class AudioSourcePitchPatch +/// AudioSource.time getter (): clip position in seconds while playing, 0 while not playing. +[HarmonyPatch(typeof(AudioSource), "time", MethodType.Getter)] +public static class AudioSourceTimeGetterPatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, float value) + public static bool Prefix(AudioSource __instance, ref float __result) { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - voice.SetPitch(value, RenderManager.Instance.AudioTime); - } - } - } -} - -[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))] -public static class AudioSourceSetScheduledEndTimePatch -{ - [HarmonyPostfix] - // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, double time) - { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - voice.SetEndTime(time - RenderManager.Instance.dspTimeBase, RenderManager.Instance.AudioTime); - } - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + __result = state.GetTime(now); + return false; } } /// -/// AudioSource.time setter: moves the playback position of the mapped voice (clip seconds) from the current render time. +/// AudioSource.time setter (): moves the playback position (clip seconds) from the current render time. /// Set before the scheduled start (ffxPlaySound sets it right after AudioManager.Play), it becomes the start position. /// [HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)] public static class AudioSourceTimePatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, float value) + public static bool Prefix(AudioSource __instance, float value) { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - voice.SetPosition(value, RenderManager.Instance.AudioTime); - } - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetTime(value, now); + return false; + } +} + +/// AudioSource.timeSamples getter (): clip position in samples of the clip. +[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Getter)] +public static class AudioSourceTimeSamplesGetterPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, ref int __result) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + __result = state.GetTimeSamples(now); + return false; + } +} + +/// AudioSource.timeSamples setter (): moves the playback position (samples of the clip). +[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Setter)] +public static class AudioSourceTimeSamplesPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, int value) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetTimeSamples(value, now); + return false; + } +} + +/// AudioSource.pitch getter (): the last pitch set. +[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Getter)] +public static class AudioSourcePitchGetterPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, ref float __result) + { + if (!EffectSource.TryGet(__instance, out var state, out _)) return true; + __result = state.GetPitch(); + return false; + } +} + +/// AudioSource.pitch setter (): changes the playback rate from the current render time. +[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] +public static class AudioSourcePitchPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, float value) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetPitch(value, now); + return false; + } +} + +/// AudioSource.SetScheduledEndTime (): DSP time (absolute) at which playback stops. +[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))] +public static class AudioSourceSetScheduledEndTimePatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, double time) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetEndTime(time - RenderManager.Instance.dspTimeBase, now); + return false; } } /// -/// AudioSource.Stop(): stops the mapped voice at the current render time and removes the mapping. -/// AudioManager.Update calls Stop() on sources it recycles, which are finished (). +/// AudioSource.Stop() (): stops playback at the current render time. The mapping stays, so the source +/// keeps reporting the stopped state. AudioManager.Update calls Stop() on sources it returns to the pool, which are no longer +/// playing, so their voices are left unchanged. /// [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])] public static class AudioSourceStopPatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance) + public static bool Prefix(AudioSource __instance) { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer == null) return; - if (mixer.TryGetVoice(__instance, out var voice)) - { - voice.Stop(RenderManager.Instance.AudioTime); - } - mixer.RemoveSourceMapping(__instance); - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.Stop(now); + return false; } } diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index bee91d9..3f5ef28 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -50,7 +50,7 @@ internal sealed class OfflineAudioMixer : IDisposable private readonly int _latencySamples; // Main thread. - private ConditionalWeakTable _sourceVoices = new(); + private ConditionalWeakTable _sourceStates = new(); private readonly List _effectVoices = new(); private int _effectVoicePruneAt = EffectVoicePruneCount; private AudioClipSource? _bgmSource; @@ -191,9 +191,9 @@ internal sealed class OfflineAudioMixer : IDisposable /// /// Starts an effect sound at . An of 0 or more makes it loop until then. - /// (when given) is mapped to the voice for later changes through the AudioSource. + /// (when given) is mapped to the returned state, which reports and applies later changes through the AudioSource. /// - public Voice RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) + public SourceState RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) { var clipSource = GetOrAddSource(clipName); var loop = endTime >= 0; @@ -207,26 +207,30 @@ internal sealed class OfflineAudioMixer : IDisposable } _effectVoices.Add(voice); - if (source) MapSource(source!, voice); - return voice; + var state = new SourceState(voice, Math.Max(startTime, now), pitch, loop, loop ? endTime : double.PositiveInfinity, + clipSource?.Duration ?? 0.0, _sampleRate); + if (source) MapSource(source!, state); + return state; } - /// Maps to for later changes through the AudioSource (replaces an earlier mapping). - public void MapSource(AudioSource source, Voice voice) + /// + /// Maps to (replaces an earlier mapping) and takes the clip length and + /// sample rate from the clip of the source. + /// + public void MapSource(AudioSource source, SourceState state) { - _sourceVoices.Remove(source); - _sourceVoices.Add(source, voice); + state.BindClip(source.clip); + _sourceStates.Remove(source); + _sourceStates.Add(source, state); } - /// Voice started for by . - public bool TryGetVoice(AudioSource? source, out Voice voice) + /// State of the effect sound last started for . The mapping stays after the sound ends or stops. + public bool TryGetSourceState(AudioSource? source, out SourceState state) { - voice = null!; - return source != null && _sourceVoices.TryGetValue(source, out voice); + state = null!; + return source != null && _sourceStates.TryGetValue(source, out state); } - public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source); - /// /// Stops every effect sound at , including ones scheduled to start later /// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of @@ -237,7 +241,7 @@ internal sealed class OfflineAudioMixer : IDisposable foreach (var voice in _effectVoices) voice.Stop(now); _effectVoices.Clear(); - _sourceVoices = new ConditionalWeakTable(); + _sourceStates = new ConditionalWeakTable(); } /// @@ -348,7 +352,7 @@ internal sealed class OfflineAudioMixer : IDisposable foreach (var filter in _filters) filter.Reset(); } - _sourceVoices = new ConditionalWeakTable(); + _sourceStates = new ConditionalWeakTable(); _effectVoices.Clear(); } } diff --git a/Util/Audio/SourceState.cs b/Util/Audio/SourceState.cs new file mode 100644 index 0000000..161c3ad --- /dev/null +++ b/Util/Audio/SourceState.cs @@ -0,0 +1,140 @@ +using System; +using ADOFAIRenderer.Util.Audio.Mixer; +using UnityEngine; + +namespace ADOFAIRenderer.Util.Audio; + +/// +/// Playback state of a game AudioSource mapped to an effect , as the AudioSource reports it to the game +/// (isPlaying, time, timeSamples, pitch). Main thread only. Times are seconds relative to the render start (the render clock that +/// also drives AudioSettings.dspTime during rendering); clip positions are seconds into the clip. +/// +/// Reported values: +/// - Playing: from the scheduled start until Stop(), the scheduled end time, or the natural end of the clip (not looping). +/// Before the start, isPlaying is true and time is the position playback will start from. +/// - time while playing: start position + elapsed time * pitch since the last change, wrapped at the clip length when looping. +/// time is 0 while not playing. +/// - timeSamples while playing: floor(time * clip frequency). While not playing: the last value set through time or timeSamples, +/// reset to 0 by Stop(). +/// - pitch: the last finite value set (NaN and infinity are ignored). Playback uses max(0, pitch). +/// Changes made while not playing are stored and do not reach the voice. +/// +internal sealed class SourceState +{ + private readonly double _start; + private double _anchorTime; + private double _anchorPosition; + private double _pitch; + private readonly bool _loop; + private double _endTime = double.PositiveInfinity; + private bool _stopped; + private int _offsetSamples; + private double _length; + private int _frequency; + + /// Voice that plays the sound. + /// Start of playback (seconds relative to the render start, not before the time it was scheduled). + /// Pitch at the start. + /// Repeats the clip. + /// Scheduled end time (seconds relative to the render start), or infinity. + /// Clip length in seconds (replaced by the game clip in ). + /// Clip sample rate for timeSamples (replaced by the game clip in ). + public SourceState(Voice voice, double start, double pitch, bool loop, double endTime, double length, int frequency) + { + Voice = voice; + _start = start; + _anchorTime = start; + _pitch = pitch; + _loop = loop; + _endTime = endTime; + _length = length; + _frequency = Math.Max(1, frequency); + } + + public Voice Voice { get; } + + /// Takes the clip length and sample rate from the clip of the game AudioSource. + public void BindClip(AudioClip? clip) + { + if (!clip) return; + _length = clip!.length; + _frequency = Math.Max(1, clip.frequency); + } + + /// Scheduled end time (see AudioSource.SetScheduledEndTime), seconds relative to the render start. + public void SetEndTime(double endTime, double now) + { + if (!IsPlaying(now)) return; + _endTime = endTime; + Voice.SetEndTime(endTime, now); + } + + public bool IsPlaying(double now) => !_stopped && now < Math.Min(_endTime, NaturalEnd()); + + public float GetTime(double now) => IsPlaying(now) ? (float)PositionAt(now) : 0f; + + public int GetTimeSamples(double now) => IsPlaying(now) ? (int)Math.Floor(PositionAt(now) * _frequency) : _offsetSamples; + + public float GetPitch() => (float)_pitch; + + public void SetTime(float seconds, double now) + { + _offsetSamples = (int)Math.Floor((double)seconds * _frequency); + Seek(seconds, now); + } + + public void SetTimeSamples(int samples, double now) + { + _offsetSamples = samples; + Seek((double)samples / _frequency, now); + } + + public void SetPitch(float pitch, double now) + { + if (float.IsNaN(pitch) || float.IsInfinity(pitch)) return; + var playing = IsPlaying(now); + if (playing) Reanchor(now); + _pitch = pitch; + if (playing) Voice.SetPitch(pitch, now); + } + + public void Stop(double now) + { + if (IsPlaying(now)) Voice.Stop(now); + _stopped = true; + _offsetSamples = 0; + } + + private double PlaybackPitch => Math.Max(0.0, _pitch); + + private void Seek(double seconds, double now) + { + if (!IsPlaying(now)) return; + var position = Math.Max(0.0, seconds); + _anchorTime = Math.Max(now, _start); + _anchorPosition = position; + Voice.SetPosition(position, now); + } + + private void Reanchor(double now) + { + var time = Math.Max(now, _start); + _anchorPosition = PositionAt(time); + _anchorTime = time; + } + + /// Clip position at (the start position before the start), wrapped when looping. + private double PositionAt(double now) + { + var position = _anchorPosition + (Math.Max(now, _anchorTime) - _anchorTime) * PlaybackPitch; + if (_loop && _length > 0 && position >= _length) position %= _length; + return position; + } + + /// Time at which the clip position reaches the clip length (infinity when looping or at pitch 0). + private double NaturalEnd() + { + if (_loop || PlaybackPitch <= 0) return double.PositiveInfinity; + return _anchorTime + Math.Max(0.0, _length - _anchorPosition) / PlaybackPitch; + } +} From ab8164a73561d349a5960838ca71a8b00d3fb0c457fe7a6e6009e2e522b1a6b8 Mon Sep 17 00:00:00 2001 From: copecone Date: Wed, 14 Oct 2026 22:12:00 +0900 Subject: [PATCH 14/14] Deactivate Mapped Effect Sources During Render --- Core/RenderManager.cs | 2 ++ Patch/Engine/AudioPatches.cs | 47 +++++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index da1900d..a2c48d4 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -9,6 +9,7 @@ using System.Linq; using ADOFAIRenderer.API; using ADOFAIRenderer.Core.Data; using ADOFAIRenderer.Data; +using ADOFAIRenderer.Patch.Engine; using ADOFAIRenderer.UI; using ADOFAIRenderer.UI.Window; using ADOFAIRenderer.Util; @@ -424,6 +425,7 @@ internal class RenderManager : MonoBehaviour { AudioMixer.UpdateBGM(AudioTime, dspTimeBase); AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS); + EffectSource.DeactivateMappedSources(AudioMixer); } if (!texture) return; diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 0202995..9d9c2f1 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using ADOFAIRenderer.Core; using ADOFAIRenderer.Util.Audio; using ADOFAIRenderer.Util.Audio.Clip; @@ -51,7 +52,10 @@ public static class AudioManagerPlayPatch double relativeStartTime = time - RenderManager.Instance.dspTimeBase; // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource - RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f, RenderManager.Instance.AudioTime); + var mixer = RenderManager.Instance.AudioMixer; + if (mixer == null) return; + mixer.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f, RenderManager.Instance.AudioTime); + EffectSource.DeactivateAtFrameEnd(__result); } } } @@ -103,18 +107,53 @@ public static class AudioManagerMakeSourcePatch var state = PlayWithEndTimePatch.PendingState; if (state == null || !__result) return; PlayWithEndTimePatch.PendingState = null; - RenderManager.Instance.AudioMixer?.MapSource(__result, state); + var mixer = RenderManager.Instance.AudioMixer; + if (mixer == null) return; + mixer.MapSource(__result, state); + EffectSource.DeactivateAtFrameEnd(__result); } } /// /// AudioSource members of an effect source mapped to a during rendering: the patches below skip the /// Unity implementation and read or change the state instead (isPlaying, time, timeSamples, pitch, SetScheduledEndTime, Stop). -/// Sources without a mapping (the song, menu sounds) use the Unity implementation. +/// The patches apply only while returns true: rendering is capturing (RenderManager.IsCapturingActive, +/// stage Render), the render has an audio mixer (created when the render starts, cleared when it ends), and the source is mapped +/// in that mixer (mappings are created only when RenderAudio is on). Otherwise they run the Unity implementation, so sources +/// without a mapping (the song, menu sounds) and all sources outside rendering behave as in the game. +/// +/// The game object of a mapped source is deactivated at the end of the frame it was mapped in: the mixer produces its sound and the +/// state answers every member above, so Unity does not process the source. The game activates it again when AudioManager.MakeSource +/// reuses it. Active AudioSources add main-thread time per frame in proportion to their count, and dense levels keep thousands of +/// scheduled effect sources alive. /// internal static class EffectSource { - /// State mapped to while capturing, with the current render time in . + /// Sources mapped during the current frame, deactivated by . Main thread only. + private static readonly List PendingDeactivation = new(); + + /// Deactivates the game object of at the end of the current frame (). + public static void DeactivateAtFrameEnd(AudioSource source) => PendingDeactivation.Add(source); + + /// + /// Deactivates the game objects of the sources mapped this frame that are still mapped and not destroyed. + /// Called by RenderManager.OnPostFrame after the audio commands of the frame are committed, which runs only in stage Render with + /// an audio mixer; sources are added only by the render-time registrations (AudioManager._Play and scrConductor.PlayWithEndTime + /// patches with RenderAudio on). + /// + public static void DeactivateMappedSources(OfflineAudioMixer mixer) + { + foreach (var source in PendingDeactivation) + { + if (source && mixer.TryGetSourceState(source, out _)) source.gameObject.SetActive(false); + } + PendingDeactivation.Clear(); + } + + /// + /// State mapped to in the mixer of the current render, with the current render time in . + /// False outside rendering (stage below Render), without a mixer, or for a source that is not mapped. + /// public static bool TryGet(AudioSource source, out SourceState state, out double now) { state = null!;