From 0a12c12ff584f65540c66b3de47349f98095586251aa3f91016eaeeda0b35e9b Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 00:30:02 +0900 Subject: [PATCH] Add Chunked Audio Clip Store --- Util/Audio/Clip/AudioClipSource.cs | 608 +++++++++++++++++++++++++++++ Util/Audio/Clip/AudioClipStore.cs | 337 ++++++++++++++++ Util/Audio/Clip/DecodeStream.cs | 181 +++++++++ Util/Audio/Clip/IPCMReader.cs | 42 ++ Util/Audio/Clip/MemoryPCMReader.cs | 83 ++++ Util/Audio/Clip/PCMResampler.cs | 106 +++++ Util/Audio/Clip/UnityClipLoader.cs | 55 +++ 7 files changed, 1412 insertions(+) create mode 100644 Util/Audio/Clip/AudioClipSource.cs create mode 100644 Util/Audio/Clip/AudioClipStore.cs create mode 100644 Util/Audio/Clip/DecodeStream.cs create mode 100644 Util/Audio/Clip/IPCMReader.cs create mode 100644 Util/Audio/Clip/MemoryPCMReader.cs create mode 100644 Util/Audio/Clip/PCMResampler.cs create mode 100644 Util/Audio/Clip/UnityClipLoader.cs diff --git a/Util/Audio/Clip/AudioClipSource.cs b/Util/Audio/Clip/AudioClipSource.cs new file mode 100644 index 0000000..9cb11ae --- /dev/null +++ b/Util/Audio/Clip/AudioClipSource.cs @@ -0,0 +1,608 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using MelonLoader; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Read position registered on an by one consumer (a playing voice or the spectrum generator). +/// For a windowed source, the chunk containing and the following chunks +/// ( chunks in total) are kept loaded. +/// +internal sealed class AudioCursor +{ + private readonly AudioClipSource _source; + private long _position; + private volatile bool _loop; + + internal AudioCursor(AudioClipSource source, long position, bool loop) + { + _source = source; + _position = position; + _loop = loop; + } + + /// + /// Frame in the source at the output sample rate (0 ~ TotalFrames - 1). + /// Written by the consumer that reads the source; read by the decoder thread. + /// Moving into another chunk wakes the decoder thread. + /// + public long Position + { + get => Interlocked.Read(ref _position); + set + { + var previous = Interlocked.Exchange(ref _position, value); + if (previous / _source.FramesPerChunk != value / _source.FramesPerChunk) + _source.OnCursorMoved(); + } + } + + /// When true, the chunk after the last chunk is chunk 0. + public bool Loop + { + get => _loop; + set + { + _loop = value; + _source.OnCursorMoved(); + } + } +} + +/// +/// One loaded chunk of an , held by a consumer between +/// and . +/// The chunk is not freed while held. +/// +internal readonly struct ChunkView +{ + /// Interleaved samples. Frame f of the source is at (f - StartFrame) * Channels. + public readonly float[] Data; + + /// Source frame of the first frame in . + public readonly long StartFrame; + + /// Valid frames in this chunk. Frames past the end of the source are not included. + public readonly int FrameCount; + + /// Channel count of the stored samples (same as the source clip). + public readonly int Channels; + + public readonly int ChunkIndex; + + internal ChunkView(float[] data, long startFrame, int frameCount, int channels, int chunkIndex) + { + Data = data; + StartFrame = startFrame; + FrameCount = frameCount; + Channels = channels; + ChunkIndex = chunkIndex; + } + + public bool IsValid => Data != null; + + public long EndFrame => StartFrame + FrameCount; + + public bool Contains(long frame) => frame >= StartFrame && frame < StartFrame + FrameCount; + + /// + /// Sample of mapped to output channel of . + /// Mapping: mono output averages all source channels; a mono source is copied to every output channel; + /// otherwise output channel c takes source channel c, and output channels beyond the source take source channel 0. + /// must be inside this chunk. + /// + public float GetSample(long frame, int outChannel, int outChannels) + { + var baseIndex = (int)(frame - StartFrame) * Channels; + if (Channels == 1) + return Data[baseIndex]; + + if (outChannels == 1) + { + var sum = 0f; + for (var c = 0; c < Channels; c++) + sum += Data[baseIndex + c]; + return sum / Channels; + } + + return Data[baseIndex + (outChannel < Channels ? outChannel : 0)]; + } +} + +/// +/// How an keeps its chunks. +/// +internal enum ChunkPolicy +{ + /// + /// Only the chunks required by registered cursors (current chunk and the next SlotCount - 1 chunks per cursor), + /// chunks requested by a waiting consumer, and chunks currently held are kept. Other chunks are freed and reloaded on demand. + /// Used for readers that can seek (files decoded with FFmpeg) when the clip is longer than one chunk. + /// + Windowed, + + /// + /// All chunks are loaded once, in order from chunk 0, starting when the source is created, and kept until the store is disposed. + /// Used for forward-only readers (clip data copied from Unity, released as it is consumed) and for clips of one chunk or less. + /// + Resident +} + +/// +/// Decoded audio of one clip, resampled to the output sample rate and stored in fixed-size chunks. +/// Channel count stays as in the clip. +/// +/// Chunks: chunk k covers output frames [k * FramesPerChunk, (k + 1) * FramesPerChunk), where +/// = / Channels (rounded down; a frame never spans two chunks). +/// Windowed chunks use pooled arrays of ChunkFloats floats; resident chunks use arrays of their exact length. +/// See for which chunks are kept. +/// +/// Threads: cursors and TryAcquire/Release may be used from any thread. +/// Loading and freeing run only on the store's decoder thread (), through +/// , and . +/// +internal sealed class AudioClipSource +{ + /// Floats per chunk array: 2^22 floats = 16 MiB. + public const int ChunkFloats = 1 << 22; + + /// Chunks kept per cursor of a windowed source: the chunk containing the cursor and the next SlotCount - 1 chunks. + public const int SlotCount = 3; + + /// Load priority of a chunk a consumer is waiting for (lowest value is loaded first). + internal const int PriorityDemanded = 0; + + /// Load priority of slot s of a cursor window is PriorityWindow + s. + internal const int PriorityWindow = 1; + + /// Load priority of resident chunks nobody is waiting for. + internal const int PriorityResident = PriorityWindow + SlotCount; + + /// + /// Most decode streams (reader + resampler) a windowed source opens. A load continues the stream positioned at the chunk start, + /// otherwise opens another stream up to this count, otherwise restarts the least recently used stream. + /// Each stream holds one open reader (file handle and decoder state, measured 0.3 ~ 2.7 MiB for ogg/flac/mp3 with the clip harness). + /// + public const int MaxDecodeStreams = 4; + + /// Decode stream limit of this source ( unless changed for tests). + internal int DecodeStreamLimit { get; set; } = MaxDecodeStreams; + + private readonly AudioClipStore _store; + private readonly IPCMReader _firstReader; + private readonly Func? _openReader; + + // Guarded by _sync: chunk table, hold counts, cursors, demand set, loaded set. + private readonly object _sync = new(); + private readonly float[]?[] _chunks; + private readonly int[] _holdCounts; + private readonly List _cursors = new(); + private readonly HashSet _demanded = new(); + private readonly HashSet _loaded = new(); + private int _residentLoadedCount; // Resident: chunks [0, _residentLoadedCount) are loaded + + // Guarded by _loadLock: decode streams. + private readonly object _loadLock = new(); + private readonly List _streams = new(); + private long _streamUseCounter; + private bool _openFailed; + + private volatile bool _isFailed; + + public string Key { get; } + + /// Channel count of the stored samples (same as the clip). + public int Channels { get; } + + /// Sample rate of the stored samples in Hz (the output sample rate). + public int Frequency { get; } + + /// Frame count at : ceil(clip frames * Frequency / clip rate). + public long TotalFrames { get; } + + /// Clip length in seconds: clip frames / clip rate. + public double Duration { get; } + + public int FramesPerChunk { get; } + + public int ChunkCount { get; } + + public ChunkPolicy Policy { get; } + + public bool IsResident => Policy == ChunkPolicy.Resident; + + /// True after loading failed. A failed source returns no chunks. + public bool IsFailed => _isFailed; + + /// True when every chunk of a resident source is loaded. + public bool IsFullyLoaded + { + get + { + lock (_sync) return _loaded.Count == ChunkCount; + } + } + + /// Number of chunk loads performed (for diagnostics). + public int LoadCount { get; private set; } + + /// Decode streams opened so far (for diagnostics). + public int StreamCount + { + get + { + lock (_loadLock) return _streams.Count; + } + } + + /// Store that owns this source. + /// Clip name or path the source is registered under. + /// First reader; also defines the clip format. + /// Opens another reader of the same clip for an additional decode stream, or null when not possible. + /// Output sample rate in Hz. + internal AudioClipSource(AudioClipStore store, string key, IPCMReader reader, Func? openReader, int outputRate) + { + if (reader.Channels <= 0) throw new ArgumentException("Reader channel count must be positive.", nameof(reader)); + if (reader.SampleRate <= 0) throw new ArgumentException("Reader sample rate must be positive.", nameof(reader)); + if (reader.TotalFrames < 0) throw new ArgumentException("Reader frame count must not be negative.", nameof(reader)); + + _store = store; + _firstReader = reader; + _openReader = openReader; + Key = key; + Channels = reader.Channels; + Frequency = outputRate; + Duration = (double)reader.TotalFrames / reader.SampleRate; + TotalFrames = CeilDiv(reader.TotalFrames * outputRate, reader.SampleRate); + FramesPerChunk = ChunkFloats / Channels; + ChunkCount = (int)CeilDiv(TotalFrames, FramesPerChunk); + Policy = reader.IsForwardOnly || ChunkCount <= 1 ? ChunkPolicy.Resident : ChunkPolicy.Windowed; + + _chunks = new float[ChunkCount][]; + _holdCounts = new int[ChunkCount]; + _streams.Add(new DecodeStream(reader, outputRate)); + } + + // ── Cursors ── + + /// Registers a read position. The decoder thread loads the chunks it requires. + public AudioCursor AddCursor(long position, bool loop) + { + var cursor = new AudioCursor(this, position, loop); + lock (_sync) + { + _cursors.Add(cursor); + } + _store.RequestService(); + return cursor; + } + + public void RemoveCursor(AudioCursor cursor) + { + lock (_sync) + { + _cursors.Remove(cursor); + } + _store.RequestService(); + } + + internal void OnCursorMoved() + { + if (!IsResident) _store.RequestService(); + } + + // ── Consumer access ── + + /// Chunk index containing (0 ~ TotalFrames - 1). + public int ChunkIndexOf(long frame) => (int)(frame / FramesPerChunk); + + /// + /// Holds the chunk containing when it is loaded. + /// When it is not loaded, records a load request (kept until the chunk is acquired), wakes the decoder thread and returns false. + /// Every successful call must be paired with . + /// + public bool TryAcquire(long frame, out ChunkView view) + { + view = default; + if (_isFailed || frame < 0 || frame >= TotalFrames) return false; + + var index = ChunkIndexOf(frame); + lock (_sync) + { + var data = _chunks[index]; + if (data != null) + { + _demanded.Remove(index); + _holdCounts[index]++; + view = CreateView(index, data); + return true; + } + _demanded.Add(index); + } + _store.RequestService(); + return false; + } + + public void Release(in ChunkView view) + { + if (!view.IsValid) return; + lock (_sync) + { + if (_holdCounts[view.ChunkIndex] > 0) + _holdCounts[view.ChunkIndex]--; + } + } + + private ChunkView CreateView(int index, float[] data) + { + var start = (long)index * FramesPerChunk; + var count = (int)Math.Min(FramesPerChunk, TotalFrames - start); + return new ChunkView(data, start, count, Channels, index); + } + + // ── Decoder thread ── + + /// + /// Windowed: frees loaded chunks that are not required (see ) and not held. + /// Resident: does nothing. Decoder thread only. + /// + internal void FreeUnused() + { + if (IsResident) return; + + List? freed = null; + lock (_sync) + { + if (_loaded.Count == 0) return; + List? remove = null; + foreach (var index in _loaded) + { + if (_holdCounts[index] > 0 || IsRequired(index)) continue; + (remove ??= new List()).Add(index); + } + if (remove == null) return; + + foreach (var index in remove) + { + (freed ??= new List()).Add(_chunks[index]!); + _chunks[index] = null; + _loaded.Remove(index); + } + } + + foreach (var data in freed!) + _store.ReturnChunkArray(data); + } + + /// + /// Next chunk to load and its priority (lower loads first), or false when nothing is missing. + /// Windowed: demanded chunks, then each cursor's window by slot. + /// Resident: the next chunk in order; priority is while a consumer is waiting on this source. + /// Decoder thread only. + /// + internal bool TryGetNextLoad(out int index, out int priority) + { + index = -1; + priority = int.MaxValue; + if (_isFailed) return false; + + lock (_sync) + { + if (IsResident) + { + if (_residentLoadedCount >= ChunkCount) return false; + index = _residentLoadedCount; + priority = _demanded.Count > 0 ? PriorityDemanded : PriorityResident; + return true; + } + + foreach (var demanded in _demanded) + { + if (_chunks[demanded] != null) continue; + index = demanded; + priority = PriorityDemanded; + return true; + } + + foreach (var cursor in _cursors) + { + if (!TryGetWindowStart(cursor, out var first, out var loop)) continue; + for (var slot = 0; slot < SlotCount; slot++) + { + if (!TryWindowIndex(first, slot, loop, out var candidate)) break; + if (_chunks[candidate] != null) continue; + if (PriorityWindow + slot < priority) + { + index = candidate; + priority = PriorityWindow + slot; + } + break; + } + } + return index >= 0; + } + } + + /// True when a demand or a cursor window covers . Caller holds _sync. + private bool IsRequired(int index) + { + if (_demanded.Contains(index)) return true; + foreach (var cursor in _cursors) + { + if (!TryGetWindowStart(cursor, out var first, out var loop)) continue; + for (var slot = 0; slot < SlotCount; slot++) + { + if (!TryWindowIndex(first, slot, loop, out var candidate)) break; + if (candidate == index) return true; + } + } + return false; + } + + private bool TryGetWindowStart(AudioCursor cursor, out int first, out bool loop) + { + var position = cursor.Position; + loop = cursor.Loop; + first = 0; + if (position < 0) position = 0; + if (position >= TotalFrames) + { + if (!loop || TotalFrames == 0) return false; + position %= TotalFrames; + } + first = ChunkIndexOf(position); + return true; + } + + private bool TryWindowIndex(int first, int slot, bool loop, out int index) + { + index = first + slot; + if (index < ChunkCount) return true; + if (!loop) return false; + index %= ChunkCount; + return true; + } + + /// + /// Loads chunk if it is not loaded. Resident sources load only the next chunk in order. + /// Decoder thread only. + /// + /// True when this call loaded the chunk. + internal bool LoadChunk(int index) + { + if (_isFailed || index < 0 || index >= ChunkCount) return false; + + lock (_loadLock) + { + lock (_sync) + { + if (_chunks[index] != null) return false; + if (IsResident && index != _residentLoadedCount) + throw new InvalidOperationException($"Resident clip '{Key}' loads chunks in order: expected {_residentLoadedCount}, got {index}."); + } + + float[]? data = null; + try + { + var start = (long)index * FramesPerChunk; + var frames = (int)Math.Min(FramesPerChunk, TotalFrames - start); + data = IsResident ? _store.AllocateExactArray(frames * Channels) : _store.RentChunkArray(); + var stream = SelectStream(start); + stream.LastUse = ++_streamUseCounter; + stream.Fill(start, frames, data); + } + catch (OutOfMemoryException e) + { + ReturnArray(data); + Fail(); + _store.ReportFatal(this, e); + return false; + } + catch (Exception e) + { + ReturnArray(data); + Fail(); + _store.ReportSourceFailure(this, e); + return false; + } + + lock (_sync) + { + _chunks[index] = data; + _loaded.Add(index); + if (IsResident) _residentLoadedCount++; + } + LoadCount++; + + // A resident source never reads again after its last chunk; the reader releases its data now. + if (IsResident && _residentLoadedCount == ChunkCount) + DisposeStreams(); + } + + _store.NotifyChunkLoaded(); + return true; + } + + /// + /// Decode stream for a load starting at output frame : + /// the stream positioned at ; otherwise a new stream while fewer than + /// are open (windowed sources only); otherwise the least recently used stream. Caller holds _loadLock. + /// + private DecodeStream SelectStream(long start) + { + DecodeStream? oldest = null; + foreach (var stream in _streams) + { + if (stream.NextOutFrame == start) return stream; + if (oldest == null || stream.LastUse < oldest.LastUse) oldest = stream; + } + + if (!IsResident && !_openFailed && _openReader != null && _streams.Count < DecodeStreamLimit) + { + IPCMReader? reader = null; + try + { + reader = _openReader(); + if (reader != null) + { + var stream = new DecodeStream(reader, Frequency); + _streams.Add(stream); + return stream; + } + } + catch (Exception e) + { + reader?.Dispose(); + Melon.Logger.Warning($"[AudioClipSource] Could not open another decode stream for '{Key}', reusing open streams: {e.Message}"); + } + _openFailed = true; + } + + return oldest!; + } + + private void Fail() + { + _isFailed = true; + if (IsResident) DisposeStreams(); + } + + private void DisposeStreams() + { + foreach (var stream in _streams) + stream.Dispose(); + _streams.Clear(); + } + + private void ReturnArray(float[]? data) + { + if (data == null) return; + if (IsResident) _store.FreeExactArray(data); + else _store.ReturnChunkArray(data); + } + + /// Releases all chunk arrays and readers. Called by the store after the decoder thread stopped. + internal void DisposeInternal() + { + lock (_loadLock) + { + lock (_sync) + { + foreach (var index in _loaded) + { + ReturnArray(_chunks[index]); + _chunks[index] = null; + } + _loaded.Clear(); + _cursors.Clear(); + _demanded.Clear(); + } + DisposeStreams(); + _firstReader.Dispose(); + } + } + + private static long CeilDiv(long a, long b) => (a + b - 1) / b; +} diff --git a/Util/Audio/Clip/AudioClipStore.cs b/Util/Audio/Clip/AudioClipStore.cs new file mode 100644 index 0000000..7c24eba --- /dev/null +++ b/Util/Audio/Clip/AudioClipStore.cs @@ -0,0 +1,337 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using MelonLoader; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Clip sources of one render, keyed by clip name or path, with a shared pool of chunk arrays and one decoder thread. +/// +/// Threads: +/// - Decoder thread (): the only thread that decodes, resamples, loads and frees chunks. +/// Each step loads one chunk, the one with the lowest priority value over all sources +/// (a chunk a consumer waits for, then cursor windows by slot, then resident chunks in order). +/// - Consumers (render thread, main thread for the spectrum): waits for the decoder thread and never loads. +/// - Main thread: creates readers (for Unity clips this copies the clip data). +/// +/// Memory has no limit; current and peak usage are tracked and logged by . +/// An while loading is kept in and rethrown by . +/// +internal sealed class AudioClipStore : IDisposable +{ + /// Longest wait of the decoder thread between checks when nothing signals it, in milliseconds. + private const int DecoderIdleWaitMs = 50; + + /// Longest wait of a consumer between checks when no chunk is loaded, in milliseconds. + private const int ConsumerWaitMs = 50; + + private readonly Dictionary _sources = new(); + private readonly List _sourceList = new(); + private readonly Stack _chunkPool = new(); + private readonly object _poolLock = new(); + private readonly object _signal = new(); + private readonly AutoResetEvent _serviceRequested = new(false); + + private Thread? _decoderThread; + private volatile bool _stopRequested; + private long _allocatedBytes; + private long _peakBytes; + private volatile Exception? _fatalError; + private volatile bool _disposed; + + /// Sample rate every source is resampled to, in Hz. + public int OutputRate { get; } + + /// Bytes of sample arrays currently allocated (loaded chunks, pooled chunk arrays and copied Unity clip data). + public long AllocatedBytes => Interlocked.Read(ref _allocatedBytes); + + /// Highest value of since creation. + public long PeakBytes => Interlocked.Read(ref _peakBytes); + + /// Out-of-memory error raised while loading, or null. + public Exception? FatalError => _fatalError; + + public AudioClipStore(int outputRate) + { + if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate)); + OutputRate = outputRate; + } + + // ── Decoder thread ── + + /// Starts the decoder thread. Called once. + public void Start() + { + if (_decoderThread != null) throw new InvalidOperationException("Decoder thread already started."); + _decoderThread = new Thread(DecoderLoop) + { + Name = "Audio Clip Decoder", + IsBackground = true + }; + _decoderThread.Start(); + } + + private void DecoderLoop() + { + try + { + while (!_stopRequested) + { + if (!LoadNext()) + _serviceRequested.WaitOne(DecoderIdleWaitMs); + } + } + catch (Exception e) + { + Melon.Logger.Error($"[AudioClipStore] Decoder thread stopped: {e}"); + _fatalError ??= e; + NotifyChunkLoaded(); + } + } + + /// + /// Frees unused windowed chunks, then loads the single chunk with the lowest priority value over all sources. + /// Runs on the decoder thread (tests may call it directly while the thread is not started). + /// + /// True when a chunk was loaded or a load was attempted. + internal bool LoadNext() + { + AudioClipSource[] sources; + lock (_sources) + { + if (_disposed) return false; + sources = _sourceList.ToArray(); + } + + AudioClipSource? best = null; + var bestIndex = -1; + var bestPriority = int.MaxValue; + foreach (var source in sources) + { + source.FreeUnused(); + if (!source.TryGetNextLoad(out var index, out var priority) || priority >= bestPriority) continue; + best = source; + bestIndex = index; + bestPriority = priority; + } + + if (best == null) return false; + best.LoadChunk(bestIndex); + return true; + } + + internal void RequestService() => _serviceRequested.Set(); + + // ── Sources ── + + /// + /// Returns the source registered under , creating it with on first use. + /// A key whose reader could not be created is remembered as null and not retried. + /// For readers that can seek, is kept and called again on the decoder thread + /// to open additional decode streams (see ). + /// + public AudioClipSource? GetOrAdd(string key, Func createReader) + { + lock (_sources) + { + if (_disposed) return null; + if (_sources.TryGetValue(key, out var existing)) return existing; + } + + AudioClipSource? source = null; + try + { + var reader = createReader(); + if (reader != null) + { + try + { + source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : createReader, OutputRate); + } + catch + { + reader.Dispose(); + throw; + } + } + } + catch (OutOfMemoryException e) + { + Melon.Logger.Error($"[AudioClipStore] Out of memory while opening clip '{key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {e}"); + _fatalError ??= e; + } + catch (Exception e) + { + Melon.Logger.Error($"[AudioClipStore] Failed to open clip '{key}': {e}"); + } + + lock (_sources) + { + if (_disposed || _sources.TryGetValue(key, out var raced)) + { + source?.DisposeInternal(); + return _disposed ? null : _sources[key]; + } + _sources[key] = source; + if (source != null) _sourceList.Add(source); + } + + if (source != null) + { + Melon.Logger.Msg($"[AudioClipStore] Opened clip '{key}': {source.Channels}ch, {source.Duration:F3}s, {source.TotalFrames} frames at {OutputRate}Hz, {source.ChunkCount} chunk(s), {source.Policy}"); + RequestService(); + } + return source; + } + + public bool TryGet(string key, out AudioClipSource? source) + { + lock (_sources) + { + return _sources.TryGetValue(key, out source); + } + } + + // ── Consumers ── + + /// + /// Holds the chunk containing , waiting for the decoder thread until it is loaded. + /// Returns an invalid view when the source failed, the frame is outside the source, or the store was disposed. + /// + /// Loading ran out of memory or the decoder thread stopped. + /// was canceled while waiting. + public ChunkView Acquire(AudioClipSource source, long frame, CancellationToken cancellationToken = default) + { + if (frame < 0 || frame >= source.TotalFrames) return default; + + while (true) + { + ThrowIfFatal(); + if (source.IsFailed || _disposed) return default; + if (source.TryAcquire(frame, out var view)) return view; + + lock (_signal) + { + if (source.TryAcquire(frame, out view)) return view; + if (source.IsFailed || _fatalError != null || _disposed) continue; + Monitor.Wait(_signal, ConsumerWaitMs); + } + cancellationToken.ThrowIfCancellationRequested(); + } + } + + private void ThrowIfFatal() + { + var error = _fatalError; + if (error != null) + throw new InvalidOperationException("Audio clip loading failed: " + error.Message, error); + } + + internal void NotifyChunkLoaded() + { + lock (_signal) + { + Monitor.PulseAll(_signal); + } + } + + internal void ReportSourceFailure(AudioClipSource source, Exception error) + { + Melon.Logger.Error($"[AudioClipStore] Failed to load clip '{source.Key}', it will be silent: {error}"); + NotifyChunkLoaded(); + } + + internal void ReportFatal(AudioClipSource source, OutOfMemoryException error) + { + Melon.Logger.Error($"[AudioClipStore] Out of memory while loading clip '{source.Key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {error}"); + _fatalError ??= error; + NotifyChunkLoaded(); + } + + // ── Arrays ── + + internal float[] RentChunkArray() + { + lock (_poolLock) + { + if (_chunkPool.Count > 0) return _chunkPool.Pop(); + } + var data = new float[AudioClipSource.ChunkFloats]; + AddAllocated(data.LongLength * sizeof(float)); + return data; + } + + internal void ReturnChunkArray(float[] data) + { + if (_disposed) + { + AddAllocated(-data.LongLength * sizeof(float)); + return; + } + lock (_poolLock) + { + _chunkPool.Push(data); + } + } + + internal float[] AllocateExactArray(int floats) + { + var data = new float[floats]; + AddAllocated(data.LongLength * sizeof(float)); + return data; + } + + internal void FreeExactArray(float[] data) => AddAllocated(-data.LongLength * sizeof(float)); + + private void AddAllocated(long bytes) + { + var current = Interlocked.Add(ref _allocatedBytes, bytes); + long peak; + while (current > (peak = Interlocked.Read(ref _peakBytes))) + { + if (Interlocked.CompareExchange(ref _peakBytes, current, peak) == peak) break; + } + } + + public void LogMemoryUsage() + { + int sourceCount; + lock (_sources) sourceCount = _sourceList.Count; + Melon.Logger.Msg($"[AudioClipStore] Clips: {sourceCount}, sample memory now {AllocatedBytes / 1048576.0:F1} MiB, peak {PeakBytes / 1048576.0:F1} MiB"); + } + + /// Stops the decoder thread, then releases every source and pooled array. + public void Dispose() + { + AudioClipSource[] sources; + lock (_sources) + { + if (_disposed) return; + _disposed = true; + sources = _sourceList.ToArray(); + } + + _stopRequested = true; + _serviceRequested.Set(); + NotifyChunkLoaded(); + _decoderThread?.Join(); + + LogMemoryUsage(); + + lock (_sources) + { + _sourceList.Clear(); + _sources.Clear(); + } + + foreach (var source in sources) + source.DisposeInternal(); + + lock (_poolLock) + { + while (_chunkPool.Count > 0) + AddAllocated(-_chunkPool.Pop().LongLength * sizeof(float)); + } + } +} diff --git a/Util/Audio/Clip/DecodeStream.cs b/Util/Audio/Clip/DecodeStream.cs new file mode 100644 index 0000000..9ff5bf8 --- /dev/null +++ b/Util/Audio/Clip/DecodeStream.cs @@ -0,0 +1,181 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// One reader plus its resampler, producing output-rate frames of a clip as a continuous stream. +/// Consecutive calls whose ranges follow each other continue the stream (reader position and resampler state); +/// any other range starts a new stream near the requested position. +/// +/// Restart: the input position is moved back by a preroll and down to a multiple of in_rate / gcd(in_rate, out_rate), +/// where input frame k * (in_rate / gcd) maps exactly to output frame k * (out_rate / gcd); +/// the resampler is reset and output before the requested frame is discarded. +/// Measured with the clip harness: output is bit-identical to one continuous conversion of the whole clip. +/// Used by one thread at a time (the store's decoder thread). +/// +internal sealed class DecodeStream : IDisposable +{ + /// Input frames read from the reader per resampler call. + private const int ReadBlockFrames = 4096; + + private readonly PCMResampler? _resampler; + private readonly int _channels; + private readonly int _outputRate; + private readonly float[] _readBuffer; + private float[]? _discardBuffer; + private bool _flushing; + private bool _ended; + + private readonly long _inputStep; + private readonly long _outputStep; + private readonly long _prerollInputFrames; + + public IPCMReader Reader { get; } + + /// Output frame the stream continues at, or -1 when the stream is not positioned. + public long NextOutFrame { get; private set; } = -1; + + /// Value of the owner's use counter at the last (for least-recently-used selection). + public long LastUse { get; set; } + + /// Number of times the stream started anew (seek + resampler reset). + public int RestartCount { get; private set; } + + public DecodeStream(IPCMReader reader, int outputRate) + { + Reader = reader; + _channels = reader.Channels; + _outputRate = outputRate; + _readBuffer = new float[ReadBlockFrames * _channels]; + + if (reader.SampleRate != outputRate) + { + _resampler = new PCMResampler(_channels, reader.SampleRate, outputRate); + var gcd = GCD(reader.SampleRate, outputRate); + _inputStep = reader.SampleRate / gcd; + _outputStep = outputRate / gcd; + + // History fed before the first kept output frame when a stream starts mid-clip. + // The filter spans about filter_size input frames per output frame at ratios up to 1:1 + // and filter_size * (input rate / output rate) input frames when downsampling. + var ratio = Math.Max(1L, (reader.SampleRate + outputRate - 1L) / outputRate); + _prerollInputFrames = 4L * _resampler.FilterSize * ratio + _inputStep; + } + else + { + _inputStep = 1; + _outputStep = 1; + } + } + + /// + /// Writes output frames [start, start + frames) into and zeros the rest of the array. + /// Frames past the end of the decoded input are zero. + /// + public unsafe void Fill(long start, int frames, float[] data) + { + var channels = _channels; + var filled = 0; + + if (_resampler == null) + { + if (Reader.Position != start) + { + Reader.Seek(start); + RestartCount++; + } + while (filled < frames) + { + var request = Math.Min(ReadBlockFrames, frames - filled); + var read = Reader.Read(_readBuffer, request); + if (read <= 0) break; + Array.Copy(_readBuffer, 0, data, filled * channels, read * channels); + filled += read; + } + Array.Clear(data, filled * channels, data.Length - filled * channels); + NextOutFrame = start + frames; + return; + } + + long discard = 0; + if (NextOutFrame != start) + { + // Input frame of the requested output frame, moved back by the preroll and down to a multiple of _inputStep. + var inputFrame = start * Reader.SampleRate / _outputRate - _prerollInputFrames; + var step = inputFrame > 0 ? inputFrame / _inputStep : 0; + + _resampler.Reset(); + Reader.Seek(step * _inputStep); + _flushing = false; + _ended = false; + discard = start - step * _outputStep; + RestartCount++; + } + + fixed (float* pIn = _readBuffer) + fixed (float* pOut = data) + { + while (discard > 0 && !_ended) + { + _discardBuffer ??= new float[ReadBlockFrames * channels]; + fixed (float* pDiscard = _discardBuffer) + { + var capacity = (int)Math.Min(ReadBlockFrames, discard); + discard -= Pull(pIn, pDiscard, capacity); + } + } + + while (filled < frames && !_ended) + filled += Pull(pIn, pOut + (long)filled * channels, frames - filled); + } + + Array.Clear(data, filled * channels, data.Length - filled * channels); + NextOutFrame = _ended ? -1 : start + frames; + } + + /// + /// Writes 1 ~ resampled frames to , reading input as needed. + /// Returns 0 and sets _ended once the input and the resampler's flushed tail are exhausted. + /// + private unsafe int Pull(float* input, float* output, int capacity) + { + var resampler = _resampler!; + if (!_flushing) + { + while (true) + { + var produced = resampler.Convert(input, 0, output, capacity); + if (produced > 0) return produced; + + var read = Reader.Read(_readBuffer, ReadBlockFrames); + if (read <= 0) + { + _flushing = true; + break; + } + + produced = resampler.Convert(input, read, output, capacity); + if (produced > 0) return produced; + } + } + + // Input is exhausted: take the resampler's remaining tail until it returns 0. + var flushed = resampler.Flush(output, capacity); + if (flushed > 0) return flushed; + + _ended = true; + return 0; + } + + public void Dispose() + { + _resampler?.Dispose(); + Reader.Dispose(); + } + + private static int GCD(int a, int b) + { + while (b != 0) (a, b) = (b, a % b); + return a; + } +} diff --git a/Util/Audio/Clip/IPCMReader.cs b/Util/Audio/Clip/IPCMReader.cs new file mode 100644 index 0000000..6561218 --- /dev/null +++ b/Util/Audio/Clip/IPCMReader.cs @@ -0,0 +1,42 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Sequential reader of decoded PCM at the clip's own sample rate and channel count. +/// Samples are interleaved 32-bit float. Positions and counts are in frames (one frame = one sample per channel). +/// Used by one thread at a time (the store's decoder thread). may be called more than once. +/// +internal interface IPCMReader : IDisposable +{ + /// Channel count of the decoded audio (1 or more). + int Channels { get; } + + /// Sample rate of the decoded audio in Hz. + int SampleRate { get; } + + /// Total frame count of the clip at . + long TotalFrames { get; } + + /// + /// True when the data can be read only once, from frame 0 to the end (data is released as it is read). + /// The source then keeps every chunk (). + /// + bool IsForwardOnly { get; } + + /// Frame index that the next starts at. + long Position { get; } + + /// + /// Moves the read position to (0 ~ ). + /// A forward-only reader accepts only frames whose data has not been released. + /// + void Seek(long frame); + + /// + /// Reads up to frames into starting at index 0 and advances the position. + /// must hold at least frameCount * Channels floats. + /// + /// Frames read. 0 at the end of the clip. + int Read(float[] buffer, int frameCount); +} diff --git a/Util/Audio/Clip/MemoryPCMReader.cs b/Util/Audio/Clip/MemoryPCMReader.cs new file mode 100644 index 0000000..4d62840 --- /dev/null +++ b/Util/Audio/Clip/MemoryPCMReader.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Forward-only over PCM held in memory as a list of blocks. +/// Block b holds frames [b * FramesPerBlock, (b + 1) * FramesPerBlock) interleaved; the last block may be shorter. +/// Each block is released (passed to the release callback and dropped) as soon as the read position passes its end, +/// and all remaining blocks are released on . +/// +internal sealed class MemoryPCMReader : IPCMReader +{ + private readonly float[]?[] _blocks; + private readonly Action? _onBlockReleased; + private int _releasedCount; // Blocks [0, _releasedCount) are released + + public int Channels { get; } + public int SampleRate { get; } + public long TotalFrames { get; } + public bool IsForwardOnly => true; + public long Position { get; private set; } + + /// Frames per block (all blocks except the last). + public int FramesPerBlock { get; } + + public MemoryPCMReader(IReadOnlyList blocks, int framesPerBlock, int channels, int sampleRate, long totalFrames, + Action? onBlockReleased) + { + if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); + if (framesPerBlock <= 0) throw new ArgumentOutOfRangeException(nameof(framesPerBlock)); + if (totalFrames > (long)blocks.Count * framesPerBlock) throw new ArgumentException("Blocks hold fewer frames than totalFrames.", nameof(blocks)); + + _blocks = new float[blocks.Count][]; + for (var i = 0; i < blocks.Count; i++) _blocks[i] = blocks[i]; + _onBlockReleased = onBlockReleased; + FramesPerBlock = framesPerBlock; + Channels = channels; + SampleRate = sampleRate; + TotalFrames = totalFrames; + } + + public void Seek(long frame) + { + if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); + if (frame < (long)_releasedCount * FramesPerBlock) + throw new InvalidOperationException($"Frame {frame} was already released by the forward-only reader."); + Position = frame; + } + + public int Read(float[] buffer, int frameCount) + { + var total = (int)Math.Min(frameCount, TotalFrames - Position); + if (total <= 0) return 0; + + var written = 0; + while (written < total) + { + var blockIndex = (int)(Position / FramesPerBlock); + var block = _blocks[blockIndex] ?? throw new InvalidOperationException($"Block {blockIndex} was already released."); + var offset = (int)(Position - (long)blockIndex * FramesPerBlock); + var count = Math.Min(total - written, FramesPerBlock - offset); + Array.Copy(block, offset * Channels, buffer, written * Channels, count * Channels); + written += count; + Position += count; + ReleaseBlocksBefore((int)(Position / FramesPerBlock)); + } + return written; + } + + private void ReleaseBlocksBefore(int blockIndex) + { + while (_releasedCount < blockIndex && _releasedCount < _blocks.Length) + { + var block = _blocks[_releasedCount]; + _blocks[_releasedCount] = null; + _releasedCount++; + if (block != null) _onBlockReleased?.Invoke(block); + } + } + + public void Dispose() => ReleaseBlocksBefore(_blocks.Length); +} diff --git a/Util/Audio/Clip/PCMResampler.cs b/Util/Audio/Clip/PCMResampler.cs new file mode 100644 index 0000000..c9d4ef0 --- /dev/null +++ b/Util/Audio/Clip/PCMResampler.cs @@ -0,0 +1,106 @@ +using System; +using FFmpeg.AutoGen; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Sample rate converter for interleaved 32-bit float PCM (libswresample). +/// Input and output have the same channel count; only the sample rate changes. +/// One instance keeps filter state between calls, so consecutive calls form one continuous stream. +/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html +/// +internal sealed unsafe class PCMResampler : IDisposable +{ + private SwrContext* _swr; + private bool _flushed; + + public int Channels { get; } + public int InputRate { get; } + public int OutputRate { get; } + + /// Value of the SwrContext "filter_size" option (filter length per phase in taps at a 1:1 ratio). + public int FilterSize { get; } + + public PCMResampler(int channels, int inputRate, int outputRate) + { + if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels)); + if (inputRate <= 0) throw new ArgumentOutOfRangeException(nameof(inputRate)); + if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate)); + + Channels = channels; + InputRate = inputRate; + OutputRate = outputRate; + + AVChannelLayout layout; + ffmpeg.av_channel_layout_default(&layout, channels); + + SwrContext* swr = null; + var ret = ffmpeg.swr_alloc_set_opts2( + &swr, + &layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, outputRate, + &layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, inputRate, + 0, null); + ffmpeg.av_channel_layout_uninit(&layout); + if (ret < 0 || swr == null) + throw new InvalidOperationException($"swr_alloc_set_opts2 failed ({ret})."); + _swr = swr; + + long filterSize; + FilterSize = ffmpeg.av_opt_get_int(_swr, "filter_size", 0, &filterSize) >= 0 ? (int)filterSize : 32; + + Reset(); + } + + /// Discards all buffered input and filter history. The next call starts a new stream. + public void Reset() + { + // swr_init on an initialized context re-initializes it. + // Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_init, swr_close) + var ret = ffmpeg.swr_init(_swr); + if (ret < 0) + throw new InvalidOperationException($"swr_init failed ({ret})."); + _flushed = false; + } + + /// + /// Feeds frames and writes up to frames. + /// Input that does not fit in the output is buffered inside the context and returned by later calls. + /// may be 0 to take only buffered output. + /// + /// Frames written to . + public int Convert(float* input, int inputFrames, float* output, int outputCapacity) + { + if (_flushed) throw new InvalidOperationException("Resampler was flushed. Call Reset first."); + + // swr_convert treats a null input array as a flush request, so a non-null pointer is passed even for 0 frames. + float dummy = 0; + var inPlane = (byte*)(input != null ? input : &dummy); + var outPlane = (byte*)output; + var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, &inPlane, inputFrames); + if (ret < 0) + throw new InvalidOperationException($"swr_convert failed ({ret})."); + return ret; + } + + /// + /// Ends the input stream and writes up to remaining frames. + /// Call repeatedly until it returns 0. is not allowed afterwards until . + /// + public int Flush(float* output, int outputCapacity) + { + _flushed = true; + var outPlane = (byte*)output; + var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, null, 0); + if (ret < 0) + throw new InvalidOperationException($"swr_convert (flush) failed ({ret})."); + return ret; + } + + public void Dispose() + { + if (_swr == null) return; + var swr = _swr; + ffmpeg.swr_free(&swr); + _swr = null; + } +} diff --git a/Util/Audio/Clip/UnityClipLoader.cs b/Util/Audio/Clip/UnityClipLoader.cs new file mode 100644 index 0000000..c9acbe5 --- /dev/null +++ b/Util/Audio/Clip/UnityClipLoader.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using UnityEngine; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Copies a Unity into memory for clips without a file on disk +/// (built-in hit sounds from Resources, official/DLC level audio). +/// runs on the main thread only, so the whole clip is copied once on the main thread +/// and the decoder thread reads the copy through a . +/// Streaming clips () are rejected because GetData does not read them. +/// Reference: https://docs.unity3d.com/ScriptReference/AudioClip.GetData.html +/// +internal static class UnityClipLoader +{ + /// + /// Copies in blocks of floats or less + /// (FramesPerBlock = ChunkFloats / channels frames per GetData call). Main thread only. + /// Block memory is counted in and released block by block as the decoder thread reads it. + /// + public static MemoryPCMReader Load(AudioClip clip, AudioClipStore store) + { + if (!clip) throw new ArgumentNullException(nameof(clip)); + if (clip.loadType == AudioClipLoadType.Streaming) + throw new NotSupportedException($"AudioClip '{clip.name}' is a streaming clip; GetData cannot read it."); + + if (clip.loadState == AudioDataLoadState.Unloaded) + clip.LoadAudioData(); + + var channels = clip.channels; + var totalFrames = (long)clip.samples; + var framesPerBlock = AudioClipSource.ChunkFloats / channels; + var blocks = new List(); + try + { + for (long start = 0; start < totalFrames; start += framesPerBlock) + { + var frames = (int)Math.Min(framesPerBlock, totalFrames - start); + var block = store.AllocateExactArray(frames * channels); + blocks.Add(block); + // GetData fills the whole array; the offset is in frames (samples per channel). + if (!clip.GetData(block, (int)start)) + throw new InvalidOperationException($"AudioClip.GetData failed for '{clip.name}' at frame {start} (load state {clip.loadState})."); + } + } + catch + { + foreach (var block in blocks) store.FreeExactArray(block); + throw; + } + + return new MemoryPCMReader(blocks, framesPerBlock, channels, clip.frequency, totalFrames, store.FreeExactArray); + } +}