From a21d86d3a612896e44f5dfb72d1b6d62ec9bc607051ea98d97f4bcef32419258 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 09:43:07 +0900 Subject: [PATCH] Add Voice Mixer With Timed Changes --- Util/Audio/Mixer/Voice.cs | 156 +++++++++++++++ Util/Audio/Mixer/VoiceMixer.cs | 341 +++++++++++++++++++++++++++++++++ 2 files changed, 497 insertions(+) create mode 100644 Util/Audio/Mixer/Voice.cs create mode 100644 Util/Audio/Mixer/VoiceMixer.cs diff --git a/Util/Audio/Mixer/Voice.cs b/Util/Audio/Mixer/Voice.cs new file mode 100644 index 0000000..a5206ce --- /dev/null +++ b/Util/Audio/Mixer/Voice.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using ADOFAIRenderer.Util.Audio.Clip; + +namespace ADOFAIRenderer.Util.Audio.Mixer; + +/// +/// One playing sound of a (the song, or one effect sound), created by . +/// +/// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start, +/// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order. +/// Render thread: applies each change at sample round(time * OutputRate). +/// +/// Playback model (output samples n, clip positions in frames of the source at the output sample rate): +/// - The voice is silent before its start sample and from its end sample on. +/// - Clip position at sample n = anchor position + (n - anchor sample) * pitch. A position or pitch change sets a new anchor +/// at the sample where it takes effect, so a pitch change keeps the position continuous. +/// - Looping: the position wraps at the clip length ( * output rate, fractional). +/// Not looping: the voice ends when the position reaches . +/// +internal sealed class Voice +{ + internal enum ChangeKind + { + Pitch, + Volume, + Loop, + Position, + EndTime, + Stop + } + + internal readonly struct Change + { + public readonly ChangeKind Kind; + public readonly long Sample; + public readonly double Value; + + public Change(ChangeKind kind, long sample, double value) + { + Kind = kind; + Sample = sample; + Value = value; + } + } + + private readonly VoiceMixer _mixer; + + // ── Render thread state ── + + /// Changes not yet applied, in time order. Render thread only. + internal readonly Queue Pending = new(); + + /// First output sample of playback. + internal long StartSample; + + /// Output sample at which playback stops (exclusive); long.MaxValue when not set. + internal long EndSample = long.MaxValue; + + internal long AnchorSample; + internal double AnchorPosition; + internal double Pitch; + internal double Volume; + internal bool Loop; + internal bool Finished; + + /// Loop length in frames at the output sample rate (source duration * output rate). + internal double Period; + + internal AudioCursor? Cursor; + internal ChunkView View; + + internal Voice(VoiceMixer mixer, AudioClipSource? source, string name) + { + _mixer = mixer; + Source = source; + Name = name; + } + + /// Clip played by this voice; null plays silence (the clip could not be loaded). + public AudioClipSource? Source { get; } + + /// Name for logs (the game's clip key). + public string Name { get; } + + /// Sets the playback rate (1 = original speed). Negative values are treated as 0. + public void SetPitch(double pitch, double time) => _mixer.Enqueue(this, ChangeKind.Pitch, time, Math.Max(0.0, pitch)); + + /// Sets the linear gain applied to the clip samples. + public void SetVolume(double volume, double time) => _mixer.Enqueue(this, ChangeKind.Volume, time, volume); + + /// Turns looping on or off. Turning it off lets the current repetition play to the end of the clip. + public void SetLoop(bool loop, double time) => _mixer.Enqueue(this, ChangeKind.Loop, time, loop ? 1.0 : 0.0); + + /// + /// Moves playback to seconds into the clip. + /// Before the voice starts, this sets the position playback starts from. + /// + public void SetPosition(double clipSeconds, double time) => _mixer.Enqueue(this, ChangeKind.Position, time, Math.Max(0.0, clipSeconds)); + + /// Stops playback at (seconds relative to the render start). + public void SetEndTime(double endTime, double time) => _mixer.Enqueue(this, ChangeKind.EndTime, time, endTime); + + /// Stops playback at . + public void Stop(double time) => _mixer.Enqueue(this, ChangeKind.Stop, time, 0.0); + + /// Unwrapped clip position at output sample (sample at or after the anchor). + internal double PositionAt(long sample) => AnchorPosition + (sample - AnchorSample) * Pitch; + + /// Clip position at , wrapped into the loop when looping. + internal double WrappedPositionAt(long sample) + { + var position = PositionAt(sample); + if (Loop && Period > 0 && position >= Period) position %= Period; + return position; + } + + /// + /// Applies at output sample (its own sample, or a later one when the change + /// arrived after its sample was mixed). Before the start sample, changes replace the starting state. Render thread only. + /// + internal void Apply(in Change change, long sample, int outputRate) + { + var at = Math.Max(sample, StartSample); + switch (change.Kind) + { + case ChangeKind.Pitch: + Reanchor(at); + Pitch = change.Value; + break; + case ChangeKind.Volume: + Volume = change.Value; + break; + case ChangeKind.Loop: + Reanchor(at); + Loop = change.Value != 0.0; + break; + case ChangeKind.Position: + AnchorSample = at; + AnchorPosition = change.Value * outputRate; + break; + case ChangeKind.EndTime: + EndSample = VoiceMixer.ToSample(change.Value, outputRate); + break; + case ChangeKind.Stop: + EndSample = Math.Min(EndSample, sample); + break; + } + } + + private void Reanchor(long sample) + { + AnchorPosition = WrappedPositionAt(sample); + AnchorSample = sample; + } +} diff --git a/Util/Audio/Mixer/VoiceMixer.cs b/Util/Audio/Mixer/VoiceMixer.cs new file mode 100644 index 0000000..9be63e6 --- /dev/null +++ b/Util/Audio/Mixer/VoiceMixer.cs @@ -0,0 +1,341 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using ADOFAIRenderer.Util.Audio.Clip; + +namespace ADOFAIRenderer.Util.Audio.Mixer; + +/// +/// Mixes every into interleaved double samples at the output sample rate and channel count. +/// +/// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed +/// to and the Voice Set methods). A time t maps to sample round(t * OutputRate). +/// +/// Threads: +/// - Main thread: and the Voice Set methods queue commands under a lock. Commands must be queued in +/// non-decreasing time order per voice. +/// - Render thread: takes the queued commands and mixes a block of samples. A command takes effect at its sample; +/// a command whose sample is before the block being mixed takes effect at the start of that block. +/// Chunks are read through , which waits for the decoder thread. +/// +/// Sample reading: clip positions are in frames of the source at the output rate (sources are resampled when loaded). +/// An integer position copies the stored sample; a fractional position interpolates linearly between the frame and the next one +/// (the next frame of a looping voice after the last frame is frame 0; after the end of a non-looping clip it is 0). +/// Channel mapping follows . +/// +internal sealed class VoiceMixer : IDisposable +{ + private enum CommandKind + { + Play, + Change + } + + private readonly struct Command + { + public readonly CommandKind Kind; + public readonly Voice Voice; + public readonly Voice.Change Change; + + public Command(CommandKind kind, Voice voice, Voice.Change change) + { + Kind = kind; + Voice = voice; + Change = change; + } + } + + private readonly AudioClipStore _store; + private readonly CancellationToken _cancellationToken; + + private readonly object _commandLock = new(); + private List _commands = new(); + private List _takenCommands = new(); + + // Render thread only. + private readonly List _voices = new(); + private bool _disposed; + + /// Store the voices' sources belong to; its OutputRate is the mixer's sample rate. + /// Interleaved channel count of the mixed output. + /// Cancels waits for chunks in . + public VoiceMixer(AudioClipStore store, int outputChannels, CancellationToken cancellationToken = default) + { + if (outputChannels < 1) throw new ArgumentOutOfRangeException(nameof(outputChannels)); + _store = store; + OutputRate = store.OutputRate; + OutputChannels = outputChannels; + _cancellationToken = cancellationToken; + } + + public int OutputRate { get; } + + public int OutputChannels { get; } + + /// Voices currently mixed (started or waiting to start). Render thread only. + public int ActiveVoiceCount => _voices.Count; + + /// Output sample nearest to . + public static long ToSample(double seconds, int rate) => (long)Math.Round(seconds * rate, MidpointRounding.AwayFromZero); + + // ── Main thread ── + + /// + /// Schedules to start at . + /// A start time before (the current time) starts the voice at . + /// + /// Clip to play; null plays silence. + /// Name for logs. + /// Start of playback, seconds relative to the render start. + /// Linear gain. + /// Playback rate (1 = original speed); negative values are treated as 0. + /// Repeats the clip until the voice is stopped. + /// Current time, seconds relative to the render start. + public Voice Play(AudioClipSource? source, string name, double startTime, double volume, double pitch, bool loop, double time) + { + var voice = new Voice(this, source, name) + { + StartSample = Math.Max(ToSample(startTime, OutputRate), ToSample(time, OutputRate)), + Volume = volume, + Pitch = Math.Max(0.0, pitch), + Loop = loop, + Period = source != null ? source.Duration * OutputRate : 0.0 + }; + voice.AnchorSample = voice.StartSample; + + lock (_commandLock) + { + _commands.Add(new Command(CommandKind.Play, voice, default)); + } + return voice; + } + + internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value) + { + var change = new Voice.Change(kind, ToSample(time, OutputRate), value); + lock (_commandLock) + { + _commands.Add(new Command(CommandKind.Change, voice, change)); + } + } + + // ── Render thread ── + + /// + /// Adds the mix of output samples [, firstSample + frames) to + /// (interleaved, frames = destination.Length / OutputChannels). Blocks are expected in increasing sample order. + /// + /// Chunk loading failed fatally (see ). + /// The cancellation token was canceled while waiting for a chunk. + public void Mix(long firstSample, Span destination) + { + if (_disposed) throw new ObjectDisposedException(nameof(VoiceMixer)); + var frames = destination.Length / OutputChannels; + var endSample = firstSample + frames; + + TakeCommands(); + + for (var i = _voices.Count - 1; i >= 0; i--) + { + var voice = _voices[i]; + var sample = firstSample; + while (!voice.Finished) + { + var segmentEnd = endSample; + if (voice.Pending.Count > 0) + { + var next = voice.Pending.Peek(); + if (next.Sample <= sample) + { + voice.Apply(voice.Pending.Dequeue(), sample, OutputRate); + continue; + } + segmentEnd = Math.Min(segmentEnd, next.Sample); + } + + if (segmentEnd > sample) + MixSegment(voice, sample, segmentEnd, destination.Slice((int)(sample - firstSample) * OutputChannels)); + sample = segmentEnd; + if (sample >= endSample) break; + } + + if (!voice.Finished && endSample >= voice.EndSample) + voice.Finished = true; + if (voice.Finished) + { + ReleaseVoice(voice); + _voices.RemoveAt(i); + } + } + } + + private void TakeCommands() + { + lock (_commandLock) + { + (_commands, _takenCommands) = (_takenCommands, _commands); + } + + foreach (var command in _takenCommands) + { + var voice = command.Voice; + if (command.Kind == CommandKind.Play) + { + if (voice.Source is { IsFailed: false } source) + voice.Cursor = source.AddCursor(0, voice.Loop); + _voices.Add(voice); + } + else if (!voice.Finished) + { + voice.Pending.Enqueue(command.Change); + } + } + _takenCommands.Clear(); + } + + /// Mixes over samples [from, to) with its current state into (starting at sample from). + private void MixSegment(Voice voice, long from, long to, Span destination) + { + var start = Math.Max(from, voice.StartSample); + var end = Math.Min(to, voice.EndSample); + if (start >= end) return; + + var source = voice.Source; + if (source == null || source.IsFailed) + { + voice.Finished = true; + return; + } + + var totalFrames = source.TotalFrames; + if (voice.Volume == 0.0) + { + // Nothing to add; a non-looping voice still ends at the end of the clip. + if (!voice.Loop && voice.PositionAt(end - 1) >= totalFrames) voice.Finished = true; + return; + } + + var channels = OutputChannels; + var volume = voice.Volume; + var offset = (int)(start - from) * channels; + + UpdateCursor(voice, (long)voice.WrappedPositionAt(start)); + + for (var n = start; n < end; n++, offset += channels) + { + var position = voice.WrappedPositionAt(n); + if (position >= totalFrames) + { + if (!voice.Loop) + { + voice.Finished = true; + return; + } + position = totalFrames - 1; + } + + var index = (long)position; + var fraction = position - index; + + if (!EnsureView(voice, index)) return; + var view = voice.View; + + if (fraction == 0.0) + { + for (var c = 0; c < channels; c++) + destination[offset + c] += view.GetSample(index, c, channels) * volume; + continue; + } + + var nextIndex = index + 1; + var nextInView = view.Contains(nextIndex); + for (var c = 0; c < channels; c++) + { + double low = view.GetSample(index, c, channels); + double high = nextInView ? view.GetSample(nextIndex, c, channels) : ReadOutside(voice, nextIndex, c); + destination[offset + c] += (low + fraction * (high - low)) * volume; + } + } + } + + /// Sample of frame outside the voice's current view (next chunk, loop start, or past the end). + private double ReadOutside(Voice voice, long frame, int channel) + { + var source = voice.Source!; + if (frame >= source.TotalFrames) + { + if (!voice.Loop) return 0.0; + frame = 0; + } + + var view = _store.Acquire(source, frame, _cancellationToken); + if (!view.IsValid) return 0.0; + try + { + return view.GetSample(frame, channel, OutputChannels); + } + finally + { + source.Release(view); + } + } + + /// Makes voice.View the chunk containing . False when the source cannot provide it (voice finished). + private bool EnsureView(Voice voice, long frame) + { + if (voice.View.IsValid && voice.View.Contains(frame)) return true; + + var source = voice.Source!; + if (voice.View.IsValid) + { + source.Release(voice.View); + voice.View = default; + } + + UpdateCursor(voice, frame); + voice.View = _store.Acquire(source, frame, _cancellationToken); + if (voice.View.IsValid) return true; + + voice.Finished = true; + return false; + } + + private static void UpdateCursor(Voice voice, long frame) + { + var cursor = voice.Cursor; + if (cursor == null) return; + if (cursor.Loop != voice.Loop) cursor.Loop = voice.Loop; + if (cursor.Position != frame) cursor.Position = frame; + } + + private static void ReleaseVoice(Voice voice) + { + var source = voice.Source; + if (source == null) return; + if (voice.View.IsValid) + { + source.Release(voice.View); + voice.View = default; + } + if (voice.Cursor != null) + { + source.RemoveCursor(voice.Cursor); + voice.Cursor = null; + } + voice.Pending.Clear(); + } + + /// Releases every voice's chunk and cursor. Render thread (or after the render thread stopped). + public void Dispose() + { + if (_disposed) return; + _disposed = true; + foreach (var voice in _voices) + ReleaseVoice(voice); + _voices.Clear(); + lock (_commandLock) + { + _commands.Clear(); + } + } +}