Add Voice Mixer With Timed Changes

Util/Audio/Mixer/Voice.cs

@@ -0,0 +1,156 @@
+using System;
+using System.Collections.Generic;
+using ADOFAIRenderer.Util.Audio.Clip;
+
+namespace ADOFAIRenderer.Util.Audio.Mixer;
+
+/// <summary>
+/// One playing sound of a <see cref="VoiceMixer"/> (the song, or one effect sound), created by <see cref="VoiceMixer.Play"/>.
+///
+/// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start,
+/// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order.
+/// Render thread: <see cref="VoiceMixer.Mix"/> applies each change at sample round(time * OutputRate).
+///
+/// Playback model (output samples n, clip positions in frames of the source at the output sample rate):
+/// - The voice is silent before its start sample and from its end sample on.
+/// - Clip position at sample n = anchor position + (n - anchor sample) * pitch. A position or pitch change sets a new anchor
+///   at the sample where it takes effect, so a pitch change keeps the position continuous.
+/// - Looping: the position wraps at the clip length (<see cref="AudioClipSource.Duration"/> * output rate, fractional).
+///   Not looping: the voice ends when the position reaches <see cref="AudioClipSource.TotalFrames"/>.
+/// </summary>
+internal sealed class Voice
+{
+    internal enum ChangeKind
+    {
+        Pitch,
+        Volume,
+        Loop,
+        Position,
+        EndTime,
+        Stop
+    }
+
+    internal readonly struct Change
+    {
+        public readonly ChangeKind Kind;
+        public readonly long Sample;
+        public readonly double Value;
+
+        public Change(ChangeKind kind, long sample, double value)
+        {
+            Kind = kind;
+            Sample = sample;
+            Value = value;
+        }
+    }
+
+    private readonly VoiceMixer _mixer;
+
+    // ── Render thread state ──
+
+    /// <summary>Changes not yet applied, in time order. Render thread only.</summary>
+    internal readonly Queue<Change> Pending = new();
+
+    /// <summary>First output sample of playback.</summary>
+    internal long StartSample;
+
+    /// <summary>Output sample at which playback stops (exclusive); long.MaxValue when not set.</summary>
+    internal long EndSample = long.MaxValue;
+
+    internal long AnchorSample;
+    internal double AnchorPosition;
+    internal double Pitch;
+    internal double Volume;
+    internal bool Loop;
+    internal bool Finished;
+
+    /// <summary>Loop length in frames at the output sample rate (source duration * output rate).</summary>
+    internal double Period;
+
+    internal AudioCursor? Cursor;
+    internal ChunkView View;
+
+    internal Voice(VoiceMixer mixer, AudioClipSource? source, string name)
+    {
+        _mixer = mixer;
+        Source = source;
+        Name = name;
+    }
+
+    /// <summary>Clip played by this voice; null plays silence (the clip could not be loaded).</summary>
+    public AudioClipSource? Source { get; }
+
+    /// <summary>Name for logs (the game's clip key).</summary>
+    public string Name { get; }
+
+    /// <summary>Sets the playback rate (1 = original speed). Negative values are treated as 0.</summary>
+    public void SetPitch(double pitch, double time) => _mixer.Enqueue(this, ChangeKind.Pitch, time, Math.Max(0.0, pitch));
+
+    /// <summary>Sets the linear gain applied to the clip samples.</summary>
+    public void SetVolume(double volume, double time) => _mixer.Enqueue(this, ChangeKind.Volume, time, volume);
+
+    /// <summary>Turns looping on or off. Turning it off lets the current repetition play to the end of the clip.</summary>
+    public void SetLoop(bool loop, double time) => _mixer.Enqueue(this, ChangeKind.Loop, time, loop ? 1.0 : 0.0);
+
+    /// <summary>
+    /// Moves playback to <paramref name="clipSeconds"/> seconds into the clip.
+    /// Before the voice starts, this sets the position playback starts from.
+    /// </summary>
+    public void SetPosition(double clipSeconds, double time) => _mixer.Enqueue(this, ChangeKind.Position, time, Math.Max(0.0, clipSeconds));
+
+    /// <summary>Stops playback at <paramref name="endTime"/> (seconds relative to the render start).</summary>
+    public void SetEndTime(double endTime, double time) => _mixer.Enqueue(this, ChangeKind.EndTime, time, endTime);
+
+    /// <summary>Stops playback at <paramref name="time"/>.</summary>
+    public void Stop(double time) => _mixer.Enqueue(this, ChangeKind.Stop, time, 0.0);
+
+    /// <summary>Unwrapped clip position at output sample <paramref name="sample"/> (sample at or after the anchor).</summary>
+    internal double PositionAt(long sample) => AnchorPosition + (sample - AnchorSample) * Pitch;
+
+    /// <summary>Clip position at <paramref name="sample"/>, wrapped into the loop when looping.</summary>
+    internal double WrappedPositionAt(long sample)
+    {
+        var position = PositionAt(sample);
+        if (Loop && Period > 0 && position >= Period) position %= Period;
+        return position;
+    }
+
+    /// <summary>
+    /// Applies <paramref name="change"/> at output sample <paramref name="sample"/> (its own sample, or a later one when the change
+    /// arrived after its sample was mixed). Before the start sample, changes replace the starting state. Render thread only.
+    /// </summary>
+    internal void Apply(in Change change, long sample, int outputRate)
+    {
+        var at = Math.Max(sample, StartSample);
+        switch (change.Kind)
+        {
+            case ChangeKind.Pitch:
+                Reanchor(at);
+                Pitch = change.Value;
+                break;
+            case ChangeKind.Volume:
+                Volume = change.Value;
+                break;
+            case ChangeKind.Loop:
+                Reanchor(at);
+                Loop = change.Value != 0.0;
+                break;
+            case ChangeKind.Position:
+                AnchorSample = at;
+                AnchorPosition = change.Value * outputRate;
+                break;
+            case ChangeKind.EndTime:
+                EndSample = VoiceMixer.ToSample(change.Value, outputRate);
+                break;
+            case ChangeKind.Stop:
+                EndSample = Math.Min(EndSample, sample);
+                break;
+        }
+    }
+
+    private void Reanchor(long sample)
+    {
+        AnchorPosition = WrappedPositionAt(sample);
+        AnchorSample = sample;
+    }
+}

Util/Audio/Mixer/VoiceMixer.cs

@@ -0,0 +1,341 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using ADOFAIRenderer.Util.Audio.Clip;
+
+namespace ADOFAIRenderer.Util.Audio.Mixer;
+
+/// <summary>
+/// Mixes every <see cref="Voice"/> into interleaved double samples at the output sample rate and channel count.
+///
+/// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed
+/// to <see cref="Play"/> and the Voice Set methods). A time t maps to sample round(t * OutputRate).
+///
+/// Threads:
+/// - Main thread: <see cref="Play"/> and the Voice Set methods queue commands under a lock. Commands must be queued in
+///   non-decreasing time order per voice.
+/// - Render thread: <see cref="Mix"/> takes the queued commands and mixes a block of samples. A command takes effect at its sample;
+///   a command whose sample is before the block being mixed takes effect at the start of that block.
+///   Chunks are read through <see cref="AudioClipStore.Acquire"/>, which waits for the decoder thread.
+///
+/// Sample reading: clip positions are in frames of the source at the output rate (sources are resampled when loaded).
+/// An integer position copies the stored sample; a fractional position interpolates linearly between the frame and the next one
+/// (the next frame of a looping voice after the last frame is frame 0; after the end of a non-looping clip it is 0).
+/// Channel mapping follows <see cref="ChunkView.GetSample"/>.
+/// </summary>
+internal sealed class VoiceMixer : IDisposable
+{
+    private enum CommandKind
+    {
+        Play,
+        Change
+    }
+
+    private readonly struct Command
+    {
+        public readonly CommandKind Kind;
+        public readonly Voice Voice;
+        public readonly Voice.Change Change;
+
+        public Command(CommandKind kind, Voice voice, Voice.Change change)
+        {
+            Kind = kind;
+            Voice = voice;
+            Change = change;
+        }
+    }
+
+    private readonly AudioClipStore _store;
+    private readonly CancellationToken _cancellationToken;
+
+    private readonly object _commandLock = new();
+    private List<Command> _commands = new();
+    private List<Command> _takenCommands = new();
+
+    // Render thread only.
+    private readonly List<Voice> _voices = new();
+    private bool _disposed;
+
+    /// <param name="store">Store the voices' sources belong to; its OutputRate is the mixer's sample rate.</param>
+    /// <param name="outputChannels">Interleaved channel count of the mixed output.</param>
+    /// <param name="cancellationToken">Cancels waits for chunks in <see cref="Mix"/>.</param>
+    public VoiceMixer(AudioClipStore store, int outputChannels, CancellationToken cancellationToken = default)
+    {
+        if (outputChannels < 1) throw new ArgumentOutOfRangeException(nameof(outputChannels));
+        _store = store;
+        OutputRate = store.OutputRate;
+        OutputChannels = outputChannels;
+        _cancellationToken = cancellationToken;
+    }
+
+    public int OutputRate { get; }
+
+    public int OutputChannels { get; }
+
+    /// <summary>Voices currently mixed (started or waiting to start). Render thread only.</summary>
+    public int ActiveVoiceCount => _voices.Count;
+
+    /// <summary>Output sample nearest to <paramref name="seconds"/>.</summary>
+    public static long ToSample(double seconds, int rate) => (long)Math.Round(seconds * rate, MidpointRounding.AwayFromZero);
+
+    // ── Main thread ──
+
+    /// <summary>
+    /// Schedules <paramref name="source"/> to start at <paramref name="startTime"/>.
+    /// A start time before <paramref name="time"/> (the current time) starts the voice at <paramref name="time"/>.
+    /// </summary>
+    /// <param name="source">Clip to play; null plays silence.</param>
+    /// <param name="name">Name for logs.</param>
+    /// <param name="startTime">Start of playback, seconds relative to the render start.</param>
+    /// <param name="volume">Linear gain.</param>
+    /// <param name="pitch">Playback rate (1 = original speed); negative values are treated as 0.</param>
+    /// <param name="loop">Repeats the clip until the voice is stopped.</param>
+    /// <param name="time">Current time, seconds relative to the render start.</param>
+    public Voice Play(AudioClipSource? source, string name, double startTime, double volume, double pitch, bool loop, double time)
+    {
+        var voice = new Voice(this, source, name)
+        {
+            StartSample = Math.Max(ToSample(startTime, OutputRate), ToSample(time, OutputRate)),
+            Volume = volume,
+            Pitch = Math.Max(0.0, pitch),
+            Loop = loop,
+            Period = source != null ? source.Duration * OutputRate : 0.0
+        };
+        voice.AnchorSample = voice.StartSample;
+
+        lock (_commandLock)
+        {
+            _commands.Add(new Command(CommandKind.Play, voice, default));
+        }
+        return voice;
+    }
+
+    internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value)
+    {
+        var change = new Voice.Change(kind, ToSample(time, OutputRate), value);
+        lock (_commandLock)
+        {
+            _commands.Add(new Command(CommandKind.Change, voice, change));
+        }
+    }
+
+    // ── Render thread ──
+
+    /// <summary>
+    /// Adds the mix of output samples [<paramref name="firstSample"/>, firstSample + frames) to <paramref name="destination"/>
+    /// (interleaved, frames = destination.Length / OutputChannels). Blocks are expected in increasing sample order.
+    /// </summary>
+    /// <exception cref="InvalidOperationException">Chunk loading failed fatally (see <see cref="AudioClipStore.Acquire"/>).</exception>
+    /// <exception cref="OperationCanceledException">The cancellation token was canceled while waiting for a chunk.</exception>
+    public void Mix(long firstSample, Span<double> destination)
+    {
+        if (_disposed) throw new ObjectDisposedException(nameof(VoiceMixer));
+        var frames = destination.Length / OutputChannels;
+        var endSample = firstSample + frames;
+
+        TakeCommands();
+
+        for (var i = _voices.Count - 1; i >= 0; i--)
+        {
+            var voice = _voices[i];
+            var sample = firstSample;
+            while (!voice.Finished)
+            {
+                var segmentEnd = endSample;
+                if (voice.Pending.Count > 0)
+                {
+                    var next = voice.Pending.Peek();
+                    if (next.Sample <= sample)
+                    {
+                        voice.Apply(voice.Pending.Dequeue(), sample, OutputRate);
+                        continue;
+                    }
+                    segmentEnd = Math.Min(segmentEnd, next.Sample);
+                }
+
+                if (segmentEnd > sample)
+                    MixSegment(voice, sample, segmentEnd, destination.Slice((int)(sample - firstSample) * OutputChannels));
+                sample = segmentEnd;
+                if (sample >= endSample) break;
+            }
+
+            if (!voice.Finished && endSample >= voice.EndSample)
+                voice.Finished = true;
+            if (voice.Finished)
+            {
+                ReleaseVoice(voice);
+                _voices.RemoveAt(i);
+            }
+        }
+    }
+
+    private void TakeCommands()
+    {
+        lock (_commandLock)
+        {
+            (_commands, _takenCommands) = (_takenCommands, _commands);
+        }
+
+        foreach (var command in _takenCommands)
+        {
+            var voice = command.Voice;
+            if (command.Kind == CommandKind.Play)
+            {
+                if (voice.Source is { IsFailed: false } source)
+                    voice.Cursor = source.AddCursor(0, voice.Loop);
+                _voices.Add(voice);
+            }
+            else if (!voice.Finished)
+            {
+                voice.Pending.Enqueue(command.Change);
+            }
+        }
+        _takenCommands.Clear();
+    }
+
+    /// <summary>Mixes <paramref name="voice"/> over samples [from, to) with its current state into <paramref name="destination"/> (starting at sample from).</summary>
+    private void MixSegment(Voice voice, long from, long to, Span<double> destination)
+    {
+        var start = Math.Max(from, voice.StartSample);
+        var end = Math.Min(to, voice.EndSample);
+        if (start >= end) return;
+
+        var source = voice.Source;
+        if (source == null || source.IsFailed)
+        {
+            voice.Finished = true;
+            return;
+        }
+
+        var totalFrames = source.TotalFrames;
+        if (voice.Volume == 0.0)
+        {
+            // Nothing to add; a non-looping voice still ends at the end of the clip.
+            if (!voice.Loop && voice.PositionAt(end - 1) >= totalFrames) voice.Finished = true;
+            return;
+        }
+
+        var channels = OutputChannels;
+        var volume = voice.Volume;
+        var offset = (int)(start - from) * channels;
+
+        UpdateCursor(voice, (long)voice.WrappedPositionAt(start));
+
+        for (var n = start; n < end; n++, offset += channels)
+        {
+            var position = voice.WrappedPositionAt(n);
+            if (position >= totalFrames)
+            {
+                if (!voice.Loop)
+                {
+                    voice.Finished = true;
+                    return;
+                }
+                position = totalFrames - 1;
+            }
+
+            var index = (long)position;
+            var fraction = position - index;
+
+            if (!EnsureView(voice, index)) return;
+            var view = voice.View;
+
+            if (fraction == 0.0)
+            {
+                for (var c = 0; c < channels; c++)
+                    destination[offset + c] += view.GetSample(index, c, channels) * volume;
+                continue;
+            }
+
+            var nextIndex = index + 1;
+            var nextInView = view.Contains(nextIndex);
+            for (var c = 0; c < channels; c++)
+            {
+                double low = view.GetSample(index, c, channels);
+                double high = nextInView ? view.GetSample(nextIndex, c, channels) : ReadOutside(voice, nextIndex, c);
+                destination[offset + c] += (low + fraction * (high - low)) * volume;
+            }
+        }
+    }
+
+    /// <summary>Sample of frame <paramref name="frame"/> outside the voice's current view (next chunk, loop start, or past the end).</summary>
+    private double ReadOutside(Voice voice, long frame, int channel)
+    {
+        var source = voice.Source!;
+        if (frame >= source.TotalFrames)
+        {
+            if (!voice.Loop) return 0.0;
+            frame = 0;
+        }
+
+        var view = _store.Acquire(source, frame, _cancellationToken);
+        if (!view.IsValid) return 0.0;
+        try
+        {
+            return view.GetSample(frame, channel, OutputChannels);
+        }
+        finally
+        {
+            source.Release(view);
+        }
+    }
+
+    /// <summary>Makes voice.View the chunk containing <paramref name="frame"/>. False when the source cannot provide it (voice finished).</summary>
+    private bool EnsureView(Voice voice, long frame)
+    {
+        if (voice.View.IsValid && voice.View.Contains(frame)) return true;
+
+        var source = voice.Source!;
+        if (voice.View.IsValid)
+        {
+            source.Release(voice.View);
+            voice.View = default;
+        }
+
+        UpdateCursor(voice, frame);
+        voice.View = _store.Acquire(source, frame, _cancellationToken);
+        if (voice.View.IsValid) return true;
+
+        voice.Finished = true;
+        return false;
+    }
+
+    private static void UpdateCursor(Voice voice, long frame)
+    {
+        var cursor = voice.Cursor;
+        if (cursor == null) return;
+        if (cursor.Loop != voice.Loop) cursor.Loop = voice.Loop;
+        if (cursor.Position != frame) cursor.Position = frame;
+    }
+
+    private static void ReleaseVoice(Voice voice)
+    {
+        var source = voice.Source;
+        if (source == null) return;
+        if (voice.View.IsValid)
+        {
+            source.Release(voice.View);
+            voice.View = default;
+        }
+        if (voice.Cursor != null)
+        {
+            source.RemoveCursor(voice.Cursor);
+            voice.Cursor = null;
+        }
+        voice.Pending.Clear();
+    }
+
+    /// <summary>Releases every voice's chunk and cursor. Render thread (or after the render thread stopped).</summary>
+    public void Dispose()
+    {
+        if (_disposed) return;
+        _disposed = true;
+        foreach (var voice in _voices)
+            ReleaseVoice(voice);
+        _voices.Clear();
+        lock (_commandLock)
+        {
+            _commands.Clear();
+        }
+    }
+}