Apply AudioSource Playback Control To Voices

Patch/Engine/AudioPatches.cs

@@ -1,6 +1,7 @@
 using System;
 using ADOFAIRenderer.Core;
 using ADOFAIRenderer.Util.Audio.Clip;
+using ADOFAIRenderer.Util.Audio.Mixer;
 using HarmonyLib;
 using UnityEngine;
 using UnityEngine.Audio;
@@ -55,9 +56,19 @@ public static class AudioManagerPlayPatch
     }
 }
 
+/// <summary>
+/// scrConductor.PlayWithEndTime: registers a looping effect voice until the end time. The AudioSource it creates with
+/// AudioManager.MakeSource is mapped to that voice (<see cref="AudioManagerMakeSourcePatch"/>).
+/// </summary>
 [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")]
 public static class PlayWithEndTimePatch
 {
+    /// <summary>Voice registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only.</summary>
+    internal static Voice? PendingVoice;
+
+    [HarmonyFinalizer]
+    public static void Finalizer() => PendingVoice = null;
+
     [HarmonyPrefix]
     public static bool Prefix(string snd, double time, double endTime, float volume, int priority)
     {
@@ -74,13 +85,51 @@ public static class PlayWithEndTimePatch
                 double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
 
                 // Register looping sound event to the offline mixer (default pitch = 1.0f)
-                mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime);
+                PendingVoice = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime);
             }
         }
         return true; // Let the original method execute
     }
 }
 
+/// <summary>Maps the AudioSource created inside scrConductor.PlayWithEndTime to its voice (<see cref="PlayWithEndTimePatch.PendingVoice"/>).</summary>
+[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))]
+public static class AudioManagerMakeSourcePatch
+{
+    [HarmonyPostfix]
+    // ReSharper disable once InconsistentNaming
+    public static void Postfix(AudioSource __result)
+    {
+        var voice = PlayWithEndTimePatch.PendingVoice;
+        if (voice == null || !__result) return;
+        PlayWithEndTimePatch.PendingVoice = null;
+        RenderManager.Instance.AudioMixer?.MapSource(__result, voice);
+    }
+}
+
+/// <summary>
+/// AudioSource.isPlaying getter: a source mapped to a voice reports true until the voice has finished, including before its
+/// scheduled start, as a source started with PlayScheduled does. During rendering PlayScheduled is blocked, so without this
+/// the game sees every effect source as stopped and AudioManager.Update returns it to the reusable pool on the next frame.
+/// Voice completion is set by the render thread, which mixes behind the main thread, so the value turns false at or after
+/// the render time the voice ends.
+/// </summary>
+[HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)]
+public static class AudioSourceIsPlayingPatch
+{
+    [HarmonyPostfix]
+    // ReSharper disable once InconsistentNaming
+    public static void Postfix(AudioSource __instance, ref bool __result)
+    {
+        if (__result || !RenderManager.IsCapturingActive) return;
+        var mixer = RenderManager.Instance.AudioMixer;
+        if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
+        {
+            __result = !voice.IsFinished;
+        }
+    }
+}
+
 [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)]
 public static class AudioSourcePitchPatch
 {
@@ -117,6 +166,32 @@ public static class AudioSourceSetScheduledEndTimePatch
     }
 }
 
+/// <summary>
+/// AudioSource.time setter: moves the playback position of the mapped voice (clip seconds) from the current render time.
+/// Set before the scheduled start (ffxPlaySound sets it right after AudioManager.Play), it becomes the start position.
+/// </summary>
+[HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)]
+public static class AudioSourceTimePatch
+{
+    [HarmonyPostfix]
+    // ReSharper disable once InconsistentNaming
+    public static void Postfix(AudioSource __instance, float value)
+    {
+        if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
+        {
+            var mixer = RenderManager.Instance.AudioMixer;
+            if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
+            {
+                voice.SetPosition(value, RenderManager.Instance.AudioTime);
+            }
+        }
+    }
+}
+
+/// <summary>
+/// AudioSource.Stop(): stops the mapped voice at the current render time and removes the mapping.
+/// AudioManager.Update calls Stop() on sources it recycles, which are finished (<see cref="AudioSourceIsPlayingPatch"/>).
+/// </summary>
 [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])]
 public static class AudioSourceStopPatch
 {
@@ -126,7 +201,32 @@ public static class AudioSourceStopPatch
     {
         if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
         {
-            RenderManager.Instance.AudioMixer?.RemoveSourceMapping(__instance);
+            var mixer = RenderManager.Instance.AudioMixer;
+            if (mixer == null) return;
+            if (mixer.TryGetVoice(__instance, out var voice))
+            {
+                voice.Stop(RenderManager.Instance.AudioTime);
+            }
+            mixer.RemoveSourceMapping(__instance);
+        }
+    }
+}
+
+/// <summary>
+/// AudioManager.StopAllSounds destroys every source in liveSources (all AudioManager._Play and scrConductor.PlayWithEndTime sources),
+/// so every effect voice stops at the current render time, including ones scheduled to start later.
+/// Callers include scrConductor.PlayHitTimes (from its second call), ScrubMusicToTile/Time, KillAllSoundsCoroutine,
+/// scrController.FailAction and scnGame.Play.
+/// </summary>
+[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.StopAllSounds))]
+public static class AudioManagerStopAllSoundsPatch
+{
+    [HarmonyPrefix]
+    public static void Prefix()
+    {
+        if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
+        {
+            RenderManager.Instance.AudioMixer?.StopAllEffects(RenderManager.Instance.AudioTime);
         }
     }
 }

Util/Audio/Mixer/Voice.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using ADOFAIRenderer.Util.Audio.Clip;
 
 namespace ADOFAIRenderer.Util.Audio.Mixer;
@@ -64,6 +65,12 @@ internal sealed class Voice
     internal bool Loop;
     internal bool Finished;
 
+    /// <summary>
+    /// Playback has ended and the voice was released by the render thread. Readable from any thread;
+    /// the value can lag behind the render thread.
+    /// </summary>
+    public bool IsFinished => Volatile.Read(ref Finished);
+
     /// <summary>Loop length in frames at the output sample rate (source duration * output rate).</summary>
     internal double Period;
 

Util/Audio/OfflineAudioMixer.cs

@@ -32,6 +32,9 @@ internal sealed class OfflineAudioMixer : IDisposable
     /// <summary>Additional gain of effect sounds (registered with <see cref="RegisterSoundEvent"/>).</summary>
     private const double EffectVolume = 2.0 / 3.0;
 
+    /// <summary>Number of tracked effect voices at which finished ones are removed from the list used by <see cref="StopAllEffects"/>.</summary>
+    private const int EffectVoicePruneCount = 1024;
+
     /// <summary>Longest wait for a commit before checking for cancellation again, in milliseconds.</summary>
     private const int CommitWaitMs = 50;
 
@@ -45,6 +48,7 @@ internal sealed class OfflineAudioMixer : IDisposable
 
     // Main thread.
     private ConditionalWeakTable<AudioSource, Voice> _sourceVoices = new();
+    private readonly List<Voice> _effectVoices = new();
     private AudioClipSource? _bgmSource;
     private string _bgmKey = "";
     private Voice? _bgmVoice;
@@ -182,16 +186,26 @@ internal sealed class OfflineAudioMixer : IDisposable
     /// Starts an effect sound at <paramref name="startTime"/>. An <paramref name="endTime"/> of 0 or more makes it loop until then.
     /// <paramref name="source"/> (when given) is mapped to the voice for later changes through the AudioSource.
     /// </summary>
-    public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
+    public Voice RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
     {
         var clipSource = GetOrAddSource(clipName);
         var loop = endTime >= 0;
         var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now);
         if (loop) voice.SetEndTime(endTime, now);
 
-        if (!source) return;
-        _sourceVoices.Remove(source!);
-        _sourceVoices.Add(source!, voice);
+        if (_effectVoices.Count >= EffectVoicePruneCount)
+            _effectVoices.RemoveAll(v => v.IsFinished);
+        _effectVoices.Add(voice);
+
+        if (source) MapSource(source!, voice);
+        return voice;
+    }
+
+    /// <summary>Maps <paramref name="source"/> to <paramref name="voice"/> for later changes through the AudioSource (replaces an earlier mapping).</summary>
+    public void MapSource(AudioSource source, Voice voice)
+    {
+        _sourceVoices.Remove(source);
+        _sourceVoices.Add(source, voice);
     }
 
     /// <summary>Voice started for <paramref name="source"/> by <see cref="RegisterSoundEvent"/>.</summary>
@@ -203,6 +217,19 @@ internal sealed class OfflineAudioMixer : IDisposable
 
     public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source);
 
+    /// <summary>
+    /// Stops every effect sound at <paramref name="now"/>, including ones scheduled to start later
+    /// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of
+    /// AudioManager._Play and scrConductor.PlayWithEndTime). The song voice keeps playing.
+    /// </summary>
+    public void StopAllEffects(double now)
+    {
+        foreach (var voice in _effectVoices)
+            voice.Stop(now);
+        _effectVoices.Clear();
+        _sourceVoices = new ConditionalWeakTable<AudioSource, Voice>();
+    }
+
     /// <summary>
     /// Marks every command with a time before <paramref name="time"/> as queued
     /// (called after each frame with the time of the next frame).
@@ -312,5 +339,6 @@ internal sealed class OfflineAudioMixer : IDisposable
                 filter.Reset();
         }
         _sourceVoices = new ConditionalWeakTable<AudioSource, Voice>();
+        _effectVoices.Clear();
     }
 }