Emulate AudioSource Playback State For Effect Sounds

Patch/Engine/AudioPatches.cs

@@ -1,7 +1,7 @@
 using System;
 using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Util.Audio;
 using ADOFAIRenderer.Util.Audio.Clip;
-using ADOFAIRenderer.Util.Audio.Mixer;
 using HarmonyLib;
 using UnityEngine;
 using UnityEngine.Audio;
@@ -63,11 +63,11 @@ public static class AudioManagerPlayPatch
 [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")]
 public static class PlayWithEndTimePatch
 {
-    /// <summary>Voice registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only.</summary>
-    internal static Voice? PendingVoice;
+    /// <summary>State registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only.</summary>
+    internal static SourceState? PendingState;
 
     [HarmonyFinalizer]
-    public static void Finalizer() => PendingVoice = null;
+    public static void Finalizer() => PendingState = null;
 
     [HarmonyPrefix]
     public static bool Prefix(string snd, double time, double endTime, float volume, int priority)
@@ -85,14 +85,14 @@ public static class PlayWithEndTimePatch
                 double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
 
                 // Register looping sound event to the offline mixer (default pitch = 1.0f)
-                PendingVoice = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime);
+                PendingState = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime);
             }
         }
         return true; // Let the original method execute
     }
 }
 
-/// <summary>Maps the AudioSource created inside scrConductor.PlayWithEndTime to its voice (<see cref="PlayWithEndTimePatch.PendingVoice"/>).</summary>
+/// <summary>Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (<see cref="PlayWithEndTimePatch.PendingState"/>).</summary>
 [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))]
 public static class AudioManagerMakeSourcePatch
 {
@@ -100,115 +100,166 @@ public static class AudioManagerMakeSourcePatch
     // ReSharper disable once InconsistentNaming
     public static void Postfix(AudioSource __result)
     {
-        var voice = PlayWithEndTimePatch.PendingVoice;
-        if (voice == null || !__result) return;
-        PlayWithEndTimePatch.PendingVoice = null;
-        RenderManager.Instance.AudioMixer?.MapSource(__result, voice);
+        var state = PlayWithEndTimePatch.PendingState;
+        if (state == null || !__result) return;
+        PlayWithEndTimePatch.PendingState = null;
+        RenderManager.Instance.AudioMixer?.MapSource(__result, state);
     }
 }
 
 /// <summary>
-/// AudioSource.isPlaying getter: a source mapped to a voice reports true until the voice has finished, including before its
-/// scheduled start, as a source started with PlayScheduled does. During rendering PlayScheduled is blocked, so without this
-/// the game sees every effect source as stopped and AudioManager.Update returns it to the reusable pool on the next frame.
-/// Voice completion is set by the render thread, which mixes behind the main thread, so the value turns false at or after
-/// the render time the voice ends.
+/// AudioSource members of an effect source mapped to a <see cref="SourceState"/> during rendering: the patches below skip the
+/// Unity implementation and read or change the state instead (isPlaying, time, timeSamples, pitch, SetScheduledEndTime, Stop).
+/// Sources without a mapping (the song, menu sounds) use the Unity implementation.
+/// </summary>
+internal static class EffectSource
+{
+    /// <summary>State mapped to <paramref name="source"/> while capturing, with the current render time in <paramref name="now"/>.</summary>
+    public static bool TryGet(AudioSource source, out SourceState state, out double now)
+    {
+        state = null!;
+        now = 0;
+        if (!RenderManager.IsCapturingActive) return false;
+        var mixer = RenderManager.Instance.AudioMixer;
+        if (mixer == null || !mixer.TryGetSourceState(source, out state)) return false;
+        now = RenderManager.Instance.AudioTime;
+        return true;
+    }
+}
+
+/// <summary>
+/// AudioSource.isPlaying getter (<see cref="EffectSource"/>): true from the scheduling call until Stop(), the scheduled end time or
+/// the end of the clip, including before the scheduled start. AudioManager.Update returns a source to its reusable pool once this is false.
 /// </summary>
 [HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)]
 public static class AudioSourceIsPlayingPatch
 {
-    [HarmonyPostfix]
+    [HarmonyPrefix]
     // ReSharper disable once InconsistentNaming
-    public static void Postfix(AudioSource __instance, ref bool __result)
+    public static bool Prefix(AudioSource __instance, ref bool __result)
     {
-        if (__result || !RenderManager.IsCapturingActive) return;
-        var mixer = RenderManager.Instance.AudioMixer;
-        if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
-        {
-            __result = !voice.IsFinished;
-        }
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        __result = state.IsPlaying(now);
+        return false;
     }
 }
 
-[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)]
-public static class AudioSourcePitchPatch
+/// <summary>AudioSource.time getter (<see cref="EffectSource"/>): clip position in seconds while playing, 0 while not playing.</summary>
+[HarmonyPatch(typeof(AudioSource), "time", MethodType.Getter)]
+public static class AudioSourceTimeGetterPatch
 {
-    [HarmonyPostfix]
+    [HarmonyPrefix]
     // ReSharper disable once InconsistentNaming
-    public static void Postfix(AudioSource __instance, float value)
+    public static bool Prefix(AudioSource __instance, ref float __result)
     {
-        if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
-        {
-            var mixer = RenderManager.Instance.AudioMixer;
-            if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
-            {
-                voice.SetPitch(value, RenderManager.Instance.AudioTime);
-            }
-        }
-    }
-}
-
-[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))]
-public static class AudioSourceSetScheduledEndTimePatch
-{
-    [HarmonyPostfix]
-    // ReSharper disable once InconsistentNaming
-    public static void Postfix(AudioSource __instance, double time)
-    {
-        if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
-        {
-            var mixer = RenderManager.Instance.AudioMixer;
-            if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
-            {
-                voice.SetEndTime(time - RenderManager.Instance.dspTimeBase, RenderManager.Instance.AudioTime);
-            }
-        }
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        __result = state.GetTime(now);
+        return false;
     }
 }
 
 /// <summary>
-/// AudioSource.time setter: moves the playback position of the mapped voice (clip seconds) from the current render time.
+/// AudioSource.time setter (<see cref="EffectSource"/>): moves the playback position (clip seconds) from the current render time.
 /// Set before the scheduled start (ffxPlaySound sets it right after AudioManager.Play), it becomes the start position.
 /// </summary>
 [HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)]
 public static class AudioSourceTimePatch
 {
-    [HarmonyPostfix]
+    [HarmonyPrefix]
     // ReSharper disable once InconsistentNaming
-    public static void Postfix(AudioSource __instance, float value)
+    public static bool Prefix(AudioSource __instance, float value)
     {
-        if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
-        {
-            var mixer = RenderManager.Instance.AudioMixer;
-            if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
-            {
-                voice.SetPosition(value, RenderManager.Instance.AudioTime);
-            }
-        }
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        state.SetTime(value, now);
+        return false;
+    }
+}
+
+/// <summary>AudioSource.timeSamples getter (<see cref="EffectSource"/>): clip position in samples of the clip.</summary>
+[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Getter)]
+public static class AudioSourceTimeSamplesGetterPatch
+{
+    [HarmonyPrefix]
+    // ReSharper disable once InconsistentNaming
+    public static bool Prefix(AudioSource __instance, ref int __result)
+    {
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        __result = state.GetTimeSamples(now);
+        return false;
+    }
+}
+
+/// <summary>AudioSource.timeSamples setter (<see cref="EffectSource"/>): moves the playback position (samples of the clip).</summary>
+[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Setter)]
+public static class AudioSourceTimeSamplesPatch
+{
+    [HarmonyPrefix]
+    // ReSharper disable once InconsistentNaming
+    public static bool Prefix(AudioSource __instance, int value)
+    {
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        state.SetTimeSamples(value, now);
+        return false;
+    }
+}
+
+/// <summary>AudioSource.pitch getter (<see cref="EffectSource"/>): the last pitch set.</summary>
+[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Getter)]
+public static class AudioSourcePitchGetterPatch
+{
+    [HarmonyPrefix]
+    // ReSharper disable once InconsistentNaming
+    public static bool Prefix(AudioSource __instance, ref float __result)
+    {
+        if (!EffectSource.TryGet(__instance, out var state, out _)) return true;
+        __result = state.GetPitch();
+        return false;
+    }
+}
+
+/// <summary>AudioSource.pitch setter (<see cref="EffectSource"/>): changes the playback rate from the current render time.</summary>
+[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)]
+public static class AudioSourcePitchPatch
+{
+    [HarmonyPrefix]
+    // ReSharper disable once InconsistentNaming
+    public static bool Prefix(AudioSource __instance, float value)
+    {
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        state.SetPitch(value, now);
+        return false;
+    }
+}
+
+/// <summary>AudioSource.SetScheduledEndTime (<see cref="EffectSource"/>): DSP time (absolute) at which playback stops.</summary>
+[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))]
+public static class AudioSourceSetScheduledEndTimePatch
+{
+    [HarmonyPrefix]
+    // ReSharper disable once InconsistentNaming
+    public static bool Prefix(AudioSource __instance, double time)
+    {
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        state.SetEndTime(time - RenderManager.Instance.dspTimeBase, now);
+        return false;
     }
 }
 
 /// <summary>
-/// AudioSource.Stop(): stops the mapped voice at the current render time and removes the mapping.
-/// AudioManager.Update calls Stop() on sources it recycles, which are finished (<see cref="AudioSourceIsPlayingPatch"/>).
+/// AudioSource.Stop() (<see cref="EffectSource"/>): stops playback at the current render time. The mapping stays, so the source
+/// keeps reporting the stopped state. AudioManager.Update calls Stop() on sources it returns to the pool, which are no longer
+/// playing, so their voices are left unchanged.
 /// </summary>
 [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])]
 public static class AudioSourceStopPatch
 {
-    [HarmonyPostfix]
+    [HarmonyPrefix]
     // ReSharper disable once InconsistentNaming
-    public static void Postfix(AudioSource __instance)
+    public static bool Prefix(AudioSource __instance)
     {
-        if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
-        {
-            var mixer = RenderManager.Instance.AudioMixer;
-            if (mixer == null) return;
-            if (mixer.TryGetVoice(__instance, out var voice))
-            {
-                voice.Stop(RenderManager.Instance.AudioTime);
-            }
-            mixer.RemoveSourceMapping(__instance);
-        }
+        if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
+        state.Stop(now);
+        return false;
     }
 }
 

Util/Audio/OfflineAudioMixer.cs

@@ -50,7 +50,7 @@ internal sealed class OfflineAudioMixer : IDisposable
     private readonly int _latencySamples;
 
     // Main thread.
-    private ConditionalWeakTable<AudioSource, Voice> _sourceVoices = new();
+    private ConditionalWeakTable<AudioSource, SourceState> _sourceStates = new();
     private readonly List<Voice> _effectVoices = new();
     private int _effectVoicePruneAt = EffectVoicePruneCount;
     private AudioClipSource? _bgmSource;
@@ -191,9 +191,9 @@ internal sealed class OfflineAudioMixer : IDisposable
 
     /// <summary>
     /// Starts an effect sound at <paramref name="startTime"/>. An <paramref name="endTime"/> of 0 or more makes it loop until then.
-    /// <paramref name="source"/> (when given) is mapped to the voice for later changes through the AudioSource.
+    /// <paramref name="source"/> (when given) is mapped to the returned state, which reports and applies later changes through the AudioSource.
     /// </summary>
-    public Voice RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
+    public SourceState RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
     {
         var clipSource = GetOrAddSource(clipName);
         var loop = endTime >= 0;
@@ -207,26 +207,30 @@ internal sealed class OfflineAudioMixer : IDisposable
         }
         _effectVoices.Add(voice);
 
-        if (source) MapSource(source!, voice);
-        return voice;
+        var state = new SourceState(voice, Math.Max(startTime, now), pitch, loop, loop ? endTime : double.PositiveInfinity,
+            clipSource?.Duration ?? 0.0, _sampleRate);
+        if (source) MapSource(source!, state);
+        return state;
     }
 
-    /// <summary>Maps <paramref name="source"/> to <paramref name="voice"/> for later changes through the AudioSource (replaces an earlier mapping).</summary>
-    public void MapSource(AudioSource source, Voice voice)
+    /// <summary>
+    /// Maps <paramref name="source"/> to <paramref name="state"/> (replaces an earlier mapping) and takes the clip length and
+    /// sample rate from the clip of the source.
+    /// </summary>
+    public void MapSource(AudioSource source, SourceState state)
     {
-        _sourceVoices.Remove(source);
-        _sourceVoices.Add(source, voice);
+        state.BindClip(source.clip);
+        _sourceStates.Remove(source);
+        _sourceStates.Add(source, state);
     }
 
-    /// <summary>Voice started for <paramref name="source"/> by <see cref="RegisterSoundEvent"/>.</summary>
-    public bool TryGetVoice(AudioSource? source, out Voice voice)
+    /// <summary>State of the effect sound last started for <paramref name="source"/>. The mapping stays after the sound ends or stops.</summary>
+    public bool TryGetSourceState(AudioSource? source, out SourceState state)
     {
-        voice = null!;
-        return source != null && _sourceVoices.TryGetValue(source, out voice);
+        state = null!;
+        return source != null && _sourceStates.TryGetValue(source, out state);
     }
 
-    public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source);
-
     /// <summary>
     /// Stops every effect sound at <paramref name="now"/>, including ones scheduled to start later
     /// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of
@@ -237,7 +241,7 @@ internal sealed class OfflineAudioMixer : IDisposable
         foreach (var voice in _effectVoices)
             voice.Stop(now);
         _effectVoices.Clear();
-        _sourceVoices = new ConditionalWeakTable<AudioSource, Voice>();
+        _sourceStates = new ConditionalWeakTable<AudioSource, SourceState>();
     }
 
     /// <summary>
@@ -348,7 +352,7 @@ internal sealed class OfflineAudioMixer : IDisposable
             foreach (var filter in _filters)
                 filter.Reset();
         }
-        _sourceVoices = new ConditionalWeakTable<AudioSource, Voice>();
+        _sourceStates = new ConditionalWeakTable<AudioSource, SourceState>();
         _effectVoices.Clear();
     }
 }

Util/Audio/SourceState.cs

@@ -0,0 +1,140 @@
+using System;
+using ADOFAIRenderer.Util.Audio.Mixer;
+using UnityEngine;
+
+namespace ADOFAIRenderer.Util.Audio;
+
+/// <summary>
+/// Playback state of a game AudioSource mapped to an effect <see cref="Voice"/>, as the AudioSource reports it to the game
+/// (isPlaying, time, timeSamples, pitch). Main thread only. Times are seconds relative to the render start (the render clock that
+/// also drives AudioSettings.dspTime during rendering); clip positions are seconds into the clip.
+///
+/// Reported values:
+/// - Playing: from the scheduled start until Stop(), the scheduled end time, or the natural end of the clip (not looping).
+///   Before the start, isPlaying is true and time is the position playback will start from.
+/// - time while playing: start position + elapsed time * pitch since the last change, wrapped at the clip length when looping.
+///   time is 0 while not playing.
+/// - timeSamples while playing: floor(time * clip frequency). While not playing: the last value set through time or timeSamples,
+///   reset to 0 by Stop().
+/// - pitch: the last finite value set (NaN and infinity are ignored). Playback uses max(0, pitch).
+/// Changes made while not playing are stored and do not reach the voice.
+/// </summary>
+internal sealed class SourceState
+{
+    private readonly double _start;
+    private double _anchorTime;
+    private double _anchorPosition;
+    private double _pitch;
+    private readonly bool _loop;
+    private double _endTime = double.PositiveInfinity;
+    private bool _stopped;
+    private int _offsetSamples;
+    private double _length;
+    private int _frequency;
+
+    /// <param name="voice">Voice that plays the sound.</param>
+    /// <param name="start">Start of playback (seconds relative to the render start, not before the time it was scheduled).</param>
+    /// <param name="pitch">Pitch at the start.</param>
+    /// <param name="loop">Repeats the clip.</param>
+    /// <param name="endTime">Scheduled end time (seconds relative to the render start), or infinity.</param>
+    /// <param name="length">Clip length in seconds (replaced by the game clip in <see cref="BindClip"/>).</param>
+    /// <param name="frequency">Clip sample rate for timeSamples (replaced by the game clip in <see cref="BindClip"/>).</param>
+    public SourceState(Voice voice, double start, double pitch, bool loop, double endTime, double length, int frequency)
+    {
+        Voice = voice;
+        _start = start;
+        _anchorTime = start;
+        _pitch = pitch;
+        _loop = loop;
+        _endTime = endTime;
+        _length = length;
+        _frequency = Math.Max(1, frequency);
+    }
+
+    public Voice Voice { get; }
+
+    /// <summary>Takes the clip length and sample rate from the clip of the game AudioSource.</summary>
+    public void BindClip(AudioClip? clip)
+    {
+        if (!clip) return;
+        _length = clip!.length;
+        _frequency = Math.Max(1, clip.frequency);
+    }
+
+    /// <summary>Scheduled end time (see AudioSource.SetScheduledEndTime), seconds relative to the render start.</summary>
+    public void SetEndTime(double endTime, double now)
+    {
+        if (!IsPlaying(now)) return;
+        _endTime = endTime;
+        Voice.SetEndTime(endTime, now);
+    }
+
+    public bool IsPlaying(double now) => !_stopped && now < Math.Min(_endTime, NaturalEnd());
+
+    public float GetTime(double now) => IsPlaying(now) ? (float)PositionAt(now) : 0f;
+
+    public int GetTimeSamples(double now) => IsPlaying(now) ? (int)Math.Floor(PositionAt(now) * _frequency) : _offsetSamples;
+
+    public float GetPitch() => (float)_pitch;
+
+    public void SetTime(float seconds, double now)
+    {
+        _offsetSamples = (int)Math.Floor((double)seconds * _frequency);
+        Seek(seconds, now);
+    }
+
+    public void SetTimeSamples(int samples, double now)
+    {
+        _offsetSamples = samples;
+        Seek((double)samples / _frequency, now);
+    }
+
+    public void SetPitch(float pitch, double now)
+    {
+        if (float.IsNaN(pitch) || float.IsInfinity(pitch)) return;
+        var playing = IsPlaying(now);
+        if (playing) Reanchor(now);
+        _pitch = pitch;
+        if (playing) Voice.SetPitch(pitch, now);
+    }
+
+    public void Stop(double now)
+    {
+        if (IsPlaying(now)) Voice.Stop(now);
+        _stopped = true;
+        _offsetSamples = 0;
+    }
+
+    private double PlaybackPitch => Math.Max(0.0, _pitch);
+
+    private void Seek(double seconds, double now)
+    {
+        if (!IsPlaying(now)) return;
+        var position = Math.Max(0.0, seconds);
+        _anchorTime = Math.Max(now, _start);
+        _anchorPosition = position;
+        Voice.SetPosition(position, now);
+    }
+
+    private void Reanchor(double now)
+    {
+        var time = Math.Max(now, _start);
+        _anchorPosition = PositionAt(time);
+        _anchorTime = time;
+    }
+
+    /// <summary>Clip position at <paramref name="now"/> (the start position before the start), wrapped when looping.</summary>
+    private double PositionAt(double now)
+    {
+        var position = _anchorPosition + (Math.Max(now, _anchorTime) - _anchorTime) * PlaybackPitch;
+        if (_loop && _length > 0 && position >= _length) position %= _length;
+        return position;
+    }
+
+    /// <summary>Time at which the clip position reaches the clip length (infinity when looping or at pitch 0).</summary>
+    private double NaturalEnd()
+    {
+        if (_loop || PlaybackPitch <= 0) return double.PositiveInfinity;
+        return _anchorTime + Math.Max(0.0, _length - _anchorPosition) / PlaybackPitch;
+    }
+}