diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 0a518fa..0202995 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -1,7 +1,7 @@ using System; using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util.Audio; using ADOFAIRenderer.Util.Audio.Clip; -using ADOFAIRenderer.Util.Audio.Mixer; using HarmonyLib; using UnityEngine; using UnityEngine.Audio; @@ -63,11 +63,11 @@ public static class AudioManagerPlayPatch [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] public static class PlayWithEndTimePatch { - /// Voice registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. - internal static Voice? PendingVoice; + /// State registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only. + internal static SourceState? PendingState; [HarmonyFinalizer] - public static void Finalizer() => PendingVoice = null; + public static void Finalizer() => PendingState = null; [HarmonyPrefix] public static bool Prefix(string snd, double time, double endTime, float volume, int priority) @@ -85,14 +85,14 @@ public static class PlayWithEndTimePatch double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; // Register looping sound event to the offline mixer (default pitch = 1.0f) - PendingVoice = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); + PendingState = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); } } return true; // Let the original method execute } } -/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its voice (). +/// Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (). [HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))] public static class AudioManagerMakeSourcePatch { @@ -100,115 +100,166 @@ public static class AudioManagerMakeSourcePatch // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __result) { - var voice = PlayWithEndTimePatch.PendingVoice; - if (voice == null || !__result) return; - PlayWithEndTimePatch.PendingVoice = null; - RenderManager.Instance.AudioMixer?.MapSource(__result, voice); + var state = PlayWithEndTimePatch.PendingState; + if (state == null || !__result) return; + PlayWithEndTimePatch.PendingState = null; + RenderManager.Instance.AudioMixer?.MapSource(__result, state); } } /// -/// AudioSource.isPlaying getter: a source mapped to a voice reports true until the voice has finished, including before its -/// scheduled start, as a source started with PlayScheduled does. During rendering PlayScheduled is blocked, so without this -/// the game sees every effect source as stopped and AudioManager.Update returns it to the reusable pool on the next frame. -/// Voice completion is set by the render thread, which mixes behind the main thread, so the value turns false at or after -/// the render time the voice ends. +/// AudioSource members of an effect source mapped to a during rendering: the patches below skip the +/// Unity implementation and read or change the state instead (isPlaying, time, timeSamples, pitch, SetScheduledEndTime, Stop). +/// Sources without a mapping (the song, menu sounds) use the Unity implementation. +/// +internal static class EffectSource +{ + /// State mapped to while capturing, with the current render time in . + public static bool TryGet(AudioSource source, out SourceState state, out double now) + { + state = null!; + now = 0; + if (!RenderManager.IsCapturingActive) return false; + var mixer = RenderManager.Instance.AudioMixer; + if (mixer == null || !mixer.TryGetSourceState(source, out state)) return false; + now = RenderManager.Instance.AudioTime; + return true; + } +} + +/// +/// AudioSource.isPlaying getter (): true from the scheduling call until Stop(), the scheduled end time or +/// the end of the clip, including before the scheduled start. AudioManager.Update returns a source to its reusable pool once this is false. /// [HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)] public static class AudioSourceIsPlayingPatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, ref bool __result) + public static bool Prefix(AudioSource __instance, ref bool __result) { - if (__result || !RenderManager.IsCapturingActive) return; - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - __result = !voice.IsFinished; - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + __result = state.IsPlaying(now); + return false; } } -[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] -public static class AudioSourcePitchPatch +/// AudioSource.time getter (): clip position in seconds while playing, 0 while not playing. +[HarmonyPatch(typeof(AudioSource), "time", MethodType.Getter)] +public static class AudioSourceTimeGetterPatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, float value) + public static bool Prefix(AudioSource __instance, ref float __result) { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - voice.SetPitch(value, RenderManager.Instance.AudioTime); - } - } - } -} - -[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))] -public static class AudioSourceSetScheduledEndTimePatch -{ - [HarmonyPostfix] - // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, double time) - { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - voice.SetEndTime(time - RenderManager.Instance.dspTimeBase, RenderManager.Instance.AudioTime); - } - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + __result = state.GetTime(now); + return false; } } /// -/// AudioSource.time setter: moves the playback position of the mapped voice (clip seconds) from the current render time. +/// AudioSource.time setter (): moves the playback position (clip seconds) from the current render time. /// Set before the scheduled start (ffxPlaySound sets it right after AudioManager.Play), it becomes the start position. /// [HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)] public static class AudioSourceTimePatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance, float value) + public static bool Prefix(AudioSource __instance, float value) { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) - { - voice.SetPosition(value, RenderManager.Instance.AudioTime); - } - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetTime(value, now); + return false; + } +} + +/// AudioSource.timeSamples getter (): clip position in samples of the clip. +[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Getter)] +public static class AudioSourceTimeSamplesGetterPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, ref int __result) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + __result = state.GetTimeSamples(now); + return false; + } +} + +/// AudioSource.timeSamples setter (): moves the playback position (samples of the clip). +[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Setter)] +public static class AudioSourceTimeSamplesPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, int value) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetTimeSamples(value, now); + return false; + } +} + +/// AudioSource.pitch getter (): the last pitch set. +[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Getter)] +public static class AudioSourcePitchGetterPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, ref float __result) + { + if (!EffectSource.TryGet(__instance, out var state, out _)) return true; + __result = state.GetPitch(); + return false; + } +} + +/// AudioSource.pitch setter (): changes the playback rate from the current render time. +[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] +public static class AudioSourcePitchPatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, float value) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetPitch(value, now); + return false; + } +} + +/// AudioSource.SetScheduledEndTime (): DSP time (absolute) at which playback stops. +[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))] +public static class AudioSourceSetScheduledEndTimePatch +{ + [HarmonyPrefix] + // ReSharper disable once InconsistentNaming + public static bool Prefix(AudioSource __instance, double time) + { + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.SetEndTime(time - RenderManager.Instance.dspTimeBase, now); + return false; } } /// -/// AudioSource.Stop(): stops the mapped voice at the current render time and removes the mapping. -/// AudioManager.Update calls Stop() on sources it recycles, which are finished (). +/// AudioSource.Stop() (): stops playback at the current render time. The mapping stays, so the source +/// keeps reporting the stopped state. AudioManager.Update calls Stop() on sources it returns to the pool, which are no longer +/// playing, so their voices are left unchanged. /// [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])] public static class AudioSourceStopPatch { - [HarmonyPostfix] + [HarmonyPrefix] // ReSharper disable once InconsistentNaming - public static void Postfix(AudioSource __instance) + public static bool Prefix(AudioSource __instance) { - if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) - { - var mixer = RenderManager.Instance.AudioMixer; - if (mixer == null) return; - if (mixer.TryGetVoice(__instance, out var voice)) - { - voice.Stop(RenderManager.Instance.AudioTime); - } - mixer.RemoveSourceMapping(__instance); - } + if (!EffectSource.TryGet(__instance, out var state, out var now)) return true; + state.Stop(now); + return false; } } diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index bee91d9..3f5ef28 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -50,7 +50,7 @@ internal sealed class OfflineAudioMixer : IDisposable private readonly int _latencySamples; // Main thread. - private ConditionalWeakTable _sourceVoices = new(); + private ConditionalWeakTable _sourceStates = new(); private readonly List _effectVoices = new(); private int _effectVoicePruneAt = EffectVoicePruneCount; private AudioClipSource? _bgmSource; @@ -191,9 +191,9 @@ internal sealed class OfflineAudioMixer : IDisposable /// /// Starts an effect sound at . An of 0 or more makes it loop until then. - /// (when given) is mapped to the voice for later changes through the AudioSource. + /// (when given) is mapped to the returned state, which reports and applies later changes through the AudioSource. /// - public Voice RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) + public SourceState RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) { var clipSource = GetOrAddSource(clipName); var loop = endTime >= 0; @@ -207,26 +207,30 @@ internal sealed class OfflineAudioMixer : IDisposable } _effectVoices.Add(voice); - if (source) MapSource(source!, voice); - return voice; + var state = new SourceState(voice, Math.Max(startTime, now), pitch, loop, loop ? endTime : double.PositiveInfinity, + clipSource?.Duration ?? 0.0, _sampleRate); + if (source) MapSource(source!, state); + return state; } - /// Maps to for later changes through the AudioSource (replaces an earlier mapping). - public void MapSource(AudioSource source, Voice voice) + /// + /// Maps to (replaces an earlier mapping) and takes the clip length and + /// sample rate from the clip of the source. + /// + public void MapSource(AudioSource source, SourceState state) { - _sourceVoices.Remove(source); - _sourceVoices.Add(source, voice); + state.BindClip(source.clip); + _sourceStates.Remove(source); + _sourceStates.Add(source, state); } - /// Voice started for by . - public bool TryGetVoice(AudioSource? source, out Voice voice) + /// State of the effect sound last started for . The mapping stays after the sound ends or stops. + public bool TryGetSourceState(AudioSource? source, out SourceState state) { - voice = null!; - return source != null && _sourceVoices.TryGetValue(source, out voice); + state = null!; + return source != null && _sourceStates.TryGetValue(source, out state); } - public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source); - /// /// Stops every effect sound at , including ones scheduled to start later /// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of @@ -237,7 +241,7 @@ internal sealed class OfflineAudioMixer : IDisposable foreach (var voice in _effectVoices) voice.Stop(now); _effectVoices.Clear(); - _sourceVoices = new ConditionalWeakTable(); + _sourceStates = new ConditionalWeakTable(); } /// @@ -348,7 +352,7 @@ internal sealed class OfflineAudioMixer : IDisposable foreach (var filter in _filters) filter.Reset(); } - _sourceVoices = new ConditionalWeakTable(); + _sourceStates = new ConditionalWeakTable(); _effectVoices.Clear(); } } diff --git a/Util/Audio/SourceState.cs b/Util/Audio/SourceState.cs new file mode 100644 index 0000000..161c3ad --- /dev/null +++ b/Util/Audio/SourceState.cs @@ -0,0 +1,140 @@ +using System; +using ADOFAIRenderer.Util.Audio.Mixer; +using UnityEngine; + +namespace ADOFAIRenderer.Util.Audio; + +/// +/// Playback state of a game AudioSource mapped to an effect , as the AudioSource reports it to the game +/// (isPlaying, time, timeSamples, pitch). Main thread only. Times are seconds relative to the render start (the render clock that +/// also drives AudioSettings.dspTime during rendering); clip positions are seconds into the clip. +/// +/// Reported values: +/// - Playing: from the scheduled start until Stop(), the scheduled end time, or the natural end of the clip (not looping). +/// Before the start, isPlaying is true and time is the position playback will start from. +/// - time while playing: start position + elapsed time * pitch since the last change, wrapped at the clip length when looping. +/// time is 0 while not playing. +/// - timeSamples while playing: floor(time * clip frequency). While not playing: the last value set through time or timeSamples, +/// reset to 0 by Stop(). +/// - pitch: the last finite value set (NaN and infinity are ignored). Playback uses max(0, pitch). +/// Changes made while not playing are stored and do not reach the voice. +/// +internal sealed class SourceState +{ + private readonly double _start; + private double _anchorTime; + private double _anchorPosition; + private double _pitch; + private readonly bool _loop; + private double _endTime = double.PositiveInfinity; + private bool _stopped; + private int _offsetSamples; + private double _length; + private int _frequency; + + /// Voice that plays the sound. + /// Start of playback (seconds relative to the render start, not before the time it was scheduled). + /// Pitch at the start. + /// Repeats the clip. + /// Scheduled end time (seconds relative to the render start), or infinity. + /// Clip length in seconds (replaced by the game clip in ). + /// Clip sample rate for timeSamples (replaced by the game clip in ). + public SourceState(Voice voice, double start, double pitch, bool loop, double endTime, double length, int frequency) + { + Voice = voice; + _start = start; + _anchorTime = start; + _pitch = pitch; + _loop = loop; + _endTime = endTime; + _length = length; + _frequency = Math.Max(1, frequency); + } + + public Voice Voice { get; } + + /// Takes the clip length and sample rate from the clip of the game AudioSource. + public void BindClip(AudioClip? clip) + { + if (!clip) return; + _length = clip!.length; + _frequency = Math.Max(1, clip.frequency); + } + + /// Scheduled end time (see AudioSource.SetScheduledEndTime), seconds relative to the render start. + public void SetEndTime(double endTime, double now) + { + if (!IsPlaying(now)) return; + _endTime = endTime; + Voice.SetEndTime(endTime, now); + } + + public bool IsPlaying(double now) => !_stopped && now < Math.Min(_endTime, NaturalEnd()); + + public float GetTime(double now) => IsPlaying(now) ? (float)PositionAt(now) : 0f; + + public int GetTimeSamples(double now) => IsPlaying(now) ? (int)Math.Floor(PositionAt(now) * _frequency) : _offsetSamples; + + public float GetPitch() => (float)_pitch; + + public void SetTime(float seconds, double now) + { + _offsetSamples = (int)Math.Floor((double)seconds * _frequency); + Seek(seconds, now); + } + + public void SetTimeSamples(int samples, double now) + { + _offsetSamples = samples; + Seek((double)samples / _frequency, now); + } + + public void SetPitch(float pitch, double now) + { + if (float.IsNaN(pitch) || float.IsInfinity(pitch)) return; + var playing = IsPlaying(now); + if (playing) Reanchor(now); + _pitch = pitch; + if (playing) Voice.SetPitch(pitch, now); + } + + public void Stop(double now) + { + if (IsPlaying(now)) Voice.Stop(now); + _stopped = true; + _offsetSamples = 0; + } + + private double PlaybackPitch => Math.Max(0.0, _pitch); + + private void Seek(double seconds, double now) + { + if (!IsPlaying(now)) return; + var position = Math.Max(0.0, seconds); + _anchorTime = Math.Max(now, _start); + _anchorPosition = position; + Voice.SetPosition(position, now); + } + + private void Reanchor(double now) + { + var time = Math.Max(now, _start); + _anchorPosition = PositionAt(time); + _anchorTime = time; + } + + /// Clip position at (the start position before the start), wrapped when looping. + private double PositionAt(double now) + { + var position = _anchorPosition + (Math.Max(now, _anchorTime) - _anchorTime) * PlaybackPitch; + if (_loop && _length > 0 && position >= _length) position %= _length; + return position; + } + + /// Time at which the clip position reaches the clip length (infinity when looping or at pitch 0). + private double NaturalEnd() + { + if (_loop || PlaybackPitch <= 0) return double.PositiveInfinity; + return _anchorTime + Math.Max(0.0, _length - _anchorPosition) / PlaybackPitch; + } +}