From cca027ed66884f6716e7b89bb515e6378c8d0f9525901dff673fdcfd4bd8de48 Mon Sep 17 00:00:00 2001 From: copecone Date: Thu, 9 Jul 2026 12:45:29 +0900 Subject: [PATCH] Refactor Audio Calculation --- Core/RenderManager.cs | 31 ++++--- Patch/Engine/AudioPatches.cs | 33 ++++---- Util/Audio/AudioFilter.cs | 7 ++ Util/Audio/AudioSyncCalculator.cs | 39 +++++++++ Util/Audio/OfflineAudioMixer.cs | 131 ++++++++++++++++-------------- 5 files changed, 155 insertions(+), 86 deletions(-) create mode 100644 Util/Audio/AudioFilter.cs create mode 100644 Util/Audio/AudioSyncCalculator.cs diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index d59fb16..5205c0c 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -49,6 +49,9 @@ namespace ADOFAIRenderer.Core public double unscaledTimeBase; public double dspTimeBase; public int frameCountBase; + + public OfflineAudioMixer? AudioMixer { get; private set; } + public AudioSyncCalculator SyncCalculator { get; private set; } = null!; private FrameReader? _frameReader; public bool renderCheck = true; @@ -432,9 +435,14 @@ namespace ADOFAIRenderer.Core { AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip); } + if (Configuration.RenderAudio) { - OfflineAudioMixer.InitializeBGM(scrConductor.instance.song.clip, Configuration.AudioChannels, Configuration.AudioSampleRate); + AudioMixer = new OfflineAudioMixer(Configuration.AudioChannels, Configuration.AudioSampleRate); + AudioMixer.InitializeBGM(scrConductor.instance.song.clip); + + var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); + SyncCalculator = new AudioSyncCalculator(Configuration.AudioSampleRate, effectiveVideoFps); } scrConductor.instance.song.clip = null; @@ -581,19 +589,17 @@ namespace ADOFAIRenderer.Core if (Configuration.RenderAudio) { - int frameIdx = (int)(passCount - 1); - double effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); - double frameStartTime = frameIdx / effectiveVideoFps; - double frameEndTime = (frameIdx + 1) / effectiveVideoFps; + var frameIdx = (int)(passCount - 1); + var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); + var frameTime = frameIdx / effectiveVideoFps; - int sampleRate = Configuration.AudioSampleRate; - int channels = Configuration.AudioChannels; - int samplesNeeded = (int)(Math.Floor(frameEndTime * sampleRate) - Math.Floor(frameStartTime * sampleRate)); + var channels = Configuration.AudioChannels; + var samplesNeeded = SyncCalculator.GetNextFrameSamples(); - float[] audioBuffer = ArrayPool.Shared.Rent(samplesNeeded * channels); + var audioBuffer = ArrayPool.Shared.Rent(samplesNeeded * channels); try { - OfflineAudioMixer.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameStartTime, dspTimeBase); + AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase); videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded); } finally @@ -798,9 +804,10 @@ namespace ADOFAIRenderer.Core } _frameReader?.Dispose(); - OfflineAudioMixer.Reset(); - ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor"); + AudioMixer?.Reset(); AudioMixer = null; + SyncCalculator.Reset(); + ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor"); Cursor.visible = true; } diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index d7c73fa..599864a 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -1,7 +1,5 @@ using System; using ADOFAIRenderer.Core; -using ADOFAIRenderer.Util; -using ADOFAIRenderer.Util.Audio; using HarmonyLib; using UnityEngine; using UnityEngine.Audio; @@ -17,7 +15,7 @@ namespace ADOFAIRenderer.Patch.Engine if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) { // Cache the clip on the main thread - AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true); + RenderManager.Instance.AudioMixer?.CacheClip(snd); } return true; // Let the original method execute to prevent NullReferenceException } @@ -31,7 +29,7 @@ namespace ADOFAIRenderer.Patch.Engine double relativeStartTime = time - RenderManager.Instance.dspTimeBase; // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource - OfflineAudioMixer.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f); + RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f); } } } @@ -44,15 +42,19 @@ namespace ADOFAIRenderer.Patch.Engine { if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) { - // Cache the clip on the main thread - AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true); + var mixer = RenderManager.Instance.AudioMixer; + if (mixer != null) + { + // Cache the clip on the main thread + mixer.CacheClip(snd); - // Convert absolute DSP times to relative times in seconds from render start - double relativeStartTime = time - RenderManager.Instance.dspTimeBase; - double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; + // Convert absolute DSP times to relative times in seconds from render start + double relativeStartTime = time - RenderManager.Instance.dspTimeBase; + double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; - // Register looping sound event to the offline mixer (default pitch = 1.0f) - OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f); + // Register looping sound event to the offline mixer (default pitch = 1.0f) + mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f); + } } return true; // Let the original method execute } @@ -62,11 +64,13 @@ namespace ADOFAIRenderer.Patch.Engine public static class AudioSourcePitchPatch { [HarmonyPostfix] + // ReSharper disable once InconsistentNaming public static void Postfix(AudioSource __instance, float value) { if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) { - if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent)) + var mixer = RenderManager.Instance.AudioMixer; + if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent)) { soundEvent.Pitch = value; } @@ -84,7 +88,8 @@ namespace ADOFAIRenderer.Patch.Engine { if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) { - if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent)) + var mixer = RenderManager.Instance.AudioMixer; + if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent)) { soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase; } @@ -100,7 +105,7 @@ namespace ADOFAIRenderer.Patch.Engine { if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) { - OfflineAudioMixer.RemoveSourceMapping(__instance); + RenderManager.Instance.AudioMixer?.RemoveSourceMapping(__instance); } } } diff --git a/Util/Audio/AudioFilter.cs b/Util/Audio/AudioFilter.cs new file mode 100644 index 0000000..4a8bfed --- /dev/null +++ b/Util/Audio/AudioFilter.cs @@ -0,0 +1,7 @@ +namespace ADOFAIRenderer.Util.Audio +{ + public class AudioFilter + { + + } +} \ No newline at end of file diff --git a/Util/Audio/AudioSyncCalculator.cs b/Util/Audio/AudioSyncCalculator.cs new file mode 100644 index 0000000..1f519ce --- /dev/null +++ b/Util/Audio/AudioSyncCalculator.cs @@ -0,0 +1,39 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio +{ + public class AudioSyncCalculator + { + private readonly long _sampleRate; + private readonly (long Numerator, long Denominator) _fpsRational; + + private long _currentSample; + private long _remainder; + + public AudioSyncCalculator(long sampleRate, double fps) + { + _sampleRate = sampleRate; + _fpsRational = RationalUtil.Double2MaxPrecisionInt(fps); + + if (Math.Abs(fps - 29.97) < 0.001) _fpsRational = (30000, 1001); + if (Math.Abs(fps - 59.94) < 0.001) _fpsRational = (60000, 1001); + if (Math.Abs(fps - 23.976) < 0.001 || Math.Abs(fps - 23.98) < 0.001) _fpsRational = (24000, 1001); + } + + public void Reset() + { + _currentSample = 0; + _remainder = 0; + } + + public int GetNextFrameSamples() + { + var numerator = _sampleRate * _fpsRational.Denominator + _remainder; + var samplesNeeded = numerator / _fpsRational.Numerator; + + _remainder = numerator % _fpsRational.Numerator; + _currentSample += samplesNeeded; + return (int)samplesNeeded; + } + } +} \ No newline at end of file diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index ddea03d..1141fd3 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -4,9 +4,9 @@ using UnityEngine; namespace ADOFAIRenderer.Util.Audio { - public static class AudioClipCache + public class AudioClipCache { - private static readonly Dictionary Cache = new(); + private readonly Dictionary _cache = new(); public class CachedClipData { @@ -14,31 +14,31 @@ namespace ADOFAIRenderer.Util.Audio public double Duration; } - public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true) + public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true) { if (string.IsNullOrEmpty(clipName)) return null; - lock (Cache) + lock (_cache) { - if (Cache.TryGetValue(clipName, out var cached)) + if (_cache.TryGetValue(clipName, out var cached)) return cached; if (!allowLoad) return null; - if (AudioManager.Instance == null) return null; + if (!AudioManager.Instance) return null; - AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName); - if (clip == null) return null; + var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName); + if (!clip) return null; try { - int totalSamples = clip.samples; - int channels = clip.channels; - float[] rawSamples = new float[totalSamples * channels]; + var totalSamples = clip.samples; + var channels = clip.channels; + var rawSamples = new float[totalSamples * channels]; clip.GetData(rawSamples, 0); // Auto-resample during cache generation - float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate); + var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate); var data = new CachedClipData { @@ -46,7 +46,7 @@ namespace ADOFAIRenderer.Util.Audio Duration = (double)resampled.Length / (targetChannels * targetSampleRate) }; - Cache[clipName] = data; + _cache[clipName] = data; return data; } catch (Exception e) @@ -57,16 +57,16 @@ namespace ADOFAIRenderer.Util.Audio } } - public static void Clear() + public void Clear() { - lock (Cache) + lock (_cache) { - Cache.Clear(); + _cache.Clear(); } } } - public static class OfflineAudioMixer + public class OfflineAudioMixer { public class SoundEvent { @@ -77,26 +77,32 @@ namespace ADOFAIRenderer.Util.Audio public float Pitch; } - private static readonly List ActiveEvents = new(); - private static System.Runtime.CompilerServices.ConditionalWeakTable _sourceMappings = new(); - private static readonly object Lock = new(); + private readonly List _activeEvents = new(); + private System.Runtime.CompilerServices.ConditionalWeakTable _sourceMappings = new(); + private readonly object _lock = new(); // Target Audio Settings - private static int _targetChannels = 2; - private static int _targetSampleRate = 44100; - private static readonly float MasterVolume = 0.2f; + private readonly int _targetChannels; + private readonly int _targetSampleRate; + private const float MasterVolume = 0.05f; // BGM Cached Samples (Pre-resampled) - private static float[]? _bgmSamples; - private static double _bgmDuration; + private float[]? _bgmSamples; + private double _bgmDuration; - public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate) + // Clip Cache + private readonly AudioClipCache _clipCache = new(); + + public OfflineAudioMixer(int targetChannels, int targetSampleRate) { - lock (Lock) - { - _targetChannels = targetChannels; - _targetSampleRate = targetSampleRate; + _targetChannels = targetChannels; + _targetSampleRate = targetSampleRate; + } + public void InitializeBGM(AudioClip? clip) + { + lock (_lock) + { if (!clip) { _bgmSamples = null; @@ -112,10 +118,10 @@ namespace ADOFAIRenderer.Util.Audio clip.GetData(rawSamples, 0); // Pre-resample the BGM to the target format - _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate); - _bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate); + _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate); + _bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate); - MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}"); + MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}"); } catch (Exception e) { @@ -126,9 +132,14 @@ namespace ADOFAIRenderer.Util.Audio } } - public static void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch) + public void CacheClip(string clipName) { - lock (Lock) + _clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true); + } + + public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch) + { + lock (_lock) { var ev = new SoundEvent { @@ -138,7 +149,7 @@ namespace ADOFAIRenderer.Util.Audio Volume = volume, Pitch = pitch }; - ActiveEvents.Add(ev); + _activeEvents.Add(ev); if (source != null) { _sourceMappings.Remove(source); @@ -147,50 +158,50 @@ namespace ADOFAIRenderer.Util.Audio } } - public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch) + public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch) { RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch); } - public static bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev) + public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev) { if (source == null) { ev = null!; return false; } - lock (Lock) + lock (_lock) { return _sourceMappings.TryGetValue(source, out ev); } } - public static bool RemoveSourceMapping(AudioSource source) + public bool RemoveSourceMapping(AudioSource source) { - lock (Lock) + lock (_lock) { return _sourceMappings.Remove(source); } } - public static void Reset() + public void Reset() { - lock (Lock) + lock (_lock) { - ActiveEvents.Clear(); + _activeEvents.Clear(); _sourceMappings = new System.Runtime.CompilerServices.ConditionalWeakTable(); _bgmSamples = null; _bgmDuration = 0; - AudioClipCache.Clear(); + _clipCache.Clear(); } } - public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) + public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) { // mixedBuffer size must be at least samplesNeeded * _targetChannels Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels); - lock (Lock) + lock (_lock) { // 1. Mix BGM MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase); @@ -207,7 +218,7 @@ namespace ADOFAIRenderer.Util.Audio } } - private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) + private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) { if (_bgmSamples == null || _bgmSamples.Length == 0) return; @@ -251,24 +262,24 @@ namespace ADOFAIRenderer.Util.Audio } } - private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime) + private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime) { - for (int i = ActiveEvents.Count - 1; i >= 0; i--) + for (var i = _activeEvents.Count - 1; i >= 0; i--) { - var ev = ActiveEvents[i]; - var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false); + var ev = _activeEvents[i]; + var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false); if (clipData == null) { - ActiveEvents.RemoveAt(i); + _activeEvents.RemoveAt(i); continue; } - bool isSoundFinished = false; - int totalSamples = clipData.Samples.Length / _targetChannels; + var isSoundFinished = false; + var totalSamples = clipData.Samples.Length / _targetChannels; - for (int s = 0; s < samplesNeeded; s++) + for (var s = 0; s < samplesNeeded; s++) { - double sampleTime = frameStartTime + (double)s / _targetSampleRate; + var sampleTime = frameStartTime + (double)s / _targetSampleRate; // If it is a looping sound and has reached or exceeded its end time, stop playing. if (ev.EndTime >= 0 && sampleTime >= ev.EndTime) @@ -277,10 +288,10 @@ namespace ADOFAIRenderer.Util.Audio break; } - double elapsed = sampleTime - ev.StartTime; + var elapsed = sampleTime - ev.StartTime; if (elapsed < 0) continue; // Sound hasn't started yet - double soundTime = elapsed * ev.Pitch; + var soundTime = elapsed * ev.Pitch; if (soundTime >= clipData.Duration) { @@ -314,7 +325,7 @@ namespace ADOFAIRenderer.Util.Audio // Clean up finished sound events (both one-shot and looping) if (isSoundFinished) { - ActiveEvents.RemoveAt(i); + _activeEvents.RemoveAt(i); } } }