Refactor Audio Calculation

Core/RenderManager.cs

@@ -49,6 +49,9 @@ namespace ADOFAIRenderer.Core
         public double unscaledTimeBase;
         public double dspTimeBase;
         public int frameCountBase;
+        
+        public OfflineAudioMixer? AudioMixer { get; private set; }
+        public AudioSyncCalculator SyncCalculator { get; private set; } = null!;
 
         private FrameReader? _frameReader;
         public bool renderCheck = true;
@@ -432,9 +435,14 @@ namespace ADOFAIRenderer.Core
             {
                 AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
             }
+            
             if (Configuration.RenderAudio)
             {
-                OfflineAudioMixer.InitializeBGM(scrConductor.instance.song.clip, Configuration.AudioChannels, Configuration.AudioSampleRate);
+                AudioMixer = new OfflineAudioMixer(Configuration.AudioChannels, Configuration.AudioSampleRate);
+                AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
+                
+                var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
+                SyncCalculator = new AudioSyncCalculator(Configuration.AudioSampleRate, effectiveVideoFps);
             }
             
             scrConductor.instance.song.clip = null;
@@ -581,19 +589,17 @@ namespace ADOFAIRenderer.Core
 
                             if (Configuration.RenderAudio)
                             {
-                                int frameIdx = (int)(passCount - 1);
+                                var frameIdx = (int)(passCount - 1);
-                                double effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
+                                var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
-                                double frameStartTime = frameIdx / effectiveVideoFps;
+                                var frameTime = frameIdx / effectiveVideoFps;
-                                double frameEndTime = (frameIdx + 1) / effectiveVideoFps;
                                 
-                                int sampleRate = Configuration.AudioSampleRate;
+                                var channels = Configuration.AudioChannels;
-                                int channels = Configuration.AudioChannels;
+                                var samplesNeeded = SyncCalculator.GetNextFrameSamples();
-                                int samplesNeeded = (int)(Math.Floor(frameEndTime * sampleRate) - Math.Floor(frameStartTime * sampleRate));
 
-                                float[] audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
+                                var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
                                 try
                                 {
-                                    OfflineAudioMixer.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameStartTime, dspTimeBase);
+                                    AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
                                     videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
                                 }
                                 finally
@@ -798,9 +804,10 @@ namespace ADOFAIRenderer.Core
             }
 
             _frameReader?.Dispose();
-            OfflineAudioMixer.Reset();
+            AudioMixer?.Reset(); AudioMixer = null;
-            ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
+            SyncCalculator.Reset();
             
+            ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
             Cursor.visible = true;
         }
 

Patch/Engine/AudioPatches.cs

@@ -1,7 +1,5 @@
 using System;
 using ADOFAIRenderer.Core;
-using ADOFAIRenderer.Util;
-using ADOFAIRenderer.Util.Audio;
 using HarmonyLib;
 using UnityEngine;
 using UnityEngine.Audio;
@@ -17,7 +15,7 @@ namespace ADOFAIRenderer.Patch.Engine
             if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
             {
                 // Cache the clip on the main thread
-                AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
+                RenderManager.Instance.AudioMixer?.CacheClip(snd);
             }
             return true; // Let the original method execute to prevent NullReferenceException
         }
@@ -31,7 +29,7 @@ namespace ADOFAIRenderer.Patch.Engine
                 double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
 
                 // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource
-                OfflineAudioMixer.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
+                RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
             }
         }
     }
@@ -44,15 +42,19 @@ namespace ADOFAIRenderer.Patch.Engine
         {
             if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
             {
-                // Cache the clip on the main thread
+                var mixer = RenderManager.Instance.AudioMixer;
-                AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
+                if (mixer != null)
+                {
+                    // Cache the clip on the main thread
+                    mixer.CacheClip(snd);
 
-                // Convert absolute DSP times to relative times in seconds from render start
+                    // Convert absolute DSP times to relative times in seconds from render start
-                double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
+                    double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
-                double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
+                    double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
 
-                // Register looping sound event to the offline mixer (default pitch = 1.0f)
+                    // Register looping sound event to the offline mixer (default pitch = 1.0f)
-                OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
+                    mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
+                }
             }
             return true; // Let the original method execute
         }
@@ -62,11 +64,13 @@ namespace ADOFAIRenderer.Patch.Engine
     public static class AudioSourcePitchPatch
     {
         [HarmonyPostfix]
+        // ReSharper disable once InconsistentNaming
         public static void Postfix(AudioSource __instance, float value)
         {
             if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
             {
-                if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent))
+                var mixer = RenderManager.Instance.AudioMixer;
+                if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
                 {
                     soundEvent.Pitch = value;
                 }
@@ -84,7 +88,8 @@ namespace ADOFAIRenderer.Patch.Engine
         {
             if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
             {
-                if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent))
+                var mixer = RenderManager.Instance.AudioMixer;
+                if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
                 {
                     soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase;
                 }
@@ -100,7 +105,7 @@ namespace ADOFAIRenderer.Patch.Engine
         {
             if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
             {
-                OfflineAudioMixer.RemoveSourceMapping(__instance);
+                RenderManager.Instance.AudioMixer?.RemoveSourceMapping(__instance);
             }
         }
     }

Util/Audio/AudioFilter.cs

@@ -0,0 +1,7 @@
+namespace ADOFAIRenderer.Util.Audio
+{
+    public class AudioFilter
+    {
+        
+    }
+}

Util/Audio/AudioSyncCalculator.cs

@@ -0,0 +1,39 @@
+using System;
+
+namespace ADOFAIRenderer.Util.Audio
+{
+    public class AudioSyncCalculator
+    {
+        private readonly long _sampleRate;
+        private readonly (long Numerator, long Denominator) _fpsRational;
+
+        private long _currentSample;
+        private long _remainder;
+        
+        public AudioSyncCalculator(long sampleRate, double fps)
+        {
+            _sampleRate = sampleRate;
+            _fpsRational = RationalUtil.Double2MaxPrecisionInt(fps);
+
+            if (Math.Abs(fps - 29.97) < 0.001) _fpsRational = (30000, 1001);
+            if (Math.Abs(fps - 59.94) < 0.001) _fpsRational = (60000, 1001);
+            if (Math.Abs(fps - 23.976) < 0.001 || Math.Abs(fps - 23.98) < 0.001) _fpsRational = (24000, 1001);
+        }
+
+        public void Reset()
+        {
+            _currentSample = 0;
+            _remainder = 0;
+        }
+
+        public int GetNextFrameSamples()
+        {
+            var numerator = _sampleRate * _fpsRational.Denominator + _remainder;
+            var samplesNeeded = numerator / _fpsRational.Numerator;
+
+            _remainder = numerator % _fpsRational.Numerator;
+            _currentSample += samplesNeeded;
+            return (int)samplesNeeded;
+        }
+    }
+}

Util/Audio/OfflineAudioMixer.cs

@@ -4,9 +4,9 @@ using UnityEngine;
 
 namespace ADOFAIRenderer.Util.Audio
 {
-    public static class AudioClipCache
+    public class AudioClipCache
     {
-        private static readonly Dictionary<string, CachedClipData> Cache = new();
+        private readonly Dictionary<string, CachedClipData> _cache = new();
 
         public class CachedClipData
         {
@@ -14,31 +14,31 @@ namespace ADOFAIRenderer.Util.Audio
             public double Duration;
         }
 
-        public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
+        public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
         {
             if (string.IsNullOrEmpty(clipName)) return null;
 
-            lock (Cache)
+            lock (_cache)
             {
-                if (Cache.TryGetValue(clipName, out var cached))
+                if (_cache.TryGetValue(clipName, out var cached))
                     return cached;
 
                 if (!allowLoad) return null;
 
-                if (AudioManager.Instance == null) return null;
+                if (!AudioManager.Instance) return null;
 
-                AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
+                var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
-                if (clip == null) return null;
+                if (!clip) return null;
 
                 try
                 {
-                    int totalSamples = clip.samples;
+                    var totalSamples = clip.samples;
-                    int channels = clip.channels;
+                    var channels = clip.channels;
-                    float[] rawSamples = new float[totalSamples * channels];
+                    var rawSamples = new float[totalSamples * channels];
                     clip.GetData(rawSamples, 0);
 
                     // Auto-resample during cache generation
-                    float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
+                    var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
 
                     var data = new CachedClipData
                     {
@@ -46,7 +46,7 @@ namespace ADOFAIRenderer.Util.Audio
                         Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
                     };
 
-                    Cache[clipName] = data;
+                    _cache[clipName] = data;
                     return data;
                 }
                 catch (Exception e)
@@ -57,16 +57,16 @@ namespace ADOFAIRenderer.Util.Audio
             }
         }
 
-        public static void Clear()
+        public void Clear()
         {
-            lock (Cache)
+            lock (_cache)
             {
-                Cache.Clear();
+                _cache.Clear();
             }
         }
     }
 
-    public static class OfflineAudioMixer
+    public class OfflineAudioMixer
     {
         public class SoundEvent
         {
@@ -77,26 +77,32 @@ namespace ADOFAIRenderer.Util.Audio
             public float Pitch;
         }
 
-        private static readonly List<SoundEvent> ActiveEvents = new();
+        private readonly List<SoundEvent> _activeEvents = new();
-        private static System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
+        private System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
-        private static readonly object Lock = new();
+        private readonly object _lock = new();
 
         // Target Audio Settings
-        private static int _targetChannels = 2;
+        private readonly int _targetChannels;
-        private static int _targetSampleRate = 44100;
+        private readonly int _targetSampleRate;
-        private static readonly float MasterVolume = 0.2f;
+        private const float MasterVolume = 0.05f;
 
         // BGM Cached Samples (Pre-resampled)
-        private static float[]? _bgmSamples;
+        private float[]? _bgmSamples;
-        private static double _bgmDuration;
+        private double _bgmDuration;
 
-        public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate)
+        // Clip Cache
+        private readonly AudioClipCache _clipCache = new();
+
+        public OfflineAudioMixer(int targetChannels, int targetSampleRate)
         {
-            lock (Lock)
+            _targetChannels = targetChannels;
-            {
+            _targetSampleRate = targetSampleRate;
-                _targetChannels = targetChannels;
+        }
-                _targetSampleRate = targetSampleRate;
 
+        public void InitializeBGM(AudioClip? clip)
+        {
+            lock (_lock)
+            {
                 if (!clip)
                 {
                     _bgmSamples = null;
@@ -112,10 +118,10 @@ namespace ADOFAIRenderer.Util.Audio
                     clip.GetData(rawSamples, 0);
 
                     // Pre-resample the BGM to the target format
-                    _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate);
+                    _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate);
-                    _bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate);
+                    _bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate);
 
-                    MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}");
+                    MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}");
                 }
                 catch (Exception e)
                 {
@@ -126,9 +132,14 @@ namespace ADOFAIRenderer.Util.Audio
             }
         }
 
-        public static void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
+        public void CacheClip(string clipName)
         {
-            lock (Lock)
+            _clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true);
+        }
+
+        public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
+        {
+            lock (_lock)
             {
                 var ev = new SoundEvent
                 {
@@ -138,7 +149,7 @@ namespace ADOFAIRenderer.Util.Audio
                     Volume = volume,
                     Pitch = pitch
                 };
-                ActiveEvents.Add(ev);
+                _activeEvents.Add(ev);
                 if (source != null)
                 {
                     _sourceMappings.Remove(source);
@@ -147,50 +158,50 @@ namespace ADOFAIRenderer.Util.Audio
             }
         }
 
-        public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
+        public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
         {
             RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
         }
 
-        public static bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
+        public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
         {
             if (source == null)
             {
                 ev = null!;
                 return false;
             }
-            lock (Lock)
+            lock (_lock)
             {
                 return _sourceMappings.TryGetValue(source, out ev);
             }
         }
 
-        public static bool RemoveSourceMapping(AudioSource source)
+        public bool RemoveSourceMapping(AudioSource source)
         {
-            lock (Lock)
+            lock (_lock)
             {
                 return _sourceMappings.Remove(source);
             }
         }
 
-        public static void Reset()
+        public void Reset()
         {
-            lock (Lock)
+            lock (_lock)
             {
-                ActiveEvents.Clear();
+                _activeEvents.Clear();
                 _sourceMappings = new System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent>();
                 _bgmSamples = null;
                 _bgmDuration = 0;
-                AudioClipCache.Clear();
+                _clipCache.Clear();
             }
         }
 
-        public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
+        public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
         {
             // mixedBuffer size must be at least samplesNeeded * _targetChannels
             Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
 
-            lock (Lock)
+            lock (_lock)
             {
                 // 1. Mix BGM
                 MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase);
@@ -207,7 +218,7 @@ namespace ADOFAIRenderer.Util.Audio
             }
         }
 
-        private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
+        private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
         {
             if (_bgmSamples == null || _bgmSamples.Length == 0) return;
 
@@ -251,24 +262,24 @@ namespace ADOFAIRenderer.Util.Audio
             }
         }
 
-        private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
+        private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
         {
-            for (int i = ActiveEvents.Count - 1; i >= 0; i--)
+            for (var i = _activeEvents.Count - 1; i >= 0; i--)
             {
-                var ev = ActiveEvents[i];
+                var ev = _activeEvents[i];
-                var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
+                var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
                 if (clipData == null)
                 {
-                    ActiveEvents.RemoveAt(i);
+                    _activeEvents.RemoveAt(i);
                     continue;
                 }
 
-                bool isSoundFinished = false;
+                var isSoundFinished = false;
-                int totalSamples = clipData.Samples.Length / _targetChannels;
+                var totalSamples = clipData.Samples.Length / _targetChannels;
 
-                for (int s = 0; s < samplesNeeded; s++)
+                for (var s = 0; s < samplesNeeded; s++)
                 {
-                    double sampleTime = frameStartTime + (double)s / _targetSampleRate;
+                    var sampleTime = frameStartTime + (double)s / _targetSampleRate;
 
                     // If it is a looping sound and has reached or exceeded its end time, stop playing.
                     if (ev.EndTime >= 0 && sampleTime >= ev.EndTime)
@@ -277,10 +288,10 @@ namespace ADOFAIRenderer.Util.Audio
                         break;
                     }
 
-                    double elapsed = sampleTime - ev.StartTime;
+                    var elapsed = sampleTime - ev.StartTime;
                     if (elapsed < 0) continue; // Sound hasn't started yet
 
-                    double soundTime = elapsed * ev.Pitch;
+                    var soundTime = elapsed * ev.Pitch;
 
                     if (soundTime >= clipData.Duration)
                     {
@@ -314,7 +325,7 @@ namespace ADOFAIRenderer.Util.Audio
                 // Clean up finished sound events (both one-shot and looping)
                 if (isSoundFinished)
                 {
-                    ActiveEvents.RemoveAt(i);
+                    _activeEvents.RemoveAt(i);
                 }
             }
         }