Add Sync Between AudioSource and SoundEvent

Core/RenderManager.cs

@@ -7,6 +7,7 @@ using System.Buffers;
 using ADOFAIRenderer.Data;
 using ADOFAIRenderer.UI;
 using ADOFAIRenderer.Util;
+using ADOFAIRenderer.Util.Audio;
 using ADOFAIRenderer.Util.FFmpeg;
 using FFmpeg.AutoGen;
 using HarmonyLib;

Patch/Engine/AudioPatches.cs

@@ -1,6 +1,9 @@
+using System;
 using ADOFAIRenderer.Core;
 using ADOFAIRenderer.Util;
+using ADOFAIRenderer.Util.Audio;
 using HarmonyLib;
+using UnityEngine;
 using UnityEngine.Audio;
 
 namespace ADOFAIRenderer.Patch.Engine
@@ -15,14 +18,21 @@ namespace ADOFAIRenderer.Patch.Engine
             {
                 // Cache the clip on the main thread
                 AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
+            }
+            return true; // Let the original method execute to prevent NullReferenceException
+        }
 
+        [HarmonyPostfix]
+        public static void Postfix(AudioSource __result, string snd, double time, AudioMixerGroup group, float volume, int priority)
+        {
+            if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio && __result != null)
+            {
                 // Convert absolute DSP time to relative time in seconds from render start
                 double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
 
-                // Register event to the offline mixer (default pitch = 1.0f)
+                // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource
-                OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, -1.0, volume, 1.0f);
+                OfflineAudioMixer.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
             }
-            return true; // Let the original method execute to prevent NullReferenceException
         }
     }
 
@@ -47,4 +57,51 @@ namespace ADOFAIRenderer.Patch.Engine
             return true; // Let the original method execute
         }
     }
+
+    [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)]
+    public static class AudioSourcePitchPatch
+    {
+        [HarmonyPostfix]
+        public static void Postfix(AudioSource __instance, float value)
+        {
+            if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
+            {
+                if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent))
+                {
+                    soundEvent.Pitch = value;
+                }
+            }
+        }
+    }
+
+
+
+    [HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))]
+    public static class AudioSourceSetScheduledEndTimePatch
+    {
+        [HarmonyPostfix]
+        public static void Postfix(AudioSource __instance, double time)
+        {
+            if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
+            {
+                if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent))
+                {
+                    soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase;
+                }
+            }
+        }
+    }
+
+    [HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])]
+    public static class AudioSourceStopPatch
+    {
+        [HarmonyPostfix]
+        public static void Postfix(AudioSource __instance)
+        {
+            if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
+            {
+                OfflineAudioMixer.RemoveSourceMapping(__instance);
+            }
+        }
+    }
 }

Patch/Engine/RenderEnvironmentPatches.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics.CodeAnalysis;
 using ADOFAIRenderer.Core;
 using ADOFAIRenderer.Util;
+using ADOFAIRenderer.Util.Audio;
 using HarmonyLib;
 using UnityEngine;
 

RenderMod.cs

@@ -1,5 +1,6 @@
 using ADOFAIRenderer;
 using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Patch.Engine;
 using ADOFAIRenderer.Util;
 using FFmpeg.AutoGen;
 using MelonLoader;

Util/Audio/AudioResampler.cs

@@ -0,0 +1,101 @@
+using System;
+
+namespace ADOFAIRenderer.Util.Audio
+{
+    public static class AudioResampler
+    {
+        public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
+        {
+            // ── Safety Validation ──
+            if (inputSamples.Length == 0 ||
+                inputChannels <= 0 || targetChannels <= 0 ||
+                inputFrequency <= 0 || targetFrequency <= 0)
+            {
+                return Array.Empty<float>();
+            }
+
+            // If format is identical, return original samples
+            if (inputFrequency == targetFrequency && inputChannels == targetChannels)
+                return inputSamples;
+
+            int inputLength = inputSamples.Length / inputChannels;
+            int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
+            float[] outputSamples = new float[targetLength * targetChannels];
+
+            for (int i = 0; i < targetLength; i++)
+            {
+                double srcIndex = (double)i * inputFrequency / targetFrequency;
+                long idxLow = (long)Math.Floor(srcIndex);
+                long idxHigh = idxLow + 1;
+                float t = (float)(srcIndex - idxLow);
+
+                if (idxHigh >= inputLength) idxHigh = inputLength - 1;
+                if (idxLow >= inputLength) idxLow = inputLength - 1;
+
+                if (targetChannels == 1) // 1. Mono Output
+                {
+                    float valLow, valHigh;
+                    if (inputChannels == 1) // Mono -> Mono
+                    {
+                        valLow = inputSamples[idxLow];
+                        valHigh = inputSamples[idxHigh];
+                    }
+                    else // Multi -> Mono (Downmix: Average all channels)
+                    {
+                        float sumLow = 0f;
+                        float sumHigh = 0f;
+                        for (int c = 0; c < inputChannels; c++)
+                        {
+                            sumLow += inputSamples[idxLow * inputChannels + c];
+                            sumHigh += inputSamples[idxHigh * inputChannels + c];
+                        }
+                        valLow = sumLow / inputChannels;
+                        valHigh = sumHigh / inputChannels;
+                    }
+                    outputSamples[i] = valLow + t * (valHigh - valLow);
+                }
+                else if (targetChannels == 2) // 2. Stereo Output
+                {
+                    if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
+                    {
+                        float valLow = inputSamples[idxLow];
+                        float valHigh = inputSamples[idxHigh];
+                        float val = valLow + t * (valHigh - valLow);
+                        outputSamples[i * 2] = val;
+                        outputSamples[i * 2 + 1] = val;
+                    }
+                    else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
+                    {
+                        float valLowL = inputSamples[idxLow * inputChannels];
+                        float valHighL = inputSamples[idxHigh * inputChannels];
+                        outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
+
+                        float valLowR = inputSamples[idxLow * inputChannels + 1];
+                        float valHighR = inputSamples[idxHigh * inputChannels + 1];
+                        outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
+                    }
+                }
+                else // 3. Multi-channel Output (e.g., 5.1 Surround)
+                {
+                    for (int c = 0; c < targetChannels; c++)
+                    {
+                        float valLow, valHigh;
+                        if (c < inputChannels)
+                        {
+                            valLow = inputSamples[idxLow * inputChannels + c];
+                            valHigh = inputSamples[idxHigh * inputChannels + c];
+                        }
+                        else
+                        {
+                            valLow = inputSamples[idxLow * inputChannels];
+                            valHigh = inputSamples[idxHigh * inputChannels];
+                        }
+                        outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
+                    }
+                }
+            }
+
+            return outputSamples;
+        }
+    }
+}

Util/Audio/AudioSpectrumGenerator.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using ADOFAIRenderer.Core;
 using UnityEngine;
 
-namespace ADOFAIRenderer.Util
+namespace ADOFAIRenderer.Util.Audio
 {
     public static class AudioSpectrumGenerator
     {

Util/Audio/OfflineAudioMixer.cs

@@ -2,105 +2,8 @@ using System;
 using System.Collections.Generic;
 using UnityEngine;
 
-namespace ADOFAIRenderer.Util
+namespace ADOFAIRenderer.Util.Audio
 {
-    public static class AudioResampler
-    {
-        public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
-        {
-            // ── Safety Validation ──
-            if (inputSamples.Length == 0 ||
-                inputChannels <= 0 || targetChannels <= 0 ||
-                inputFrequency <= 0 || targetFrequency <= 0)
-            {
-                return Array.Empty<float>();
-            }
-
-            // If format is identical, return original samples
-            if (inputFrequency == targetFrequency && inputChannels == targetChannels)
-                return inputSamples;
-
-            int inputLength = inputSamples.Length / inputChannels;
-            int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
-            float[] outputSamples = new float[targetLength * targetChannels];
-
-            for (int i = 0; i < targetLength; i++)
-            {
-                double srcIndex = (double)i * inputFrequency / targetFrequency;
-                long idxLow = (long)Math.Floor(srcIndex);
-                long idxHigh = idxLow + 1;
-                float t = (float)(srcIndex - idxLow);
-
-                if (idxHigh >= inputLength) idxHigh = inputLength - 1;
-                if (idxLow >= inputLength) idxLow = inputLength - 1;
-
-                if (targetChannels == 1) // 1. Mono Output
-                {
-                    float valLow, valHigh;
-                    if (inputChannels == 1) // Mono -> Mono
-                    {
-                        valLow = inputSamples[idxLow];
-                        valHigh = inputSamples[idxHigh];
-                    }
-                    else // Multi -> Mono (Downmix: Average all channels)
-                    {
-                        float sumLow = 0f;
-                        float sumHigh = 0f;
-                        for (int c = 0; c < inputChannels; c++)
-                        {
-                            sumLow += inputSamples[idxLow * inputChannels + c];
-                            sumHigh += inputSamples[idxHigh * inputChannels + c];
-                        }
-                        valLow = sumLow / inputChannels;
-                        valHigh = sumHigh / inputChannels;
-                    }
-                    outputSamples[i] = valLow + t * (valHigh - valLow);
-                }
-                else if (targetChannels == 2) // 2. Stereo Output
-                {
-                    if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
-                    {
-                        float valLow = inputSamples[idxLow];
-                        float valHigh = inputSamples[idxHigh];
-                        float val = valLow + t * (valHigh - valLow);
-                        outputSamples[i * 2] = val;
-                        outputSamples[i * 2 + 1] = val;
-                    }
-                    else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
-                    {
-                        float valLowL = inputSamples[idxLow * inputChannels];
-                        float valHighL = inputSamples[idxHigh * inputChannels];
-                        outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
-
-                        float valLowR = inputSamples[idxLow * inputChannels + 1];
-                        float valHighR = inputSamples[idxHigh * inputChannels + 1];
-                        outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
-                    }
-                }
-                else // 3. Multi-channel Output (e.g., 5.1 Surround)
-                {
-                    for (int c = 0; c < targetChannels; c++)
-                    {
-                        float valLow, valHigh;
-                        if (c < inputChannels)
-                        {
-                            valLow = inputSamples[idxLow * inputChannels + c];
-                            valHigh = inputSamples[idxHigh * inputChannels + c];
-                        }
-                        else
-                        {
-                            valLow = inputSamples[idxLow * inputChannels];
-                            valHigh = inputSamples[idxHigh * inputChannels];
-                        }
-                        outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
-                    }
-                }
-            }
-
-            return outputSamples;
-        }
-    }
-
     public static class AudioClipCache
     {
         private static readonly Dictionary<string, CachedClipData> Cache = new();
@@ -165,9 +68,9 @@ namespace ADOFAIRenderer.Util
 
     public static class OfflineAudioMixer
     {
-        public struct SoundEvent
+        public class SoundEvent
         {
-            public string ClipName;
+            public string ClipName = null!;
             public double StartTime; // In seconds relative to dspTimeBase
             public double EndTime;   // In seconds relative to dspTimeBase (-1 if one-shot)
             public float Volume;
@@ -175,11 +78,13 @@ namespace ADOFAIRenderer.Util
         }
 
         private static readonly List<SoundEvent> ActiveEvents = new();
+        private static System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
         private static readonly object Lock = new();
 
         // Target Audio Settings
         private static int _targetChannels = 2;
         private static int _targetSampleRate = 44100;
+        private static readonly float MasterVolume = 0.2f;
 
         // BGM Cached Samples (Pre-resampled)
         private static float[]? _bgmSamples;
@@ -221,18 +126,50 @@ namespace ADOFAIRenderer.Util
             }
         }
 
-        public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
+        public static void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
         {
             lock (Lock)
             {
-                ActiveEvents.Add(new SoundEvent
+                var ev = new SoundEvent
                 {
                     ClipName = clipName,
                     StartTime = startTime,
                     EndTime = endTime,
                     Volume = volume,
                     Pitch = pitch
-                });
+                };
+                ActiveEvents.Add(ev);
+                if (source != null)
+                {
+                    _sourceMappings.Remove(source);
+                    _sourceMappings.Add(source, ev);
+                }
+            }
+        }
+
+        public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
+        {
+            RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
+        }
+
+        public static bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
+        {
+            if (source == null)
+            {
+                ev = null!;
+                return false;
+            }
+            lock (Lock)
+            {
+                return _sourceMappings.TryGetValue(source, out ev);
+            }
+        }
+
+        public static bool RemoveSourceMapping(AudioSource source)
+        {
+            lock (Lock)
+            {
+                return _sourceMappings.Remove(source);
             }
         }
 
@@ -241,6 +178,7 @@ namespace ADOFAIRenderer.Util
             lock (Lock)
             {
                 ActiveEvents.Clear();
+                _sourceMappings = new System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent>();
                 _bgmSamples = null;
                 _bgmDuration = 0;
                 AudioClipCache.Clear();
@@ -307,7 +245,7 @@ namespace ADOFAIRenderer.Util
                     {
                         float valLow = _bgmSamples[idxLow * _targetChannels + c];
                         float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
-                        mixedBuffer[s * _targetChannels + c] += valLow + t * (valHigh - valLow);
+                        mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * MasterVolume;
                     }
                 }
             }
@@ -319,7 +257,11 @@ namespace ADOFAIRenderer.Util
             {
                 var ev = ActiveEvents[i];
                 var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
-                if (clipData == null) continue;
+                if (clipData == null)
+                {
+                    ActiveEvents.RemoveAt(i);
+                    continue;
+                }
 
                 bool isSoundFinished = false;
                 int totalSamples = clipData.Samples.Length / _targetChannels;
@@ -365,7 +307,7 @@ namespace ADOFAIRenderer.Util
                     {
                         float valLow = clipData.Samples[idxLow * _targetChannels + c];
                         float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
-                        mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume;
+                        mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume * MasterVolume;
                     }
                 }