Add First Prototype of Audio Render Feature

Core/RenderManager.cs

@@ -384,6 +384,11 @@ namespace ADOFAIRenderer.Core
 
         private IEnumerator SetupPlay()
         {
+            if (Configuration.RenderAudio && GetOutputExtension() != ".mp4")
+            {
+                Configuration.AudioSampleRate = 48000;
+            }
+
             for (; countdown > 0; countdown--)
             {
                 yield return new WaitForSecondsRealtime(1);
@@ -426,6 +431,10 @@ namespace ADOFAIRenderer.Core
             {
                 AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
             }
+            if (Configuration.RenderAudio)
+            {
+                OfflineAudioMixer.InitializeBGM(scrConductor.instance.song.clip, Configuration.AudioChannels, Configuration.AudioSampleRate);
+            }
             
             scrConductor.instance.song.clip = null;
             scrConductor.instance.song.time = -1000;
@@ -459,13 +468,7 @@ namespace ADOFAIRenderer.Core
             var videoFPS = Configuration.VideoFPS;
             
             var selectedCodec = Configuration.Codec;
-
+            var ext = GetOutputExtension();
-            var ext = selectedCodec switch
-            {
-                Codec.AV1 or Codec.VP9 => ".webm",
-                Codec.VVC => ".mkv",
-                _ => ".mp4"
-            };
 
             var startTime = DateTime.Now;
             var fileName = startTime.ToString("yyyy-MM-dd HH-mm-ss") + ext;
@@ -475,6 +478,7 @@ namespace ADOFAIRenderer.Core
             try
             {
                 var effectiveVideoFps = Math.Min(videoFPS, renderFPS);
+                var codecId = GetAudioCodecId(ext);
                 var settings = new FFmpegNativeEncoder.EncoderSettings
                 {
                     Width = Configuration.RenderWidth,
@@ -486,7 +490,15 @@ namespace ADOFAIRenderer.Core
                     TargetBitrate = Configuration.EncodingConfig.Bitrate,
                     MaxBitrate = Configuration.EncodingConfig.MaxBitrate,
                     GopSize = Configuration.EncodingConfig.KeyframeInterval,
-                    ForceSoftware = Configuration.EncodingConfig.ForceSoftwareEncoder
+                    ForceSoftware = Configuration.EncodingConfig.ForceSoftwareEncoder,
+                    Audio = Configuration.RenderAudio ? new FFmpegNativeEncoder.AudioSettings
+                    {
+                        SampleRate = Configuration.AudioSampleRate,
+                        Channels = Configuration.AudioChannels,
+                        InputSampleFormat = AVSampleFormat.AV_SAMPLE_FMT_FLT,
+                        AudioCodecId = codecId,
+                        Bitrate = Configuration.AudioBitrate
+                    } : null
                 };
 
                 videoFFmpeg = new FFmpegNativeEncoder(settings);
@@ -565,6 +577,29 @@ namespace ADOFAIRenderer.Core
                                     _frameReader?.RecycleState(frame.ReadbackState);
                                 }
                             }
+
+                            if (Configuration.RenderAudio)
+                            {
+                                int frameIdx = (int)(passCount - 1);
+                                double effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
+                                double frameStartTime = frameIdx / effectiveVideoFps;
+                                double frameEndTime = (frameIdx + 1) / effectiveVideoFps;
+                                
+                                int sampleRate = Configuration.AudioSampleRate;
+                                int channels = Configuration.AudioChannels;
+                                int samplesNeeded = (int)(Math.Floor(frameEndTime * sampleRate) - Math.Floor(frameStartTime * sampleRate));
+
+                                float[] audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
+                                try
+                                {
+                                    OfflineAudioMixer.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameStartTime, dspTimeBase);
+                                    videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
+                                }
+                                finally
+                                {
+                                    ArrayPool<float>.Shared.Return(audioBuffer);
+                                }
+                            }
                             continue;
                         }
 
@@ -762,9 +797,26 @@ namespace ADOFAIRenderer.Core
             }
 
             _frameReader?.Dispose();
+            OfflineAudioMixer.Reset();
             ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
             
             Cursor.visible = true;
         }
+
+        private string GetOutputExtension()
+        {
+            var selectedCodec = Configuration.Codec;
+            return selectedCodec switch
+            {
+                Codec.AV1 or Codec.VP9 => ".webm",
+                Codec.VVC => ".mkv",
+                _ => ".mp4"
+            };
+        }
+
+        private AVCodecID GetAudioCodecId(string ext)
+        {
+            return ext == ".mp4" ? AVCodecID.AV_CODEC_ID_AAC : AVCodecID.AV_CODEC_ID_OPUS;
+        }
     }
 }

Data/RenderConfig.cs

@@ -18,6 +18,10 @@ namespace ADOFAIRenderer.Data
 
         public int? RandomSeed { get; set; } = null;
         public bool ProcessAudio { get; set; }
+        public bool RenderAudio { get; set; } = true;
+        public int AudioSampleRate { get; set; } = 44100;
+        public int AudioChannels { get; set; } = 2;
+        public int AudioBitrate { get; set; } = 192000;
     }
 
     public class EncodingConfig

Patch/Engine/AudioPatches.cs

@@ -0,0 +1,50 @@
+using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Util;
+using HarmonyLib;
+using UnityEngine.Audio;
+
+namespace ADOFAIRenderer.Patch.Engine
+{
+    [HarmonyPatch(typeof(AudioManager), "_Play")]
+    public static class AudioManagerPlayPatch
+    {
+        [HarmonyPrefix]
+        public static bool Prefix(string snd, double time, AudioMixerGroup group, float volume, int priority)
+        {
+            if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
+            {
+                // Cache the clip on the main thread
+                AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
+
+                // Convert absolute DSP time to relative time in seconds from render start
+                double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
+
+                // Register event to the offline mixer (default pitch = 1.0f)
+                OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, -1.0, volume, 1.0f);
+            }
+            return true; // Let the original method execute to prevent NullReferenceException
+        }
+    }
+
+    [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")]
+    public static class PlayWithEndTimePatch
+    {
+        [HarmonyPrefix]
+        public static bool Prefix(string snd, double time, double endTime, float volume, int priority)
+        {
+            if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
+            {
+                // Cache the clip on the main thread
+                AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
+
+                // Convert absolute DSP times to relative times in seconds from render start
+                double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
+                double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
+
+                // Register looping sound event to the offline mixer (default pitch = 1.0f)
+                OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
+            }
+            return true; // Let the original method execute
+        }
+    }
+}

Util/OfflineAudioMixer.cs

@@ -0,0 +1,379 @@
+using System;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace ADOFAIRenderer.Util
+{
+    public static class AudioResampler
+    {
+        public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
+        {
+            // ── Safety Validation ──
+            if (inputSamples.Length == 0 ||
+                inputChannels <= 0 || targetChannels <= 0 ||
+                inputFrequency <= 0 || targetFrequency <= 0)
+            {
+                return Array.Empty<float>();
+            }
+
+            // If format is identical, return original samples
+            if (inputFrequency == targetFrequency && inputChannels == targetChannels)
+                return inputSamples;
+
+            int inputLength = inputSamples.Length / inputChannels;
+            int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
+            float[] outputSamples = new float[targetLength * targetChannels];
+
+            for (int i = 0; i < targetLength; i++)
+            {
+                double srcIndex = (double)i * inputFrequency / targetFrequency;
+                long idxLow = (long)Math.Floor(srcIndex);
+                long idxHigh = idxLow + 1;
+                float t = (float)(srcIndex - idxLow);
+
+                if (idxHigh >= inputLength) idxHigh = inputLength - 1;
+                if (idxLow >= inputLength) idxLow = inputLength - 1;
+
+                if (targetChannels == 1) // 1. Mono Output
+                {
+                    float valLow, valHigh;
+                    if (inputChannels == 1) // Mono -> Mono
+                    {
+                        valLow = inputSamples[idxLow];
+                        valHigh = inputSamples[idxHigh];
+                    }
+                    else // Multi -> Mono (Downmix: Average all channels)
+                    {
+                        float sumLow = 0f;
+                        float sumHigh = 0f;
+                        for (int c = 0; c < inputChannels; c++)
+                        {
+                            sumLow += inputSamples[idxLow * inputChannels + c];
+                            sumHigh += inputSamples[idxHigh * inputChannels + c];
+                        }
+                        valLow = sumLow / inputChannels;
+                        valHigh = sumHigh / inputChannels;
+                    }
+                    outputSamples[i] = valLow + t * (valHigh - valLow);
+                }
+                else if (targetChannels == 2) // 2. Stereo Output
+                {
+                    if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
+                    {
+                        float valLow = inputSamples[idxLow];
+                        float valHigh = inputSamples[idxHigh];
+                        float val = valLow + t * (valHigh - valLow);
+                        outputSamples[i * 2] = val;
+                        outputSamples[i * 2 + 1] = val;
+                    }
+                    else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
+                    {
+                        float valLowL = inputSamples[idxLow * inputChannels];
+                        float valHighL = inputSamples[idxHigh * inputChannels];
+                        outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
+
+                        float valLowR = inputSamples[idxLow * inputChannels + 1];
+                        float valHighR = inputSamples[idxHigh * inputChannels + 1];
+                        outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
+                    }
+                }
+                else // 3. Multi-channel Output (e.g., 5.1 Surround)
+                {
+                    for (int c = 0; c < targetChannels; c++)
+                    {
+                        float valLow, valHigh;
+                        if (c < inputChannels)
+                        {
+                            valLow = inputSamples[idxLow * inputChannels + c];
+                            valHigh = inputSamples[idxHigh * inputChannels + c];
+                        }
+                        else
+                        {
+                            valLow = inputSamples[idxLow * inputChannels];
+                            valHigh = inputSamples[idxHigh * inputChannels];
+                        }
+                        outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
+                    }
+                }
+            }
+
+            return outputSamples;
+        }
+    }
+
+    public static class AudioClipCache
+    {
+        private static readonly Dictionary<string, CachedClipData> Cache = new();
+
+        public class CachedClipData
+        {
+            public float[] Samples = null!; // Pre-resampled to target channels and sample rate
+            public double Duration;
+        }
+
+        public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
+        {
+            if (string.IsNullOrEmpty(clipName)) return null;
+
+            lock (Cache)
+            {
+                if (Cache.TryGetValue(clipName, out var cached))
+                    return cached;
+
+                if (!allowLoad) return null;
+
+                if (AudioManager.Instance == null) return null;
+
+                AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
+                if (clip == null) return null;
+
+                try
+                {
+                    int totalSamples = clip.samples;
+                    int channels = clip.channels;
+                    float[] rawSamples = new float[totalSamples * channels];
+                    clip.GetData(rawSamples, 0);
+
+                    // Auto-resample during cache generation
+                    float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
+
+                    var data = new CachedClipData
+                    {
+                        Samples = resampled,
+                        Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
+                    };
+
+                    Cache[clipName] = data;
+                    return data;
+                }
+                catch (Exception e)
+                {
+                    MelonLoader.MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}");
+                    return null;
+                }
+            }
+        }
+
+        public static void Clear()
+        {
+            lock (Cache)
+            {
+                Cache.Clear();
+            }
+        }
+    }
+
+    public static class OfflineAudioMixer
+    {
+        public struct SoundEvent
+        {
+            public string ClipName;
+            public double StartTime; // In seconds relative to dspTimeBase
+            public double EndTime;   // In seconds relative to dspTimeBase (-1 if one-shot)
+            public float Volume;
+            public float Pitch;
+        }
+
+        private static readonly List<SoundEvent> ActiveEvents = new();
+        private static readonly object Lock = new();
+
+        // Target Audio Settings
+        private static int _targetChannels = 2;
+        private static int _targetSampleRate = 44100;
+
+        // BGM Cached Samples (Pre-resampled)
+        private static float[]? _bgmSamples;
+        private static double _bgmDuration;
+
+        public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate)
+        {
+            lock (Lock)
+            {
+                _targetChannels = targetChannels;
+                _targetSampleRate = targetSampleRate;
+
+                if (!clip)
+                {
+                    _bgmSamples = null;
+                    _bgmDuration = 0;
+                    return;
+                }
+
+                try
+                {
+                    int originalSamples = clip.samples;
+                    int originalChannels = clip.channels;
+                    float[] rawSamples = new float[originalSamples * originalChannels];
+                    clip.GetData(rawSamples, 0);
+
+                    // Pre-resample the BGM to the target format
+                    _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate);
+                    _bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate);
+
+                    MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}");
+                }
+                catch (Exception e)
+                {
+                    MelonLoader.MelonLogger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}");
+                    _bgmSamples = null;
+                    _bgmDuration = 0;
+                }
+            }
+        }
+
+        public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
+        {
+            lock (Lock)
+            {
+                ActiveEvents.Add(new SoundEvent
+                {
+                    ClipName = clipName,
+                    StartTime = startTime,
+                    EndTime = endTime,
+                    Volume = volume,
+                    Pitch = pitch
+                });
+            }
+        }
+
+        public static void Reset()
+        {
+            lock (Lock)
+            {
+                ActiveEvents.Clear();
+                _bgmSamples = null;
+                _bgmDuration = 0;
+                AudioClipCache.Clear();
+            }
+        }
+
+        public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
+        {
+            // mixedBuffer size must be at least samplesNeeded * _targetChannels
+            Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
+
+            lock (Lock)
+            {
+                // 1. Mix BGM
+                MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase);
+
+                // 2. Mix Hitsounds/Effect Sounds
+                MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime);
+
+                // 3. Saturation Guard (Clamping)
+                for (int i = 0; i < samplesNeeded * _targetChannels; i++)
+                {
+                    if (mixedBuffer[i] > 1.0f) mixedBuffer[i] = 1.0f;
+                    else if (mixedBuffer[i] < -1.0f) mixedBuffer[i] = -1.0f;
+                }
+            }
+        }
+
+        private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
+        {
+            if (_bgmSamples == null || _bgmSamples.Length == 0) return;
+
+            var conductor = scrConductor.instance;
+            if (!conductor) return;
+
+            double pitch = conductor.song.pitch;
+            double dspTimeSong = conductor.dspTimeSong;
+            double separateCountdownOffset = conductor.separateCountdownTime
+                ? (conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks)
+                : 0.0;
+
+            // BGM Playback start time relative to dspTimeBase
+            double time1 = dspTimeSong + separateCountdownOffset;
+            double relativeTime1 = time1 - dspTimeBase;
+
+            int totalSamples = _bgmSamples.Length / _targetChannels;
+
+            for (int s = 0; s < samplesNeeded; s++)
+            {
+                double sampleTime = frameStartTime + (double)s / _targetSampleRate;
+                double clipTime = (sampleTime - relativeTime1) * pitch;
+
+                if (clipTime >= 0 && clipTime < _bgmDuration)
+                {
+                    double sampleIdxDouble = clipTime * _targetSampleRate;
+                    long idxLow = (long)Math.Floor(sampleIdxDouble);
+                    long idxHigh = idxLow + 1;
+                    float t = (float)(sampleIdxDouble - idxLow);
+
+                    if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
+                    if (idxLow >= totalSamples) idxLow = totalSamples - 1;
+
+                    for (int c = 0; c < _targetChannels; c++)
+                    {
+                        float valLow = _bgmSamples[idxLow * _targetChannels + c];
+                        float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
+                        mixedBuffer[s * _targetChannels + c] += valLow + t * (valHigh - valLow);
+                    }
+                }
+            }
+        }
+
+        private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
+        {
+            for (int i = ActiveEvents.Count - 1; i >= 0; i--)
+            {
+                var ev = ActiveEvents[i];
+                var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
+                if (clipData == null) continue;
+
+                bool isSoundFinished = false;
+                int totalSamples = clipData.Samples.Length / _targetChannels;
+
+                for (int s = 0; s < samplesNeeded; s++)
+                {
+                    double sampleTime = frameStartTime + (double)s / _targetSampleRate;
+                    double elapsed = sampleTime - ev.StartTime;
+
+                    if (elapsed < 0) continue; // Sound hasn't started yet
+
+                    double soundTime = elapsed * ev.Pitch;
+
+                    if (soundTime >= clipData.Duration)
+                    {
+                        if (ev.EndTime < 0) // One-shot
+                        {
+                            isSoundFinished = true;
+                            break;
+                        }
+                        else // Looping
+                        {
+                            if (sampleTime >= ev.EndTime)
+                            {
+                                isSoundFinished = true;
+                                break;
+                            }
+                            soundTime %= clipData.Duration;
+                        }
+                    }
+
+                    // Linear interpolation for pitch shifting in the mixer loop
+                    double sampleIdxDouble = soundTime * _targetSampleRate;
+                    long idxLow = (long)Math.Floor(sampleIdxDouble);
+                    long idxHigh = idxLow + 1;
+                    float t = (float)(sampleIdxDouble - idxLow);
+
+                    if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
+                    if (idxLow >= totalSamples) idxLow = totalSamples - 1;
+
+                    for (int c = 0; c < _targetChannels; c++)
+                    {
+                        float valLow = clipData.Samples[idxLow * _targetChannels + c];
+                        float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
+                        mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume;
+                    }
+                }
+
+                // Clean up finished one-shot sounds
+                if (isSoundFinished && ev.EndTime < 0)
+                {
+                    ActiveEvents.RemoveAt(i);
+                }
+            }
+        }
+    }
+}