Add First Prototype of Audio Render Feature

Core/RenderManager.cs

@@ -384,6 +384,11 @@ namespace ADOFAIRenderer.Core
 
         private IEnumerator SetupPlay()
         {
+            if (Configuration.RenderAudio && GetOutputExtension() != ".mp4")
+            {
+                Configuration.AudioSampleRate = 48000;
+            }
+
             for (; countdown > 0; countdown--)
             {
                 yield return new WaitForSecondsRealtime(1);
@@ -426,6 +431,10 @@ namespace ADOFAIRenderer.Core
             {
                 AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
             }
+            if (Configuration.RenderAudio)
+            {
+                OfflineAudioMixer.InitializeBGM(scrConductor.instance.song.clip, Configuration.AudioChannels, Configuration.AudioSampleRate);
+            }
             
             scrConductor.instance.song.clip = null;
             scrConductor.instance.song.time = -1000;
@@ -459,13 +468,7 @@ namespace ADOFAIRenderer.Core
             var videoFPS = Configuration.VideoFPS;
             
             var selectedCodec = Configuration.Codec;
-
-            var ext = selectedCodec switch
-            {
-                Codec.AV1 or Codec.VP9 => ".webm",
-                Codec.VVC => ".mkv",
-                _ => ".mp4"
-            };
+            var ext = GetOutputExtension();
 
             var startTime = DateTime.Now;
             var fileName = startTime.ToString("yyyy-MM-dd HH-mm-ss") + ext;
@@ -475,6 +478,7 @@ namespace ADOFAIRenderer.Core
             try
             {
                 var effectiveVideoFps = Math.Min(videoFPS, renderFPS);
+                var codecId = GetAudioCodecId(ext);
                 var settings = new FFmpegNativeEncoder.EncoderSettings
                 {
                     Width = Configuration.RenderWidth,
@@ -486,7 +490,15 @@ namespace ADOFAIRenderer.Core
                     TargetBitrate = Configuration.EncodingConfig.Bitrate,
                     MaxBitrate = Configuration.EncodingConfig.MaxBitrate,
                     GopSize = Configuration.EncodingConfig.KeyframeInterval,
-                    ForceSoftware = Configuration.EncodingConfig.ForceSoftwareEncoder
+                    ForceSoftware = Configuration.EncodingConfig.ForceSoftwareEncoder,
+                    Audio = Configuration.RenderAudio ? new FFmpegNativeEncoder.AudioSettings
+                    {
+                        SampleRate = Configuration.AudioSampleRate,
+                        Channels = Configuration.AudioChannels,
+                        InputSampleFormat = AVSampleFormat.AV_SAMPLE_FMT_FLT,
+                        AudioCodecId = codecId,
+                        Bitrate = Configuration.AudioBitrate
+                    } : null
                 };
 
                 videoFFmpeg = new FFmpegNativeEncoder(settings);
@@ -565,6 +577,29 @@ namespace ADOFAIRenderer.Core
                                     _frameReader?.RecycleState(frame.ReadbackState);
                                 }
                             }
+
+                            if (Configuration.RenderAudio)
+                            {
+                                int frameIdx = (int)(passCount - 1);
+                                double effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
+                                double frameStartTime = frameIdx / effectiveVideoFps;
+                                double frameEndTime = (frameIdx + 1) / effectiveVideoFps;
+                                
+                                int sampleRate = Configuration.AudioSampleRate;
+                                int channels = Configuration.AudioChannels;
+                                int samplesNeeded = (int)(Math.Floor(frameEndTime * sampleRate) - Math.Floor(frameStartTime * sampleRate));
+
+                                float[] audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
+                                try
+                                {
+                                    OfflineAudioMixer.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameStartTime, dspTimeBase);
+                                    videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
+                                }
+                                finally
+                                {
+                                    ArrayPool<float>.Shared.Return(audioBuffer);
+                                }
+                            }
                             continue;
                         }
 
@@ -762,9 +797,26 @@ namespace ADOFAIRenderer.Core
             }
 
             _frameReader?.Dispose();
+            OfflineAudioMixer.Reset();
             ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
             
             Cursor.visible = true;
         }
+
+        private string GetOutputExtension()
+        {
+            var selectedCodec = Configuration.Codec;
+            return selectedCodec switch
+            {
+                Codec.AV1 or Codec.VP9 => ".webm",
+                Codec.VVC => ".mkv",
+                _ => ".mp4"
+            };
+        }
+
+        private AVCodecID GetAudioCodecId(string ext)
+        {
+            return ext == ".mp4" ? AVCodecID.AV_CODEC_ID_AAC : AVCodecID.AV_CODEC_ID_OPUS;
+        }
     }
 }