Merge pull request 'Rewrite Audio' (#5) from feature/audio_rewrite into main

Reviewed-on: #5

Core/RenderManager.cs

@@ -8,6 +8,7 @@ using System.Diagnostics;
 using ADOFAIRenderer.API;
 using ADOFAIRenderer.Core.Data;
 using ADOFAIRenderer.Data;
+using ADOFAIRenderer.Patch.Engine;
 using ADOFAIRenderer.UI;
 using ADOFAIRenderer.UI.Window;
 using ADOFAIRenderer.Util;
@@ -165,6 +166,9 @@ internal class RenderManager : MonoBehaviour
     public int frameCountBase;
         
     public OfflineAudioMixer? AudioMixer { get; private set; }
+
+    /// <summary>Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands.</summary>
+    public double AudioTime => frameCount / Configuration.RenderFPS;
     private AudioSyncCalculator? SyncCalculator { get; set; }
 
     private FrameReader? _frameReader;
@@ -276,8 +280,10 @@ internal class RenderManager : MonoBehaviour
         if (stage != RenderStage.None)
             MarkFailed("the game exited or RenderManager was destroyed during rendering.");
         ResetRenderState(success: false);
-            
+
         AudioSpectrumGenerator.Clear();
+        // Cancels the render thread's audio waits, then releases the clip store (decoder thread, readers, chunks).
+        AudioMixer?.Dispose(); AudioMixer = null;
 
         ResetFrameHookState();
 
@@ -476,7 +482,17 @@ internal class RenderManager : MonoBehaviour
     // ReSharper disable once Unity.IncorrectMethodSignature
     public void OnPostFrame(RenderTexture? texture)
     {
-        if (Instance.stage < RenderStage.Render || !texture) return;
+        if (Instance.stage < RenderStage.Render) return;
+
+        // Audio commands of this frame are queued: update the song voice and let the render thread mix up to the next frame.
+        if (AudioMixer != null)
+        {
+            AudioMixer.UpdateBGM(AudioTime, dspTimeBase);
+            AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS);
+            EffectSource.DeactivateMappedSources(AudioMixer);
+        }
+
+        if (!texture) return;
 
         // [Backpressure Throttling]
         // The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread
@@ -651,19 +667,19 @@ internal class RenderManager : MonoBehaviour
         scrController.instance.levelWasSkipped = true;
             
         if (!scrConductor.instance || !scrConductor.instance.song) yield break;
-        if (Configuration.AdvancedConfig.ProcessAudio)
-        {
-            AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
-        }
-            
         if (Configuration.AudioConfig.RenderAudio)
         {
             AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate);
             AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
-                
+
             var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
             SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps);
         }
+
+        if (Configuration.AdvancedConfig.ProcessAudio)
+        {
+            AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer);
+        }
             
         scrConductor.instance.song.clip = null;
         scrConductor.instance.song.time = -1000;
@@ -786,23 +802,35 @@ internal class RenderManager : MonoBehaviour
 
             endSuccessful = !IsRenderAborted;
         }
+        catch (OperationCanceledException) when (IsRenderAborted)
+        {
+            // Audio waits end with OperationCanceledException when the render is cancelled or has failed (OfflineAudioMixer.Cancel).
+            Melon<RenderMod>.Logger.Msg("Render thread stopped: the render was aborted.");
+            DisposeEncoderAfterStop(videoFFmpeg);
+        }
         catch (Exception ex)
         {
             Melon<RenderMod>.Logger.Error($"FFmpeg rendering thread error: {ex}");
 
             MarkFailed("encoder thread error. See the log for details.");
-            try
-            {
-                videoFFmpeg.Dispose();
-            }
-            catch (Exception disposeEx)
-            {
-                Melon<RenderMod>.Logger.Error($"Encoder dispose after thread error also failed: {disposeEx.Message}");
-            }
-            UnregisterEncoder(videoFFmpeg);
+            DisposeEncoderAfterStop(videoFFmpeg);
         }
     }
 
+    /// <summary>Releases the encoder after the render thread stopped early (abort or error) and removes it from the encoder list.</summary>
+    private void DisposeEncoderAfterStop(FFmpegNativeEncoder videoFFmpeg)
+    {
+        try
+        {
+            videoFFmpeg.Dispose();
+        }
+        catch (Exception disposeEx)
+        {
+            Melon<RenderMod>.Logger.Error($"Encoder dispose after the render thread stopped also failed: {disposeEx.Message}");
+        }
+        UnregisterEncoder(videoFFmpeg);
+    }
+
     /// <summary>
     /// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio
     /// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready.
@@ -892,17 +920,13 @@ internal class RenderManager : MonoBehaviour
     {
         if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return;
 
-        var frameIdx = (int)(passCount - 1);
-        var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
-        var frameTime = frameIdx / effectiveVideoFps;
-
         var channels = Configuration.AudioConfig.AudioChannels;
         var samplesNeeded = syncCalculator.GetNextFrameSamples();
 
         var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
         try
         {
-            AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
+            AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded);
             videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
         }
         finally
@@ -945,6 +969,9 @@ internal class RenderManager : MonoBehaviour
         {
             MarkCancelled();
         }
+        // The abort state is set first: the render thread reads it when its audio wait is cancelled.
+        if (success) AudioMixer?.Complete();
+        else AudioMixer?.Cancel();
         FrameAvailableEvent.Set();
         skipPressToStart = false;
         _renderSetup = false;
@@ -1087,10 +1114,10 @@ internal class RenderManager : MonoBehaviour
 
         _frameReader?.Dispose();
         ResetFrameHookState();
-        AudioMixer?.Reset(); AudioMixer = null;
-        SyncCalculator = null;
-        
         AudioSpectrumGenerator.Clear();
+        AudioMixer?.Dispose(); AudioMixer = null;
+        SyncCalculator = null;
+
         RenderContext.Reset();
             
     }