Fix Render State Cleanup on Initialization Failure

Core/RenderManager.cs

@@ -145,7 +145,7 @@ internal class RenderManager : MonoBehaviour
     public int frameCountBase;
         
     public OfflineAudioMixer? AudioMixer { get; private set; }
-    private AudioSyncCalculator SyncCalculator { get; set; } = null!;
+    private AudioSyncCalculator? SyncCalculator { get; set; }
 
     private FrameReader? _frameReader;
     private FrameHookPipeline? _frameHookPipeline;
@@ -499,6 +499,9 @@ internal class RenderManager : MonoBehaviour
             yield return new WaitForSecondsRealtime(1);
         }
 
+        // Cancelled or failed during the countdown: do not start the encoder
+        if (stage != RenderStage.OnScene) yield break;
+
         frameCount = 0; framesSent = 0;
         _renderStopwatch.Restart();
             
@@ -521,13 +524,20 @@ internal class RenderManager : MonoBehaviour
         catch (Exception ex)
         {
             Melon<RenderMod>.Logger.Error($"Encoder initialization error: {ex}");
-            ResetFrameHookState();
+
+            // Release the encoder/thread and restore time, input offset, stage and hooks before showing the dialog
+            MarkFailed($"failed to initialize the video encoder: {ex.Message}");
+            ResetRenderState(success: false);
+            StopFFmpeg(false);
+            RestoreRenderEnvironment();
+
             CommonWarningDialogUI.ShowDialog(
                 "Encoder Error",
                 $"Failed to initialize video encoder.\nError: {ex.Message}",
                 new DialogButton("OK", () =>
                 {
                     ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
+                    Cursor.visible = true;
                 })
             );
             yield break;
@@ -691,14 +701,14 @@ internal class RenderManager : MonoBehaviour
                             }
                         }
 
-                        if (Configuration.AudioConfig.RenderAudio)
+                        if (Configuration.AudioConfig.RenderAudio && SyncCalculator is { } syncCalculator)
                         {
                             var frameIdx = (int)(passCount - 1);
                             var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
                             var frameTime = frameIdx / effectiveVideoFps;
                                 
                             var channels = Configuration.AudioConfig.AudioChannels;
-                            var samplesNeeded = SyncCalculator.GetNextFrameSamples();
+                            var samplesNeeded = syncCalculator.GetNextFrameSamples();
 
                             var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
                             try
@@ -888,6 +898,15 @@ internal class RenderManager : MonoBehaviour
     }
 
     private void EndRender()
+    {
+        RestoreRenderEnvironment();
+
+        ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
+        Cursor.visible = true;
+    }
+
+    // Undoes what the render setup changed (time, input offset, auto, queues, session hooks), without leaving the scene.
+    private void RestoreRenderEnvironment()
     {
         Time.timeScale = 1f;
         Time.captureDeltaTime = 0f;
@@ -922,13 +941,11 @@ internal class RenderManager : MonoBehaviour
         _frameReader?.Dispose();
         ResetFrameHookState();
         AudioMixer?.Reset(); AudioMixer = null;
-        SyncCalculator.Reset();
+        SyncCalculator = null;
         
         AudioSpectrumGenerator.Clear();
         RenderContext.Reset();
             
-        ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
-        Cursor.visible = true;
     }
 
     private RenderResult BuildResult(bool success, string? errorMessage = null)