Fix Render Freeze on Encoder Thread Failure

.gitignore

@@ -36,6 +36,7 @@ desktop.ini
 /.agent/
 /GEMINI.md
 /.mcp.json
+/CLAUDE.md
 
 # Decompiled Game Code
 *.cs.txt
@@ -43,4 +44,7 @@ desktop.ini
 # Developer Description Files
 /AsyncInputAutoplayFix.md
 /AudioMixerExplanation.md
-/O5Theme.md
+/O5Theme.md
+
+# Developer Utility Files
+/REVIEW_CHECKLIST.md

Core/RenderManager.cs

@@ -91,7 +91,50 @@ internal class RenderManager : MonoBehaviour
     public long frameCount;
     public bool skipPressToStart;
     public string levelPath = "";
-    public bool IsRenderCancelled { get; private set; }
+    private readonly object _abortStateLock = new();
+    private volatile bool _isRenderCancelled;
+    private volatile bool _renderFailed;
+    private volatile string? _failureReason;
+
+    // ReSharper disable once MemberCanBePrivate.Global
+    public bool IsRenderCancelled => _isRenderCancelled;
+    public bool IsRenderFailed => _renderFailed;
+    
+    // Cancelled and failed are mutually exclusive; this is true for either (render must stop).
+    public bool IsRenderAborted => _isRenderCancelled || _renderFailed;
+
+    private void MarkCancelled()
+    {
+        lock (_abortStateLock)
+        {
+            if (!_renderFailed) _isRenderCancelled = true;
+        }
+    }
+
+    // The first failure reason wins; ignored once the render is already cancelled or failed.
+    private void MarkFailed(string reason)
+    {
+        lock (_abortStateLock)
+        {
+            if (_isRenderCancelled || _renderFailed) return;
+            _failureReason = reason;
+            _renderFailed = true;
+        }
+    }
+
+    private void ClearAbortState()
+    {
+        lock (_abortStateLock)
+        {
+            _isRenderCancelled = false;
+            _renderFailed = false;
+            _failureReason = null;
+        }
+    }
+
+    private string BuildAbortMessage() => IsRenderFailed
+        ? $"Rendering failed: {_failureReason ?? "unknown error"}"
+        : "Rendering was cancelled by user.";
     public RenderConfig Configuration = null!;
 
     // Render Sessions State (OOP Instance Fields)
@@ -151,6 +194,7 @@ internal class RenderManager : MonoBehaviour
 
     public long frameCountForVideo;
     public long lastVideoFrameIndex = -1;
+    // long cannot be volatile: access via Volatile.Read/Write (written on main thread, read on render thread)
     public long lastRequestedVideoFrameIndex = -1;
     public int readbackFailureCount;
     public int throttleCount;
@@ -170,8 +214,8 @@ internal class RenderManager : MonoBehaviour
     internal AutoResetEvent FrameAvailableEvent = new(false);
     public Thread? RenderThread;
     public long passCount;
-    public bool endRequested;
-    public bool endSuccessful;
+    public volatile bool endRequested;
+    public volatile bool endSuccessful;
     public long endFrame = -1;
 
     // Private Instance Fields
@@ -195,12 +239,15 @@ internal class RenderManager : MonoBehaviour
 
     public void ResetRenderState(bool success)
     {
-        IsRenderCancelled = !success;
+        if (success) _isRenderCancelled = false;
+        else MarkCancelled();
         stage = RenderStage.None;
     }
 
     public void CleanUpActiveRender()
     {
+        if (stage != RenderStage.None)
+            MarkFailed("the game exited or RenderManager was destroyed during rendering.");
         ResetRenderState(success: false);
             
         AudioSpectrumGenerator.Clear();
@@ -283,8 +330,15 @@ internal class RenderManager : MonoBehaviour
         }
 
         if (Instance.stage < RenderStage.OnScene) return;
+        if (stage >= RenderStage.Render && RenderThread is not { IsAlive: true })
+        {
+            MarkFailed("the render thread terminated unexpectedly.");
+            CancelRender();
+            return;
+        }
         if (!scrController.instance || scrController.instance.state == States.Fail || scrController.instance.state == States.Fail2)
         {
+            MarkFailed("the level failed or the game controller was lost during rendering.");
             ResetRenderState(success: false);
             StopFFmpeg(false);
             EndRender();
@@ -333,6 +387,12 @@ internal class RenderManager : MonoBehaviour
         
         while (Instance.stage >= RenderStage.Render)
         {
+            if (RenderThread is not { IsAlive: true })
+            {
+                isThrottling = false;
+                return;
+            }
+
             int count;
             lock (ImageQueue)
             {
@@ -362,7 +422,7 @@ internal class RenderManager : MonoBehaviour
         if (videoFrameIndex > lastVideoFrameIndex)
         {
             lastVideoFrameIndex = videoFrameIndex;
-            lastRequestedVideoFrameIndex = videoFrameIndex;
+            Volatile.Write(ref lastRequestedVideoFrameIndex, videoFrameIndex);
 
             var targetTexture = texture;
             if (HasActiveFrameHooks && _frameHookPipeline != null)
@@ -403,7 +463,6 @@ internal class RenderManager : MonoBehaviour
 
             Application.wantsToQuit -= quitDelegate;
             RenderConfigWindow.CreateWindow();
-            // ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "Assets/Scenes/RenderSettingsScene.unity");
         });
     }
 
@@ -511,10 +570,10 @@ internal class RenderManager : MonoBehaviour
             ImageQueue.Clear();
         }
         passCount = 0;
-        IsRenderCancelled = false;
+        ClearAbortState();
         frameCountForVideo = 0;
         lastVideoFrameIndex = -1;
-        lastRequestedVideoFrameIndex = -1;
+        Volatile.Write(ref lastRequestedVideoFrameIndex, -1);
         endRequested = false;
         endSuccessful = false;
         FrameAvailableEvent.Reset();
@@ -655,17 +714,17 @@ internal class RenderManager : MonoBehaviour
                         continue;
                     }
 
-                    if (IsRenderCancelled) break;
+                    if (IsRenderAborted) break;
 
                     if (endRequested)
                     {
-                        if (passCount > lastRequestedVideoFrameIndex) break;
+                        if (passCount > Volatile.Read(ref lastRequestedVideoFrameIndex)) break;
                     }
 
                     FrameAvailableEvent.WaitOne(50);
                 }
 
-                if (!IsRenderCancelled) videoFFmpeg.End();
+                if (!IsRenderAborted) videoFFmpeg.End();
                 videoFFmpeg.Dispose();
 
                 lock (_allVideoFFmpegInstances)
@@ -674,12 +733,22 @@ internal class RenderManager : MonoBehaviour
                 }
 
                 _activeVideoFFmpeg = null;
-                endSuccessful = !IsRenderCancelled;
+                endSuccessful = !IsRenderAborted;
             }
             catch (Exception ex)
             {
                 Melon<RenderMod>.Logger.Error($"FFmpeg rendering thread error: {ex}");
-                videoFFmpeg.Dispose();
+
+                MarkFailed("encoder thread error. See the log for details.");
+                try
+                {
+                    videoFFmpeg.Dispose();
+                }
+                catch (Exception disposeEx)
+                {
+                    Melon<RenderMod>.Logger.Error($"Encoder dispose after thread error also failed: {disposeEx.Message}");
+                }
+
                 lock (_allVideoFFmpegInstances)
                 {
                     _allVideoFFmpegInstances.Remove(videoFFmpeg);
@@ -708,7 +777,7 @@ internal class RenderManager : MonoBehaviour
         endRequested = true;
         if (!success)
         {
-            IsRenderCancelled = true;
+            MarkCancelled();
         }
         FrameAvailableEvent.Set();
         skipPressToStart = false;
@@ -722,7 +791,8 @@ internal class RenderManager : MonoBehaviour
         if (success) StartCoroutine(StopRenderCoroutine());
         else
         {
-            Melon<RenderMod>.Logger.Error("Rendering failed or was cancelled. Safely cleaning up state and resources.");
+            var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED";
+            Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources.");
 
             // 1. Wait for render background thread to finish processing remaining queued frames and terminate
             if (RenderThread is { IsAlive: true })
@@ -796,15 +866,20 @@ internal class RenderManager : MonoBehaviour
                 CurrentOutPath = null;
             }
 
-            var result = BuildResult(false,
-                IsRenderCancelled ? "Rendering was cancelled by user." : "Rendering failed.");
+            var result = BuildResult(false, BuildAbortMessage());
             TriggerOnComplete(result);
         }
     }
 
     private IEnumerator StopRenderCoroutine()
     {
-        yield return new WaitUntil(() => endSuccessful);
+        yield return new WaitUntil(() => endSuccessful || RenderThread is not { IsAlive: true });
+        if (!endSuccessful)
+        {
+            MarkFailed("the render thread terminated unexpectedly.");
+            CancelRender();
+            yield break;
+        }
 
         var result = BuildResult(success: true);
         TriggerOnComplete(result);
@@ -848,7 +923,8 @@ internal class RenderManager : MonoBehaviour
         ResetFrameHookState();
         AudioMixer?.Reset(); AudioMixer = null;
         SyncCalculator.Reset();
-
+        
+        AudioSpectrumGenerator.Clear();
         RenderContext.Reset();
             
         ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");

Util/Audio/AudioSpectrumGenerator.cs

@@ -108,7 +108,7 @@ internal static class AudioSpectrumGenerator
             return;
         }
 
-        var currentFrame = (RenderManager.Instance != null) ? RenderManager.Instance.frameCount : -1;
+        var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1;
 
         lock (CachedSpectrums)
         {

Util/FrameReader.cs

@@ -140,7 +140,7 @@ namespace ADOFAIRenderer.Util
                 }
             }
 
-            if (RenderManager.Instance && RenderManager.Instance.IsRenderCancelled)
+            if (RenderManager.Instance && RenderManager.Instance.IsRenderAborted)
             {
                 RecycleState(state);
                 return;