using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Threading; using System.Buffers; using System.Diagnostics; using ADOFAIRenderer.API; using ADOFAIRenderer.Core.Data; using ADOFAIRenderer.Data; using ADOFAIRenderer.Patch.Engine; using ADOFAIRenderer.UI; using ADOFAIRenderer.UI.Window; using ADOFAIRenderer.Util; using ADOFAIRenderer.Util.Audio; using ADOFAIRenderer.Util.FFmpeg; using ADOFAIRenderer.Util.OS; using FFmpeg.AutoGen; using HarmonyLib; using MelonLoader; using UnityEngine; using UnityEngine.Rendering; using Random = UnityEngine.Random; using ThreadPriority = System.Threading.ThreadPriority; namespace ADOFAIRenderer.Core; internal enum RenderStage { None, Load, OnScene, Render } internal static class RenderContext { public static bool Auto = true; public static bool TimePatch = true; public static bool ScreenPatch = true; public static string? OutputPath; public static bool Notification = true; public static readonly List> SessionFrameHooks = new(); private static readonly Func EndConditionDefault = () => scrController.instance.state == States.Won; public static Func EndCondition = EndConditionDefault; private static readonly Action OnCompleteDefault = result => { if (result.Success && Notification) { var fileString = !string.IsNullOrEmpty(RenderManager.Instance.CurrentOutPath) ? $"'{Path.GetFileName(RenderManager.Instance.CurrentOutPath)}'" : "Video"; var renderFolder = Path.GetDirectoryName(RenderManager.Instance.CurrentOutPath); OSNotification.Show("Rendering Complete!", $"{fileString} saved successfully.", renderFolder); } }; public static Action OnComplete = OnCompleteDefault; private static readonly Action OnLastUpdateDefault = () => { }; public static Action OnLastUpdate = OnLastUpdateDefault; public static void Reset() { Auto = true; TimePatch = true; ScreenPatch = true; OutputPath = null; Notification = true; SessionFrameHooks.Clear(); EndCondition = EndConditionDefault; OnComplete = OnCompleteDefault; OnLastUpdate = OnLastUpdateDefault; } } internal class RenderManager : MonoBehaviour { // Singleton Accessors public static RenderManager Instance { get; private set; } = null!; public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene; public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render; /// /// Maximum number of frames in ImageQueue. The main thread waits (throttles) while the queue holds more frames. /// public const int MaxQueuedFrames = 64; /// /// Upper bound of the readback buffer bytes of the frames in ImageQueue (4 GiB). /// The frame limit becomes min(MaxQueuedFrames, MaxQueuedBytes / bytes per frame) (see ComputeQueueFrameLimit), /// so it only lowers the frame count at resolutions where MaxQueuedFrames frames exceed this size. /// public const long MaxQueuedBytes = 4L * 1024 * 1024 * 1024; /// /// Wait time of the throttle loop (OnPostFrame) for the render thread to take a frame from ImageQueue. /// When it elapses without a dequeue, AsyncGPUReadback.WaitAllRequests() is called once so the readbacks /// still in flight complete (the render thread may be waiting for one of them to fill a gap in ImageQueue). /// private const int ThrottleWaitMilliseconds = 10; /// Frame limit of ImageQueue applied by the last throttle check (MaxQueuedFrames or lower). public int QueueFrameLimit { get; private set; } = MaxQueuedFrames; // Statistics public long framesSent; private readonly Stopwatch _renderStopwatch = new(); // Core Render Configuration and Path public int countdown; public long frameCount; public bool skipPressToStart; public string levelPath = ""; private readonly object _abortStateLock = new(); private volatile bool _isRenderCancelled; private volatile bool _renderFailed; private volatile string? _failureReason; // ReSharper disable once MemberCanBePrivate.Global public bool IsRenderCancelled => _isRenderCancelled; public bool IsRenderFailed => _renderFailed; // Cancelled and failed are mutually exclusive; this is true for either (render must stop). public bool IsRenderAborted => _isRenderCancelled || _renderFailed; private void MarkCancelled() { lock (_abortStateLock) { if (!_renderFailed) _isRenderCancelled = true; } } // The first failure reason wins; ignored once the render is already cancelled or failed. private void MarkFailed(string reason) { lock (_abortStateLock) { if (_isRenderCancelled || _renderFailed) return; _failureReason = reason; _renderFailed = true; } } private void ClearAbortState() { lock (_abortStateLock) { _isRenderCancelled = false; _renderFailed = false; _failureReason = null; } } private string BuildAbortMessage() => IsRenderFailed ? $"Rendering failed: {_failureReason ?? "unknown error"}" : "Rendering was cancelled by user."; public RenderConfig Configuration = null!; // Render Sessions State (OOP Instance Fields) public RenderStage stage = RenderStage.None; public double timeBase; public double unscaledTimeBase; public double dspTimeBase; public int frameCountBase; public OfflineAudioMixer? AudioMixer { get; private set; } /// Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands. public double AudioTime => frameCount / Configuration.RenderFPS; private AudioSyncCalculator? SyncCalculator { get; set; } private FrameReader? _frameReader; private FrameHookPipeline? _frameHookPipeline; private readonly List> _activeHooksCache = new(); private int _cachedGlobalHooksVersion = -1; private bool _hasActiveHooksCached; public bool HasActiveFrameHooks { get { EnsureActiveHooksUpToDate(); return _hasActiveHooksCached; } } private void EnsureActiveHooksUpToDate() { var globalVersion = RenderSession.GlobalHooksVersion; if (_cachedGlobalHooksVersion == globalVersion) return; _cachedGlobalHooksVersion = globalVersion; _activeHooksCache.Clear(); if (RenderContext.SessionFrameHooks.Count > 0) { _activeHooksCache.AddRange(RenderContext.SessionFrameHooks); } RenderSession.CopyGlobalFrameHooksTo(_activeHooksCache); _hasActiveHooksCached = _activeHooksCache.Count > 0; } private List> GetActiveFrameHooks() { EnsureActiveHooksUpToDate(); return _activeHooksCache; } private void ResetFrameHookState() { _frameHookPipeline?.Dispose(); _frameHookPipeline = null; _activeHooksCache.Clear(); _cachedGlobalHooksVersion = -1; _hasActiveHooksCached = false; } public long frameCountForVideo; public long lastVideoFrameIndex = -1; // long cannot be volatile: access via Volatile.Read/Write (written on main thread, read on render thread) public long lastRequestedVideoFrameIndex = -1; public int readbackFailureCount; public int throttleCount; public bool isThrottling; internal struct CapturedFrame { public IntPtr NativePointer; public int Length; public byte[]? ManagedBuffer; // Non-null if it's a managed buffer (e.g. black frame) public ReadbackRequestState? ReadbackState; // Associated state to recycle public bool IsColorConverted; } // Thread and Queue State internal SortedList ImageQueue = new(); internal AutoResetEvent FrameAvailableEvent = new(false); // Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it. internal AutoResetEvent FrameDequeuedEvent = new(false); public Thread? RenderThread; public long passCount; public volatile bool endRequested; public volatile bool endSuccessful; public long endFrame = -1; // Private Instance Fields private FFmpegNativeEncoder? _activeVideoFFmpeg; private readonly List _allVideoFFmpegInstances = new(); private int _originalInputOffset; private GameObject? _debugMonitorObj; private GameObject? _escapeHandlerObj; internal string? CurrentOutPath; // Output file of the current render that is not finished yet. // Set just before the encoder opens the file, cleared by the render thread after the encoder finished the file (End). // Only this file is deleted when a render is cancelled, fails or the game exits (DeleteIncompleteOutputFile). private string? _incompleteOutputPath; private bool _renderSetup; private void Awake() { Instance = this; } private void OnDestroy() { CleanUpActiveRender(); } public void ResetRenderState(bool success) { if (success) _isRenderCancelled = false; else MarkCancelled(); stage = RenderStage.None; } public void CleanUpActiveRender() { if (stage != RenderStage.None) MarkFailed("the game exited or RenderManager was destroyed during rendering."); ResetRenderState(success: false); AudioSpectrumGenerator.Clear(); // Cancels the render thread's audio waits, then releases the clip store (decoder thread, readers, chunks). AudioMixer?.Dispose(); AudioMixer = null; ResetFrameHookState(); if (_activeVideoFFmpeg != null) { Melon.Logger.Msg("Game exit or RenderManager destruction detected: Disposing FFmpeg native encoder resources."); _activeVideoFFmpeg.Dispose(); _activeVideoFFmpeg = null; } lock (_allVideoFFmpegInstances) { if (_allVideoFFmpegInstances.Count > 0) { Melon.Logger.Msg($"Game exit detected: Cleaning up and disposing all created FFmpeg encoder instances ({_allVideoFFmpegInstances.Count})."); foreach (var instance in _allVideoFFmpegInstances) { try { instance.Dispose(); } catch { // ignored } } _allVideoFFmpegInstances.Clear(); } } if (RenderThread is { IsAlive: true }) { RenderThread.Join(200); } // The encoders are disposed above, which closes the output file. An unfinished file of this render is deleted. DeleteIncompleteOutputFile(); } /// /// Deletes the output file of the current render when the render did not finish it (_incompleteOutputPath). /// Files of earlier, finished renders are never deleted. /// Returns true when a file was deleted. /// private bool DeleteIncompleteOutputFile() { var path = Interlocked.Exchange(ref _incompleteOutputPath, null); if (string.IsNullOrEmpty(path) || !File.Exists(path)) return false; try { File.Delete(path); Melon.Logger.Msg($"[File Cleanup] Deleted incomplete video: {path}"); return true; } catch (Exception ex) { Melon.Logger.Error($"[File Cleanup] Failed to delete incomplete video: {ex.Message}"); return false; } } private void Update() { if (_renderSetup && ADOBase.lm) { ADOBase.lm.hideDifficultyUI = true; } if (_renderSetup && scnGame.instance && !scnGame.instance.isLoading) { _renderSetup = false; scrCamera.instance.forceRTCam = true; if (!scrCamera.instance.gameObject.GetComponent()) scrCamera.instance.gameObject.AddComponent(); _originalInputOffset = scrConductor.currentPreset.inputOffset; scrConductor.currentPreset.inputOffset = 0; RDC.auto = RenderContext.Auto; AccessTools.Field(typeof(scrController), "disableCongratsMessage") .SetValue(scrController.instance, true); if (scrController.instance) { scrController.instance.noFail = true; scrController.instance.noFailInfiniteMargin = true; } stage = RenderStage.OnScene; countdown = 3; endFrame = -1; if (!_escapeHandlerObj) { _escapeHandlerObj = new GameObject("EscapeHandlerCanvas"); _escapeHandlerObj.AddComponent(); DontDestroyOnLoad(_escapeHandlerObj); } Time.captureDeltaTime = (float)(1 / Configuration.RenderFPS); scrController.instance!.StartCoroutine(SetupPlay()); } if (Instance.stage < RenderStage.OnScene) return; if (stage >= RenderStage.Render && RenderThread is not { IsAlive: true }) { MarkFailed("the render thread terminated unexpectedly."); CancelRender(); return; } if (!scrController.instance || scrController.instance.state == States.Fail || scrController.instance.state == States.Fail2) { MarkFailed("the level failed or the game controller was lost during rendering."); ResetRenderState(success: false); StopFFmpeg(false); EndRender(); } else if (RenderContext.EndCondition()) { if (endFrame == -1) { endFrame = frameCount; } var extraFrames = (long)Math.Ceiling(Configuration.DelayAfterEnd * Configuration.RenderFPS); if (frameCount - endFrame < extraFrames) return; ResetRenderState(success: true); StopFFmpeg(true); } } public void OnPreUpdate() { if (Instance.stage < RenderStage.OnScene) return; frameCount++; } // Floors that own a hold renderer, collected from scrLevelMaker.listFloors (see RefreshHoldFloors). private readonly List _holdFloors = new(); private List? _holdFloorsSource; private int _holdFloorsSourceCount = -1; public void OnLastUpdate() { if (Instance.stage < RenderStage.Load) return; if (Configuration.AdvancedConfig.NonTile) { // scrHoldRenderer.UpdateColor sets m_meshRenderer.enabled = true every time it runs // (scrHoldRenderer.cs, called from scrPlanet/scrPlayer), so the hold meshes are disabled again every frame. RefreshHoldFloors(); foreach (var floor in _holdFloors) { if (!floor || !floor.holdRenderer) continue; var meshRenderer = floor.holdRenderer.m_meshRenderer; if (meshRenderer && meshRenderer.enabled) meshRenderer.enabled = false; } } TriggerOnLastUpdate(); } /// /// Rebuilds the hold floor list when scrLevelMaker.listFloors is a different list or has a different count /// (hold renderers are created together with the floors in scrLevelMaker). /// private void RefreshHoldFloors() { var floors = scrLevelMaker.instance.listFloors; if (ReferenceEquals(floors, _holdFloorsSource) && floors.Count == _holdFloorsSourceCount) return; _holdFloors.Clear(); foreach (var floor in floors) { if (floor && floor.holdRenderer) _holdFloors.Add(floor); } _holdFloorsSource = floors; _holdFloorsSourceCount = floors.Count; } /// /// Frame limit of ImageQueue for the given capture texture: MaxQueuedBytes divided by the readback bytes of /// one frame (YUV 4:2:0 when the GPU conversion is active, RGBA otherwise), within 4 to MaxQueuedFrames. /// private int ComputeQueueFrameLimit(RenderTexture texture) { var frameBytes = _frameReader?.GetFrameBufferBytes(texture.width, texture.height) ?? (long)texture.width * texture.height * 4; if (frameBytes <= 0) return MaxQueuedFrames; return (int)Math.Min(MaxQueuedFrames, Math.Max(4, MaxQueuedBytes / frameBytes)); } // ReSharper disable once Unity.IncorrectMethodSignature public void OnPostFrame(RenderTexture? texture) { if (Instance.stage < RenderStage.Render) return; // Audio commands of this frame are queued: update the song voice and let the render thread mix up to the next frame. if (AudioMixer != null) { AudioMixer.UpdateBGM(AudioTime, dspTimeBase); AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS); EffectSource.DeactivateMappedSources(AudioMixer); } if (!texture) return; // [Backpressure Throttling] // The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread // removes frames, and it signals FrameDequeuedEvent for each one. The wait has a timeout so that the loop // re-checks the render thread state and pumps pending readback completions (see ThrottleWaitMilliseconds). isThrottling = false; var frameLimit = QueueFrameLimit = ComputeQueueFrameLimit(texture!); while (Instance.stage >= RenderStage.Render) { if (RenderThread is not { IsAlive: true }) { isThrottling = false; return; } int count; lock (ImageQueue) { count = ImageQueue.Count; } if (count <= frameLimit) break; if (!isThrottling) { isThrottling = true; throttleCount++; } if (!FrameDequeuedEvent.WaitOne(ThrottleWaitMilliseconds)) { AsyncGPUReadback.WaitAllRequests(); } } isThrottling = false; var currentFrame = frameCountForVideo++; var renderFPS = Configuration.RenderFPS; var videoFps = Configuration.VideoFPS; var effectiveVideoFps = Math.Min(videoFps, renderFPS); var videoFrameIndex = (long)(currentFrame * effectiveVideoFps / renderFPS); if (videoFrameIndex > lastVideoFrameIndex) { lastVideoFrameIndex = videoFrameIndex; Volatile.Write(ref lastRequestedVideoFrameIndex, videoFrameIndex); var targetTexture = texture; if (HasActiveFrameHooks && _frameHookPipeline != null) { var videoTime = effectiveVideoFps > 0 ? videoFrameIndex / effectiveVideoFps : 0.0; targetTexture = _frameHookPipeline.Process( texture, videoFrameIndex, currentFrame, videoTime, renderFPS, effectiveVideoFps, GetActiveFrameHooks() ) ?? texture; } _frameReader?.RequestFrameCapture(targetTexture, videoFrameIndex); } } public static void StartRender() { var targetMethod = typeof(scnEditor).GetMethod("TryApplicationQuit", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public); if (targetMethod == null || !scnEditor.instance) return; if (ADOBase.levelPath.IsNullOrEmpty()) { scnEditor.instance.TryQuitToMenu(); return; } if (Instance) Instance.levelPath = ADOBase.levelPath; scnEditor.instance.CheckUnsavedChanges(() => { var quitDelegate = (Func)Delegate.CreateDelegate(typeof(Func), scnEditor.instance, targetMethod); Application.wantsToQuit -= quitDelegate; RenderConfigWindow.CreateWindow(); }); } public void SetupCaptureDirectory() { var renderDir = GlobalSettings.EffectiveOutputDirectory; FileUtil.CreateDirectory(renderDir); } public void SetupGameScene() { GCS.customLevelPaths = new[] { levelPath }; GCS.checkpointNum = 0; GCS.customLevelId = ""; GCS.nextSpeedRun = 1; GCS.customLevelIndex = 0; if (Configuration.AdvancedConfig.RandomSeed.HasValue) Random.InitState(Configuration.AdvancedConfig.RandomSeed.Value); ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnGame"); skipPressToStart = true; _renderSetup = true; } private IEnumerator SetupPlay() { if (Configuration.AudioConfig.RenderAudio) { if (!CodecUtil.AvailableSampleRate[CodecUtil.AudioCodecMatch[Configuration.VideoConfig.Codec]].Contains(Configuration.AudioConfig.AudioSampleRate)) Configuration.AudioConfig.AudioSampleRate = 48000; // most compatible value } for (; countdown > 0; countdown--) { yield return new WaitForSecondsRealtime(1); } // Cancelled or failed during the countdown: do not start the encoder if (stage != RenderStage.OnScene) yield break; frameCount = 0; framesSent = 0; _renderStopwatch.Restart(); timeBase = Time.timeAsDouble; unscaledTimeBase = Time.unscaledTimeAsDouble; dspTimeBase = AudioSettings.dspTime; frameCountBase = Time.frameCount; _frameReader?.Dispose(); _frameReader = null; _frameHookPipeline?.Dispose(); _frameHookPipeline = new FrameHookPipeline(); _cachedGlobalHooksVersion = -1; EnsureActiveHooksUpToDate(); try { StartFFmpegThread(); _frameReader = _activeVideoFFmpeg != null ? new FrameReader(_activeVideoFFmpeg.SelectedPixelFormat) : new FrameReader(AVPixelFormat.AV_PIX_FMT_YUV420P); } catch (Exception ex) { Melon.Logger.Error($"Encoder initialization error: {ex}"); // Release the encoder/thread and restore time, input offset, stage and hooks before showing the dialog MarkFailed($"failed to initialize the video encoder: {ex.Message}"); ResetRenderState(success: false); StopFFmpeg(false); RestoreRenderEnvironment(); CommonWarningDialogUI.ShowDialog( "Encoder Error", $"Failed to initialize video encoder.\nError: {ex.Message}", new DialogButton("OK", () => { ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor"); Cursor.visible = true; }) ); yield break; } Instance.stage = RenderStage.Render; scrController.instance.paused = scrController.instance.audioPaused = false; scrController.instance.levelWasSkipped = true; if (!scrConductor.instance || !scrConductor.instance.song) yield break; if (Configuration.AudioConfig.RenderAudio) { AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate); AudioMixer.InitializeBGM(scrConductor.instance.song.clip); var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps); } if (Configuration.AdvancedConfig.ProcessAudio) { AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer); } scrConductor.instance.song.clip = null; scrConductor.instance.song.time = -1000; } private void StartFFmpegThread() { if (!_debugMonitorObj) { _debugMonitorObj = new GameObject("DebugMonitorCanvas"); _debugMonitorObj.AddComponent(); DontDestroyOnLoad(_debugMonitorObj); } lock (ImageQueue) { ImageQueue.Clear(); } passCount = 0; ClearAbortState(); frameCountForVideo = 0; lastVideoFrameIndex = -1; Volatile.Write(ref lastRequestedVideoFrameIndex, -1); endRequested = false; endSuccessful = false; FrameAvailableEvent.Reset(); FrameDequeuedEvent.Reset(); var renderFPS = Configuration.RenderFPS; var videoFPS = Configuration.VideoFPS; var startTime = DateTime.Now; if (RenderContext.OutputPath != null) CurrentOutPath = RenderContext.OutputPath; else { var fileName = startTime.ToString("yyyy-MM-dd HH-mm-ss") + $".{Configuration.AdvancedConfig.FileExtension.TrimStart('.').ToLower()}"; CurrentOutPath = Path.Combine(GlobalSettings.EffectiveOutputDirectory, fileName); } FFmpegNativeEncoder? videoFFmpeg = null; try { var effectiveVideoFps = Math.Min(videoFPS, renderFPS); var codecId = CodecUtil.AudioCodecMatch[Configuration.VideoConfig.Codec]; var settings = new FFmpegNativeEncoder.EncoderSettings { Width = Configuration.RenderWidth, Height = Configuration.RenderHeight, FPS = effectiveVideoFps, Codec = Configuration.VideoConfig.Codec, Mode = Configuration.VideoConfig.RateControlMode, QualityValue = Configuration.VideoConfig.QP, TargetBitrate = Configuration.VideoConfig.Bitrate, MaxBitrate = Configuration.VideoConfig.MaxBitrate, GopSize = Configuration.VideoConfig.KeyframeInterval, ForceSoftware = Configuration.VideoConfig.ForceSoftwareEncoder, Audio = Configuration.AudioConfig.RenderAudio ? new FFmpegNativeEncoder.AudioSettings { SampleRate = Configuration.AudioConfig.AudioSampleRate, Channels = Configuration.AudioConfig.AudioChannels, InputSampleFormat = AVSampleFormat.AV_SAMPLE_FMT_FLT, AudioCodecId = codecId, Bitrate = Configuration.AudioConfig.AudioBitrate } : null }; videoFFmpeg = new FFmpegNativeEncoder(settings); _activeVideoFFmpeg = videoFFmpeg; lock (_allVideoFFmpegInstances) { _allVideoFFmpegInstances.Add(videoFFmpeg); } Volatile.Write(ref _incompleteOutputPath, CurrentOutPath); videoFFmpeg.Start(CurrentOutPath); } catch (Exception) { if (videoFFmpeg != null) { lock (_allVideoFFmpegInstances) { _allVideoFFmpegInstances.Remove(videoFFmpeg); } videoFFmpeg.Dispose(); } _activeVideoFFmpeg = null; throw; } var encoder = videoFFmpeg; RenderThread = new Thread(() => RunRenderThread(encoder)) { IsBackground = true, Priority = ThreadPriority.AboveNormal }; RenderThread.Start(); } /// /// Body of the render thread. Sends queued frames and their audio to the encoder until the render ends, /// finishes the output file, and releases the encoder. An exception is handled as a render failure. /// private void RunRenderThread(FFmpegNativeEncoder videoFFmpeg) { try { EncodeQueuedFrames(videoFFmpeg); if (!IsRenderAborted) { // End throws when the file cannot be finished (e.g. the trailer cannot be written), // which is handled as a render failure by the catch block below. videoFFmpeg.End(); Volatile.Write(ref _incompleteOutputPath, null); } videoFFmpeg.Dispose(); UnregisterEncoder(videoFFmpeg); endSuccessful = !IsRenderAborted; } catch (OperationCanceledException) when (IsRenderAborted) { // Audio waits end with OperationCanceledException when the render is cancelled or has failed (OfflineAudioMixer.Cancel). Melon.Logger.Msg("Render thread stopped: the render was aborted."); DisposeEncoderAfterStop(videoFFmpeg); } catch (Exception ex) { Melon.Logger.Error($"FFmpeg rendering thread error: {ex}"); MarkFailed("encoder thread error. See the log for details."); DisposeEncoderAfterStop(videoFFmpeg); } } /// Releases the encoder after the render thread stopped early (abort or error) and removes it from the encoder list. private void DisposeEncoderAfterStop(FFmpegNativeEncoder videoFFmpeg) { try { videoFFmpeg.Dispose(); } catch (Exception disposeEx) { Melon.Logger.Error($"Encoder dispose after the render thread stopped also failed: {disposeEx.Message}"); } UnregisterEncoder(videoFFmpeg); } /// /// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio /// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready. /// Returns when the render is aborted (checked before every frame, frames left in ImageQueue are released /// by the cleanup after the thread ends) or all requested frames have been sent after endRequested. /// private void EncodeQueuedFrames(FFmpegNativeEncoder videoFFmpeg) { while (true) { if (IsRenderAborted) return; if (TryDequeueNextFrame(out var frame)) { FrameDequeuedEvent.Set(); SendVideoFrame(videoFFmpeg, frame); WriteAudioForFrame(videoFFmpeg); continue; } if (ShouldStopEncodeLoop()) return; FrameAvailableEvent.WaitOne(50); } } /// /// Removes the frame whose key equals passCount from ImageQueue and advances passCount. /// Before the Render stage, passCount jumps to the first queued key (the first captured frame index is not 0). /// Returns false when the queue is empty or the next frame has not arrived yet. /// private bool TryDequeueNextFrame(out CapturedFrame frame) { frame = default; lock (ImageQueue) { if (ImageQueue.Count == 0) return false; var firstKey = ImageQueue.Keys[0]; if (firstKey > passCount && Instance.stage < RenderStage.Render) { passCount = firstKey; } if (firstKey != passCount) return false; frame = ImageQueue[firstKey]; ImageQueue.RemoveAt(0); passCount++; return true; } } /// /// Sends one captured frame to the encoder and releases its buffer (managed: ArrayPool, native: readback state). /// A frame without data is skipped. /// private void SendVideoFrame(FFmpegNativeEncoder videoFFmpeg, in CapturedFrame frame) { if (frame.ManagedBuffer != null) { if (frame.ManagedBuffer.Length == 0) return; videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length, frame.IsColorConverted); Interlocked.Increment(ref framesSent); ArrayPool.Shared.Return(frame.ManagedBuffer); return; } if (frame.NativePointer == IntPtr.Zero || frame.Length <= 0) return; videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length, frame.IsColorConverted); Interlocked.Increment(ref framesSent); if (frame.ReadbackState != null) { _frameReader?.RecycleState(frame.ReadbackState); } } /// /// Mixes the audio of the frame that was just sent (index passCount - 1) and writes it to the encoder. /// Does nothing when audio rendering is off. /// private void WriteAudioForFrame(FFmpegNativeEncoder videoFFmpeg) { if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return; var channels = Configuration.AudioConfig.AudioChannels; var samplesNeeded = syncCalculator.GetNextFrameSamples(); var audioBuffer = ArrayPool.Shared.Rent(samplesNeeded * channels); try { AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded); videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded); } finally { ArrayPool.Shared.Return(audioBuffer); } } /// /// True when the render is aborted, or when the end was requested and every requested frame has been sent. /// private bool ShouldStopEncodeLoop() { if (IsRenderAborted) return true; return endRequested && passCount > Volatile.Read(ref lastRequestedVideoFrameIndex); } private void UnregisterEncoder(FFmpegNativeEncoder videoFFmpeg) { lock (_allVideoFFmpegInstances) { _allVideoFFmpegInstances.Remove(videoFFmpeg); } _activeVideoFFmpeg = null; } public void CancelRender() { ResetRenderState(success: false); StopFFmpeg(false); EndRender(); } private void StopFFmpeg(bool success) { endRequested = true; if (!success) { MarkCancelled(); } // The abort state is set first: the render thread reads it when its audio wait is cancelled. if (success) AudioMixer?.Complete(); else AudioMixer?.Cancel(); FrameAvailableEvent.Set(); skipPressToStart = false; _renderSetup = false; if (_escapeHandlerObj) { Destroy(_escapeHandlerObj); _escapeHandlerObj = null; } if (success) StartCoroutine(StopRenderCoroutine()); else { var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED"; Melon.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources."); // 1. Wait (up to 2s) for the render background thread to see the abort flag and terminate if (RenderThread is { IsAlive: true }) { RenderThread.Join(2000); } // 2. After ensuring thread termination, safely release FFmpeg native encoder resources if (_activeVideoFFmpeg != null) { _activeVideoFFmpeg.Dispose(); _activeVideoFFmpeg = null; } // 3. Memory leak prevention: Return rented temporary frame buffer pool in queue and clear lock (ImageQueue) { foreach (var kvp in ImageQueue.Values) { if (kvp.ManagedBuffer is { Length: > 0 }) { ArrayPool.Shared.Return(kvp.ManagedBuffer); } if (kvp.ReadbackState != null) { _frameReader?.RecycleState(kvp.ReadbackState); } } ImageQueue.Clear(); } _frameReader?.Dispose(); ResetFrameHookState(); // Clean up Debug Monitor UI if (_debugMonitorObj) { Destroy(_debugMonitorObj); _debugMonitorObj = null; } // 4. Safely restore in-game camera and RTCam state for rendering if (scrCamera.instance) { scrCamera.instance.forceRTCam = false; scrCamera.instance.SetupRTCam(false); var hook = scrCamera.instance.GetComponent(); if (hook) Destroy(hook); } // 5. Restore in-game no-fail flags if (scrController.instance) { scrController.instance.noFail = false; scrController.instance.noFailInfiniteMargin = false; } // 6. Immediately delete the incomplete video file of this render if (DeleteIncompleteOutputFile()) { CurrentOutPath = null; } var result = BuildResult(false, BuildAbortMessage()); TriggerOnComplete(result); } } private IEnumerator StopRenderCoroutine() { yield return new WaitUntil(() => endSuccessful || RenderThread is not { IsAlive: true }); if (!endSuccessful) { MarkFailed("the render thread terminated unexpectedly."); CancelRender(); yield break; } var result = BuildResult(success: true); TriggerOnComplete(result); EndRender(); } private void EndRender() { RestoreRenderEnvironment(); ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor"); Cursor.visible = true; } // Undoes what the render setup changed (time, input offset, auto, queues, session hooks), without leaving the scene. private void RestoreRenderEnvironment() { Time.timeScale = 1f; Time.captureDeltaTime = 0f; if (_debugMonitorObj) { Destroy(_debugMonitorObj); _debugMonitorObj = null; } RDC.auto = false; scrConductor.currentPreset.inputOffset = _originalInputOffset; GCS.customLevelPaths = null; GCS.customLevelId = null; lock (ImageQueue) { foreach (var kvp in ImageQueue.Values) { if (kvp.ManagedBuffer is { Length: > 0 }) { ArrayPool.Shared.Return(kvp.ManagedBuffer); } if (kvp.ReadbackState != null) { _frameReader?.RecycleState(kvp.ReadbackState); } } ImageQueue.Clear(); } _frameReader?.Dispose(); ResetFrameHookState(); AudioSpectrumGenerator.Clear(); AudioMixer?.Dispose(); AudioMixer = null; SyncCalculator = null; RenderContext.Reset(); } private RenderResult BuildResult(bool success, string? errorMessage = null) { _renderStopwatch.Stop(); var effectiveFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); var sent = Interlocked.Read(ref framesSent); return success ? RenderResult.CreateSuccess(CurrentOutPath ?? string.Empty, sent, effectiveFps, _renderStopwatch.Elapsed) : RenderResult.CreateFailure(errorMessage, sent, effectiveFps, _renderStopwatch.Elapsed); } private static void TriggerOnComplete(RenderResult result) { try { RenderContext.OnComplete.Invoke(result); } catch (Exception err) { Melon.Logger.Error($"OnComplete callback caused error: {err}"); } } private static void TriggerOnLastUpdate() { try { RenderContext.OnLastUpdate.Invoke(); } catch (Exception err) { Melon.Logger.Error($"OnLastUpdate callback caused error: {err}"); } } }