using System; using System.Buffers; using System.Collections.Generic; using System.Linq; using ADOFAIRenderer.Core; using ADOFAIRenderer.UI; using FFmpeg.AutoGen; using MelonLoader; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; using UnityEngine; using UnityEngine.Rendering; namespace ADOFAIRenderer.Util { /// /// GPU capture of rendered frames for the encoder. For each requested frame it optionally converts the RGBA texture to /// the encoder's YUV 4:2:0 layout with a compute shader (ColorConversion.compute from the mod's asset bundle, kernels /// CSMain_NV12 / CSMain_YUV420P), then reads the result back asynchronously (AsyncGPUReadback.RequestIntoNativeArray) /// into a persistent native buffer and adds it to RenderManager.ImageQueue under its video frame index. /// When the shader or the kernel for the pixel format is unavailable, the RGBA texture is read back unchanged and the /// encoder converts it (IsColorConverted = false). /// /// Each request uses a from a pool (16 created up front, more allocated when the pool /// is empty). A state stays out of the pool from the request until the render thread has sent its frame /// (RenderManager.SendVideoFrame, then ) or the frame was dropped. /// /// /// Threads: requests and readback completions run on the main Unity thread; is also called by /// the render thread. The pool and the disposed flag are guarded by _poolLock. One instance per render, created in /// RenderManager.SetupPlay with the pixel format of the opened encoder. /// /// internal class FrameReader : IDisposable { // Property IDs of the compute shader parameters: source texture, output buffer and frame size in pixels. private static readonly int Height = Shader.PropertyToID("Height"); private static readonly int Width = Shader.PropertyToID("Width"); private static readonly int DestinationBuffer = Shader.PropertyToID("DestinationBuffer"); private static readonly int SourceTexture = Shader.PropertyToID("SourceTexture"); /// Number of readback states created by InitializePool. private const int InitialBufferSize = 16; /// States that are free for a new request. Guarded by _poolLock. private readonly Stack _statePool = new(); /// Every state created by this reader (free, in flight or holding a queued frame), for Dispose. Guarded by _poolLock. private readonly List _allocatedStates = new(); /// Lock for _statePool, _allocatedStates, _isInitialized, _isDisposed and ReadbackRequestState.IsInFlight. private readonly object _poolLock = new(); /// True after InitializePool; false again after Dispose (RecycleState then disposes returned states). private bool _isInitialized; /// Set by Dispose. Completions and recycles that arrive afterwards dispose their state. private volatile bool _isDisposed; /// Color conversion compute shader from the asset bundle (ModAssets.ColorConversionShader); null when not loaded. private readonly ComputeShader? _computeShader; /// Kernel index for the target pixel format; -1 when the format has no kernel or the kernel was not found. private readonly int _kernelIndex = -1; /// True when frames are converted to YUV on the GPU (shader and kernel available). private readonly bool _useGPUConversion; /// Pixel format of the read-back data: the target format with GPU conversion, AV_PIX_FMT_RGBA otherwise. private readonly AVPixelFormat _expectedFormat; /// /// Selects the conversion kernel for and fills the state pool. /// /// /// Pixel format of the encoder input (FFmpegNativeEncoder.SelectedPixelFormat). NV12 and YUV420P use GPU conversion; /// any other format falls back to RGBA readback. /// public FrameReader(AVPixelFormat targetFormat) { _computeShader = ModAssets.ColorConversionShader; if (_computeShader) { var kernelName = targetFormat switch { AVPixelFormat.AV_PIX_FMT_NV12 => "CSMain_NV12", AVPixelFormat.AV_PIX_FMT_YUV420P => "CSMain_YUV420P", _ => null }; if (kernelName != null) { try { _kernelIndex = _computeShader.FindKernel(kernelName); } catch { _kernelIndex = -1; } } } _useGPUConversion = _kernelIndex >= 0; _expectedFormat = _useGPUConversion ? targetFormat : AVPixelFormat.AV_PIX_FMT_RGBA; InitializePool(); } /// Creates InitialBufferSize states once. Their buffers are allocated on first use (ReadbackRequestState.Setup). private void InitializePool() { lock (_poolLock) { if (_isInitialized) return; for (var i = 0; i < InitialBufferSize; i++) { var state = new ReadbackRequestState(this); _statePool.Push(state); _allocatedStates.Add(state); } _isInitialized = true; } } /// /// Bytes of the readback buffer of one frame: Width x Height x 3 / 2 when the GPU conversion to NV12 / YUV420P /// is used, Width x Height x 4 (RGBA) otherwise. The same size as ReadbackRequestState.Setup uses. /// public long GetFrameBufferBytes(int width, int height) { var pixels = (long)width * height; return _useGPUConversion ? pixels * 3 / 2 : pixels * 4; } /// /// Requests the capture of as video frame . Does nothing /// outside the Render stage. Main thread. /// With GPU conversion, the compute kernel writes the YUV data of the texture into the state's ComputeBuffer and that /// buffer is read back; otherwise the texture itself is read back as RGBA. The buffers are sized from the texture /// dimensions of this call. The result arrives in OnReadbackComplete. /// /// Frame to capture (game frame or the output of the frame hooks). /// Video frame index; the key of the frame in ImageQueue. public void RequestFrameCapture(Texture texture, long currentFrame) { if (!RenderManager.IsCapturingActive) return; ReadbackRequestState state; lock (_poolLock) { if (_statePool.Count == 0) { state = new ReadbackRequestState(this); _allocatedStates.Add(state); } else { state = _statePool.Pop(); } state.IsInFlight = true; } var bufferSize = texture.width * texture.height * 4; // RGBA = 4 channel state.Setup(currentFrame, bufferSize, _useGPUConversion, _expectedFormat, texture.width, texture.height); if (_useGPUConversion && state.GPUBuffer != null && _computeShader) { _computeShader.SetTexture(_kernelIndex, SourceTexture, texture); _computeShader.SetBuffer(_kernelIndex, DestinationBuffer, state.GPUBuffer); _computeShader.SetInt(Width, texture.width); _computeShader.SetInt(Height, texture.height); // Thread group counts cover the frame in tiles of 32 x 16 pixels (NV12) or 64 x 16 pixels (YUV420P), rounded up. // The tile size is the area one thread group of the ColorConversion.compute kernels processes. var threadGroupsX = _expectedFormat == AVPixelFormat.AV_PIX_FMT_NV12 ? (texture.width + 31) / 32 : (texture.width + 63) / 64; var threadGroupsY = (texture.height + 15) / 16; _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1); // The native buffer must not be disposed until the request completes; Dispose skips states that are in flight. // Reference: https://docs.unity3d.com/ScriptReference/Rendering.AsyncGPUReadback.RequestIntoNativeArray.html AsyncGPUReadback.RequestIntoNativeArray( ref state.NativeBuffer, state.GPUBuffer, state.CachedCallback ); /*AsyncGPUReadback.Request( state.GPUBuffer, state.CachedCallback );*/ } else { AsyncGPUReadback.RequestIntoNativeArray( ref state.NativeBuffer, texture, 0, state.CachedCallback ); } } /// /// Completion of a readback request (through ReadbackRequestState.CachedCallback). /// /// Reader disposed: the state is disposed. /// Render aborted: the state is returned to the pool and nothing is queued. /// Readback error: a black frame (managed buffer) is queued under the same index, so the render thread does not /// wait forever for the missing index, and the state is returned to the pool. /// Success: the state's native buffer is queued without copying; the state stays out of the pool until the /// render thread has encoded the frame. /// /// internal void OnReadbackComplete(ReadbackRequestState state, AsyncGPUReadbackRequest request) { var frame = state.FrameIndex; lock (_poolLock) { state.IsInFlight = false; if (_isDisposed) { state.Dispose(); return; } } if (RenderManager.Instance && RenderManager.Instance.IsRenderAborted) { RecycleState(state); return; } if (request.hasError) { if (RenderManager.Instance) { RenderManager.Instance.readbackFailureCount++; } Melon.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame."); var blackFrame = CreateBlackFrame(state, out var byteSize); var capFrame = new RenderManager.CapturedFrame { ManagedBuffer = blackFrame, Length = byteSize, IsColorConverted = state.IsGPUConversion }; PushToQueue(frame, capFrame); RecycleState(state); } else { unsafe { /* if (state.IsGPUConversion) { var data = request.GetData(); // Copy async request buffer data to persistent NativeBuffer memory NativeArray.Copy(data, state.NativeBuffer, data.Length); } */ var ptr = (IntPtr)state.NativeBuffer.GetUnsafeReadOnlyPtr(); var capFrame = new RenderManager.CapturedFrame { NativePointer = ptr, Length = state.NativeBuffer.Length, ReadbackState = state, IsColorConverted = state.IsGPUConversion }; PushToQueue(frame, capFrame); } } } /// /// Creates a black frame in the same format as the frames of this reader, from ArrayPool<byte>.Shared /// (returned to the pool by the render thread after encoding). /// /// GPU conversion (NV12 / YUV420P, full range): Width × Height × 3 / 2 bytes. /// Y plane (Width × Height bytes) = 0, U/V data (the rest) = 128. /// 128 is the zero chroma value of the 8-bit full range: the U/V offset 0.5 of RGB2YUV_BT709_Full /// in ColorConversion.compute, which the encoder input of this path uses. /// Both NV12 (interleaved UV) and YUV420P (U plane, then V plane) store all chroma after the Y plane, /// so the same fill applies to both. /// No GPU conversion (RGBA input, converted by the encoder): Width × Height × 4 bytes, all 0. /// private static byte[] CreateBlackFrame(ReadbackRequestState state, out int byteSize) { var lumaSize = state.Width * state.Height; byteSize = state.IsGPUConversion ? lumaSize * 3 / 2 : lumaSize * 4; var frame = ArrayPool.Shared.Rent(byteSize); if (state.IsGPUConversion) { Array.Clear(frame, 0, lumaSize); frame.AsSpan(lumaSize, byteSize - lumaSize).Fill(128); } else { Array.Clear(frame, 0, byteSize); } return frame; } /// /// Returns a state to the pool after its frame was sent or dropped, or disposes it when the reader is disposed. /// Called by the render thread (after sending) and by the main Unity thread (abort, errors, queue cleanup). /// internal void RecycleState(ReadbackRequestState state) { lock (_poolLock) { if (_isDisposed || !_isInitialized) { state.Dispose(); return; } state.Recycle(); _statePool.Push(state); } } /// /// Adds the frame to RenderManager.ImageQueue under (replacing a frame with the same index) /// and wakes the render thread. /// private static void PushToQueue(long frame, RenderManager.CapturedFrame capturedFrame) { if (!RenderManager.Instance) return; lock (RenderManager.Instance.ImageQueue) { RenderManager.Instance.ImageQueue[frame] = capturedFrame; } RenderManager.Instance.FrameAvailableEvent.Set(); } /// /// Disposes every state that is not in flight and clears the pool. States in flight are disposed when their /// readback completes. States still referenced by frames in ImageQueue are disposed too, so callers clear the queue /// first (RenderManager.StopFFmpeg / RestoreRenderEnvironment). Main thread. /// public void Dispose() { lock (_poolLock) { if (_isDisposed) return; _isDisposed = true; foreach (var state in _allocatedStates.Where(state => !state.IsInFlight)) { state.Dispose(); } _statePool.Clear(); _allocatedStates.Clear(); _isInitialized = false; } } } /// /// One reusable capture slot of : the persistent native buffer the readback writes into, the /// ComputeBuffer the conversion kernel writes (GPU conversion only), and the metadata of the frame it currently holds. /// The buffers are kept between frames and reallocated only when the frame size or conversion mode changes. /// internal class ReadbackRequestState : IDisposable { /// Reader that receives the completion and owns the pool. private readonly FrameReader _owner; /// Video frame index of the current request; -1 while the state is free. public long FrameIndex { get; private set; } /// /// Persistent native buffer receiving the readback: Width x Height x 3 / 2 bytes of YUV 4:2:0 with GPU conversion, /// Width x Height x 4 bytes of RGBA otherwise. Read by the render thread through CapturedFrame.NativePointer. /// public NativeArray NativeBuffer; /// True between the request and its completion. Written under FrameReader._poolLock. public bool IsInFlight { get; internal set; } /// Raw ComputeBuffer (4-byte elements) the conversion kernel writes; null without GPU conversion. public ComputeBuffer? GPUBuffer; /// True when the current frame is converted to YUV on the GPU. public bool IsGPUConversion { get; private set; } /// Width of the current frame in pixels. public int Width { get; private set; } /// Height of the current frame in pixels. public int Height { get; private set; } /// Completion delegate created once per state, so a request does not allocate a new delegate. public readonly Action CachedCallback; /// Creates a free state without buffers. public ReadbackRequestState(FrameReader owner) { _owner = owner; CachedCallback = OnComplete; FrameIndex = -1; } /// /// Prepares the state for a request: stores the frame metadata and (re)allocates the buffers when their size differs. /// The native buffer holds width x height x 3 / 2 bytes with GPU conversion, otherwise . /// The ComputeBuffer holds the same number of bytes as 4-byte elements (byte count / 4, rounded down). /// /// Video frame index of the request. /// Readback size in bytes for RGBA readback (width x height x 4). /// Whether the kernel converts the frame to YUV. /// Pixel format of the read-back data (not used by this method). /// Frame width in pixels. /// Frame height in pixels. public void Setup(long frameIndex, int bufferSize, bool isGPUConversion, AVPixelFormat pixelFormat, int width, int height) { FrameIndex = frameIndex; IsGPUConversion = isGPUConversion; Width = width; Height = height; var readbackSize = isGPUConversion ? (width * height * 3 / 2) : bufferSize; if (!NativeBuffer.IsCreated || NativeBuffer.Length != readbackSize) { if (NativeBuffer.IsCreated) NativeBuffer.Dispose(); NativeBuffer = new NativeArray(readbackSize, Allocator.Persistent, NativeArrayOptions.UninitializedMemory); } if (isGPUConversion) { var computeBufferSize = readbackSize / 4; if (GPUBuffer != null && GPUBuffer.count == computeBufferSize) return; GPUBuffer?.Dispose(); GPUBuffer = new ComputeBuffer(computeBufferSize, 4, ComputeBufferType.Raw); } else { GPUBuffer?.Dispose(); GPUBuffer = null; } } /// Readback completion; forwards to FrameReader.OnReadbackComplete. private void OnComplete(AsyncGPUReadbackRequest request) { _owner.OnReadbackComplete(this, request); } /// Marks the state as free (FrameIndex -1). The buffers are kept for the next request. public void Recycle() { FrameIndex = -1; } /// Releases the native buffer and the ComputeBuffer. public void Dispose() { if (NativeBuffer.IsCreated) NativeBuffer.Dispose(); GPUBuffer?.Dispose(); GPUBuffer = null; } } }