Optimize Frame Readback and Encoding Pipeline

Core/RenderManager.cs

@@ -117,6 +117,7 @@ internal class RenderManager : MonoBehaviour
         public int Length;
         public byte[]? ManagedBuffer; // Non-null if it's a managed buffer (e.g. black frame)
         public ReadbackRequestState? ReadbackState; // Associated state to recycle
+        public bool IsColorConverted;
     }
 
     // Thread and Queue State
@@ -546,7 +547,7 @@ internal class RenderManager : MonoBehaviour
                         {
                             if (frame.ManagedBuffer.Length > 0)
                             {
-                                videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length);
+                                videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length, frame.IsColorConverted);
                                 Interlocked.Increment(ref framesSent);
                                     
                                 ArrayPool<byte>.Shared.Return(frame.ManagedBuffer);
@@ -554,7 +555,7 @@ internal class RenderManager : MonoBehaviour
                         }
                         else if (frame.NativePointer != IntPtr.Zero && frame.Length > 0)
                         {
-                            videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length);
+                            videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length, frame.IsColorConverted);
                             Interlocked.Increment(ref framesSent);
 
                             if (frame.ReadbackState != null)

Util/FFmpeg/FFmpegNativeEncoder.cs

@@ -974,29 +974,22 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// Encodes a frame from IntPtr (native pointer).
         /// Can be used from external native plugins or custom capture sources.
         /// </summary>
-        public void SendDataFromNativeBuffer(IntPtr pointer, int length)
+        public void SendDataFromNativeBuffer(IntPtr pointer, int length, bool isColorConverted)
         {
             lock (_lock)
             {
                 if (_disposed || !_isInitialized) return;
-                var preConvertedSize = _settings.Width * _settings.Height * 3 / 2;
                 
                 if (!_loggedConversionMode)
                 {
                     _loggedConversionMode = true;
-                    Melon<RenderMod>.Logger.Msg(length == preConvertedSize
-                        ? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
+                    Melon<RenderMod>.Logger.Msg(isColorConverted
+                        ? "[Encoder] GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
                         : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
                 }
 
-                if (length == preConvertedSize)
-                {
-                    WritePreConvertedFrame((byte*)pointer.ToPointer());
-                }
-                else
-                {
-                    WriteFrameFromBuffer((byte*)pointer.ToPointer());
-                }
+                if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
+                else WriteFrameFromBuffer((byte*)pointer.ToPointer());
             }
         }
 
@@ -1004,31 +997,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// Encodes a frame from a managed byte[].
         /// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer.
         /// </summary>
-        public void SendDataFromBuffer(byte[] buffer, int length)
+        public void SendDataFromBuffer(byte[] buffer, int length, bool isColorConverted)
         {
             lock (_lock)
             {
                 if (_disposed || !_isInitialized) return;
                 fixed (byte* pBuffer = buffer)
                 {
-                    var preConvertedSize = _settings.Width * _settings.Height * 3 / 2;
-                    
                     if (!_loggedConversionMode)
                     {
                         _loggedConversionMode = true;
-                        Melon<RenderMod>.Logger.Msg(length == preConvertedSize
+                        Melon<RenderMod>.Logger.Msg(isColorConverted
                             ? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
                             : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
                     }
 
-                    if (length == preConvertedSize)
-                    {
-                        WritePreConvertedFrame(pBuffer);
-                    }
-                    else
-                    {
-                        WriteFrameFromBuffer(pBuffer);
-                    }
+                    if (isColorConverted) WritePreConvertedFrame(pBuffer);
+                    else WriteFrameFromBuffer(pBuffer);
                 }
             }
         }
@@ -1145,33 +1130,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                     throw new NotSupportedException($"This pixel format ({_dstPixFmt}) does not support direct copying of pre-converted buffers.");
             }
 
-            // QSV acceleration handling or encoder send
-            if (_isQsvEncoder)
-            {
-                ffmpeg.av_frame_unref(_pHwFrame);
-                try
-                {
-                    var ret = ffmpeg.av_hwframe_get_buffer(_pCodecContext->hw_frames_ctx, _pHwFrame, 0);
-                    if (ret < 0)
-                        throw new Exception($"Failed to allocate QSV hardware frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
-
-                    ret = ffmpeg.av_hwframe_transfer_data(_pHwFrame, _pDstFrame, 0);
-                    if (ret < 0)
-                        throw new Exception($"Failed to upload QSV frame — {GetFFmpegErrorString(ret)} (Code: {ret})");
-
-                    _pHwFrame->pts = _frameIndex++;
-                    SendFrameAndWritePackets(_pHwFrame);
-                }
-                finally
-                {
-                    ffmpeg.av_frame_unref(_pHwFrame);
-                }
-            }
-            else
-            {
-                _pDstFrame->pts = _frameIndex++;
-                SendFrameAndWritePackets(_pDstFrame);
-            }
+            SubmitDstFrame();
         }
 
         /// <summary>
@@ -1209,6 +1168,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
                 _pDstFrame->data, _pDstFrame->linesize
             );
 
+            SubmitDstFrame();
+        }
+
+        private void SubmitDstFrame()
+        {
             if (_isQsvEncoder)
             {
                 // ── QSV hw_frames_ctx mode: System memory → GPU surface upload ──

Util/FrameReader.cs

@@ -71,36 +71,30 @@ namespace ADOFAIRenderer.Util
                 state.IsInFlight = true;
             }
 
-            // Check if GPU color conversion is enabled
-            bool useGPUConversion = ModAssets.ColorConversionShader;
-            AVPixelFormat expectedFmt = AVPixelFormat.AV_PIX_FMT_RGBA;
+            // Check if GPU color conversion is available
+            var expectedFmt = AVPixelFormat.AV_PIX_FMT_RGBA;
 
-            if (useGPUConversion)
+            var kernelIndex = -1;
+            if (ModAssets.ColorConversionShader)
             {
-                expectedFmt = _targetFormat;
-                // Compute Shader only supports NV12 and YUV420P; fallback to CPU for other formats
-                if (expectedFmt != AVPixelFormat.AV_PIX_FMT_NV12 && expectedFmt != AVPixelFormat.AV_PIX_FMT_YUV420P)
+                var kernalName = _targetFormat switch
                 {
-                    useGPUConversion = false;
-                }
+                    AVPixelFormat.AV_PIX_FMT_NV12 => "CSMain_NV12",
+                    AVPixelFormat.AV_PIX_FMT_YUV420P => "CSMain_YUV420P",
+                    _ => null
+                };
+
+                if (kernalName != null)
+                    kernelIndex = ModAssets.ColorConversionShader.FindKernel(kernalName);
             }
 
+            var useGPUConversion = kernelIndex >= 0;
+            if (useGPUConversion) expectedFmt = _targetFormat;
+
             var bufferSize = texture.width * texture.height * 4; // RGBA = 4 channel
             state.Setup(currentFrame, bufferSize, useGPUConversion, expectedFmt, texture.width, texture.height);
 
             if (useGPUConversion && state.GPUBuffer != null && ModAssets.ColorConversionShader)
-            {
-                int kernelIndex = -1;
-                if (expectedFmt == AVPixelFormat.AV_PIX_FMT_NV12)
-                {
-                    kernelIndex = ModAssets.ColorConversionShader.FindKernel("CSMain_NV12");
-                }
-                else if (expectedFmt == AVPixelFormat.AV_PIX_FMT_YUV420P)
-                {
-                    kernelIndex = ModAssets.ColorConversionShader.FindKernel("CSMain_YUV420P");
-                }
-
-                if (kernelIndex >= 0)
             {
                 ModAssets.ColorConversionShader.SetTexture(kernelIndex, SourceTexture, texture);
                 ModAssets.ColorConversionShader.SetBuffer(kernelIndex, DestinationBuffer, state.GPUBuffer);
@@ -114,29 +108,18 @@ namespace ADOFAIRenderer.Util
 
                 ModAssets.ColorConversionShader.Dispatch(kernelIndex, threadGroupsX, threadGroupsY, 1);
 
-                    AsyncGPUReadback.Request(
+                AsyncGPUReadback.RequestIntoNativeArray(
+                    ref state.NativeBuffer, state.GPUBuffer, state.CachedCallback
+                );
+                /*AsyncGPUReadback.Request(
                     state.GPUBuffer,
                     state.CachedCallback
-                    );
-                }
-                else
-                {
-                    // If kernel is not found, fallback to standard RGBA texture readback
-                    AsyncGPUReadback.RequestIntoNativeArray(
-                        ref state.NativeBuffer,
-                        texture,
-                        0,
-                        state.CachedCallback
-                    );
-                }
+                );*/
             }
             else
             {
                 AsyncGPUReadback.RequestIntoNativeArray(
-                    ref state.NativeBuffer,
-                    texture,
-                    0,
-                    state.CachedCallback
+                    ref state.NativeBuffer, texture, 0, state.CachedCallback
                 );
             }
         }
@@ -169,16 +152,14 @@ namespace ADOFAIRenderer.Util
                 }
                 Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame.");
 
-                var byteSize = state.IsGPUConversion
-                    ? state.Width * state.Height * 3 / 2
-                    : state.Width * state.Height * 4;
+                var byteSize = state.Width * state.Height * 3 / 2;
                 var blackFrame = ArrayPool<byte>.Shared.Rent(byteSize);
                 Array.Clear(blackFrame, 0, byteSize);
                 
                 var capFrame = new RenderManager.CapturedFrame
                 {
-                    ManagedBuffer = blackFrame,
-                    Length = byteSize
+                    ManagedBuffer = blackFrame, Length = byteSize,
+                    IsColorConverted = true
                 };
                 PushToQueue(frame, capFrame);
                 RecycleState(state);
@@ -187,19 +168,20 @@ namespace ADOFAIRenderer.Util
             {
                 unsafe
                 {
+                    /*
                     if (state.IsGPUConversion)
                     {
                         var data = request.GetData<byte>();
                         // Copy async request buffer data to persistent NativeBuffer memory
                         NativeArray<byte>.Copy(data, state.NativeBuffer, data.Length);
                     }
+                    */
                     
                     var ptr = (IntPtr)state.NativeBuffer.GetUnsafeReadOnlyPtr();
                     var capFrame = new RenderManager.CapturedFrame
                     {
-                        NativePointer = ptr,
-                        Length = state.NativeBuffer.Length,
-                        ReadbackState = state
+                        NativePointer = ptr, Length = state.NativeBuffer.Length,
+                        ReadbackState = state, IsColorConverted = state.IsGPUConversion
                     };
                     PushToQueue(frame, capFrame);
                 }