Fix FFmpegNativeDecoder to Consider Color Range When Decoding Video

Patch/Game/scnGamePatch.cs

@@ -0,0 +1,73 @@
+using System.IO;
+using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Util;
+using HarmonyLib;
+
+namespace ADOFAIRenderer.Patch.Game
+{
+    [HarmonyPatch(typeof(scnGame))]
+    // ReSharper disable once InconsistentNaming
+    public static class scnGamePatch
+    {
+        [HarmonyPrefix]
+        [HarmonyPatch("UpdateVideo")]
+        public static bool Prefix(scnGame __instance)
+        {
+            // Only replace when rendering is active
+            if (!RenderManager.IsActive)
+            {
+                // Remove FFmpeg player if it was attached previously
+                var oldPlayer = __instance.videoBG.gameObject.GetComponent<FFmpegVideoBGPlayer>();
+                if (oldPlayer != null)
+                {
+                    UnityEngine.Object.Destroy(oldPlayer);
+                }
+                // Re-enable native player component
+                __instance.videoBG.enabled = true;
+                return true; // Execute original
+            }
+
+            string text = __instance.levelData.miscSettings.GetString("bgVideo");
+            if (string.IsNullOrEmpty(text))
+            {
+                __instance.videoBG.gameObject.SetActive(false);
+                return false; // Skip original
+            }
+
+            __instance.videoBG.gameObject.SetActive(true);
+
+            // We can only play external video files via FFmpegNativeDecoder.
+            // Internal/Bundle level background videos are built-in VideoClips, so we fallback to native Unity VideoPlayer.
+            if (ADOBase.isInternalLevel || ADOBase.isBundleLevel)
+            {
+                __instance.videoBG.enabled = true;
+                return true; // Execute original
+            }
+
+            // Custom level video path
+            string videoPath = Path.Combine(Path.GetDirectoryName(ADOBase.levelPath), text);
+            if (!File.Exists(videoPath))
+            {
+                return true; // Let the original print error
+            }
+
+            // Disable native VideoPlayer component (to prevent it from attempting real-time playback and sync errors)
+            __instance.videoBG.enabled = false;
+
+            // Add or retrieve our custom FFmpeg video background player
+            var player = __instance.videoBG.gameObject.GetComponent<FFmpegVideoBGPlayer>();
+            if (player == null)
+            {
+                player = __instance.videoBG.gameObject.AddComponent<FFmpegVideoBGPlayer>();
+            }
+
+            bool isLooping = __instance.levelData.miscSettings.GetBool("loopVideo");
+            
+            // In custom levels, ADOFAI's videoBG targetTexture is pre-allocated by the game to render background filters.
+            // We initialize the player with the target RenderTexture.
+            player.Init(videoPath, __instance.videoBG.targetTexture, isLooping);
+
+            return false; // Skip original UpdateVideo
+        }
+    }
+}

Util/FFmpeg/FFmpegNativeDecoder.cs

@@ -60,6 +60,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// <summary>비디오 원본 코덱의 픽셀 포맷.</summary>
         public AVPixelFormat PixelFormat => _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE;
 
+        /// <summary>비디오 원본 코덱의 색상 범위.</summary>
+        public AVColorRange ColorRange => _pCodecContext != null ? _pCodecContext->color_range : AVColorRange.AVCOL_RANGE_UNSPECIFIED;
+
         public FFmpegNativeDecoder(string videoPath)
         {
             Open(videoPath);
@@ -255,7 +258,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// <param name="vflip">Unity 텍스처 시스템에 맞게 상하 flip을 적용할지 여부</param>
         /// <param name="timestamp">출력된 프레임의 타임스탬프 (초)</param>
         /// <returns>프레임을 성공적으로 디코딩하여 복사했으면 true, 재생이 끝났거나 실패했으면 false</returns>
-        public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
+        private bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
         {
             timestamp = 0;
             lock (_lock)
@@ -362,7 +365,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// 비디오 원본이 YUV420p 계열(YUV420P, YUVJ420P)이면 CPU 컬러 변환(sws_scale)을 우회하고 메모리를 다이렉트로 복사하여 속도가 매우 빠릅니다.
         /// 그 외 포맷일 경우 자동으로 YUV420p로 변환 후 복사합니다.
         /// </summary>
-        public bool TryReadYUV420PFrame(
+        // ReSharper disable once InconsistentNaming
+        private bool TryReadYUV420PFrame(
             byte* yBuffer, int yStride,
             byte* uBuffer, int uStride,
             byte* vBuffer, int vStride,
@@ -373,7 +377,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             {
                 if (!_isOpened || _disposed) return false;
 
-                AVFrame* frame = null;
+                AVFrame* frame;
 
                 if (_hasCachedFrame)
                 {
@@ -477,6 +481,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             }
         }
 
+        // ReSharper disable once InconsistentNaming
         public bool TryReadYUV420PFrame(
             NativeArray<byte> yBuffer,
             NativeArray<byte> uBuffer,
@@ -506,7 +511,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// 비디오 원본이 NV12 계열이면 CPU 컬러 변환(sws_scale)을 우회하고 메모리를 다이렉트로 복사하여 속도가 매우 빠릅니다.
         /// 그 외 포맷일 경우 자동으로 NV12로 변환 후 복사합니다.
         /// </summary>
-        public bool TryReadNV12Frame(
+        // ReSharper disable once InconsistentNaming
+        private bool TryReadNV12Frame(
             byte* yBuffer, int yStride,
             byte* uvBuffer, int uvStride,
             out double timestamp)
@@ -516,7 +522,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             {
                 if (!_isOpened || _disposed) return false;
 
-                AVFrame* frame = null;
+                AVFrame* frame;
 
                 if (_hasCachedFrame)
                 {
@@ -607,6 +613,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             }
         }
 
+        // ReSharper disable once InconsistentNaming
         public bool TryReadNV12Frame(
             NativeArray<byte> yBuffer,
             NativeArray<byte> uvBuffer,
@@ -633,12 +640,12 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// </summary>
         /// <param name="count">버릴 프레임 개수</param>
         /// <returns>실제로 건너뛴 프레임 개수</returns>
-        public int SkipFrames(int count)
+        private void SkipFrames(int count)
         {
             lock (_lock)
             {
-                if (!_isOpened || _disposed) return 0;
-                if (count <= 0) return 0;
+                if (!_isOpened || _disposed) return;
+                if (count <= 0) return;
 
                 int skipped = 0;
                 if (_hasCachedFrame)
@@ -678,7 +685,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
                         throw new Exception($"프레임 건너뛰기 중 오류 발생: {GetFFmpegErrorString(ret)}");
                     }
                 }
-                return skipped;
             }
         }
 
@@ -696,7 +702,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                 double targetFrameIndex = timeSeconds * FPS;
 
                 // 이미 해당 인덱스이고 매우 인접한 미래라면 seek 대신 skip 처리 (I-프레임 찾기 오버헤드 방지)
-                bool useSkip = targetFrameIndex >= _currentFrameIndex && (targetFrameIndex - _currentFrameIndex) < 60;
+                bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60;
                 if (useSkip)
                 {
                     SkipFrames((int)(targetFrameIndex - _currentFrameIndex));

Util/FFmpegVideoBGPlayer.cs

@@ -0,0 +1,331 @@
+using System;
+using System.IO;
+using ADOFAIRenderer.Util.FFmpeg;
+using MelonLoader;
+using UnityEngine;
+using FFmpeg.AutoGen;
+
+namespace ADOFAIRenderer.Util
+{
+    // ReSharper disable once InconsistentNaming
+    public class FFmpegVideoBGPlayer : MonoBehaviour
+    {
+        private enum GPUMode
+        {
+            None,
+            YUV420P,
+            NV12
+        }
+
+        private FFmpegNativeDecoder? _decoder;
+        
+        // CPU fallback rendering
+        private Texture2D? _textureRGBA;
+
+        // GPU conversion textures
+        private Texture2D? _textureY;
+        private Texture2D? _textureU;
+        private Texture2D? _textureV;
+        private Texture2D? _textureUV;
+
+        // Compute Shader variables
+        private ComputeShader? _computeShader;
+        private RenderTexture? _renderTexture;
+        private int _kernelIndex = -1;
+
+        private double _currentFrameTimestamp;
+        private double _nextFrameTimestamp;
+        private GPUMode _gpuMode = GPUMode.None;
+        private RenderTexture? _targetTexture;
+        private bool _isLooping;
+        private bool _isInitialized;
+
+        public void Init(string videoPath, RenderTexture targetTexture, bool loop)
+        {
+            CleanUp();
+
+            _targetTexture = targetTexture;
+            _isLooping = loop;
+
+            if (!File.Exists(videoPath))
+            {
+                Melon<RenderMod>.Logger.Warning($"[FFmpegVideoBGPlayer] 비디오 파일이 존재하지 않습니다: {videoPath}");
+                return;
+            }
+
+            try
+            {
+                _decoder = new FFmpegNativeDecoder(videoPath);
+                _computeShader = RenderMod.ColorConversionShader;
+
+                int w = _decoder.Width;
+                int h = _decoder.Height;
+                AVPixelFormat pixFmt = _decoder.PixelFormat;
+
+                bool gpuInitSuccess = false;
+
+                if (_computeShader != null)
+                {
+                    if (pixFmt == AVPixelFormat.AV_PIX_FMT_NV12)
+                    {
+                        try
+                        {
+                            _kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
+                            
+                            _textureY = new Texture2D(w, h, TextureFormat.R8, false);
+                            _textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false);
+                            ConfigureTexture(_textureY);
+                            ConfigureTexture(_textureUV);
+
+                            _gpuMode = GPUMode.NV12;
+                            gpuInitSuccess = true;
+                            Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] Compute Shader NV12 모드 초기화 완료");
+                        }
+                        catch { }
+                    }
+                    else if (pixFmt == AVPixelFormat.AV_PIX_FMT_YUV420P || pixFmt == AVPixelFormat.AV_PIX_FMT_YUVJ420P)
+                    {
+                        try
+                        {
+                            _kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
+                            
+                            _textureY = new Texture2D(w, h, TextureFormat.R8, false);
+                            _textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
+                            _textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
+                            ConfigureTexture(_textureY);
+                            ConfigureTexture(_textureU);
+                            ConfigureTexture(_textureV);
+
+                            _gpuMode = GPUMode.YUV420P;
+                            gpuInitSuccess = true;
+                            Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] Compute Shader YUV420P 모드 초기화 완료");
+                        }
+                        catch { }
+                    }
+                }
+
+                if (gpuInitSuccess && _computeShader != null)
+                {
+                    _renderTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32);
+                    _renderTexture.enableRandomWrite = true;
+                    _renderTexture.Create();
+                }
+                else
+                {
+                    _textureRGBA = new Texture2D(w, h, TextureFormat.RGBA32, false);
+                    ConfigureTexture(_textureRGBA);
+                    _gpuMode = GPUMode.None;
+                    Melon<RenderMod>.Logger.Warning($"[FFmpegVideoBGPlayer] GPU 가속 미지원 또는 로드 실패로 CPU 폴백(RGBA)으로 진행합니다.");
+                }
+
+                _currentFrameTimestamp = 0.0;
+                _nextFrameTimestamp = 0.0;
+                _isInitialized = true;
+
+                Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] 초기화 성공 - 해상도: {w}x{h}, FPS: {_decoder.FPS:F2}");
+            }
+            catch (Exception ex)
+            {
+                Melon<RenderMod>.Logger.Error($"[FFmpegVideoBGPlayer] 초기화 중 에러 발생: {ex}");
+                CleanUp();
+            }
+        }
+
+        private void ConfigureTexture(Texture2D tex)
+        {
+            tex.wrapMode = TextureWrapMode.Clamp;
+            tex.filterMode = FilterMode.Bilinear;
+        }
+
+        private void Update()
+        {
+            if (!_isInitialized || _decoder == null) return;
+
+            scrConductor cond = scrConductor.instance;
+            if (cond == null || !cond.hasSongStarted) return;
+
+            double num4 = (cond.separateCountdownTime ? (cond.crotchetAtStart * (double)ADOBase.conductor.adjustedCountdownTicks) : 0.0);
+            double vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
+            double targetVideoTime = cond.songposition_minusi - num4 + vidOffset;
+
+            if (targetVideoTime < 0)
+            {
+                if (_targetTexture != null)
+                {
+                    RenderTexture active = RenderTexture.active;
+                    RenderTexture.active = _targetTexture;
+                    GL.Clear(true, true, Color.black);
+                    RenderTexture.active = active;
+                }
+                return;
+            }
+
+            try
+            {
+                if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0)
+                {
+                    _decoder.SeekToTime(targetVideoTime);
+                    _currentFrameTimestamp = targetVideoTime;
+                    _nextFrameTimestamp = targetVideoTime;
+                }
+
+                bool frameUpdated = false;
+                while (targetVideoTime >= _nextFrameTimestamp)
+                {
+                    double timestamp;
+                    bool success = false;
+
+                    if (_gpuMode == GPUMode.NV12 && _textureY != null && _textureUV != null)
+                    {
+                        success = _decoder.TryReadNV12Frame(
+                            _textureY.GetRawTextureData<byte>(),
+                            _textureUV.GetRawTextureData<byte>(),
+                            out timestamp
+                        );
+                    }
+                    else if (_gpuMode == GPUMode.YUV420P && _textureY != null && _textureU != null && _textureV != null)
+                    {
+                        success = _decoder.TryReadYUV420PFrame(
+                            _textureY.GetRawTextureData<byte>(),
+                            _textureU.GetRawTextureData<byte>(),
+                            _textureV.GetRawTextureData<byte>(),
+                            out timestamp
+                        );
+                    }
+                    else
+                    {
+                        if (_textureRGBA != null)
+                        {
+                            success = _decoder.TryReadFrame(
+                                _textureRGBA.GetRawTextureData<byte>(),
+                                vflip: true,
+                                out timestamp
+                            );
+                        }
+                    }
+
+                    if (success)
+                    {
+                        _currentFrameTimestamp = timestamp;
+                        _nextFrameTimestamp = timestamp + (1.0 / _decoder.FPS);
+                        frameUpdated = true;
+                    }
+                    else
+                    {
+                        if (_isLooping)
+                        {
+                            _decoder.SeekToTime(0);
+                            _currentFrameTimestamp = 0;
+                            _nextFrameTimestamp = 1.0 / _decoder.FPS;
+                            break;
+                        }
+                        else
+                        {
+                            break;
+                        }
+                    }
+                }
+
+                if (frameUpdated)
+                {
+                    RenderToTarget();
+                }
+            }
+            catch (Exception ex)
+            {
+                Melon<RenderMod>.Logger.Error($"[FFmpegVideoBGPlayer] Update 재생 중 에러 발생: {ex}");
+                _isInitialized = false;
+            }
+        }
+
+        private void RenderToTarget()
+        {
+            if (_decoder == null || _targetTexture == null) return;
+
+            int w = _decoder.Width;
+            int h = _decoder.Height;
+
+            if (_gpuMode == GPUMode.NV12 && _textureY != null && _textureUV != null && _computeShader != null && _renderTexture != null)
+            {
+                _textureY.Apply();
+                _textureUV.Apply();
+
+                _computeShader.SetTexture(_kernelIndex, "_YTex", _textureY);
+                _computeShader.SetTexture(_kernelIndex, "_UVTex", _textureUV);
+                _computeShader.SetTexture(_kernelIndex, "_Result", _renderTexture);
+                _computeShader.SetInt("Width", w);
+                _computeShader.SetInt("Height", h);
+
+                int threadGroupsX = (w + 7) / 8;
+                int threadGroupsY = (h + 7) / 8;
+                _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
+
+                Graphics.Blit(_renderTexture, _targetTexture);
+            }
+            else if (_gpuMode == GPUMode.YUV420P && _textureY != null && _textureU != null && _textureV != null && _computeShader != null && _renderTexture != null)
+            {
+                _textureY.Apply();
+                _textureU.Apply();
+                _textureV.Apply();
+
+                _computeShader.SetTexture(_kernelIndex, "_YTex", _textureY);
+                _computeShader.SetTexture(_kernelIndex, "_UTex", _textureU);
+                _computeShader.SetTexture(_kernelIndex, "_VTex", _textureV);
+                _computeShader.SetTexture(_kernelIndex, "_Result", _renderTexture);
+                _computeShader.SetInt("Width", w);
+                _computeShader.SetInt("Height", h);
+
+                int threadGroupsX = (w + 7) / 8;
+                int threadGroupsY = (h + 7) / 8;
+                _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
+
+                Graphics.Blit(_renderTexture, _targetTexture);
+            }
+            else if (_textureRGBA != null)
+            {
+                _textureRGBA.Apply();
+                Graphics.Blit(_textureRGBA, _targetTexture);
+            }
+        }
+
+        private void OnDestroy()
+        {
+            CleanUp();
+        }
+
+        public void CleanUp()
+        {
+            _isInitialized = false;
+
+            if (_decoder != null)
+            {
+                _decoder.Dispose();
+                _decoder = null;
+                Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] 디코더 해제 완료.");
+            }
+
+            if (_textureRGBA != null)
+            {
+                Destroy(_textureRGBA);
+                _textureRGBA = null;
+            }
+
+            if (_textureY != null) Destroy(_textureY);
+            if (_textureU != null) Destroy(_textureU);
+            if (_textureV != null) Destroy(_textureV);
+            if (_textureUV != null) Destroy(_textureUV);
+
+            _textureY = null;
+            _textureU = null;
+            _textureV = null;
+            _textureUV = null;
+
+            if (_renderTexture != null)
+            {
+                _renderTexture.Release();
+                Destroy(_renderTexture);
+                _renderTexture = null;
+            }
+        }
+    }
+}