Fix FFmpegVideoBGPlayer

ADOFAIRenderer.csproj

@@ -71,6 +71,9 @@
       <Reference Include="UnityEngine.UIModule">
         <HintPath>$(ADOFAI_PATH)\A Dance of Fire and Ice_Data\Managed\UnityEngine.UIModule.dll</HintPath>
       </Reference>
+      <Reference Include="UnityEngine.VideoModule">
+        <HintPath>$(ADOFAI_PATH)\A Dance of Fire and Ice_Data\Managed\UnityEngine.VideoModule.dll</HintPath>
+      </Reference>
     </ItemGroup>
 
     <ItemGroup>

Util/FFmpegVideoBGPlayer.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using ADOFAIRenderer.Util.FFmpeg;
 using MelonLoader;
@@ -10,6 +11,16 @@ namespace ADOFAIRenderer.Util
     // ReSharper disable once InconsistentNaming
     public class FFmpegVideoBGPlayer : MonoBehaviour
     {
+        private static readonly int YTex = Shader.PropertyToID("_YTex");
+        private static readonly int UVTex = Shader.PropertyToID("_UVTex");
+        private static readonly int Result = Shader.PropertyToID("_Result");
+        private static readonly int Width = Shader.PropertyToID("Width");
+        private static readonly int Height = Shader.PropertyToID("Height");
+        private static readonly int UTex = Shader.PropertyToID("_UTex");
+        private static readonly int VTex = Shader.PropertyToID("_VTex");
+        private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
+
+        [SuppressMessage("ReSharper", "InconsistentNaming")]
         private enum GPUMode
         {
             None,
@@ -20,6 +31,7 @@ namespace ADOFAIRenderer.Util
         private FFmpegNativeDecoder? _decoder;
         
         // CPU fallback rendering
+        // ReSharper disable once InconsistentNaming
         private Texture2D? _textureRGBA;
 
         // GPU conversion textures
@@ -40,13 +52,22 @@ namespace ADOFAIRenderer.Util
         private bool _isLooping;
         private bool _isInitialized;
 
-        public void Init(string videoPath, RenderTexture targetTexture, bool loop)
+        private GameObject? _quad;
+        private Material? _material;
+
+        // ReSharper disable once InconsistentNaming
+        public void Init(string videoPath, RenderTexture? targetTexture, bool loop, Camera? targetCamera, GameObject videoBGGo)
         {
             CleanUp();
 
             _targetTexture = targetTexture;
             _isLooping = loop;
 
+            if (_targetTexture == null && targetCamera != null)
+            {
+                SetupBackgroundQuad(targetCamera, videoBGGo);
+            }
+
             if (!File.Exists(videoPath))
             {
                 Melon<RenderMod>.Logger.Warning($"[FFmpegVideoBGPlayer] 비디오 파일이 존재하지 않습니다: {videoPath}");
@@ -58,16 +79,17 @@ namespace ADOFAIRenderer.Util
                 _decoder = new FFmpegNativeDecoder(videoPath);
                 _computeShader = RenderMod.ColorConversionShader;
 
-                int w = _decoder.Width;
+                var w = _decoder.Width;
-                int h = _decoder.Height;
+                var h = _decoder.Height;
-                AVPixelFormat pixFmt = _decoder.PixelFormat;
+                var pixFmt = _decoder.PixelFormat;
 
-                bool gpuInitSuccess = false;
+                var gpuInitSuccess = false;
 
                 if (_computeShader != null)
                 {
-                    if (pixFmt == AVPixelFormat.AV_PIX_FMT_NV12)
+                    switch (pixFmt)
                     {
+                        case AVPixelFormat.AV_PIX_FMT_NV12:
                             try
                             {
                                 _kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
@@ -81,10 +103,14 @@ namespace ADOFAIRenderer.Util
                                 gpuInitSuccess = true;
                                 Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] Compute Shader NV12 모드 초기화 완료");
                             }
-                        catch { }
+                            catch
-                    }
-                    else if (pixFmt == AVPixelFormat.AV_PIX_FMT_YUV420P || pixFmt == AVPixelFormat.AV_PIX_FMT_YUVJ420P)
                             {
+                                // ignored
+                            }
+
+                            break;
+                        case AVPixelFormat.AV_PIX_FMT_YUV420P:
+                        case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
                             try
                             {
                                 _kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
@@ -100,14 +126,23 @@ namespace ADOFAIRenderer.Util
                                 gpuInitSuccess = true;
                                 Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] Compute Shader YUV420P 모드 초기화 완료");
                             }
-                        catch { }
+                            catch
+                            {
+                                // ignored
+                            }
+
+                            break;
+                        default:
+                            throw new ArgumentOutOfRangeException();
                     }
                 }
 
                 if (gpuInitSuccess && _computeShader != null)
                 {
-                    _renderTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32);
+                    _renderTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32)
-                    _renderTexture.enableRandomWrite = true;
+                    {
+                        enableRandomWrite = true
+                    };
                     _renderTexture.Create();
                 }
                 else
@@ -131,35 +166,49 @@ namespace ADOFAIRenderer.Util
             }
         }
 
-        private void ConfigureTexture(Texture2D tex)
+        private static void ConfigureTexture(Texture2D tex)
         {
             tex.wrapMode = TextureWrapMode.Clamp;
             tex.filterMode = FilterMode.Bilinear;
         }
 
-        private void Update()
+        private void LateUpdate()
         {
             if (!_isInitialized || _decoder == null) return;
 
-            scrConductor cond = scrConductor.instance;
+            var cond = scrConductor.instance;
-            if (cond == null || !cond.hasSongStarted) return;
+            if (!cond || !cond.hasSongStarted) return;
             
-            double num4 = (cond.separateCountdownTime ? (cond.crotchetAtStart * (double)ADOBase.conductor.adjustedCountdownTicks) : 0.0);
+            var pitch = cond.song.pitch;
-            double vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
+            var separateCountdownOffset = cond.separateCountdownTime ? 
-            double targetVideoTime = cond.songposition_minusi - num4 + vidOffset;
+                cond.crotchetAtStart * cond.adjustedCountdownTicks : 0.0;
+            var songTime = (cond.dspTime - cond.dspTimeSong) * pitch - cond.addoffset - separateCountdownOffset;
+
+            var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
+            var targetVideoTime = songTime + vidOffset;
 
             if (targetVideoTime < 0)
             {
-                if (_targetTexture != null)
+                if (_targetTexture)
                 {
-                    RenderTexture active = RenderTexture.active;
+                    var active = RenderTexture.active;
                     RenderTexture.active = _targetTexture;
                     GL.Clear(true, true, Color.black);
                     RenderTexture.active = active;
                 }
+                if (_quad)
+                {
+                    _quad.SetActive(false);
+                }
                 return;
             }
 
+
+            if (ADOBase.customLevel)
+            {
+                ADOBase.customLevel.ShowTutorialBackground(false);
+            }
+
             try
             {
                 if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0)
@@ -172,10 +221,10 @@ namespace ADOFAIRenderer.Util
                 bool frameUpdated = false;
                 while (targetVideoTime >= _nextFrameTimestamp)
                 {
-                    double timestamp;
+                    double timestamp = 0.0;
                     bool success = false;
 
-                    if (_gpuMode == GPUMode.NV12 && _textureY != null && _textureUV != null)
+                    if (_gpuMode == GPUMode.NV12 && _textureY && _textureUV)
                     {
                         success = _decoder.TryReadNV12Frame(
                             _textureY.GetRawTextureData<byte>(),
@@ -183,7 +232,7 @@ namespace ADOFAIRenderer.Util
                             out timestamp
                         );
                     }
-                    else if (_gpuMode == GPUMode.YUV420P && _textureY != null && _textureU != null && _textureV != null)
+                    else if (_gpuMode == GPUMode.YUV420P && _textureY && _textureU && _textureV)
                     {
                         success = _decoder.TryReadYUV420PFrame(
                             _textureY.GetRawTextureData<byte>(),
@@ -240,52 +289,96 @@ namespace ADOFAIRenderer.Util
 
         private void RenderToTarget()
         {
-            if (_decoder == null || _targetTexture == null) return;
+            if (_decoder == null) return;
 
             int w = _decoder.Width;
             int h = _decoder.Height;
 
-            if (_gpuMode == GPUMode.NV12 && _textureY != null && _textureUV != null && _computeShader != null && _renderTexture != null)
+            if (_gpuMode == GPUMode.NV12 && _textureY && _textureUV && _computeShader && _renderTexture)
             {
                 _textureY.Apply();
                 _textureUV.Apply();
 
-                _computeShader.SetTexture(_kernelIndex, "_YTex", _textureY);
+                _computeShader.SetTexture(_kernelIndex, YTex, _textureY);
-                _computeShader.SetTexture(_kernelIndex, "_UVTex", _textureUV);
+                _computeShader.SetTexture(_kernelIndex, UVTex, _textureUV);
-                _computeShader.SetTexture(_kernelIndex, "_Result", _renderTexture);
+                _computeShader.SetTexture(_kernelIndex, Result, _renderTexture);
-                _computeShader.SetInt("Width", w);
+                _computeShader.SetInt(Width, w);
-                _computeShader.SetInt("Height", h);
+                _computeShader.SetInt(Height, h);
+                _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
 
                 int threadGroupsX = (w + 7) / 8;
                 int threadGroupsY = (h + 7) / 8;
                 _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
 
+                if (_targetTexture)
+                {
                     Graphics.Blit(_renderTexture, _targetTexture);
                 }
-            else if (_gpuMode == GPUMode.YUV420P && _textureY != null && _textureU != null && _textureV != null && _computeShader != null && _renderTexture != null)
+                else if (_material)
+                {
+                    if (_material.mainTexture != _renderTexture)
+                    {
+                        _material.mainTexture = _renderTexture;
+                    }
+                    if (_quad != null && !_quad.activeSelf)
+                    {
+                        _quad.SetActive(true);
+                    }
+                }
+            }
+            else if (_gpuMode == GPUMode.YUV420P && _textureY && _textureU && _textureV && _computeShader && _renderTexture)
             {
                 _textureY.Apply();
                 _textureU.Apply();
                 _textureV.Apply();
 
-                _computeShader.SetTexture(_kernelIndex, "_YTex", _textureY);
+                _computeShader.SetTexture(_kernelIndex, YTex, _textureY);
-                _computeShader.SetTexture(_kernelIndex, "_UTex", _textureU);
+                _computeShader.SetTexture(_kernelIndex, UTex, _textureU);
-                _computeShader.SetTexture(_kernelIndex, "_VTex", _textureV);
+                _computeShader.SetTexture(_kernelIndex, VTex, _textureV);
-                _computeShader.SetTexture(_kernelIndex, "_Result", _renderTexture);
+                _computeShader.SetTexture(_kernelIndex, Result, _renderTexture);
-                _computeShader.SetInt("Width", w);
+                _computeShader.SetInt(Width, w);
-                _computeShader.SetInt("Height", h);
+                _computeShader.SetInt(Height, h);
+                _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
 
                 int threadGroupsX = (w + 7) / 8;
                 int threadGroupsY = (h + 7) / 8;
                 _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
 
+                if (_targetTexture)
+                {
                     Graphics.Blit(_renderTexture, _targetTexture);
                 }
-            else if (_textureRGBA != null)
+                else if (_material)
+                {
+                    if (_material.mainTexture != _renderTexture)
+                    {
+                        _material.mainTexture = _renderTexture;
+                    }
+                    if (_quad != null && !_quad.activeSelf)
+                    {
+                        _quad.SetActive(true);
+                    }
+                }
+            }
+            else if (_textureRGBA)
             {
                 _textureRGBA.Apply();
+                if (_targetTexture)
+                {
                     Graphics.Blit(_textureRGBA, _targetTexture);
                 }
+                else if (_material)
+                {
+                    if (_material.mainTexture != _textureRGBA)
+                    {
+                        _material.mainTexture = _textureRGBA;
+                    }
+                    if (_quad != null && !_quad.activeSelf)
+                    {
+                        _quad.SetActive(true);
+                    }
+                }
+            }
         }
 
         private void OnDestroy()
@@ -326,6 +419,98 @@ namespace ADOFAIRenderer.Util
                 Destroy(_renderTexture);
                 _renderTexture = null;
             }
+
+            if (_quad != null)
+            {
+                Destroy(_quad);
+                _quad = null;
+            }
+
+            if (_material != null)
+            {
+                Destroy(_material);
+                _material = null;
+            }
+        }
+
+        // ReSharper disable once InconsistentNaming
+        private void SetupBackgroundQuad(Camera targetCamera, GameObject videoBGGo)
+        {
+            _quad = GameObject.CreatePrimitive(PrimitiveType.Quad);
+            _quad.name = "FFmpegVideoBGQuad";
+            
+            // ReSharper disable once Unity.UnresolvedComponentOrScriptableObject
+            var col = _quad.GetComponent("MeshCollider");
+            if (col) Destroy(col);
+            
+            _quad.transform.parent = targetCamera.transform;
+            
+            var zPos = targetCamera.farClipPlane - 0.5f;
+            _quad.transform.localPosition = new Vector3(0f, 0f, zPos);
+            _quad.transform.localRotation = Quaternion.identity;
+            
+            float height;
+            float width;
+            if (targetCamera.orthographic)
+            {
+                height = 2.0f * targetCamera.orthographicSize;
+                width = height * targetCamera.aspect;
+            }
+            else
+            {
+                height = 2.0f * zPos * Mathf.Tan(targetCamera.fieldOfView * 0.5f * Mathf.Deg2Rad);
+                width = height * targetCamera.aspect;
+            }
+            _quad.transform.localScale = new Vector3(width, height, 1.0f);
+            _quad.layer = videoBGGo.layer;
+
+            // Set vertex colors to white so Sprites/Default works properly (otherwise it may draw transparently)
+            var meshFilter = _quad.GetComponent<MeshFilter>();
+            if (meshFilter != null && meshFilter.mesh != null)
+            {
+                var mesh = meshFilter.mesh;
+                var colors = new Color[mesh.vertexCount];
+                for (var i = 0; i < colors.Length; i++)
+                {
+                    colors[i] = Color.white;
+                }
+                mesh.colors = colors;
+            }
+
+            Shader? shader = null;
+            if (scrCamera.instance != null)
+            {
+                try
+                {
+                    var camQuadMeshField = typeof(scrCamera).GetField("camQuadMesh", System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
+                    if (camQuadMeshField != null)
+                    {
+                        var renderer = camQuadMeshField.GetValue(scrCamera.instance) as MeshRenderer;
+                        if (renderer != null && renderer.sharedMaterial != null)
+                        {
+                            shader = renderer.sharedMaterial.shader;
+                        }
+                    }
+                }
+                catch (Exception)
+                {
+                    // ignored
+                }
+            }
+            
+            if (shader == null) shader = Shader.Find("Sprites/Default");
+            if (shader == null) shader = Shader.Find("Unlit/Texture");
+            if (shader == null) shader = Shader.Find("Standard");
+            
+            _material = new Material(shader)
+            {
+                color = Color.white
+            };
+
+            _quad.GetComponent<MeshRenderer>().material = _material;
+            
+            // Set inactive initially to prevent white flash
+            _quad.SetActive(false);
         }
     }
 }