From d48579435a79fb2f338f98f73c9f55b77fe895a2b09c8ebcbafcd61518efe09f Mon Sep 17 00:00:00 2001 From: copecone Date: Sun, 21 Jun 2026 23:06:57 +0900 Subject: [PATCH] Fix `FFmpegNativeDecoder` to Consider Color Range When Decoding Video --- Patch/Game/scnGamePatch.cs | 73 +++++++ Util/FFmpeg/FFmpegNativeDecoder.cs | 26 ++- Util/FFmpegVideoBGPlayer.cs | 331 +++++++++++++++++++++++++++++ 3 files changed, 420 insertions(+), 10 deletions(-) create mode 100644 Patch/Game/scnGamePatch.cs create mode 100644 Util/FFmpegVideoBGPlayer.cs diff --git a/Patch/Game/scnGamePatch.cs b/Patch/Game/scnGamePatch.cs new file mode 100644 index 0000000..5d10a4a --- /dev/null +++ b/Patch/Game/scnGamePatch.cs @@ -0,0 +1,73 @@ +using System.IO; +using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util; +using HarmonyLib; + +namespace ADOFAIRenderer.Patch.Game +{ + [HarmonyPatch(typeof(scnGame))] + // ReSharper disable once InconsistentNaming + public static class scnGamePatch + { + [HarmonyPrefix] + [HarmonyPatch("UpdateVideo")] + public static bool Prefix(scnGame __instance) + { + // Only replace when rendering is active + if (!RenderManager.IsActive) + { + // Remove FFmpeg player if it was attached previously + var oldPlayer = __instance.videoBG.gameObject.GetComponent(); + if (oldPlayer != null) + { + UnityEngine.Object.Destroy(oldPlayer); + } + // Re-enable native player component + __instance.videoBG.enabled = true; + return true; // Execute original + } + + string text = __instance.levelData.miscSettings.GetString("bgVideo"); + if (string.IsNullOrEmpty(text)) + { + __instance.videoBG.gameObject.SetActive(false); + return false; // Skip original + } + + __instance.videoBG.gameObject.SetActive(true); + + // We can only play external video files via FFmpegNativeDecoder. + // Internal/Bundle level background videos are built-in VideoClips, so we fallback to native Unity VideoPlayer. + if (ADOBase.isInternalLevel || ADOBase.isBundleLevel) + { + __instance.videoBG.enabled = true; + return true; // Execute original + } + + // Custom level video path + string videoPath = Path.Combine(Path.GetDirectoryName(ADOBase.levelPath), text); + if (!File.Exists(videoPath)) + { + return true; // Let the original print error + } + + // Disable native VideoPlayer component (to prevent it from attempting real-time playback and sync errors) + __instance.videoBG.enabled = false; + + // Add or retrieve our custom FFmpeg video background player + var player = __instance.videoBG.gameObject.GetComponent(); + if (player == null) + { + player = __instance.videoBG.gameObject.AddComponent(); + } + + bool isLooping = __instance.levelData.miscSettings.GetBool("loopVideo"); + + // In custom levels, ADOFAI's videoBG targetTexture is pre-allocated by the game to render background filters. + // We initialize the player with the target RenderTexture. + player.Init(videoPath, __instance.videoBG.targetTexture, isLooping); + + return false; // Skip original UpdateVideo + } + } +} diff --git a/Util/FFmpeg/FFmpegNativeDecoder.cs b/Util/FFmpeg/FFmpegNativeDecoder.cs index 0413b9d..f3a1ad1 100644 --- a/Util/FFmpeg/FFmpegNativeDecoder.cs +++ b/Util/FFmpeg/FFmpegNativeDecoder.cs @@ -60,6 +60,9 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 비디오 원본 코덱의 픽셀 포맷. public AVPixelFormat PixelFormat => _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE; + /// 비디오 원본 코덱의 색상 범위. + public AVColorRange ColorRange => _pCodecContext != null ? _pCodecContext->color_range : AVColorRange.AVCOL_RANGE_UNSPECIFIED; + public FFmpegNativeDecoder(string videoPath) { Open(videoPath); @@ -255,7 +258,7 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Unity 텍스처 시스템에 맞게 상하 flip을 적용할지 여부 /// 출력된 프레임의 타임스탬프 (초) /// 프레임을 성공적으로 디코딩하여 복사했으면 true, 재생이 끝났거나 실패했으면 false - public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) + private bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) { timestamp = 0; lock (_lock) @@ -362,7 +365,8 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 비디오 원본이 YUV420p 계열(YUV420P, YUVJ420P)이면 CPU 컬러 변환(sws_scale)을 우회하고 메모리를 다이렉트로 복사하여 속도가 매우 빠릅니다. /// 그 외 포맷일 경우 자동으로 YUV420p로 변환 후 복사합니다. /// - public bool TryReadYUV420PFrame( + // ReSharper disable once InconsistentNaming + private bool TryReadYUV420PFrame( byte* yBuffer, int yStride, byte* uBuffer, int uStride, byte* vBuffer, int vStride, @@ -373,7 +377,7 @@ namespace ADOFAIRenderer.Util.FFmpeg { if (!_isOpened || _disposed) return false; - AVFrame* frame = null; + AVFrame* frame; if (_hasCachedFrame) { @@ -477,6 +481,7 @@ namespace ADOFAIRenderer.Util.FFmpeg } } + // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( NativeArray yBuffer, NativeArray uBuffer, @@ -506,7 +511,8 @@ namespace ADOFAIRenderer.Util.FFmpeg /// 비디오 원본이 NV12 계열이면 CPU 컬러 변환(sws_scale)을 우회하고 메모리를 다이렉트로 복사하여 속도가 매우 빠릅니다. /// 그 외 포맷일 경우 자동으로 NV12로 변환 후 복사합니다. /// - public bool TryReadNV12Frame( + // ReSharper disable once InconsistentNaming + private bool TryReadNV12Frame( byte* yBuffer, int yStride, byte* uvBuffer, int uvStride, out double timestamp) @@ -516,7 +522,7 @@ namespace ADOFAIRenderer.Util.FFmpeg { if (!_isOpened || _disposed) return false; - AVFrame* frame = null; + AVFrame* frame; if (_hasCachedFrame) { @@ -607,6 +613,7 @@ namespace ADOFAIRenderer.Util.FFmpeg } } + // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( NativeArray yBuffer, NativeArray uvBuffer, @@ -633,12 +640,12 @@ namespace ADOFAIRenderer.Util.FFmpeg /// /// 버릴 프레임 개수 /// 실제로 건너뛴 프레임 개수 - public int SkipFrames(int count) + private void SkipFrames(int count) { lock (_lock) { - if (!_isOpened || _disposed) return 0; - if (count <= 0) return 0; + if (!_isOpened || _disposed) return; + if (count <= 0) return; int skipped = 0; if (_hasCachedFrame) @@ -678,7 +685,6 @@ namespace ADOFAIRenderer.Util.FFmpeg throw new Exception($"프레임 건너뛰기 중 오류 발생: {GetFFmpegErrorString(ret)}"); } } - return skipped; } } @@ -696,7 +702,7 @@ namespace ADOFAIRenderer.Util.FFmpeg double targetFrameIndex = timeSeconds * FPS; // 이미 해당 인덱스이고 매우 인접한 미래라면 seek 대신 skip 처리 (I-프레임 찾기 오버헤드 방지) - bool useSkip = targetFrameIndex >= _currentFrameIndex && (targetFrameIndex - _currentFrameIndex) < 60; + bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60; if (useSkip) { SkipFrames((int)(targetFrameIndex - _currentFrameIndex)); diff --git a/Util/FFmpegVideoBGPlayer.cs b/Util/FFmpegVideoBGPlayer.cs new file mode 100644 index 0000000..ec71c9e --- /dev/null +++ b/Util/FFmpegVideoBGPlayer.cs @@ -0,0 +1,331 @@ +using System; +using System.IO; +using ADOFAIRenderer.Util.FFmpeg; +using MelonLoader; +using UnityEngine; +using FFmpeg.AutoGen; + +namespace ADOFAIRenderer.Util +{ + // ReSharper disable once InconsistentNaming + public class FFmpegVideoBGPlayer : MonoBehaviour + { + private enum GPUMode + { + None, + YUV420P, + NV12 + } + + private FFmpegNativeDecoder? _decoder; + + // CPU fallback rendering + private Texture2D? _textureRGBA; + + // GPU conversion textures + private Texture2D? _textureY; + private Texture2D? _textureU; + private Texture2D? _textureV; + private Texture2D? _textureUV; + + // Compute Shader variables + private ComputeShader? _computeShader; + private RenderTexture? _renderTexture; + private int _kernelIndex = -1; + + private double _currentFrameTimestamp; + private double _nextFrameTimestamp; + private GPUMode _gpuMode = GPUMode.None; + private RenderTexture? _targetTexture; + private bool _isLooping; + private bool _isInitialized; + + public void Init(string videoPath, RenderTexture targetTexture, bool loop) + { + CleanUp(); + + _targetTexture = targetTexture; + _isLooping = loop; + + if (!File.Exists(videoPath)) + { + Melon.Logger.Warning($"[FFmpegVideoBGPlayer] 비디오 파일이 존재하지 않습니다: {videoPath}"); + return; + } + + try + { + _decoder = new FFmpegNativeDecoder(videoPath); + _computeShader = RenderMod.ColorConversionShader; + + int w = _decoder.Width; + int h = _decoder.Height; + AVPixelFormat pixFmt = _decoder.PixelFormat; + + bool gpuInitSuccess = false; + + if (_computeShader != null) + { + if (pixFmt == AVPixelFormat.AV_PIX_FMT_NV12) + { + try + { + _kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA"); + + _textureY = new Texture2D(w, h, TextureFormat.R8, false); + _textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false); + ConfigureTexture(_textureY); + ConfigureTexture(_textureUV); + + _gpuMode = GPUMode.NV12; + gpuInitSuccess = true; + Melon.Logger.Msg($"[FFmpegVideoBGPlayer] Compute Shader NV12 모드 초기화 완료"); + } + catch { } + } + else if (pixFmt == AVPixelFormat.AV_PIX_FMT_YUV420P || pixFmt == AVPixelFormat.AV_PIX_FMT_YUVJ420P) + { + try + { + _kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA"); + + _textureY = new Texture2D(w, h, TextureFormat.R8, false); + _textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false); + _textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false); + ConfigureTexture(_textureY); + ConfigureTexture(_textureU); + ConfigureTexture(_textureV); + + _gpuMode = GPUMode.YUV420P; + gpuInitSuccess = true; + Melon.Logger.Msg($"[FFmpegVideoBGPlayer] Compute Shader YUV420P 모드 초기화 완료"); + } + catch { } + } + } + + if (gpuInitSuccess && _computeShader != null) + { + _renderTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32); + _renderTexture.enableRandomWrite = true; + _renderTexture.Create(); + } + else + { + _textureRGBA = new Texture2D(w, h, TextureFormat.RGBA32, false); + ConfigureTexture(_textureRGBA); + _gpuMode = GPUMode.None; + Melon.Logger.Warning($"[FFmpegVideoBGPlayer] GPU 가속 미지원 또는 로드 실패로 CPU 폴백(RGBA)으로 진행합니다."); + } + + _currentFrameTimestamp = 0.0; + _nextFrameTimestamp = 0.0; + _isInitialized = true; + + Melon.Logger.Msg($"[FFmpegVideoBGPlayer] 초기화 성공 - 해상도: {w}x{h}, FPS: {_decoder.FPS:F2}"); + } + catch (Exception ex) + { + Melon.Logger.Error($"[FFmpegVideoBGPlayer] 초기화 중 에러 발생: {ex}"); + CleanUp(); + } + } + + private void ConfigureTexture(Texture2D tex) + { + tex.wrapMode = TextureWrapMode.Clamp; + tex.filterMode = FilterMode.Bilinear; + } + + private void Update() + { + if (!_isInitialized || _decoder == null) return; + + scrConductor cond = scrConductor.instance; + if (cond == null || !cond.hasSongStarted) return; + + double num4 = (cond.separateCountdownTime ? (cond.crotchetAtStart * (double)ADOBase.conductor.adjustedCountdownTicks) : 0.0); + double vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0; + double targetVideoTime = cond.songposition_minusi - num4 + vidOffset; + + if (targetVideoTime < 0) + { + if (_targetTexture != null) + { + RenderTexture active = RenderTexture.active; + RenderTexture.active = _targetTexture; + GL.Clear(true, true, Color.black); + RenderTexture.active = active; + } + return; + } + + try + { + if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0) + { + _decoder.SeekToTime(targetVideoTime); + _currentFrameTimestamp = targetVideoTime; + _nextFrameTimestamp = targetVideoTime; + } + + bool frameUpdated = false; + while (targetVideoTime >= _nextFrameTimestamp) + { + double timestamp; + bool success = false; + + if (_gpuMode == GPUMode.NV12 && _textureY != null && _textureUV != null) + { + success = _decoder.TryReadNV12Frame( + _textureY.GetRawTextureData(), + _textureUV.GetRawTextureData(), + out timestamp + ); + } + else if (_gpuMode == GPUMode.YUV420P && _textureY != null && _textureU != null && _textureV != null) + { + success = _decoder.TryReadYUV420PFrame( + _textureY.GetRawTextureData(), + _textureU.GetRawTextureData(), + _textureV.GetRawTextureData(), + out timestamp + ); + } + else + { + if (_textureRGBA != null) + { + success = _decoder.TryReadFrame( + _textureRGBA.GetRawTextureData(), + vflip: true, + out timestamp + ); + } + } + + if (success) + { + _currentFrameTimestamp = timestamp; + _nextFrameTimestamp = timestamp + (1.0 / _decoder.FPS); + frameUpdated = true; + } + else + { + if (_isLooping) + { + _decoder.SeekToTime(0); + _currentFrameTimestamp = 0; + _nextFrameTimestamp = 1.0 / _decoder.FPS; + break; + } + else + { + break; + } + } + } + + if (frameUpdated) + { + RenderToTarget(); + } + } + catch (Exception ex) + { + Melon.Logger.Error($"[FFmpegVideoBGPlayer] Update 재생 중 에러 발생: {ex}"); + _isInitialized = false; + } + } + + private void RenderToTarget() + { + if (_decoder == null || _targetTexture == null) return; + + int w = _decoder.Width; + int h = _decoder.Height; + + if (_gpuMode == GPUMode.NV12 && _textureY != null && _textureUV != null && _computeShader != null && _renderTexture != null) + { + _textureY.Apply(); + _textureUV.Apply(); + + _computeShader.SetTexture(_kernelIndex, "_YTex", _textureY); + _computeShader.SetTexture(_kernelIndex, "_UVTex", _textureUV); + _computeShader.SetTexture(_kernelIndex, "_Result", _renderTexture); + _computeShader.SetInt("Width", w); + _computeShader.SetInt("Height", h); + + int threadGroupsX = (w + 7) / 8; + int threadGroupsY = (h + 7) / 8; + _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1); + + Graphics.Blit(_renderTexture, _targetTexture); + } + else if (_gpuMode == GPUMode.YUV420P && _textureY != null && _textureU != null && _textureV != null && _computeShader != null && _renderTexture != null) + { + _textureY.Apply(); + _textureU.Apply(); + _textureV.Apply(); + + _computeShader.SetTexture(_kernelIndex, "_YTex", _textureY); + _computeShader.SetTexture(_kernelIndex, "_UTex", _textureU); + _computeShader.SetTexture(_kernelIndex, "_VTex", _textureV); + _computeShader.SetTexture(_kernelIndex, "_Result", _renderTexture); + _computeShader.SetInt("Width", w); + _computeShader.SetInt("Height", h); + + int threadGroupsX = (w + 7) / 8; + int threadGroupsY = (h + 7) / 8; + _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1); + + Graphics.Blit(_renderTexture, _targetTexture); + } + else if (_textureRGBA != null) + { + _textureRGBA.Apply(); + Graphics.Blit(_textureRGBA, _targetTexture); + } + } + + private void OnDestroy() + { + CleanUp(); + } + + public void CleanUp() + { + _isInitialized = false; + + if (_decoder != null) + { + _decoder.Dispose(); + _decoder = null; + Melon.Logger.Msg("[FFmpegVideoBGPlayer] 디코더 해제 완료."); + } + + if (_textureRGBA != null) + { + Destroy(_textureRGBA); + _textureRGBA = null; + } + + if (_textureY != null) Destroy(_textureY); + if (_textureU != null) Destroy(_textureU); + if (_textureV != null) Destroy(_textureV); + if (_textureUV != null) Destroy(_textureUV); + + _textureY = null; + _textureU = null; + _textureV = null; + _textureUV = null; + + if (_renderTexture != null) + { + _renderTexture.Release(); + Destroy(_renderTexture); + _renderTexture = null; + } + } + } +}