4 Commits
Author SHA256 Message Date
copecone daed6adcd7 Upgrade Version 2026-10-07 15:50:28 +09:00
copecone 86c8a87a0f Optimize Frame Readback and Encoding Pipeline 2026-10-07 15:48:09 +09:00
copecone 892f1ca7b9 Cleanup FolderDialogUtil 2026-10-07 15:47:31 +09:00
copecone e7ef081e55 Rename FFmpegVideoBGPlayer to VideoBGPlayer 2026-10-07 15:44:35 +09:00
8 changed files with 574 additions and 629 deletions

No files matched your search

+3 -2
View File
@@ -117,6 +117,7 @@ internal class RenderManager : MonoBehaviour
public int Length;
public byte[]? ManagedBuffer; // Non-null if it's a managed buffer (e.g. black frame)
public ReadbackRequestState? ReadbackState; // Associated state to recycle
public bool IsColorConverted;
}
// Thread and Queue State
@@ -546,7 +547,7 @@ internal class RenderManager : MonoBehaviour
{
if (frame.ManagedBuffer.Length > 0)
{
videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length);
videoFFmpeg.SendDataFromBuffer(frame.ManagedBuffer, frame.Length, frame.IsColorConverted);
Interlocked.Increment(ref framesSent);
ArrayPool<byte>.Shared.Return(frame.ManagedBuffer);
@@ -554,7 +555,7 @@ internal class RenderManager : MonoBehaviour
}
else if (frame.NativePointer != IntPtr.Zero && frame.Length > 0)
{
videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length);
videoFFmpeg.SendDataFromNativeBuffer(frame.NativePointer, frame.Length, frame.IsColorConverted);
Interlocked.Increment(ref framesSent);
if (frame.ReadbackState != null)
+3 -3
View File
@@ -77,7 +77,7 @@ public static class scnGamePatch
if (!RenderManager.IsActive)
{
// Remove FFmpeg player if it was attached previously
var oldPlayer = __instance.videoBG.gameObject.GetComponent<FFmpegVideoBGPlayer>();
var oldPlayer = __instance.videoBG.gameObject.GetComponent<VideoBGPlayer>();
if (oldPlayer) Object.Destroy(oldPlayer);
// Re-enable native player component
@@ -123,9 +123,9 @@ public static class scnGamePatch
__instance.videoBG.enabled = false;
// Add or retrieve our custom FFmpeg video background player
if (!__instance.videoBG.gameObject.TryGetComponent<FFmpegVideoBGPlayer>(out var player))
if (!__instance.videoBG.gameObject.TryGetComponent<VideoBGPlayer>(out var player))
{
player = __instance.videoBG.gameObject.AddComponent<FFmpegVideoBGPlayer>();
player = __instance.videoBG.gameObject.AddComponent<VideoBGPlayer>();
}
var isLooping = __instance.levelData.miscSettings.GetBool("loopVideo");
+1 -2
View File
@@ -2,7 +2,6 @@ using System;
using ADOFAIRenderer;
using ADOFAIRenderer.Core;
using ADOFAIRenderer.Core.Data;
using ADOFAIRenderer.Data;
using ADOFAIRenderer.UI;
using FFmpeg.AutoGen;
using MelonLoader;
@@ -10,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine;
using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.1-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.2-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer;
+15 -51
View File
@@ -974,29 +974,22 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources.
/// </summary>
public void SendDataFromNativeBuffer(IntPtr pointer, int length)
public void SendDataFromNativeBuffer(IntPtr pointer, int length, bool isColorConverted)
{
lock (_lock)
{
if (_disposed || !_isInitialized) return;
var preConvertedSize = _settings.Width * _settings.Height * 3 / 2;
if (!_loggedConversionMode)
{
_loggedConversionMode = true;
Melon<RenderMod>.Logger.Msg(length == preConvertedSize
? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
Melon<RenderMod>.Logger.Msg(isColorConverted
? "[Encoder] GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
}
if (length == preConvertedSize)
{
WritePreConvertedFrame((byte*)pointer.ToPointer());
}
else
{
WriteFrameFromBuffer((byte*)pointer.ToPointer());
}
if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
else WriteFrameFromBuffer((byte*)pointer.ToPointer());
}
}
@@ -1004,31 +997,23 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// Encodes a frame from a managed byte[].
/// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer.
/// </summary>
public void SendDataFromBuffer(byte[] buffer, int length)
public void SendDataFromBuffer(byte[] buffer, int length, bool isColorConverted)
{
lock (_lock)
{
if (_disposed || !_isInitialized) return;
fixed (byte* pBuffer = buffer)
{
var preConvertedSize = _settings.Width * _settings.Height * 3 / 2;
if (!_loggedConversionMode)
{
_loggedConversionMode = true;
Melon<RenderMod>.Logger.Msg(length == preConvertedSize
Melon<RenderMod>.Logger.Msg(isColorConverted
? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
}
if (length == preConvertedSize)
{
WritePreConvertedFrame(pBuffer);
}
else
{
WriteFrameFromBuffer(pBuffer);
}
if (isColorConverted) WritePreConvertedFrame(pBuffer);
else WriteFrameFromBuffer(pBuffer);
}
}
}
@@ -1145,33 +1130,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
throw new NotSupportedException($"This pixel format ({_dstPixFmt}) does not support direct copying of pre-converted buffers.");
}
// QSV acceleration handling or encoder send
if (_isQsvEncoder)
{
ffmpeg.av_frame_unref(_pHwFrame);
try
{
var ret = ffmpeg.av_hwframe_get_buffer(_pCodecContext->hw_frames_ctx, _pHwFrame, 0);
if (ret < 0)
throw new Exception($"Failed to allocate QSV hardware frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
ret = ffmpeg.av_hwframe_transfer_data(_pHwFrame, _pDstFrame, 0);
if (ret < 0)
throw new Exception($"Failed to upload QSV frame — {GetFFmpegErrorString(ret)} (Code: {ret})");
_pHwFrame->pts = _frameIndex++;
SendFrameAndWritePackets(_pHwFrame);
}
finally
{
ffmpeg.av_frame_unref(_pHwFrame);
}
}
else
{
_pDstFrame->pts = _frameIndex++;
SendFrameAndWritePackets(_pDstFrame);
}
SubmitDstFrame();
}
/// <summary>
@@ -1209,6 +1168,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pDstFrame->data, _pDstFrame->linesize
);
SubmitDstFrame();
}
private void SubmitDstFrame()
{
if (_isQsvEncoder)
{
// ── QSV hw_frames_ctx mode: System memory → GPU surface upload ──
-518
View File
@@ -1,518 +0,0 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using ADOFAIRenderer.UI;
using ADOFAIRenderer.Util.FFmpeg;
using MelonLoader;
using UnityEngine;
using FFmpeg.AutoGen;
namespace ADOFAIRenderer.Util
{
// ReSharper disable once InconsistentNaming
internal class FFmpegVideoBGPlayer : MonoBehaviour
{
private static readonly int YTex = Shader.PropertyToID("_YTex");
private static readonly int UVTex = Shader.PropertyToID("_UVTex");
private static readonly int Result = Shader.PropertyToID("_Result");
private static readonly int Width = Shader.PropertyToID("Width");
private static readonly int Height = Shader.PropertyToID("Height");
private static readonly int UTex = Shader.PropertyToID("_UTex");
private static readonly int VTex = Shader.PropertyToID("_VTex");
private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
[SuppressMessage("ReSharper", "InconsistentNaming")]
private enum GPUMode
{
None,
YUV420P,
NV12
}
private FFmpegNativeDecoder? _decoder;
// CPU fallback rendering
// ReSharper disable once InconsistentNaming
private Texture2D? _textureRGBA;
// GPU conversion textures
private Texture2D? _textureY;
private Texture2D? _textureU;
private Texture2D? _textureV;
private Texture2D? _textureUV;
// Compute Shader variables
private ComputeShader? _computeShader;
private RenderTexture? _renderTexture;
private int _kernelIndex = -1;
private double _currentFrameTimestamp;
private double _nextFrameTimestamp;
private GPUMode _gpuMode = GPUMode.None;
private RenderTexture? _targetTexture;
private bool _isLooping;
private bool _isInitialized;
private GameObject? _quad;
private Material? _material;
// ReSharper disable once InconsistentNaming
public void Init(string videoPath, RenderTexture? targetTexture, bool loop, Camera? targetCamera, GameObject videoBGGo)
{
CleanUp();
_targetTexture = targetTexture;
_isLooping = loop;
if (_targetTexture == null && targetCamera != null)
{
SetupBackgroundQuad(targetCamera, videoBGGo);
}
if (!File.Exists(videoPath))
{
Melon<RenderMod>.Logger.Warning($"[FFmpegVideoBGPlayer] Video file does not exist: {videoPath}");
return;
}
try
{
_decoder = new FFmpegNativeDecoder(videoPath);
_computeShader = ModAssets.ColorConversionShader;
var w = _decoder.Width;
var h = _decoder.Height;
var pixFmt = _decoder.PixelFormat;
var gpuInitSuccess = false;
if (_computeShader != null)
{
switch (pixFmt)
{
case AVPixelFormat.AV_PIX_FMT_NV12:
try
{
_kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureUV);
_gpuMode = GPUMode.NV12;
gpuInitSuccess = true;
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Initialized Compute Shader NV12 mode successfully");
}
catch
{
// ignored
}
break;
case AVPixelFormat.AV_PIX_FMT_YUV420P:
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
try
{
_kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
_textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureU);
ConfigureTexture(_textureV);
_gpuMode = GPUMode.YUV420P;
gpuInitSuccess = true;
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Initialized Compute Shader YUV420P mode successfully");
}
catch
{
// ignored
}
break;
default:
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Cannot use Compute Shader for colorspace conversion.");
break;
}
}
if (gpuInitSuccess && _computeShader != null)
{
_renderTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32)
{
enableRandomWrite = true
};
_renderTexture.Create();
}
else
{
_textureRGBA = new Texture2D(w, h, TextureFormat.RGBA32, false);
ConfigureTexture(_textureRGBA);
_gpuMode = GPUMode.None;
Melon<RenderMod>.Logger.Warning("[FFmpegVideoBGPlayer] GPU acceleration unsupported or failed to load, falling back to CPU (RGBA).");
}
_currentFrameTimestamp = 0.0;
_nextFrameTimestamp = 0.0;
_isInitialized = true;
Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] Successfully initialized - Resolution: {w}x{h}, FPS: {_decoder.FPS:F2}");
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[FFmpegVideoBGPlayer] Error during initialization: {ex}");
CleanUp();
}
}
private static void ConfigureTexture(Texture2D tex)
{
tex.wrapMode = TextureWrapMode.Clamp;
tex.filterMode = FilterMode.Bilinear;
}
private void LateUpdate()
{
if (!_isInitialized || _decoder == null) return;
var cond = scrConductor.instance;
if (!cond || !cond.hasSongStarted) return;
var pitch = cond.song.pitch;
var separateCountdownOffset = cond.separateCountdownTime ?
cond.crotchetAtStart * cond.adjustedCountdownTicks : 0.0;
var songTime = (cond.dspTime - cond.dspTimeSong) * pitch - cond.addoffset - separateCountdownOffset;
var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
var targetVideoTime = songTime + vidOffset;
if (targetVideoTime < 0)
{
if (_targetTexture)
{
var active = RenderTexture.active;
RenderTexture.active = _targetTexture;
GL.Clear(true, true, Color.black);
RenderTexture.active = active;
}
if (_quad)
{
_quad.SetActive(false);
}
return;
}
if (ADOBase.customLevel)
{
ADOBase.customLevel.ShowTutorialBackground(false);
}
try
{
if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0)
{
_decoder.SeekToTime(targetVideoTime);
_currentFrameTimestamp = targetVideoTime;
_nextFrameTimestamp = targetVideoTime;
}
bool frameUpdated = false;
while (targetVideoTime >= _nextFrameTimestamp)
{
double timestamp = 0.0;
bool success = false;
if (_gpuMode == GPUMode.NV12 && _textureY && _textureUV)
{
success = _decoder.TryReadNV12Frame(
_textureY.GetRawTextureData<byte>(),
_textureUV.GetRawTextureData<byte>(),
out timestamp
);
}
else if (_gpuMode == GPUMode.YUV420P && _textureY && _textureU && _textureV)
{
success = _decoder.TryReadYUV420PFrame(
_textureY.GetRawTextureData<byte>(),
_textureU.GetRawTextureData<byte>(),
_textureV.GetRawTextureData<byte>(),
out timestamp
);
}
else
{
if (_textureRGBA != null)
{
success = _decoder.TryReadFrame(
_textureRGBA.GetRawTextureData<byte>(),
vflip: true,
out timestamp
);
}
}
if (success)
{
_currentFrameTimestamp = timestamp;
_nextFrameTimestamp = timestamp + (1.0 / _decoder.FPS);
frameUpdated = true;
}
else
{
if (_isLooping)
{
_decoder.SeekToTime(0);
_currentFrameTimestamp = 0;
_nextFrameTimestamp = 1.0 / _decoder.FPS;
break;
}
else
{
break;
}
}
}
if (frameUpdated)
{
RenderToTarget();
}
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[FFmpegVideoBGPlayer] Error during playback update: {ex}");
_isInitialized = false;
}
}
private void RenderToTarget()
{
if (_decoder == null) return;
int w = _decoder.Width;
int h = _decoder.Height;
if (_gpuMode == GPUMode.NV12 && _textureY && _textureUV && _computeShader && _renderTexture)
{
_textureY.Apply();
_textureUV.Apply();
_computeShader.SetTexture(_kernelIndex, YTex, _textureY);
_computeShader.SetTexture(_kernelIndex, UVTex, _textureUV);
_computeShader.SetTexture(_kernelIndex, Result, _renderTexture);
_computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8;
_computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
if (_targetTexture)
{
Graphics.Blit(_renderTexture, _targetTexture);
}
else if (_material)
{
if (_material.mainTexture != _renderTexture)
{
_material.mainTexture = _renderTexture;
}
if (_quad != null && !_quad.activeSelf)
{
_quad.SetActive(true);
}
}
}
else if (_gpuMode == GPUMode.YUV420P && _textureY && _textureU && _textureV && _computeShader && _renderTexture)
{
_textureY.Apply();
_textureU.Apply();
_textureV.Apply();
_computeShader.SetTexture(_kernelIndex, YTex, _textureY);
_computeShader.SetTexture(_kernelIndex, UTex, _textureU);
_computeShader.SetTexture(_kernelIndex, VTex, _textureV);
_computeShader.SetTexture(_kernelIndex, Result, _renderTexture);
_computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8;
_computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
if (_targetTexture)
{
Graphics.Blit(_renderTexture, _targetTexture);
}
else if (_material)
{
if (_material.mainTexture != _renderTexture)
{
_material.mainTexture = _renderTexture;
}
if (_quad != null && !_quad.activeSelf)
{
_quad.SetActive(true);
}
}
}
else if (_textureRGBA)
{
_textureRGBA.Apply();
if (_targetTexture)
{
Graphics.Blit(_textureRGBA, _targetTexture);
}
else if (_material)
{
if (_material.mainTexture != _textureRGBA)
{
_material.mainTexture = _textureRGBA;
}
if (_quad != null && !_quad.activeSelf)
{
_quad.SetActive(true);
}
}
}
}
private void OnDestroy()
{
CleanUp();
}
public void CleanUp()
{
_isInitialized = false;
if (_decoder != null)
{
_decoder.Dispose();
_decoder = null;
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Decoder successfully disposed.");
}
if (_textureRGBA != null)
{
Destroy(_textureRGBA);
_textureRGBA = null;
}
if (_textureY != null) Destroy(_textureY);
if (_textureU != null) Destroy(_textureU);
if (_textureV != null) Destroy(_textureV);
if (_textureUV != null) Destroy(_textureUV);
_textureY = null;
_textureU = null;
_textureV = null;
_textureUV = null;
if (_renderTexture != null)
{
_renderTexture.Release();
Destroy(_renderTexture);
_renderTexture = null;
}
if (_quad != null)
{
Destroy(_quad);
_quad = null;
}
if (_material != null)
{
Destroy(_material);
_material = null;
}
}
// ReSharper disable once InconsistentNaming
private void SetupBackgroundQuad(Camera targetCamera, GameObject videoBGGo)
{
_quad = GameObject.CreatePrimitive(PrimitiveType.Quad);
_quad.name = "FFmpegVideoBGQuad";
// ReSharper disable once Unity.UnresolvedComponentOrScriptableObject
var col = _quad.GetComponent("MeshCollider");
if (col) Destroy(col);
_quad.transform.parent = targetCamera.transform;
var zPos = targetCamera.farClipPlane - 0.5f;
_quad.transform.localPosition = new Vector3(0f, 0f, zPos);
_quad.transform.localRotation = Quaternion.identity;
float height;
float width;
if (targetCamera.orthographic)
{
height = 2.0f * targetCamera.orthographicSize;
width = height * targetCamera.aspect;
}
else
{
height = 2.0f * zPos * Mathf.Tan(targetCamera.fieldOfView * 0.5f * Mathf.Deg2Rad);
width = height * targetCamera.aspect;
}
_quad.transform.localScale = new Vector3(width, height, 1.0f);
_quad.layer = videoBGGo.layer;
// Set vertex colors to white so Sprites/Default works properly (otherwise it may draw transparently)
var meshFilter = _quad.GetComponent<MeshFilter>();
if (meshFilter != null && meshFilter.mesh != null)
{
var mesh = meshFilter.mesh;
var colors = new Color[mesh.vertexCount];
for (var i = 0; i < colors.Length; i++)
{
colors[i] = Color.white;
}
mesh.colors = colors;
}
Shader? shader = null;
if (scrCamera.instance != null)
{
try
{
var camQuadMeshField = typeof(scrCamera).GetField("camQuadMesh", System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
if (camQuadMeshField != null)
{
var renderer = camQuadMeshField.GetValue(scrCamera.instance) as MeshRenderer;
if (renderer != null && renderer.sharedMaterial != null)
{
shader = renderer.sharedMaterial.shader;
}
}
}
catch (Exception)
{
// ignored
}
}
if (shader == null) shader = Shader.Find("Sprites/Default");
if (shader == null) shader = Shader.Find("Unlit/Texture");
if (shader == null) shader = Shader.Find("Standard");
_material = new Material(shader)
{
color = Color.white
};
_quad.GetComponent<MeshRenderer>().material = _material;
// Set inactive initially to prevent white flash
_quad.SetActive(false);
}
}
}
+7 -7
View File
@@ -1,5 +1,6 @@
using System;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
@@ -9,6 +10,8 @@ using MelonLoader;
namespace ADOFAIRenderer.Util;
[SuppressMessage("ReSharper", "InconsistentNaming")]
[SuppressMessage("ReSharper", "UnusedMember.Local")]
internal static class FolderDialogUtil
{
private static int _isDialogOpen;
@@ -187,14 +190,12 @@ internal static class FolderDialogUtil
private static string? OpenWindowsFolderBrowserModern(string initialDirectory, IntPtr ownerHandle)
{
IFileOpenDialog? dialog = null;
// 1. Try native CoCreateInstance first (works on Mono and .NET without class registration)
var clsid = CLSID_FileOpenDialog;
var iid = IID_IFileOpenDialog;
int createHr = CoCreateInstance(ref clsid, IntPtr.Zero, 1 /* CLSCTX_INPROC_SERVER */, ref iid, out dialog);
int createHr = CoCreateInstance(ref clsid, IntPtr.Zero, 1 /* CLSCTX_INPROC_SERVER */, ref iid, out var dialog);
if (createHr != 0 || dialog == null)
if (createHr != 0)
{
// 2. Fallback to Activator via Type.GetTypeFromCLSID
var comType = Type.GetTypeFromCLSID(CLSID_FileOpenDialog);
@@ -224,7 +225,7 @@ internal static class FolderDialogUtil
{
var shellItemGuid = IID_IShellItem;
int itemHr = SHCreateItemFromParsingName(normalized, IntPtr.Zero, ref shellItemGuid, out var initialItem);
if (itemHr == 0 && initialItem != null)
if (itemHr == 0)
{
try
{
@@ -263,7 +264,6 @@ internal static class FolderDialogUtil
}
dialog.GetResult(out var resultItem);
if (resultItem == null) return null;
try
{
@@ -319,7 +319,7 @@ internal static class FolderDialogUtil
if (Directory.Exists(normalized))
{
initialDirPtr = Marshal.StringToHGlobalUni(normalized);
callback = (hwnd, uMsg, lParam, lpData) =>
callback = (hwnd, uMsg, _, lpData) =>
{
if (uMsg == BFFM_INITIALIZED && lpData != IntPtr.Zero)
{
+28 -46
View File
@@ -71,36 +71,30 @@ namespace ADOFAIRenderer.Util
state.IsInFlight = true;
}
// Check if GPU color conversion is enabled
bool useGPUConversion = ModAssets.ColorConversionShader;
AVPixelFormat expectedFmt = AVPixelFormat.AV_PIX_FMT_RGBA;
// Check if GPU color conversion is available
var expectedFmt = AVPixelFormat.AV_PIX_FMT_RGBA;
if (useGPUConversion)
var kernelIndex = -1;
if (ModAssets.ColorConversionShader)
{
expectedFmt = _targetFormat;
// Compute Shader only supports NV12 and YUV420P; fallback to CPU for other formats
if (expectedFmt != AVPixelFormat.AV_PIX_FMT_NV12 && expectedFmt != AVPixelFormat.AV_PIX_FMT_YUV420P)
var kernalName = _targetFormat switch
{
useGPUConversion = false;
}
AVPixelFormat.AV_PIX_FMT_NV12 => "CSMain_NV12",
AVPixelFormat.AV_PIX_FMT_YUV420P => "CSMain_YUV420P",
_ => null
};
if (kernalName != null)
kernelIndex = ModAssets.ColorConversionShader.FindKernel(kernalName);
}
var useGPUConversion = kernelIndex >= 0;
if (useGPUConversion) expectedFmt = _targetFormat;
var bufferSize = texture.width * texture.height * 4; // RGBA = 4 channel
state.Setup(currentFrame, bufferSize, useGPUConversion, expectedFmt, texture.width, texture.height);
if (useGPUConversion && state.GPUBuffer != null && ModAssets.ColorConversionShader)
{
int kernelIndex = -1;
if (expectedFmt == AVPixelFormat.AV_PIX_FMT_NV12)
{
kernelIndex = ModAssets.ColorConversionShader.FindKernel("CSMain_NV12");
}
else if (expectedFmt == AVPixelFormat.AV_PIX_FMT_YUV420P)
{
kernelIndex = ModAssets.ColorConversionShader.FindKernel("CSMain_YUV420P");
}
if (kernelIndex >= 0)
{
ModAssets.ColorConversionShader.SetTexture(kernelIndex, SourceTexture, texture);
ModAssets.ColorConversionShader.SetBuffer(kernelIndex, DestinationBuffer, state.GPUBuffer);
@@ -114,29 +108,18 @@ namespace ADOFAIRenderer.Util
ModAssets.ColorConversionShader.Dispatch(kernelIndex, threadGroupsX, threadGroupsY, 1);
AsyncGPUReadback.Request(
AsyncGPUReadback.RequestIntoNativeArray(
ref state.NativeBuffer, state.GPUBuffer, state.CachedCallback
);
/*AsyncGPUReadback.Request(
state.GPUBuffer,
state.CachedCallback
);
}
else
{
// If kernel is not found, fallback to standard RGBA texture readback
AsyncGPUReadback.RequestIntoNativeArray(
ref state.NativeBuffer,
texture,
0,
state.CachedCallback
);
}
);*/
}
else
{
AsyncGPUReadback.RequestIntoNativeArray(
ref state.NativeBuffer,
texture,
0,
state.CachedCallback
ref state.NativeBuffer, texture, 0, state.CachedCallback
);
}
}
@@ -169,16 +152,14 @@ namespace ADOFAIRenderer.Util
}
Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame.");
var byteSize = state.IsGPUConversion
? state.Width * state.Height * 3 / 2
: state.Width * state.Height * 4;
var byteSize = state.Width * state.Height * 3 / 2;
var blackFrame = ArrayPool<byte>.Shared.Rent(byteSize);
Array.Clear(blackFrame, 0, byteSize);
var capFrame = new RenderManager.CapturedFrame
{
ManagedBuffer = blackFrame,
Length = byteSize
ManagedBuffer = blackFrame, Length = byteSize,
IsColorConverted = true
};
PushToQueue(frame, capFrame);
RecycleState(state);
@@ -187,19 +168,20 @@ namespace ADOFAIRenderer.Util
{
unsafe
{
/*
if (state.IsGPUConversion)
{
var data = request.GetData<byte>();
// Copy async request buffer data to persistent NativeBuffer memory
NativeArray<byte>.Copy(data, state.NativeBuffer, data.Length);
}
*/
var ptr = (IntPtr)state.NativeBuffer.GetUnsafeReadOnlyPtr();
var capFrame = new RenderManager.CapturedFrame
{
NativePointer = ptr,
Length = state.NativeBuffer.Length,
ReadbackState = state
NativePointer = ptr, Length = state.NativeBuffer.Length,
ReadbackState = state, IsColorConverted = state.IsGPUConversion
};
PushToQueue(frame, capFrame);
}
+517
View File
@@ -0,0 +1,517 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using ADOFAIRenderer.UI;
using ADOFAIRenderer.Util.FFmpeg;
using MelonLoader;
using UnityEngine;
using FFmpeg.AutoGen;
namespace ADOFAIRenderer.Util;
// ReSharper disable once InconsistentNaming
internal class VideoBGPlayer : MonoBehaviour
{
private static readonly int YTex = Shader.PropertyToID("_YTex");
private static readonly int UVTex = Shader.PropertyToID("_UVTex");
private static readonly int Result = Shader.PropertyToID("_Result");
private static readonly int Width = Shader.PropertyToID("Width");
private static readonly int Height = Shader.PropertyToID("Height");
private static readonly int UTex = Shader.PropertyToID("_UTex");
private static readonly int VTex = Shader.PropertyToID("_VTex");
private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
[SuppressMessage("ReSharper", "InconsistentNaming")]
private enum GPUMode
{
None,
YUV420P,
NV12
}
private FFmpegNativeDecoder? _decoder;
// CPU fallback rendering
// ReSharper disable once InconsistentNaming
private Texture2D? _textureRGBA;
// GPU conversion textures
private Texture2D? _textureY;
private Texture2D? _textureU;
private Texture2D? _textureV;
private Texture2D? _textureUV;
// Compute Shader variables
private ComputeShader? _computeShader;
private RenderTexture? _renderTexture;
private int _kernelIndex = -1;
private double _currentFrameTimestamp;
private double _nextFrameTimestamp;
private GPUMode _gpuMode = GPUMode.None;
private RenderTexture? _targetTexture;
private bool _isLooping;
private bool _isInitialized;
private GameObject? _quad;
private Material? _material;
// ReSharper disable once InconsistentNaming
public void Init(string videoPath, RenderTexture? targetTexture, bool loop, Camera? targetCamera, GameObject videoBGGo)
{
CleanUp();
_targetTexture = targetTexture;
_isLooping = loop;
if (_targetTexture == null && targetCamera != null)
{
SetupBackgroundQuad(targetCamera, videoBGGo);
}
if (!File.Exists(videoPath))
{
Melon<RenderMod>.Logger.Warning($"[FFmpegVideoBGPlayer] Video file does not exist: {videoPath}");
return;
}
try
{
_decoder = new FFmpegNativeDecoder(videoPath);
_computeShader = ModAssets.ColorConversionShader;
var w = _decoder.Width;
var h = _decoder.Height;
var pixFmt = _decoder.PixelFormat;
var gpuInitSuccess = false;
if (_computeShader != null)
{
switch (pixFmt)
{
case AVPixelFormat.AV_PIX_FMT_NV12:
try
{
_kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureUV);
_gpuMode = GPUMode.NV12;
gpuInitSuccess = true;
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Initialized Compute Shader NV12 mode successfully");
}
catch
{
// ignored
}
break;
case AVPixelFormat.AV_PIX_FMT_YUV420P:
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
try
{
_kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
_textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureU);
ConfigureTexture(_textureV);
_gpuMode = GPUMode.YUV420P;
gpuInitSuccess = true;
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Initialized Compute Shader YUV420P mode successfully");
}
catch
{
// ignored
}
break;
default:
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Cannot use Compute Shader for colorspace conversion.");
break;
}
}
if (gpuInitSuccess && _computeShader != null)
{
_renderTexture = new RenderTexture(w, h, 0, RenderTextureFormat.ARGB32)
{
enableRandomWrite = true
};
_renderTexture.Create();
}
else
{
_textureRGBA = new Texture2D(w, h, TextureFormat.RGBA32, false);
ConfigureTexture(_textureRGBA);
_gpuMode = GPUMode.None;
Melon<RenderMod>.Logger.Warning("[FFmpegVideoBGPlayer] GPU acceleration unsupported or failed to load, falling back to CPU (RGBA).");
}
_currentFrameTimestamp = 0.0;
_nextFrameTimestamp = 0.0;
_isInitialized = true;
Melon<RenderMod>.Logger.Msg($"[FFmpegVideoBGPlayer] Successfully initialized - Resolution: {w}x{h}, FPS: {_decoder.FPS:F2}");
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[FFmpegVideoBGPlayer] Error during initialization: {ex}");
CleanUp();
}
}
private static void ConfigureTexture(Texture2D tex)
{
tex.wrapMode = TextureWrapMode.Clamp;
tex.filterMode = FilterMode.Bilinear;
}
private void LateUpdate()
{
if (!_isInitialized || _decoder == null) return;
var cond = scrConductor.instance;
if (!cond || !cond.hasSongStarted) return;
var pitch = cond.song.pitch;
var separateCountdownOffset = cond.separateCountdownTime ?
cond.crotchetAtStart * cond.adjustedCountdownTicks : 0.0;
var songTime = (cond.dspTime - cond.dspTimeSong) * pitch - cond.addoffset - separateCountdownOffset;
var vidOffset = scrVfxPlus.instance ? scrVfxPlus.instance.vidOffset : 0.0;
var targetVideoTime = songTime + vidOffset;
if (targetVideoTime < 0)
{
if (_targetTexture)
{
var active = RenderTexture.active;
RenderTexture.active = _targetTexture;
GL.Clear(true, true, Color.black);
RenderTexture.active = active;
}
if (_quad)
{
_quad.SetActive(false);
}
return;
}
if (ADOBase.customLevel)
{
ADOBase.customLevel.ShowTutorialBackground(false);
}
try
{
if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0)
{
_decoder.SeekToTime(targetVideoTime);
_currentFrameTimestamp = targetVideoTime;
_nextFrameTimestamp = targetVideoTime;
}
bool frameUpdated = false;
while (targetVideoTime >= _nextFrameTimestamp)
{
double timestamp = 0.0;
bool success = false;
if (_gpuMode == GPUMode.NV12 && _textureY && _textureUV)
{
success = _decoder.TryReadNV12Frame(
_textureY.GetRawTextureData<byte>(),
_textureUV.GetRawTextureData<byte>(),
out timestamp
);
}
else if (_gpuMode == GPUMode.YUV420P && _textureY && _textureU && _textureV)
{
success = _decoder.TryReadYUV420PFrame(
_textureY.GetRawTextureData<byte>(),
_textureU.GetRawTextureData<byte>(),
_textureV.GetRawTextureData<byte>(),
out timestamp
);
}
else
{
if (_textureRGBA != null)
{
success = _decoder.TryReadFrame(
_textureRGBA.GetRawTextureData<byte>(),
vflip: true,
out timestamp
);
}
}
if (success)
{
_currentFrameTimestamp = timestamp;
_nextFrameTimestamp = timestamp + (1.0 / _decoder.FPS);
frameUpdated = true;
}
else
{
if (_isLooping)
{
_decoder.SeekToTime(0);
_currentFrameTimestamp = 0;
_nextFrameTimestamp = 1.0 / _decoder.FPS;
break;
}
else
{
break;
}
}
}
if (frameUpdated)
{
RenderToTarget();
}
}
catch (Exception ex)
{
Melon<RenderMod>.Logger.Error($"[FFmpegVideoBGPlayer] Error during playback update: {ex}");
_isInitialized = false;
}
}
private void RenderToTarget()
{
if (_decoder == null) return;
int w = _decoder.Width;
int h = _decoder.Height;
if (_gpuMode == GPUMode.NV12 && _textureY && _textureUV && _computeShader && _renderTexture)
{
_textureY.Apply();
_textureUV.Apply();
_computeShader.SetTexture(_kernelIndex, YTex, _textureY);
_computeShader.SetTexture(_kernelIndex, UVTex, _textureUV);
_computeShader.SetTexture(_kernelIndex, Result, _renderTexture);
_computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8;
_computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
if (_targetTexture)
{
Graphics.Blit(_renderTexture, _targetTexture);
}
else if (_material)
{
if (_material.mainTexture != _renderTexture)
{
_material.mainTexture = _renderTexture;
}
if (_quad != null && !_quad.activeSelf)
{
_quad.SetActive(true);
}
}
}
else if (_gpuMode == GPUMode.YUV420P && _textureY && _textureU && _textureV && _computeShader && _renderTexture)
{
_textureY.Apply();
_textureU.Apply();
_textureV.Apply();
_computeShader.SetTexture(_kernelIndex, YTex, _textureY);
_computeShader.SetTexture(_kernelIndex, UTex, _textureU);
_computeShader.SetTexture(_kernelIndex, VTex, _textureV);
_computeShader.SetTexture(_kernelIndex, Result, _renderTexture);
_computeShader.SetInt(Width, w);
_computeShader.SetInt(Height, h);
_computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
int threadGroupsX = (w + 7) / 8;
int threadGroupsY = (h + 7) / 8;
_computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
if (_targetTexture)
{
Graphics.Blit(_renderTexture, _targetTexture);
}
else if (_material)
{
if (_material.mainTexture != _renderTexture)
{
_material.mainTexture = _renderTexture;
}
if (_quad != null && !_quad.activeSelf)
{
_quad.SetActive(true);
}
}
}
else if (_textureRGBA)
{
_textureRGBA.Apply();
if (_targetTexture)
{
Graphics.Blit(_textureRGBA, _targetTexture);
}
else if (_material)
{
if (_material.mainTexture != _textureRGBA)
{
_material.mainTexture = _textureRGBA;
}
if (_quad != null && !_quad.activeSelf)
{
_quad.SetActive(true);
}
}
}
}
private void OnDestroy()
{
CleanUp();
}
public void CleanUp()
{
_isInitialized = false;
if (_decoder != null)
{
_decoder.Dispose();
_decoder = null;
Melon<RenderMod>.Logger.Msg("[FFmpegVideoBGPlayer] Decoder successfully disposed.");
}
if (_textureRGBA != null)
{
Destroy(_textureRGBA);
_textureRGBA = null;
}
if (_textureY != null) Destroy(_textureY);
if (_textureU != null) Destroy(_textureU);
if (_textureV != null) Destroy(_textureV);
if (_textureUV != null) Destroy(_textureUV);
_textureY = null;
_textureU = null;
_textureV = null;
_textureUV = null;
if (_renderTexture != null)
{
_renderTexture.Release();
Destroy(_renderTexture);
_renderTexture = null;
}
if (_quad != null)
{
Destroy(_quad);
_quad = null;
}
if (_material != null)
{
Destroy(_material);
_material = null;
}
}
// ReSharper disable once InconsistentNaming
private void SetupBackgroundQuad(Camera targetCamera, GameObject videoBGGo)
{
_quad = GameObject.CreatePrimitive(PrimitiveType.Quad);
_quad.name = "FFmpegVideoBGQuad";
// ReSharper disable once Unity.UnresolvedComponentOrScriptableObject
var col = _quad.GetComponent("MeshCollider");
if (col) Destroy(col);
_quad.transform.parent = targetCamera.transform;
var zPos = targetCamera.farClipPlane - 0.5f;
_quad.transform.localPosition = new Vector3(0f, 0f, zPos);
_quad.transform.localRotation = Quaternion.identity;
float height;
float width;
if (targetCamera.orthographic)
{
height = 2.0f * targetCamera.orthographicSize;
width = height * targetCamera.aspect;
}
else
{
height = 2.0f * zPos * Mathf.Tan(targetCamera.fieldOfView * 0.5f * Mathf.Deg2Rad);
width = height * targetCamera.aspect;
}
_quad.transform.localScale = new Vector3(width, height, 1.0f);
_quad.layer = videoBGGo.layer;
// Set vertex colors to white so Sprites/Default works properly (otherwise it may draw transparently)
var meshFilter = _quad.GetComponent<MeshFilter>();
if (meshFilter != null && meshFilter.mesh != null)
{
var mesh = meshFilter.mesh;
var colors = new Color[mesh.vertexCount];
for (var i = 0; i < colors.Length; i++)
{
colors[i] = Color.white;
}
mesh.colors = colors;
}
Shader? shader = null;
if (scrCamera.instance != null)
{
try
{
var camQuadMeshField = typeof(scrCamera).GetField("camQuadMesh", System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
if (camQuadMeshField != null)
{
var renderer = camQuadMeshField.GetValue(scrCamera.instance) as MeshRenderer;
if (renderer != null && renderer.sharedMaterial != null)
{
shader = renderer.sharedMaterial.shader;
}
}
}
catch (Exception)
{
// ignored
}
}
if (shader == null) shader = Shader.Find("Sprites/Default");
if (shader == null) shader = Shader.Find("Unlit/Texture");
if (shader == null) shader = Shader.Find("Standard");
_material = new Material(shader)
{
color = Color.white
};
_quad.GetComponent<MeshRenderer>().material = _material;
// Set inactive initially to prevent white flash
_quad.SetActive(false);
}
}