Document Render Core And API

Util/VideoBGPlayer.cs

@@ -9,9 +9,25 @@ using FFmpeg.AutoGen;
 
 namespace ADOFAIRenderer.Util;
 
+/// <summary>
+/// Plays a custom level's background video with FFmpeg (FFmpegNativeDecoder) in place of Unity's VideoPlayer while a
+/// render is active, so the video frame shown always matches the render clock. Attached to the scnGame.videoBG object
+/// by scnGamePatch.ReplaceVideo, which disables the native VideoPlayer; removed again by that patch when no render is
+/// active.
+/// <para>
+/// Every LateUpdate it computes the video time from the conductor (the same formula scrVfxPlus uses to start and
+/// position the native video), decodes frames up to that time and draws the newest one either into the VideoPlayer's
+/// target RenderTexture or onto a quad at the far plane of the background camera.
+/// Decoded YUV frames are converted to RGBA on the GPU (ColorConversion.compute, kernels CSMain_NV12ToRGBA /
+/// CSMain_YUV420PToRGBA); other pixel formats, or a missing shader, use the decoder's RGBA output (CPU conversion).
+/// Main thread only.
+/// </para>
+/// </summary>
 // ReSharper disable once InconsistentNaming
 internal class VideoBGPlayer : MonoBehaviour
 {
+    // Property IDs of the compute shader parameters: input planes, output texture, frame size in pixels,
+    // ColorRange (1 = full range, 0 = limited range) and ColorMatrix (FFmpegNativeDecoder.ColorMatrix).
     private static readonly int YTex = Shader.PropertyToID("_YTex");
     private static readonly int UVTex = Shader.PropertyToID("_UVTex");
     private static readonly int Result = Shader.PropertyToID("_Result");
@@ -22,6 +38,10 @@ internal class VideoBGPlayer : MonoBehaviour
     private static readonly int ColorRange = Shader.PropertyToID("ColorRange");
     private static readonly int ColorMatrix = Shader.PropertyToID("ColorMatrix");
 
+    /// <summary>
+    /// Conversion path of the current video: <c>None</c> = CPU RGBA texture, <c>YUV420P</c> = separate Y, U, V planes,
+    /// <c>NV12</c> = Y plane and interleaved UV plane.
+    /// </summary>
     [SuppressMessage("ReSharper", "InconsistentNaming")]
     private enum GPUMode
     {
@@ -30,6 +50,7 @@ internal class VideoBGPlayer : MonoBehaviour
         NV12
     }
 
+    /// <summary>Decoder of the current video; null when not initialized or after CleanUp.</summary>
     private FFmpegNativeDecoder? _decoder;
         
     // CPU fallback rendering
@@ -47,16 +68,34 @@ internal class VideoBGPlayer : MonoBehaviour
     private RenderTexture? _renderTexture;
     private int _kernelIndex = -1;
 
+    /// <summary>Presentation time in seconds of the frame currently shown (video time, 0 = start of the file).</summary>
     private double _currentFrameTimestamp;
+    /// <summary>Video time in seconds from which the next frame is decoded (current timestamp + 1 / FPS).</summary>
     private double _nextFrameTimestamp;
+    /// <summary>Conversion path selected by Init.</summary>
     private GPUMode _gpuMode = GPUMode.None;
+    /// <summary>Target RenderTexture of the native VideoPlayer; null when the video is shown on the background quad.</summary>
     private RenderTexture? _targetTexture;
+    /// <summary>Whether the level loops the video (miscSettings "loopVideo").</summary>
     private bool _isLooping;
+    /// <summary>True after a successful Init; set to false by CleanUp or a playback error (playback then stops).</summary>
     private bool _isInitialized;
 
+    /// <summary>Background quad created by SetupBackgroundQuad when there is no target texture; hidden until the first frame.</summary>
     private GameObject? _quad;
+    /// <summary>Material of the background quad; its main texture is the converted frame.</summary>
     private Material? _material;
 
+    /// <summary>
+    /// Releases any previous video and opens <paramref name="videoPath"/>. Allocates the plane textures for the GPU path
+    /// of the decoder's pixel format (chroma planes are (width + 1) / 2 x (height + 1) / 2), or an RGBA texture for the
+    /// CPU path. Errors are logged; the player then stays uninitialized and shows nothing.
+    /// </summary>
+    /// <param name="videoPath">Full path of the video file.</param>
+    /// <param name="targetTexture">RenderTexture the native VideoPlayer renders into, or null to draw on a background quad.</param>
+    /// <param name="loop">Whether the video loops.</param>
+    /// <param name="targetCamera">Camera the background quad is attached to when <paramref name="targetTexture"/> is null.</param>
+    /// <param name="videoBGGo">VideoPlayer object; its layer is given to the background quad.</param>
     // ReSharper disable once InconsistentNaming
     public void Init(string videoPath, RenderTexture? targetTexture, bool loop, Camera? targetCamera, GameObject videoBGGo)
     {
@@ -169,12 +208,19 @@ internal class VideoBGPlayer : MonoBehaviour
         }
     }
 
+    /// <summary>Sets clamp wrapping and bilinear filtering.</summary>
     private static void ConfigureTexture(Texture2D tex)
     {
         tex.wrapMode = TextureWrapMode.Clamp;
         tex.filterMode = FilterMode.Bilinear;
     }
 
+    /// <summary>
+    /// Main thread, every frame after the song has started. Computes the video time, seeks when it moved backwards or more
+    /// than 1 s ahead of the shown frame, decodes frames until the next frame lies after that time, and draws the last
+    /// decoded frame. Before the video starts (negative time) the target texture is cleared to black and the quad hidden.
+    /// When decoding fails at the end of the file, a looping video seeks to 0; otherwise the last frame stays.
+    /// </summary>
     private void LateUpdate()
     {
         if (!_isInitialized || _decoder == null) return;
@@ -182,6 +228,9 @@ internal class VideoBGPlayer : MonoBehaviour
         var cond = scrConductor.instance;
         if (!cond || !cond.hasSongStarted) return;
             
+        // Song position in seconds of song time, as scrConductor computes songposition_minusi from dspTime with the input
+        // offset calibration_i = 0 (the render sets the input offset to 0), minus the separate countdown length.
+        // scrVfxPlus starts the native video at songposition_minusi - countdown + vidOffset (scrVfxPlus.cs).
         var pitch = cond.song.pitch;
         var separateCountdownOffset = cond.separateCountdownTime ? 
             cond.crotchetAtStart * cond.adjustedCountdownTicks : 0.0;
@@ -215,6 +264,7 @@ internal class VideoBGPlayer : MonoBehaviour
         }
 
 
+        // scrVfxPlus hides the tutorial (editor) background when the native video starts (scrVfxPlus.cs).
         if (ADOBase.customLevel)
         {
             ADOBase.customLevel.ShowTutorialBackground(false);
@@ -222,6 +272,7 @@ internal class VideoBGPlayer : MonoBehaviour
 
         try
         {
+            // A seek is cheaper than decoding many frames when the time jumped forward by more than 1 s.
             if (targetVideoTime < _currentFrameTimestamp || targetVideoTime - _currentFrameTimestamp > 1.0)
             {
                 _decoder.SeekToTime(targetVideoTime);
@@ -298,6 +349,10 @@ internal class VideoBGPlayer : MonoBehaviour
         }
     }
 
+    /// <summary>
+    /// Uploads the decoded planes, converts them to RGBA with the compute shader (or uploads the RGBA texture), and shows
+    /// the result: blitted into the target texture, or set as the quad's main texture (the quad is activated).
+    /// </summary>
     private void RenderToTarget()
     {
         if (_decoder == null) return;
@@ -318,6 +373,7 @@ internal class VideoBGPlayer : MonoBehaviour
             _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
             _computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
 
+            // One thread group per 8 x 8 output pixels, rounded up (group size expected by the ColorConversion.compute kernels).
             int threadGroupsX = (w + 7) / 8;
             int threadGroupsY = (h + 7) / 8;
             _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
@@ -353,6 +409,7 @@ internal class VideoBGPlayer : MonoBehaviour
             _computeShader.SetInt(ColorRange, _decoder.ColorRange == AVColorRange.AVCOL_RANGE_JPEG ? 1 : 0);
             _computeShader.SetInt(ColorMatrix, _decoder.ColorMatrix);
 
+            // One thread group per 8 x 8 output pixels, rounded up (group size expected by the ColorConversion.compute kernels).
             int threadGroupsX = (w + 7) / 8;
             int threadGroupsY = (h + 7) / 8;
             _computeShader.Dispatch(_kernelIndex, threadGroupsX, threadGroupsY, 1);
@@ -394,11 +451,16 @@ internal class VideoBGPlayer : MonoBehaviour
         }
     }
 
+    /// <summary>Releases the decoder and GPU resources when the component is destroyed.</summary>
     private void OnDestroy()
     {
         CleanUp();
     }
 
+    /// <summary>
+    /// Disposes the decoder and destroys the textures, the RenderTexture, the background quad and its material.
+    /// The player is uninitialized afterwards. Called by Init, on errors and on destroy.
+    /// </summary>
     public void CleanUp()
     {
         _isInitialized = false;
@@ -446,6 +508,12 @@ internal class VideoBGPlayer : MonoBehaviour
         }
     }
 
+    /// <summary>
+    /// Creates a quad as a child of <paramref name="targetCamera"/>, 0.5 units in front of its far clip plane, scaled to
+    /// fill the view (orthographic size or field of view), on the layer of <paramref name="videoBGGo"/>. Its shader is
+    /// taken from scrCamera's camera quad material when available, otherwise the first of Sprites/Default, Unlit/Texture
+    /// and Standard that exists. The quad starts inactive.
+    /// </summary>
     // ReSharper disable once InconsistentNaming
     private void SetupBackgroundQuad(Camera targetCamera, GameObject videoBGGo)
     {