diff --git a/Util/FFmpeg/FFmpegNativeDecoder.cs b/Util/FFmpeg/FFmpegNativeDecoder.cs
index 3a4611e..a1b8c5f 100644
--- a/Util/FFmpeg/FFmpegNativeDecoder.cs
+++ b/Util/FFmpeg/FFmpegNativeDecoder.cs
@@ -1,23 +1,190 @@
using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
using FFmpeg.AutoGen;
+using MelonLoader;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
namespace ADOFAIRenderer.Util.FFmpeg;
///
-/// Decodes video files in a streaming fashion using FFmpeg's native C API.
-/// Unlike a standalone player, it allows reading frames sequentially or seeking/skipping forward on demand.
+/// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke.
+/// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and
+/// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback.
+///
+/// ── Architecture & Lifecycle Flow ──────────────────────────────────────────────
+///
+/// 1. Construction & Demuxing:
+/// a) avformat_open_input: Open container file and parse stream headers.
+/// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS.
+/// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO).
+///
+/// 2. Codec & Hardware Device Discovery:
+/// a) avcodec_find_decoder: Locate codec implementation by codec ID.
+/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder.
+/// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox).
+/// d) Attach hw_device_ctx and register custom get_format callback.
+/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding
+/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME).
+/// f) avcodec_open2: Initialize codec context.
+///
+/// 3. Frame Demuxing & Decoding (TryRead*):
+/// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet.
+/// b) (Decoding) avcodec_receive_frame retrieves decoded frames.
+/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data
+/// into system RAM as NV12 (av_frame_copy_props preserves timestamps).
+/// d) (Output Dispatch):
+/// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback.
+/// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback.
+/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
+///
+/// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame):
+/// a) av_seek_frame: Fast backward keyframe seek.
+/// b) avcodec_flush_buffers: Clear internal decoder state.
+/// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead;
+/// transfers and caches the first matching frame in _pCachedFrame for immediate consumption.
+///
+/// 5. Resource Disposal (Dispose):
+/// a) Free scaler context (sws_freeContext).
+/// b) Free all allocated frames and packets (av_frame_free, av_packet_free).
+/// c) Free codec context (avcodec_free_context).
+/// d) Release hardware device context (av_buffer_unref).
+/// e) Close format context (avformat_close_input).
+///
+/// ── Reference Documentation ────────────────────────────────────────────────────
+/// • FFmpeg Demuxing & Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
+/// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
+/// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
+/// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html
+/// • libswscale Scaling & Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html
///
internal sealed unsafe class FFmpegNativeDecoder : IDisposable
{
+ // ──────────────────────────────────────────────────────────────
+ // Hardware Acceleration Priorities & Candidate Structure
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Candidate hardware device configuration discovered via avcodec_get_hw_config.
+ ///
+ private struct HwConfigCandidate
+ {
+ public AVHWDeviceType DeviceType;
+ public AVPixelFormat PixelFormat;
+ }
+
+ ///
+ /// Preferred hardware device types in priority order.
+ /// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration,
+ /// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA.
+ /// Non-Windows systems fall back to VAAPI or VideoToolbox.
+ ///
+ private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
+ {
+ AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA,
+ AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2,
+ AVHWDeviceType.AV_HWDEVICE_TYPE_QSV,
+ AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA,
+ AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA,
+ AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI,
+ AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX
+ };
+
+ // ──────────────────────────────────────────────────────────────
+ // FFmpeg Core Struct Pointers
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Context managing container format demuxing (MP4, MKV, WebM, etc.).
+ /// Used for packet demuxing and seeking.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
+ ///
private AVFormatContext* _pFormatContext = null;
+
+ ///
+ /// Context managing video decoder state and parameters.
+ /// Configured with codec parameters, hardware contexts, and threading options.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
+ ///
private AVCodecContext* _pCodecContext = null;
+
+ ///
+ /// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.).
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html
+ ///
+ private AVBufferRef* _hwDeviceCtx = null;
+
+ ///
+ /// Frame structure holding raw decoded frames received directly from avcodec_receive_frame.
+ /// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView).
+ /// In SW decoding mode, this contains pixel buffers in system memory.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
+ ///
private AVFrame* _pDecodedFrame = null;
+
+ ///
+ /// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces
+ /// via av_hwframe_transfer_data.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
+ ///
+ private AVFrame* _pHwTransferFrame = null;
+
+ ///
+ /// Frame structure holding converted pixel data produced by sws_scale (e.g., RGBA or fallback YUV).
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
+ ///
private AVFrame* _pConvertedFrame = null;
- private SwsContext* _pSwsContext = null;
+
+ ///
+ /// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation.
+ /// Consumed by the next TryRead* invocation.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
+ ///
+ private AVFrame* _pCachedFrame = null;
+ private bool _hasCachedFrame;
+
+ ///
+ /// Packet structure temporarily holding compressed demuxed data read from the input container.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html
+ ///
private AVPacket* _pPacket = null;
-
+
+ ///
+ /// libswscale colorspace and format conversion context.
+ /// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
+ ///
+ private SwsContext* _pSwsContext = null;
+
+ // ──────────────────────────────────────────────────────────────
+ // Hardware Acceleration State & Delegates
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code.
+ /// Passed to _pCodecContext->get_format.
+ ///
+ private AVCodecContext_get_format? _getFormatDelegate;
+
+ /// Whether hardware-accelerated decoding is currently active.
+ private bool _isHwDecoding;
+
+ /// Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV).
+ private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
+
+ /// Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD).
+ private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
+
+ ///
+ /// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12).
+ ///
+ private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
+
+ // ──────────────────────────────────────────────────────────────
+ // Playback, Stream & Scaler Synchronization Fields
+ // ──────────────────────────────────────────────────────────────
+
private int _videoStreamIndex = -1;
private long _currentFrameIndex;
private AVRational _timeBase;
@@ -26,16 +193,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
private bool _isDraining;
private readonly object _lock = new();
- // Cached frame used to hold matching frames after seeking
- private AVFrame* _pCachedFrame = null;
- private bool _hasCachedFrame;
-
- // Field to detect SwsContext updates
+ // Fields to detect SwsContext updates
private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
private int _srcWidth;
private int _srcHeight;
+ // ──────────────────────────────────────────────────────────────
+ // Public Properties
+ // ──────────────────────────────────────────────────────────────
+
/// Video width in pixels.
public int Width { get; private set; }
@@ -52,22 +219,123 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
public long TotalFrames { get; private set; }
/// Index of the currently read frame.
- public long CurrentFrameIndex => _currentFrameIndex;
+ public long CurrentFrameIndex
+ {
+ get
+ {
+ lock (_lock)
+ {
+ return _currentFrameIndex;
+ }
+ }
+ }
/// Current playback time in seconds.
- public double CurrentTime => _currentFrameIndex / FPS;
+ public double CurrentTime
+ {
+ get
+ {
+ lock (_lock)
+ {
+ return FPS > 0 ? _currentFrameIndex / FPS : 0;
+ }
+ }
+ }
- /// Pixel format of the original video stream.
- public AVPixelFormat PixelFormat => _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE;
+ ///
+ /// Pixel format of the video frames produced by the decoder.
+ /// When hardware decoding is active, returns the pixel format of the downloaded software frame
+ /// in system RAM (typically ) rather than the GPU surface format
+ /// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines.
+ /// In software mode, returns the codec context's pixel format (e.g., YUV420P).
+ ///
+ public AVPixelFormat PixelFormat
+ {
+ get
+ {
+ lock (_lock)
+ {
+ if (_isHwDecoding)
+ {
+ return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE
+ ? _downloadedPixFmt
+ : AVPixelFormat.AV_PIX_FMT_NV12;
+ }
- /// Color range of the original video stream.
- public AVColorRange ColorRange => _pCodecContext != null ? _pCodecContext->color_range : AVColorRange.AVCOL_RANGE_UNSPECIFIED;
+ return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE;
+ }
+ }
+ }
+ ///
+ /// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]).
+ ///
+ public AVColorRange ColorRange
+ {
+ get
+ {
+ lock (_lock)
+ {
+ if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED)
+ {
+ return _pCodecContext->color_range;
+ }
+ if (_pFormatContext != null && _videoStreamIndex >= 0)
+ {
+ return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range;
+ }
+ return AVColorRange.AVCOL_RANGE_UNSPECIFIED;
+ }
+ }
+ }
+
+ ///
+ /// Gets a value indicating whether hardware-accelerated decoding is currently active.
+ ///
+ public bool IsHardwareAccelerated
+ {
+ get
+ {
+ lock (_lock)
+ {
+ return _isHwDecoding;
+ }
+ }
+ }
+
+ ///
+ /// Gets the hardware device type in use for hardware acceleration, or null if software decoding.
+ ///
+ public AVHWDeviceType? HardwareDeviceType
+ {
+ get
+ {
+ lock (_lock)
+ {
+ return _isHwDecoding ? _hwDeviceType : null;
+ }
+ }
+ }
+
+ // ──────────────────────────────────────────────────────────────
+ // Initialization & Setup
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Initializes a new instance of and opens the specified video file.
+ /// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back
+ /// to multi-threaded software decoding if unavailable.
+ ///
+ /// Absolute or relative file path to the video file.
public FFmpegNativeDecoder(string videoPath)
{
Open(videoPath);
}
+ ///
+ /// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers.
+ ///
+ /// Path to video file.
private void Open(string videoPath)
{
lock (_lock)
@@ -76,6 +344,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
try
{
+ // 1. Open media container
+ // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
fixed (AVFormatContext** ppFormatCtx = &_pFormatContext)
{
int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null);
@@ -83,9 +353,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})");
}
+ // 2. Retrieve stream information
if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0)
throw new Exception("Failed to retrieve stream info.");
+ // 3. Find primary video stream
_videoStreamIndex = -1;
for (int i = 0; i < _pFormatContext->nb_streams; i++)
{
@@ -102,7 +374,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
var stream = _pFormatContext->streams[_videoStreamIndex];
_timeBase = stream->time_base;
- // Calculate FPS
+ // 4. Calculate FPS
if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0)
{
FPS = ffmpeg.av_q2d(stream->r_frame_rate);
@@ -116,11 +388,13 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
FPS = 30.0; // Fallback default
}
+ // 5. Find decoder by codec ID
var codecId = stream->codecpar->codec_id;
var decoder = ffmpeg.avcodec_find_decoder(codecId);
if (decoder == null)
throw new Exception($"Unsupported video codec: {codecId}");
+ // 6. Allocate codec context
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
if (_pCodecContext == null)
throw new Exception("Failed to allocate codec context.");
@@ -128,17 +402,28 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
throw new Exception("Failed to copy codec parameters.");
- // Multi-threading settings for improved decoding performance (auto)
- _pCodecContext->thread_count = 0;
- _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME;
+ // 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding
+ bool hwSuccess = TryInitializeHwDecoder(decoder, stream);
+ if (!hwSuccess)
+ {
+ Melon.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding.");
+ _isHwDecoding = false;
+ _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
- if (ffmpeg.avcodec_open2(_pCodecContext, decoder, null) < 0)
- throw new Exception("Failed to open codec.");
+ // Multi-threading settings for improved software decoding performance (auto thread count)
+ // Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
+ _pCodecContext->thread_count = 0;
+ _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME;
- Width = _pCodecContext->width;
- Height = _pCodecContext->height;
+ int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
+ if (openRet < 0)
+ throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}");
+ }
- // Calculate total frames and duration
+ Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width;
+ Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height;
+
+ // 8. Calculate total frames and duration
if (stream->nb_frames > 0)
{
TotalFrames = stream->nb_frames;
@@ -160,7 +445,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE
: (stream->duration > 0 ? stream->duration * ffmpeg.av_q2d(stream->time_base) : 0);
+ // 9. Allocate frame and packet buffers
+ // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
_pDecodedFrame = ffmpeg.av_frame_alloc();
+ _pHwTransferFrame = ffmpeg.av_frame_alloc();
_pConvertedFrame = ffmpeg.av_frame_alloc();
_pCachedFrame = ffmpeg.av_frame_alloc();
_pPacket = ffmpeg.av_packet_alloc();
@@ -177,13 +465,205 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
+ ///
+ /// Searches for supported hardware device configurations via avcodec_get_hw_config,
+ /// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA),
+ /// registers the get_format callback, and attempts to open the hardware decoder.
+ ///
+ /// Target decoder.
+ /// Source video stream.
+ /// True if hardware decoding was successfully configured and opened; false to fall back.
+ private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream)
+ {
+ // Collect all supported hardware configs from decoder
+ const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01;
+ var candidates = new List();
+ for (int i = 0; ; i++)
+ {
+ var config = ffmpeg.avcodec_get_hw_config(decoder, i);
+ if (config == null) break;
+
+ if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0)
+ {
+ candidates.Add(new HwConfigCandidate
+ {
+ DeviceType = config->device_type,
+ PixelFormat = config->pix_fmt
+ });
+ }
+ }
+
+ if (candidates.Count == 0)
+ {
+ return false;
+ }
+
+ // Order candidates according to preference
+ var orderedCandidates = new List();
+ foreach (var preferred in PreferredHwDeviceTypes)
+ {
+ foreach (var candidate in candidates)
+ {
+ if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate))
+ {
+ orderedCandidates.Add(candidate);
+ }
+ }
+ }
+
+ // Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes
+ foreach (var candidate in candidates)
+ {
+ if (!orderedCandidates.Contains(candidate))
+ {
+ orderedCandidates.Add(candidate);
+ }
+ }
+
+ // Try candidate hardware configurations in priority order
+ foreach (var candidate in orderedCandidates)
+ {
+ AVBufferRef* hwDeviceCtx = null;
+ int err;
+
+ // Device creation strategy
+ // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
+ if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV)
+ {
+ var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+ string childDeviceName = isWindows ? "d3d11va" : "vaapi";
+ AVDictionary* opts = null;
+ ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0);
+
+ err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0);
+ ffmpeg.av_dict_free(&opts);
+
+ if (err < 0)
+ {
+ err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
+ }
+ }
+ else
+ {
+ err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0);
+ }
+
+ if (err < 0 || hwDeviceCtx == null)
+ {
+ if (hwDeviceCtx != null)
+ {
+ ffmpeg.av_buffer_unref(&hwDeviceCtx);
+ hwDeviceCtx = null;
+ }
+ Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate...");
+ continue;
+ }
+
+ // Assign hardware context and format selection callback
+ _hwDeviceCtx = hwDeviceCtx;
+ _hwPixFmt = candidate.PixelFormat;
+ _hwDeviceType = candidate.DeviceType;
+ _isHwDecoding = true;
+ _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12;
+
+ _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx);
+ if (_pCodecContext->hw_device_ctx == null)
+ {
+ Melon.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate...");
+ var pHw = _hwDeviceCtx;
+ ffmpeg.av_buffer_unref(&pHw);
+ _hwDeviceCtx = null;
+ _isHwDecoding = false;
+ continue;
+ }
+
+ _getFormatDelegate = GetFormat;
+ _pCodecContext->get_format = _getFormatDelegate;
+
+ // Hardware decoding requires single-threaded decoding (thread_count = 1).
+ // Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function
+ // GPU decoders and causes surface pool allocation exhaustion or crashes in get_format.
+ _pCodecContext->thread_count = 1;
+
+ // Attempt opening codec context with hardware configuration
+ int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
+ if (openRet >= 0)
+ {
+ Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}");
+ return true;
+ }
+
+ // Failed to open codec with this hardware configuration, clean up and try next
+ Melon.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate...");
+
+ if (_pCodecContext != null)
+ {
+ var pCodecCtx = _pCodecContext;
+ ffmpeg.avcodec_free_context(&pCodecCtx);
+ _pCodecContext = null;
+ }
+
+ if (_hwDeviceCtx != null)
+ {
+ var pHw = _hwDeviceCtx;
+ ffmpeg.av_buffer_unref(&pHw);
+ _hwDeviceCtx = null;
+ }
+
+ _isHwDecoding = false;
+ _getFormatDelegate = null;
+
+ // Reallocate fresh codec context for subsequent attempts
+ _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
+ if (_pCodecContext == null)
+ throw new Exception("Failed to re-allocate codec context during fallback.");
+
+ if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
+ throw new Exception("Failed to copy codec parameters during fallback.");
+ }
+
+ return false;
+ }
+
+ ///
+ /// Callback invoked by libavcodec during decoding to negotiate the output pixel format.
+ /// Confirms the hardware surface format () if offered by the decoder,
+ /// or falls back to software pixel formats via avcodec_default_get_format.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
+ ///
+ private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts)
+ {
+ for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++)
+ {
+ if (*p == _hwPixFmt)
+ {
+ return *p;
+ }
+ }
+
+ Melon.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format.");
+ _isHwDecoding = false;
+ _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
+ return ffmpeg.avcodec_default_get_format(ctx, pixFmts);
+ }
+
+ // ──────────────────────────────────────────────────────────────
+ // Demuxing & Decoding Engine
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Reads packets from the container until a packet belonging to the video stream is found,
+ /// then sends it to the decoder via avcodec_send_packet.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
+ ///
+ /// True if a video packet was sent; false on EOF.
private bool ReadAndSendPacket()
{
while (true)
{
ffmpeg.av_packet_unref(_pPacket);
int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket);
- if (ret < 0) return false; // EOF or error
+ if (ret < 0) return false; // EOF or read error
if (_pPacket->stream_index == _videoStreamIndex)
{
@@ -198,6 +678,73 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
+ ///
+ /// Receives the next decoded video frame from the codec context.
+ /// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV),
+ /// it is automatically downloaded to system RAM via av_hwframe_transfer_data into _pHwTransferFrame,
+ /// preserving presentation timestamps via av_frame_copy_props.
+ ///
+ /// The software frame containing pixel data in system RAM.
+ /// True if a frame was successfully decoded and available; false on EOF or draining.
+ private bool TryReceiveFrame(out AVFrame* outFrame)
+ {
+ outFrame = null;
+
+ while (true)
+ {
+ int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
+ if (ret == 0)
+ {
+ // Check if the received frame is on a hardware surface
+ if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
+ {
+ ffmpeg.av_frame_unref(_pHwTransferFrame);
+ int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0);
+ if (transferRet < 0)
+ {
+ ffmpeg.av_frame_unref(_pDecodedFrame);
+ throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}");
+ }
+
+ // Copy presentation timestamp and metadata properties
+ ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame);
+ ffmpeg.av_frame_unref(_pDecodedFrame);
+
+ _downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format;
+ outFrame = _pHwTransferFrame;
+ return true;
+ }
+
+ // Frame is in system memory (CPU decoding or software fallback format)
+ outFrame = _pDecodedFrame;
+ return true;
+ }
+
+ if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
+ {
+ if (_isDraining) return false;
+
+ if (!ReadAndSendPacket())
+ {
+ _isDraining = true;
+ ffmpeg.avcodec_send_packet(_pCodecContext, null);
+ }
+ }
+ else if (ret == ffmpeg.AVERROR_EOF)
+ {
+ return false;
+ }
+ else
+ {
+ throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
+ }
+ }
+ }
+
+ ///
+ /// Lazily initializes or updates the when source/destination formats or dimensions change.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
+ ///
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat)
{
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat)
@@ -224,11 +771,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (_pSwsContext == null)
{
- throw new Exception("Failed to initialize SwsContext.");
+ throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
}
}
- private double GetTimestampAndUpdateIndex(AVFrame* frame)
+ ///
+ /// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state.
+ /// Used by seeking to inspect candidate frames without falsely incrementing frame indices.
+ ///
+ private double GetFrameTimestamp(AVFrame* frame)
{
long ptsValue = frame->best_effort_timestamp;
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
@@ -236,123 +787,128 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
ptsValue = frame->pts;
}
- double timestamp;
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
{
- timestamp = _currentFrameIndex / FPS;
+ return FPS > 0 ? _currentFrameIndex / FPS : 0;
+ }
+
+ return ptsValue * ffmpeg.av_q2d(_timeBase);
+ }
+
+ ///
+ /// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter.
+ ///
+ private double GetTimestampAndUpdateIndex(AVFrame* frame)
+ {
+ double timestamp = GetFrameTimestamp(frame);
+ long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE
+ ? frame->best_effort_timestamp
+ : frame->pts;
+
+ if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
+ {
_currentFrameIndex++;
}
else
{
- timestamp = ptsValue * ffmpeg.av_q2d(_timeBase);
_currentFrameIndex = (long)Math.Round(timestamp * FPS);
}
return timestamp;
}
///
- /// Reads the next video frame and copies it in RGBA format.
+ /// Resets all data plane pointers and strides in to zero.
+ /// Prevents stale or dangling pointers from previous conversions being retained across frames.
///
- /// Pointer to the output buffer storing RGBA data.
- /// Byte stride per row of the output buffer (typically Width * 4).
- /// Whether to flip vertically to match Unity's texture coordinate system.
+ private void ClearConvertedFramePointers()
+ {
+ if (_pConvertedFrame == null) return;
+ for (uint i = 0; i < 8; i++)
+ {
+ _pConvertedFrame->data[i] = null;
+ _pConvertedFrame->linesize[i] = 0;
+ }
+ }
+
+ // ──────────────────────────────────────────────────────────────
+ // Frame Reading APIs: RGBA (CPU Fallback)
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Reads the next video frame and copies it in 32-bit RGBA format using libswscale.
+ /// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P).
+ ///
+ /// Pointer to destination memory storing RGBA data.
+ /// Byte stride per row of destination memory (Width * 4).
+ /// Whether to vertically flip the output to match Unity texture coordinates.
/// Timestamp of the output frame in seconds.
- /// True if the frame was successfully decoded and copied; false on EOF or failure.
- private bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
+ /// True if a frame was successfully retrieved; false on EOF.
+ public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
{
timestamp = 0;
lock (_lock)
{
- if (!_isOpened || _disposed) return false;
+ if (!_isOpened || _disposed || outBuffer == null) return false;
+ AVFrame* frame;
if (_hasCachedFrame)
{
- timestamp = GetTimestampAndUpdateIndex(_pCachedFrame);
- InitOrUpdateSwsContext(_pCodecContext->pix_fmt, _pCachedFrame->width, _pCachedFrame->height, AVPixelFormat.AV_PIX_FMT_RGBA);
-
- if (vflip)
- {
- _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1);
- _pConvertedFrame->linesize[0] = -outStride;
- }
- else
- {
- _pConvertedFrame->data[0] = outBuffer;
- _pConvertedFrame->linesize[0] = outStride;
- }
-
- ffmpeg.sws_scale(
- _pSwsContext,
- _pCachedFrame->data, _pCachedFrame->linesize,
- 0, _pCachedFrame->height,
- _pConvertedFrame->data, _pConvertedFrame->linesize
- );
-
- ffmpeg.av_frame_unref(_pCachedFrame);
+ frame = _pCachedFrame;
_hasCachedFrame = false;
- return true;
}
-
- while (true)
+ else
{
- int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
- if (ret == 0)
- {
- timestamp = GetTimestampAndUpdateIndex(_pDecodedFrame);
- InitOrUpdateSwsContext(_pCodecContext->pix_fmt, _pDecodedFrame->width, _pDecodedFrame->height, AVPixelFormat.AV_PIX_FMT_RGBA);
-
- if (vflip)
- {
- _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1);
- _pConvertedFrame->linesize[0] = -outStride;
- }
- else
- {
- _pConvertedFrame->data[0] = outBuffer;
- _pConvertedFrame->linesize[0] = outStride;
- }
-
- ffmpeg.sws_scale(
- _pSwsContext,
- _pDecodedFrame->data, _pDecodedFrame->linesize,
- 0, _pDecodedFrame->height,
- _pConvertedFrame->data, _pConvertedFrame->linesize
- );
-
- ffmpeg.av_frame_unref(_pDecodedFrame);
- return true;
- }
-
- if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
- {
- if (_isDraining) return false;
-
- if (!ReadAndSendPacket())
- {
- _isDraining = true;
- ffmpeg.avcodec_send_packet(_pCodecContext, null);
- }
- }
- else if (ret == ffmpeg.AVERROR_EOF)
- {
- return false;
- }
- else
- {
- throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
- }
+ if (!TryReceiveFrame(out frame)) return false;
}
+
+ if (frame == null) return false;
+
+ timestamp = GetTimestampAndUpdateIndex(frame);
+ InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA);
+
+ ClearConvertedFramePointers();
+ if (vflip)
+ {
+ _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1);
+ _pConvertedFrame->linesize[0] = -outStride;
+ }
+ else
+ {
+ _pConvertedFrame->data[0] = outBuffer;
+ _pConvertedFrame->linesize[0] = outStride;
+ }
+
+ ffmpeg.sws_scale(
+ _pSwsContext,
+ frame->data, frame->linesize,
+ 0, frame->height,
+ _pConvertedFrame->data, _pConvertedFrame->linesize
+ );
+
+ ClearConvertedFramePointers();
+ ffmpeg.av_frame_unref(frame);
+ return true;
}
}
+ ///
+ /// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer.
+ ///
public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp)
{
return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp);
}
+ ///
+ /// Reads the next video frame and copies it in RGBA format to a Unity NativeArray.
+ ///
public bool TryReadFrame(NativeArray outBuffer, bool vflip, out double timestamp)
{
- if (outBuffer.Length < Width * Height * 4)
+ if (!outBuffer.IsCreated)
+ {
+ throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer));
+ }
+ if (outBuffer.Length < (long)Width * Height * 4)
{
throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer));
}
@@ -360,14 +916,155 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp);
}
+ // ──────────────────────────────────────────────────────────────
+ // Frame Reading APIs: NV12 (Compute Shader Pipeline)
+ // ──────────────────────────────────────────────────────────────
+
///
- /// Reads the next video frame and copies its Y, U, and V channel data separately.
- /// If the source format is YUV420P family (YUV420P, YUVJ420P), CPU color conversion (sws_scale) is bypassed
- /// and memory is copied directly for maximum performance.
- /// Other formats are automatically converted to YUV420P before copying.
+ /// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format.
+ /// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed,
+ /// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing.
+ /// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale.
+ ///
+ /// Pointer to destination buffer for Y plane (Width * Height bytes).
+ /// Row stride in bytes for Y plane.
+ /// Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes).
+ /// Row stride in bytes for UV plane.
+ /// Timestamp of the output frame in seconds.
+ /// True if a frame was successfully retrieved; false on EOF.
+ // ReSharper disable once InconsistentNaming
+ public bool TryReadNV12Frame(
+ byte* yBuffer, int yStride,
+ byte* uvBuffer, int uvStride,
+ out double timestamp)
+ {
+ timestamp = 0;
+ lock (_lock)
+ {
+ if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false;
+
+ AVFrame* frame;
+ if (_hasCachedFrame)
+ {
+ frame = _pCachedFrame;
+ _hasCachedFrame = false;
+ }
+ else
+ {
+ if (!TryReceiveFrame(out frame)) return false;
+ }
+
+ if (frame == null) return false;
+
+ timestamp = GetTimestampAndUpdateIndex(frame);
+
+ if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height)
+ {
+ InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12);
+
+ ClearConvertedFramePointers();
+ _pConvertedFrame->data[0] = yBuffer;
+ _pConvertedFrame->data[1] = uvBuffer;
+ _pConvertedFrame->linesize[0] = yStride;
+ _pConvertedFrame->linesize[1] = uvStride;
+
+ ffmpeg.sws_scale(
+ _pSwsContext,
+ frame->data, frame->linesize,
+ 0, frame->height,
+ _pConvertedFrame->data, _pConvertedFrame->linesize
+ );
+ ClearConvertedFramePointers();
+ }
+ else
+ {
+ int w = Width;
+ int h = Height;
+ int uvH = (h + 1) / 2;
+ int uvRowBytes = (w / 2) * 2;
+
+ // Copy Y plane (row by row to safely skip linesize GPU padding)
+ for (int y = 0; y < h; y++)
+ {
+ Buffer.MemoryCopy(
+ frame->data[0] + y * frame->linesize[0],
+ yBuffer + y * yStride,
+ yStride, w
+ );
+ }
+
+ // Copy UV plane (interleaved U and V samples)
+ for (int y = 0; y < uvH; y++)
+ {
+ Buffer.MemoryCopy(
+ frame->data[1] + y * frame->linesize[1],
+ uvBuffer + y * uvStride,
+ uvStride, uvRowBytes
+ );
+ }
+ }
+
+ ffmpeg.av_frame_unref(frame);
+ return true;
+ }
+ }
+
+ ///
+ /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers.
+ ///
+ public bool TryReadNV12Frame(
+ IntPtr yBuffer, int yStride,
+ IntPtr uvBuffer, int uvStride,
+ out double timestamp)
+ {
+ return TryReadNV12Frame(
+ (byte*)yBuffer.ToPointer(), yStride,
+ (byte*)uvBuffer.ToPointer(), uvStride,
+ out timestamp
+ );
+ }
+
+ ///
+ /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays.
///
// ReSharper disable once InconsistentNaming
- private bool TryReadYUV420PFrame(
+ public bool TryReadNV12Frame(
+ NativeArray yBuffer,
+ NativeArray uvBuffer,
+ out double timestamp)
+ {
+ if (!yBuffer.IsCreated || !uvBuffer.IsCreated)
+ {
+ throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
+ }
+
+ int w = Width;
+ int h = Height;
+ int uvH = (h + 1) / 2;
+ int uvRowBytes = (w / 2) * 2;
+
+ if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
+ {
+ throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution.");
+ }
+
+ return TryReadNV12Frame(
+ (byte*)yBuffer.GetUnsafePtr(), w,
+ (byte*)uvBuffer.GetUnsafePtr(), uvRowBytes,
+ out timestamp
+ );
+ }
+
+ // ──────────────────────────────────────────────────────────────
+ // Frame Reading APIs: YUV420P (Compute Shader Pipeline)
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Reads the next video frame and copies its planar Y, U, and V channel data separately.
+ /// For native YUV420P sources, color conversion is bypassed and memory is copied directly.
+ /// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale.
+ ///
+ public bool TryReadYUV420PFrame(
byte* yBuffer, int yStride,
byte* uBuffer, int uStride,
byte* vBuffer, int vStride,
@@ -376,10 +1073,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
timestamp = 0;
lock (_lock)
{
- if (!_isOpened || _disposed) return false;
+ if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false;
AVFrame* frame;
-
if (_hasCachedFrame)
{
frame = _pCachedFrame;
@@ -387,44 +1083,18 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
else
{
- while (true)
- {
- int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
- if (ret == 0)
- {
- frame = _pDecodedFrame;
- break;
- }
-
- if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
- {
- if (_isDraining) return false;
-
- if (!ReadAndSendPacket())
- {
- _isDraining = true;
- ffmpeg.avcodec_send_packet(_pCodecContext, null);
- }
- }
- else if (ret == ffmpeg.AVERROR_EOF)
- {
- return false;
- }
- else
- {
- throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
- }
- }
+ if (!TryReceiveFrame(out frame)) return false;
}
if (frame == null) return false;
timestamp = GetTimestampAndUpdateIndex(frame);
- if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P)
+ if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height)
{
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P);
+ ClearConvertedFramePointers();
_pConvertedFrame->data[0] = yBuffer;
_pConvertedFrame->data[1] = uBuffer;
_pConvertedFrame->data[2] = vBuffer;
@@ -438,13 +1108,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
0, frame->height,
_pConvertedFrame->data, _pConvertedFrame->linesize
);
+ ClearConvertedFramePointers();
}
else
{
- int w = frame->width;
- int h = frame->height;
- int uvW = w / 2;
- int uvH = h / 2;
+ int w = Width;
+ int h = Height;
+ int uvW = (w + 1) / 2;
+ int uvH = (h + 1) / 2;
// Copy Y plane
for (int y = 0; y < h; y++)
@@ -452,7 +1123,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
Buffer.MemoryCopy(
frame->data[0] + y * frame->linesize[0],
yBuffer + y * yStride,
- w, w
+ yStride, w
);
}
@@ -462,7 +1133,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
Buffer.MemoryCopy(
frame->data[1] + y * frame->linesize[1],
uBuffer + y * uStride,
- uvW, uvW
+ uStride, uvW
);
}
@@ -472,7 +1143,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
Buffer.MemoryCopy(
frame->data[2] + y * frame->linesize[2],
vBuffer + y * vStride,
- uvW, uvW
+ vStride, uvW
);
}
}
@@ -482,6 +1153,27 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
+ ///
+ /// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers.
+ ///
+ // ReSharper disable once InconsistentNaming
+ public bool TryReadYUV420PFrame(
+ IntPtr yBuffer, int yStride,
+ IntPtr uBuffer, int uStride,
+ IntPtr vBuffer, int vStride,
+ out double timestamp)
+ {
+ return TryReadYUV420PFrame(
+ (byte*)yBuffer.ToPointer(), yStride,
+ (byte*)uBuffer.ToPointer(), uStride,
+ (byte*)vBuffer.ToPointer(), vStride,
+ out timestamp
+ );
+ }
+
+ ///
+ /// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays.
+ ///
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
NativeArray yBuffer,
@@ -489,12 +1181,17 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
NativeArray vBuffer,
out double timestamp)
{
+ if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated)
+ {
+ throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
+ }
+
int w = Width;
int h = Height;
- int uvW = w / 2;
- int uvH = h / 2;
+ int uvW = (w + 1) / 2;
+ int uvH = (h + 1) / 2;
- if (yBuffer.Length < w * h || uBuffer.Length < uvW * uvH || vBuffer.Length < uvW * uvH)
+ if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH)
{
throw new ArgumentException("Buffer size is smaller than video YUV plane resolution.");
}
@@ -507,138 +1204,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
);
}
- ///
- /// Reads the next video frame and copies its NV12 format Y and UV channel data separately.
- /// If the source format is NV12 family, CPU color conversion (sws_scale) is bypassed
- /// and memory is copied directly for maximum performance.
- /// Other formats are automatically converted to NV12 before copying.
- ///
- // ReSharper disable once InconsistentNaming
- private bool TryReadNV12Frame(
- byte* yBuffer, int yStride,
- byte* uvBuffer, int uvStride,
- out double timestamp)
- {
- timestamp = 0;
- lock (_lock)
- {
- if (!_isOpened || _disposed) return false;
-
- AVFrame* frame;
-
- if (_hasCachedFrame)
- {
- frame = _pCachedFrame;
- _hasCachedFrame = false;
- }
- else
- {
- while (true)
- {
- int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
- if (ret == 0)
- {
- frame = _pDecodedFrame;
- break;
- }
-
- if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
- {
- if (_isDraining) return false;
-
- if (!ReadAndSendPacket())
- {
- _isDraining = true;
- ffmpeg.avcodec_send_packet(_pCodecContext, null);
- }
- }
- else if (ret == ffmpeg.AVERROR_EOF)
- {
- return false;
- }
- else
- {
- throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
- }
- }
- }
-
- if (frame == null) return false;
-
- timestamp = GetTimestampAndUpdateIndex(frame);
-
- if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12)
- {
- InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12);
-
- _pConvertedFrame->data[0] = yBuffer;
- _pConvertedFrame->data[1] = uvBuffer;
- _pConvertedFrame->linesize[0] = yStride;
- _pConvertedFrame->linesize[1] = uvStride;
-
- ffmpeg.sws_scale(
- _pSwsContext,
- frame->data, frame->linesize,
- 0, frame->height,
- _pConvertedFrame->data, _pConvertedFrame->linesize
- );
- }
- else
- {
- int w = frame->width;
- int h = frame->height;
- int uvH = h / 2;
-
- // Copy Y plane
- for (int y = 0; y < h; y++)
- {
- Buffer.MemoryCopy(
- frame->data[0] + y * frame->linesize[0],
- yBuffer + y * yStride,
- w, w
- );
- }
-
- // Copy UV plane
- for (int y = 0; y < uvH; y++)
- {
- Buffer.MemoryCopy(
- frame->data[1] + y * frame->linesize[1],
- uvBuffer + y * uvStride,
- w, w
- );
- }
- }
-
- ffmpeg.av_frame_unref(frame);
- return true;
- }
- }
-
- // ReSharper disable once InconsistentNaming
- public bool TryReadNV12Frame(
- NativeArray yBuffer,
- NativeArray uvBuffer,
- out double timestamp)
- {
- int w = Width;
- int h = Height;
- int uvH = h / 2;
-
- if (yBuffer.Length < w * h || uvBuffer.Length < w * uvH)
- {
- throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution.");
- }
-
- return TryReadNV12Frame(
- (byte*)yBuffer.GetUnsafePtr(), w,
- (byte*)uvBuffer.GetUnsafePtr(), w,
- out timestamp
- );
- }
+ // ──────────────────────────────────────────────────────────────
+ // Seeking & Skipping
+ // ──────────────────────────────────────────────────────────────
///
- /// Decodes and discards a specified number of frames immediately (for fast forward skipping).
+ /// Decodes and discards a specified number of frames immediately without GPU-to-CPU transfer overhead.
///
/// Number of frames to discard.
private void SkipFrames(int count)
@@ -690,7 +1261,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
///
- /// Seeks to a specific playback time in seconds.
+ /// Seeks to a specific playback time in seconds using backward keyframe seek followed by forward decode.
+ /// Intermediate frames are discarded without GPU-to-host memory transfer to maximize seek speed;
+ /// only the target frame is transferred and preserved in .
///
/// Target time in seconds to seek to.
public void SeekToTime(double timeSeconds)
@@ -702,7 +1275,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (timeSeconds < 0) timeSeconds = 0;
double targetFrameIndex = timeSeconds * FPS;
- // If already at target index or very close future, skip frames instead of seek to avoid I-frame overhead
+ // If already at target index or very close future, skip frames instead of seek to avoid I-frame seek latency
bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60;
if (useSkip)
{
@@ -720,7 +1293,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
- // Fallback to start of file on seek failure
+ // Fallback to beginning of file on seek failure
ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
@@ -728,27 +1301,60 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
}
}
- // Flush decoder internal cache
+ // Flush decoder internal buffers
+ // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
ffmpeg.avcodec_flush_buffers(_pCodecContext);
_isDraining = false;
- // Decode and discard frames until the first frame past the specified time is found
+ // Decode frames until reaching the target presentation timestamp
while (true)
{
ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
if (ret == 0)
{
- double currentPts = GetTimestampAndUpdateIndex(_pDecodedFrame);
+ double currentPts = GetFrameTimestamp(_pDecodedFrame);
- // Check from half frame earlier to account for floating point errors
+ // Check with half-frame margin to accommodate floating point rounding
if (currentPts >= timeSeconds - (0.5 / FPS))
{
- // Cache the first matching frame so it will be consumed by the next TryReadFrame call
- ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame);
+ // Cache the first matching frame in system RAM for immediate consumption
+ if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
+ {
+ ffmpeg.av_frame_unref(_pCachedFrame);
+ int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0);
+ if (transferRet < 0)
+ {
+ ffmpeg.av_frame_unref(_pDecodedFrame);
+ throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}");
+ }
+
+ ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame);
+ ffmpeg.av_frame_unref(_pDecodedFrame);
+
+ _downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format;
+ }
+ else
+ {
+ ffmpeg.av_frame_unref(_pCachedFrame);
+ ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame);
+ }
+
_hasCachedFrame = true;
break;
}
+ // Discard non-matching intermediate frame without transferring to CPU
+ long ptsValue = _pDecodedFrame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE
+ ? _pDecodedFrame->best_effort_timestamp
+ : _pDecodedFrame->pts;
+ if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
+ {
+ _currentFrameIndex++;
+ }
+ else
+ {
+ _currentFrameIndex = (long)Math.Round(currentPts * FPS);
+ }
ffmpeg.av_frame_unref(_pDecodedFrame);
continue;
}
@@ -778,23 +1384,34 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
///
/// Seeks to a specific frame index.
///
- /// Target frame index to seek to.
+ /// Target frame index.
public void SeekToFrame(long frameIndex)
{
SeekToTime(frameIndex / FPS);
}
+ // ──────────────────────────────────────────────────────────────
+ // Error Handling & Disposal
+ // ──────────────────────────────────────────────────────────────
+
+ ///
+ /// Translates an FFmpeg integer error code to a readable string using av_strerror.
+ /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
+ ///
private string GetFFmpegErrorString(int errnum)
{
const ulong bufferSize = 1024;
byte* buffer = stackalloc byte[(int)bufferSize];
if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0)
{
- return System.Runtime.InteropServices.Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})";
+ return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})";
}
return $"Unknown error (Code: {errnum})";
}
+ ///
+ /// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures.
+ ///
public void Dispose()
{
lock (_lock)
@@ -802,12 +1419,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
if (_disposed) return;
_disposed = true;
+ // 1. Release scaler context
if (_pSwsContext != null)
{
ffmpeg.sws_freeContext(_pSwsContext);
_pSwsContext = null;
}
+ // 2. Release frame buffers
if (_pDecodedFrame != null)
{
var pFrame = _pDecodedFrame;
@@ -815,6 +1434,13 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_pDecodedFrame = null;
}
+ if (_pHwTransferFrame != null)
+ {
+ var pFrame = _pHwTransferFrame;
+ ffmpeg.av_frame_free(&pFrame);
+ _pHwTransferFrame = null;
+ }
+
if (_pCachedFrame != null)
{
var pFrame = _pCachedFrame;
@@ -829,6 +1455,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_pConvertedFrame = null;
}
+ // 3. Release packet buffer
if (_pPacket != null)
{
var pPacket = _pPacket;
@@ -836,6 +1463,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_pPacket = null;
}
+ // 4. Release codec context
if (_pCodecContext != null)
{
var pCodecCtx = _pCodecContext;
@@ -843,6 +1471,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_pCodecContext = null;
}
+ // 5. Release hardware device context
+ if (_hwDeviceCtx != null)
+ {
+ var pHw = _hwDeviceCtx;
+ ffmpeg.av_buffer_unref(&pHw);
+ _hwDeviceCtx = null;
+ }
+
+ // 6. Close input file and release format context
if (_pFormatContext != null)
{
var pFormatCtx = _pFormatContext;
@@ -850,7 +1487,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
_pFormatContext = null;
}
+ _getFormatDelegate = null;
_isOpened = false;
+ _isHwDecoding = false;
GC.SuppressFinalize(this);
}
}