diff --git a/Util/FFmpeg/FFmpegNativeDecoder.cs b/Util/FFmpeg/FFmpegNativeDecoder.cs index 3a4611e..a1b8c5f 100644 --- a/Util/FFmpeg/FFmpegNativeDecoder.cs +++ b/Util/FFmpeg/FFmpegNativeDecoder.cs @@ -1,23 +1,190 @@ using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; using FFmpeg.AutoGen; +using MelonLoader; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; namespace ADOFAIRenderer.Util.FFmpeg; /// -/// Decodes video files in a streaming fashion using FFmpeg's native C API. -/// Unlike a standalone player, it allows reading frames sequentially or seeking/skipping forward on demand. +/// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke. +/// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and +/// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback. +/// +/// ── Architecture & Lifecycle Flow ────────────────────────────────────────────── +/// +/// 1. Construction & Demuxing: +/// a) avformat_open_input: Open container file and parse stream headers. +/// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS. +/// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO). +/// +/// 2. Codec & Hardware Device Discovery: +/// a) avcodec_find_decoder: Locate codec implementation by codec ID. +/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder. +/// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox). +/// d) Attach hw_device_ctx and register custom get_format callback. +/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding +/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME). +/// f) avcodec_open2: Initialize codec context. +/// +/// 3. Frame Demuxing & Decoding (TryRead*): +/// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet. +/// b) (Decoding) avcodec_receive_frame retrieves decoded frames. +/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data +/// into system RAM as NV12 (av_frame_copy_props preserves timestamps). +/// d) (Output Dispatch): +/// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback. +/// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback. +/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip). +/// +/// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame): +/// a) av_seek_frame: Fast backward keyframe seek. +/// b) avcodec_flush_buffers: Clear internal decoder state. +/// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead; +/// transfers and caches the first matching frame in _pCachedFrame for immediate consumption. +/// +/// 5. Resource Disposal (Dispose): +/// a) Free scaler context (sws_freeContext). +/// b) Free all allocated frames and packets (av_frame_free, av_packet_free). +/// c) Free codec context (avcodec_free_context). +/// d) Release hardware device context (av_buffer_unref). +/// e) Close format context (avformat_close_input). +/// +/// ── Reference Documentation ──────────────────────────────────────────────────── +/// • FFmpeg Demuxing & Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html +/// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html +/// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html +/// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html +/// • libswscale Scaling & Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// internal sealed unsafe class FFmpegNativeDecoder : IDisposable { + // ────────────────────────────────────────────────────────────── + // Hardware Acceleration Priorities & Candidate Structure + // ────────────────────────────────────────────────────────────── + + /// + /// Candidate hardware device configuration discovered via avcodec_get_hw_config. + /// + private struct HwConfigCandidate + { + public AVHWDeviceType DeviceType; + public AVPixelFormat PixelFormat; + } + + /// + /// Preferred hardware device types in priority order. + /// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration, + /// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA. + /// Non-Windows systems fall back to VAAPI or VideoToolbox. + /// + private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[] + { + AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA, + AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2, + AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, + AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA, + AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA, + AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI, + AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX + }; + + // ────────────────────────────────────────────────────────────── + // FFmpeg Core Struct Pointers + // ────────────────────────────────────────────────────────────── + + /// + /// Context managing container format demuxing (MP4, MKV, WebM, etc.). + /// Used for packet demuxing and seeking. + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html + /// private AVFormatContext* _pFormatContext = null; + + /// + /// Context managing video decoder state and parameters. + /// Configured with codec parameters, hardware contexts, and threading options. + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html + /// private AVCodecContext* _pCodecContext = null; + + /// + /// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.). + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html + /// + private AVBufferRef* _hwDeviceCtx = null; + + /// + /// Frame structure holding raw decoded frames received directly from avcodec_receive_frame. + /// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView). + /// In SW decoding mode, this contains pixel buffers in system memory. + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html + /// private AVFrame* _pDecodedFrame = null; + + /// + /// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces + /// via av_hwframe_transfer_data. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html + /// + private AVFrame* _pHwTransferFrame = null; + + /// + /// Frame structure holding converted pixel data produced by sws_scale (e.g., RGBA or fallback YUV). + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html + /// private AVFrame* _pConvertedFrame = null; - private SwsContext* _pSwsContext = null; + + /// + /// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation. + /// Consumed by the next TryRead* invocation. + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html + /// + private AVFrame* _pCachedFrame = null; + private bool _hasCachedFrame; + + /// + /// Packet structure temporarily holding compressed demuxed data read from the input container. + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html + /// private AVPacket* _pPacket = null; - + + /// + /// libswscale colorspace and format conversion context. + /// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html + /// + private SwsContext* _pSwsContext = null; + + // ────────────────────────────────────────────────────────────── + // Hardware Acceleration State & Delegates + // ────────────────────────────────────────────────────────────── + + /// + /// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code. + /// Passed to _pCodecContext->get_format. + /// + private AVCodecContext_get_format? _getFormatDelegate; + + /// Whether hardware-accelerated decoding is currently active. + private bool _isHwDecoding; + + /// Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV). + private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; + + /// Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD). + private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; + + /// + /// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12). + /// + private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; + + // ────────────────────────────────────────────────────────────── + // Playback, Stream & Scaler Synchronization Fields + // ────────────────────────────────────────────────────────────── + private int _videoStreamIndex = -1; private long _currentFrameIndex; private AVRational _timeBase; @@ -26,16 +193,16 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable private bool _isDraining; private readonly object _lock = new(); - // Cached frame used to hold matching frames after seeking - private AVFrame* _pCachedFrame = null; - private bool _hasCachedFrame; - - // Field to detect SwsContext updates + // Fields to detect SwsContext updates private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; private int _srcWidth; private int _srcHeight; + // ────────────────────────────────────────────────────────────── + // Public Properties + // ────────────────────────────────────────────────────────────── + /// Video width in pixels. public int Width { get; private set; } @@ -52,22 +219,123 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable public long TotalFrames { get; private set; } /// Index of the currently read frame. - public long CurrentFrameIndex => _currentFrameIndex; + public long CurrentFrameIndex + { + get + { + lock (_lock) + { + return _currentFrameIndex; + } + } + } /// Current playback time in seconds. - public double CurrentTime => _currentFrameIndex / FPS; + public double CurrentTime + { + get + { + lock (_lock) + { + return FPS > 0 ? _currentFrameIndex / FPS : 0; + } + } + } - /// Pixel format of the original video stream. - public AVPixelFormat PixelFormat => _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE; + /// + /// Pixel format of the video frames produced by the decoder. + /// When hardware decoding is active, returns the pixel format of the downloaded software frame + /// in system RAM (typically ) rather than the GPU surface format + /// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines. + /// In software mode, returns the codec context's pixel format (e.g., YUV420P). + /// + public AVPixelFormat PixelFormat + { + get + { + lock (_lock) + { + if (_isHwDecoding) + { + return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE + ? _downloadedPixFmt + : AVPixelFormat.AV_PIX_FMT_NV12; + } - /// Color range of the original video stream. - public AVColorRange ColorRange => _pCodecContext != null ? _pCodecContext->color_range : AVColorRange.AVCOL_RANGE_UNSPECIFIED; + return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE; + } + } + } + /// + /// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]). + /// + public AVColorRange ColorRange + { + get + { + lock (_lock) + { + if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED) + { + return _pCodecContext->color_range; + } + if (_pFormatContext != null && _videoStreamIndex >= 0) + { + return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range; + } + return AVColorRange.AVCOL_RANGE_UNSPECIFIED; + } + } + } + + /// + /// Gets a value indicating whether hardware-accelerated decoding is currently active. + /// + public bool IsHardwareAccelerated + { + get + { + lock (_lock) + { + return _isHwDecoding; + } + } + } + + /// + /// Gets the hardware device type in use for hardware acceleration, or null if software decoding. + /// + public AVHWDeviceType? HardwareDeviceType + { + get + { + lock (_lock) + { + return _isHwDecoding ? _hwDeviceType : null; + } + } + } + + // ────────────────────────────────────────────────────────────── + // Initialization & Setup + // ────────────────────────────────────────────────────────────── + + /// + /// Initializes a new instance of and opens the specified video file. + /// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back + /// to multi-threaded software decoding if unavailable. + /// + /// Absolute or relative file path to the video file. public FFmpegNativeDecoder(string videoPath) { Open(videoPath); } + /// + /// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers. + /// + /// Path to video file. private void Open(string videoPath) { lock (_lock) @@ -76,6 +344,8 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable try { + // 1. Open media container + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html fixed (AVFormatContext** ppFormatCtx = &_pFormatContext) { int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null); @@ -83,9 +353,11 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})"); } + // 2. Retrieve stream information if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0) throw new Exception("Failed to retrieve stream info."); + // 3. Find primary video stream _videoStreamIndex = -1; for (int i = 0; i < _pFormatContext->nb_streams; i++) { @@ -102,7 +374,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable var stream = _pFormatContext->streams[_videoStreamIndex]; _timeBase = stream->time_base; - // Calculate FPS + // 4. Calculate FPS if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->r_frame_rate); @@ -116,11 +388,13 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable FPS = 30.0; // Fallback default } + // 5. Find decoder by codec ID var codecId = stream->codecpar->codec_id; var decoder = ffmpeg.avcodec_find_decoder(codecId); if (decoder == null) throw new Exception($"Unsupported video codec: {codecId}"); + // 6. Allocate codec context _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); if (_pCodecContext == null) throw new Exception("Failed to allocate codec context."); @@ -128,17 +402,28 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters."); - // Multi-threading settings for improved decoding performance (auto) - _pCodecContext->thread_count = 0; - _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME; + // 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding + bool hwSuccess = TryInitializeHwDecoder(decoder, stream); + if (!hwSuccess) + { + Melon.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding."); + _isHwDecoding = false; + _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; - if (ffmpeg.avcodec_open2(_pCodecContext, decoder, null) < 0) - throw new Exception("Failed to open codec."); + // Multi-threading settings for improved software decoding performance (auto thread count) + // Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html + _pCodecContext->thread_count = 0; + _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME; - Width = _pCodecContext->width; - Height = _pCodecContext->height; + int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null); + if (openRet < 0) + throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}"); + } - // Calculate total frames and duration + Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width; + Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height; + + // 8. Calculate total frames and duration if (stream->nb_frames > 0) { TotalFrames = stream->nb_frames; @@ -160,7 +445,10 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : (stream->duration > 0 ? stream->duration * ffmpeg.av_q2d(stream->time_base) : 0); + // 9. Allocate frame and packet buffers + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html _pDecodedFrame = ffmpeg.av_frame_alloc(); + _pHwTransferFrame = ffmpeg.av_frame_alloc(); _pConvertedFrame = ffmpeg.av_frame_alloc(); _pCachedFrame = ffmpeg.av_frame_alloc(); _pPacket = ffmpeg.av_packet_alloc(); @@ -177,13 +465,205 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } } + /// + /// Searches for supported hardware device configurations via avcodec_get_hw_config, + /// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA), + /// registers the get_format callback, and attempts to open the hardware decoder. + /// + /// Target decoder. + /// Source video stream. + /// True if hardware decoding was successfully configured and opened; false to fall back. + private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream) + { + // Collect all supported hardware configs from decoder + const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01; + var candidates = new List(); + for (int i = 0; ; i++) + { + var config = ffmpeg.avcodec_get_hw_config(decoder, i); + if (config == null) break; + + if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0) + { + candidates.Add(new HwConfigCandidate + { + DeviceType = config->device_type, + PixelFormat = config->pix_fmt + }); + } + } + + if (candidates.Count == 0) + { + return false; + } + + // Order candidates according to preference + var orderedCandidates = new List(); + foreach (var preferred in PreferredHwDeviceTypes) + { + foreach (var candidate in candidates) + { + if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate)) + { + orderedCandidates.Add(candidate); + } + } + } + + // Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes + foreach (var candidate in candidates) + { + if (!orderedCandidates.Contains(candidate)) + { + orderedCandidates.Add(candidate); + } + } + + // Try candidate hardware configurations in priority order + foreach (var candidate in orderedCandidates) + { + AVBufferRef* hwDeviceCtx = null; + int err; + + // Device creation strategy + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html + if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV) + { + var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + string childDeviceName = isWindows ? "d3d11va" : "vaapi"; + AVDictionary* opts = null; + ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0); + + err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0); + ffmpeg.av_dict_free(&opts); + + if (err < 0) + { + err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0); + } + } + else + { + err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0); + } + + if (err < 0 || hwDeviceCtx == null) + { + if (hwDeviceCtx != null) + { + ffmpeg.av_buffer_unref(&hwDeviceCtx); + hwDeviceCtx = null; + } + Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate..."); + continue; + } + + // Assign hardware context and format selection callback + _hwDeviceCtx = hwDeviceCtx; + _hwPixFmt = candidate.PixelFormat; + _hwDeviceType = candidate.DeviceType; + _isHwDecoding = true; + _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12; + + _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx); + if (_pCodecContext->hw_device_ctx == null) + { + Melon.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate..."); + var pHw = _hwDeviceCtx; + ffmpeg.av_buffer_unref(&pHw); + _hwDeviceCtx = null; + _isHwDecoding = false; + continue; + } + + _getFormatDelegate = GetFormat; + _pCodecContext->get_format = _getFormatDelegate; + + // Hardware decoding requires single-threaded decoding (thread_count = 1). + // Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function + // GPU decoders and causes surface pool allocation exhaustion or crashes in get_format. + _pCodecContext->thread_count = 1; + + // Attempt opening codec context with hardware configuration + int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null); + if (openRet >= 0) + { + Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}"); + return true; + } + + // Failed to open codec with this hardware configuration, clean up and try next + Melon.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate..."); + + if (_pCodecContext != null) + { + var pCodecCtx = _pCodecContext; + ffmpeg.avcodec_free_context(&pCodecCtx); + _pCodecContext = null; + } + + if (_hwDeviceCtx != null) + { + var pHw = _hwDeviceCtx; + ffmpeg.av_buffer_unref(&pHw); + _hwDeviceCtx = null; + } + + _isHwDecoding = false; + _getFormatDelegate = null; + + // Reallocate fresh codec context for subsequent attempts + _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); + if (_pCodecContext == null) + throw new Exception("Failed to re-allocate codec context during fallback."); + + if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) + throw new Exception("Failed to copy codec parameters during fallback."); + } + + return false; + } + + /// + /// Callback invoked by libavcodec during decoding to negotiate the output pixel format. + /// Confirms the hardware surface format () if offered by the decoder, + /// or falls back to software pixel formats via avcodec_default_get_format. + /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html + /// + private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts) + { + for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++) + { + if (*p == _hwPixFmt) + { + return *p; + } + } + + Melon.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format."); + _isHwDecoding = false; + _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; + return ffmpeg.avcodec_default_get_format(ctx, pixFmts); + } + + // ────────────────────────────────────────────────────────────── + // Demuxing & Decoding Engine + // ────────────────────────────────────────────────────────────── + + /// + /// Reads packets from the container until a packet belonging to the video stream is found, + /// then sends it to the decoder via avcodec_send_packet. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html + /// + /// True if a video packet was sent; false on EOF. private bool ReadAndSendPacket() { while (true) { ffmpeg.av_packet_unref(_pPacket); int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket); - if (ret < 0) return false; // EOF or error + if (ret < 0) return false; // EOF or read error if (_pPacket->stream_index == _videoStreamIndex) { @@ -198,6 +678,73 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } } + /// + /// Receives the next decoded video frame from the codec context. + /// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV), + /// it is automatically downloaded to system RAM via av_hwframe_transfer_data into _pHwTransferFrame, + /// preserving presentation timestamps via av_frame_copy_props. + /// + /// The software frame containing pixel data in system RAM. + /// True if a frame was successfully decoded and available; false on EOF or draining. + private bool TryReceiveFrame(out AVFrame* outFrame) + { + outFrame = null; + + while (true) + { + int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); + if (ret == 0) + { + // Check if the received frame is on a hardware surface + if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) + { + ffmpeg.av_frame_unref(_pHwTransferFrame); + int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0); + if (transferRet < 0) + { + ffmpeg.av_frame_unref(_pDecodedFrame); + throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}"); + } + + // Copy presentation timestamp and metadata properties + ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame); + ffmpeg.av_frame_unref(_pDecodedFrame); + + _downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format; + outFrame = _pHwTransferFrame; + return true; + } + + // Frame is in system memory (CPU decoding or software fallback format) + outFrame = _pDecodedFrame; + return true; + } + + if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) + { + if (_isDraining) return false; + + if (!ReadAndSendPacket()) + { + _isDraining = true; + ffmpeg.avcodec_send_packet(_pCodecContext, null); + } + } + else if (ret == ffmpeg.AVERROR_EOF) + { + return false; + } + else + { + throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); + } + } + } + + /// + /// Lazily initializes or updates the when source/destination formats or dimensions change. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html + /// private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat) { if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat) @@ -224,11 +771,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable if (_pSwsContext == null) { - throw new Exception("Failed to initialize SwsContext."); + throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height})."); } } - private double GetTimestampAndUpdateIndex(AVFrame* frame) + /// + /// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state. + /// Used by seeking to inspect candidate frames without falsely incrementing frame indices. + /// + private double GetFrameTimestamp(AVFrame* frame) { long ptsValue = frame->best_effort_timestamp; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) @@ -236,123 +787,128 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable ptsValue = frame->pts; } - double timestamp; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { - timestamp = _currentFrameIndex / FPS; + return FPS > 0 ? _currentFrameIndex / FPS : 0; + } + + return ptsValue * ffmpeg.av_q2d(_timeBase); + } + + /// + /// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter. + /// + private double GetTimestampAndUpdateIndex(AVFrame* frame) + { + double timestamp = GetFrameTimestamp(frame); + long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE + ? frame->best_effort_timestamp + : frame->pts; + + if (ptsValue == ffmpeg.AV_NOPTS_VALUE) + { _currentFrameIndex++; } else { - timestamp = ptsValue * ffmpeg.av_q2d(_timeBase); _currentFrameIndex = (long)Math.Round(timestamp * FPS); } return timestamp; } /// - /// Reads the next video frame and copies it in RGBA format. + /// Resets all data plane pointers and strides in to zero. + /// Prevents stale or dangling pointers from previous conversions being retained across frames. /// - /// Pointer to the output buffer storing RGBA data. - /// Byte stride per row of the output buffer (typically Width * 4). - /// Whether to flip vertically to match Unity's texture coordinate system. + private void ClearConvertedFramePointers() + { + if (_pConvertedFrame == null) return; + for (uint i = 0; i < 8; i++) + { + _pConvertedFrame->data[i] = null; + _pConvertedFrame->linesize[i] = 0; + } + } + + // ────────────────────────────────────────────────────────────── + // Frame Reading APIs: RGBA (CPU Fallback) + // ────────────────────────────────────────────────────────────── + + /// + /// Reads the next video frame and copies it in 32-bit RGBA format using libswscale. + /// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P). + /// + /// Pointer to destination memory storing RGBA data. + /// Byte stride per row of destination memory (Width * 4). + /// Whether to vertically flip the output to match Unity texture coordinates. /// Timestamp of the output frame in seconds. - /// True if the frame was successfully decoded and copied; false on EOF or failure. - private bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) + /// True if a frame was successfully retrieved; false on EOF. + public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) { timestamp = 0; lock (_lock) { - if (!_isOpened || _disposed) return false; + if (!_isOpened || _disposed || outBuffer == null) return false; + AVFrame* frame; if (_hasCachedFrame) { - timestamp = GetTimestampAndUpdateIndex(_pCachedFrame); - InitOrUpdateSwsContext(_pCodecContext->pix_fmt, _pCachedFrame->width, _pCachedFrame->height, AVPixelFormat.AV_PIX_FMT_RGBA); - - if (vflip) - { - _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); - _pConvertedFrame->linesize[0] = -outStride; - } - else - { - _pConvertedFrame->data[0] = outBuffer; - _pConvertedFrame->linesize[0] = outStride; - } - - ffmpeg.sws_scale( - _pSwsContext, - _pCachedFrame->data, _pCachedFrame->linesize, - 0, _pCachedFrame->height, - _pConvertedFrame->data, _pConvertedFrame->linesize - ); - - ffmpeg.av_frame_unref(_pCachedFrame); + frame = _pCachedFrame; _hasCachedFrame = false; - return true; } - - while (true) + else { - int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); - if (ret == 0) - { - timestamp = GetTimestampAndUpdateIndex(_pDecodedFrame); - InitOrUpdateSwsContext(_pCodecContext->pix_fmt, _pDecodedFrame->width, _pDecodedFrame->height, AVPixelFormat.AV_PIX_FMT_RGBA); - - if (vflip) - { - _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); - _pConvertedFrame->linesize[0] = -outStride; - } - else - { - _pConvertedFrame->data[0] = outBuffer; - _pConvertedFrame->linesize[0] = outStride; - } - - ffmpeg.sws_scale( - _pSwsContext, - _pDecodedFrame->data, _pDecodedFrame->linesize, - 0, _pDecodedFrame->height, - _pConvertedFrame->data, _pConvertedFrame->linesize - ); - - ffmpeg.av_frame_unref(_pDecodedFrame); - return true; - } - - if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) - { - if (_isDraining) return false; - - if (!ReadAndSendPacket()) - { - _isDraining = true; - ffmpeg.avcodec_send_packet(_pCodecContext, null); - } - } - else if (ret == ffmpeg.AVERROR_EOF) - { - return false; - } - else - { - throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); - } + if (!TryReceiveFrame(out frame)) return false; } + + if (frame == null) return false; + + timestamp = GetTimestampAndUpdateIndex(frame); + InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA); + + ClearConvertedFramePointers(); + if (vflip) + { + _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); + _pConvertedFrame->linesize[0] = -outStride; + } + else + { + _pConvertedFrame->data[0] = outBuffer; + _pConvertedFrame->linesize[0] = outStride; + } + + ffmpeg.sws_scale( + _pSwsContext, + frame->data, frame->linesize, + 0, frame->height, + _pConvertedFrame->data, _pConvertedFrame->linesize + ); + + ClearConvertedFramePointers(); + ffmpeg.av_frame_unref(frame); + return true; } } + /// + /// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer. + /// public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp) { return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp); } + /// + /// Reads the next video frame and copies it in RGBA format to a Unity NativeArray. + /// public bool TryReadFrame(NativeArray outBuffer, bool vflip, out double timestamp) { - if (outBuffer.Length < Width * Height * 4) + if (!outBuffer.IsCreated) + { + throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer)); + } + if (outBuffer.Length < (long)Width * Height * 4) { throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer)); } @@ -360,14 +916,155 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp); } + // ────────────────────────────────────────────────────────────── + // Frame Reading APIs: NV12 (Compute Shader Pipeline) + // ────────────────────────────────────────────────────────────── + /// - /// Reads the next video frame and copies its Y, U, and V channel data separately. - /// If the source format is YUV420P family (YUV420P, YUVJ420P), CPU color conversion (sws_scale) is bypassed - /// and memory is copied directly for maximum performance. - /// Other formats are automatically converted to YUV420P before copying. + /// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format. + /// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed, + /// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing. + /// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale. + /// + /// Pointer to destination buffer for Y plane (Width * Height bytes). + /// Row stride in bytes for Y plane. + /// Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes). + /// Row stride in bytes for UV plane. + /// Timestamp of the output frame in seconds. + /// True if a frame was successfully retrieved; false on EOF. + // ReSharper disable once InconsistentNaming + public bool TryReadNV12Frame( + byte* yBuffer, int yStride, + byte* uvBuffer, int uvStride, + out double timestamp) + { + timestamp = 0; + lock (_lock) + { + if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false; + + AVFrame* frame; + if (_hasCachedFrame) + { + frame = _pCachedFrame; + _hasCachedFrame = false; + } + else + { + if (!TryReceiveFrame(out frame)) return false; + } + + if (frame == null) return false; + + timestamp = GetTimestampAndUpdateIndex(frame); + + if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height) + { + InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12); + + ClearConvertedFramePointers(); + _pConvertedFrame->data[0] = yBuffer; + _pConvertedFrame->data[1] = uvBuffer; + _pConvertedFrame->linesize[0] = yStride; + _pConvertedFrame->linesize[1] = uvStride; + + ffmpeg.sws_scale( + _pSwsContext, + frame->data, frame->linesize, + 0, frame->height, + _pConvertedFrame->data, _pConvertedFrame->linesize + ); + ClearConvertedFramePointers(); + } + else + { + int w = Width; + int h = Height; + int uvH = (h + 1) / 2; + int uvRowBytes = (w / 2) * 2; + + // Copy Y plane (row by row to safely skip linesize GPU padding) + for (int y = 0; y < h; y++) + { + Buffer.MemoryCopy( + frame->data[0] + y * frame->linesize[0], + yBuffer + y * yStride, + yStride, w + ); + } + + // Copy UV plane (interleaved U and V samples) + for (int y = 0; y < uvH; y++) + { + Buffer.MemoryCopy( + frame->data[1] + y * frame->linesize[1], + uvBuffer + y * uvStride, + uvStride, uvRowBytes + ); + } + } + + ffmpeg.av_frame_unref(frame); + return true; + } + } + + /// + /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers. + /// + public bool TryReadNV12Frame( + IntPtr yBuffer, int yStride, + IntPtr uvBuffer, int uvStride, + out double timestamp) + { + return TryReadNV12Frame( + (byte*)yBuffer.ToPointer(), yStride, + (byte*)uvBuffer.ToPointer(), uvStride, + out timestamp + ); + } + + /// + /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays. /// // ReSharper disable once InconsistentNaming - private bool TryReadYUV420PFrame( + public bool TryReadNV12Frame( + NativeArray yBuffer, + NativeArray uvBuffer, + out double timestamp) + { + if (!yBuffer.IsCreated || !uvBuffer.IsCreated) + { + throw new ArgumentException("Input NativeArray buffer is not created or already disposed."); + } + + int w = Width; + int h = Height; + int uvH = (h + 1) / 2; + int uvRowBytes = (w / 2) * 2; + + if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH) + { + throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution."); + } + + return TryReadNV12Frame( + (byte*)yBuffer.GetUnsafePtr(), w, + (byte*)uvBuffer.GetUnsafePtr(), uvRowBytes, + out timestamp + ); + } + + // ────────────────────────────────────────────────────────────── + // Frame Reading APIs: YUV420P (Compute Shader Pipeline) + // ────────────────────────────────────────────────────────────── + + /// + /// Reads the next video frame and copies its planar Y, U, and V channel data separately. + /// For native YUV420P sources, color conversion is bypassed and memory is copied directly. + /// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale. + /// + public bool TryReadYUV420PFrame( byte* yBuffer, int yStride, byte* uBuffer, int uStride, byte* vBuffer, int vStride, @@ -376,10 +1073,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable timestamp = 0; lock (_lock) { - if (!_isOpened || _disposed) return false; + if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false; AVFrame* frame; - if (_hasCachedFrame) { frame = _pCachedFrame; @@ -387,44 +1083,18 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } else { - while (true) - { - int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); - if (ret == 0) - { - frame = _pDecodedFrame; - break; - } - - if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) - { - if (_isDraining) return false; - - if (!ReadAndSendPacket()) - { - _isDraining = true; - ffmpeg.avcodec_send_packet(_pCodecContext, null); - } - } - else if (ret == ffmpeg.AVERROR_EOF) - { - return false; - } - else - { - throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); - } - } + if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); - if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) + if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P); + ClearConvertedFramePointers(); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uBuffer; _pConvertedFrame->data[2] = vBuffer; @@ -438,13 +1108,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); + ClearConvertedFramePointers(); } else { - int w = frame->width; - int h = frame->height; - int uvW = w / 2; - int uvH = h / 2; + int w = Width; + int h = Height; + int uvW = (w + 1) / 2; + int uvH = (h + 1) / 2; // Copy Y plane for (int y = 0; y < h; y++) @@ -452,7 +1123,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, - w, w + yStride, w ); } @@ -462,7 +1133,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uBuffer + y * uStride, - uvW, uvW + uStride, uvW ); } @@ -472,7 +1143,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable Buffer.MemoryCopy( frame->data[2] + y * frame->linesize[2], vBuffer + y * vStride, - uvW, uvW + vStride, uvW ); } } @@ -482,6 +1153,27 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } } + /// + /// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers. + /// + // ReSharper disable once InconsistentNaming + public bool TryReadYUV420PFrame( + IntPtr yBuffer, int yStride, + IntPtr uBuffer, int uStride, + IntPtr vBuffer, int vStride, + out double timestamp) + { + return TryReadYUV420PFrame( + (byte*)yBuffer.ToPointer(), yStride, + (byte*)uBuffer.ToPointer(), uStride, + (byte*)vBuffer.ToPointer(), vStride, + out timestamp + ); + } + + /// + /// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays. + /// // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( NativeArray yBuffer, @@ -489,12 +1181,17 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable NativeArray vBuffer, out double timestamp) { + if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated) + { + throw new ArgumentException("Input NativeArray buffer is not created or already disposed."); + } + int w = Width; int h = Height; - int uvW = w / 2; - int uvH = h / 2; + int uvW = (w + 1) / 2; + int uvH = (h + 1) / 2; - if (yBuffer.Length < w * h || uBuffer.Length < uvW * uvH || vBuffer.Length < uvW * uvH) + if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH) { throw new ArgumentException("Buffer size is smaller than video YUV plane resolution."); } @@ -507,138 +1204,12 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable ); } - /// - /// Reads the next video frame and copies its NV12 format Y and UV channel data separately. - /// If the source format is NV12 family, CPU color conversion (sws_scale) is bypassed - /// and memory is copied directly for maximum performance. - /// Other formats are automatically converted to NV12 before copying. - /// - // ReSharper disable once InconsistentNaming - private bool TryReadNV12Frame( - byte* yBuffer, int yStride, - byte* uvBuffer, int uvStride, - out double timestamp) - { - timestamp = 0; - lock (_lock) - { - if (!_isOpened || _disposed) return false; - - AVFrame* frame; - - if (_hasCachedFrame) - { - frame = _pCachedFrame; - _hasCachedFrame = false; - } - else - { - while (true) - { - int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); - if (ret == 0) - { - frame = _pDecodedFrame; - break; - } - - if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) - { - if (_isDraining) return false; - - if (!ReadAndSendPacket()) - { - _isDraining = true; - ffmpeg.avcodec_send_packet(_pCodecContext, null); - } - } - else if (ret == ffmpeg.AVERROR_EOF) - { - return false; - } - else - { - throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); - } - } - } - - if (frame == null) return false; - - timestamp = GetTimestampAndUpdateIndex(frame); - - if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12) - { - InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12); - - _pConvertedFrame->data[0] = yBuffer; - _pConvertedFrame->data[1] = uvBuffer; - _pConvertedFrame->linesize[0] = yStride; - _pConvertedFrame->linesize[1] = uvStride; - - ffmpeg.sws_scale( - _pSwsContext, - frame->data, frame->linesize, - 0, frame->height, - _pConvertedFrame->data, _pConvertedFrame->linesize - ); - } - else - { - int w = frame->width; - int h = frame->height; - int uvH = h / 2; - - // Copy Y plane - for (int y = 0; y < h; y++) - { - Buffer.MemoryCopy( - frame->data[0] + y * frame->linesize[0], - yBuffer + y * yStride, - w, w - ); - } - - // Copy UV plane - for (int y = 0; y < uvH; y++) - { - Buffer.MemoryCopy( - frame->data[1] + y * frame->linesize[1], - uvBuffer + y * uvStride, - w, w - ); - } - } - - ffmpeg.av_frame_unref(frame); - return true; - } - } - - // ReSharper disable once InconsistentNaming - public bool TryReadNV12Frame( - NativeArray yBuffer, - NativeArray uvBuffer, - out double timestamp) - { - int w = Width; - int h = Height; - int uvH = h / 2; - - if (yBuffer.Length < w * h || uvBuffer.Length < w * uvH) - { - throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution."); - } - - return TryReadNV12Frame( - (byte*)yBuffer.GetUnsafePtr(), w, - (byte*)uvBuffer.GetUnsafePtr(), w, - out timestamp - ); - } + // ────────────────────────────────────────────────────────────── + // Seeking & Skipping + // ────────────────────────────────────────────────────────────── /// - /// Decodes and discards a specified number of frames immediately (for fast forward skipping). + /// Decodes and discards a specified number of frames immediately without GPU-to-CPU transfer overhead. /// /// Number of frames to discard. private void SkipFrames(int count) @@ -690,7 +1261,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } /// - /// Seeks to a specific playback time in seconds. + /// Seeks to a specific playback time in seconds using backward keyframe seek followed by forward decode. + /// Intermediate frames are discarded without GPU-to-host memory transfer to maximize seek speed; + /// only the target frame is transferred and preserved in . /// /// Target time in seconds to seek to. public void SeekToTime(double timeSeconds) @@ -702,7 +1275,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable if (timeSeconds < 0) timeSeconds = 0; double targetFrameIndex = timeSeconds * FPS; - // If already at target index or very close future, skip frames instead of seek to avoid I-frame overhead + // If already at target index or very close future, skip frames instead of seek to avoid I-frame seek latency bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60; if (useSkip) { @@ -720,7 +1293,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { - // Fallback to start of file on seek failure + // Fallback to beginning of file on seek failure ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { @@ -728,27 +1301,60 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable } } - // Flush decoder internal cache + // Flush decoder internal buffers + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html ffmpeg.avcodec_flush_buffers(_pCodecContext); _isDraining = false; - // Decode and discard frames until the first frame past the specified time is found + // Decode frames until reaching the target presentation timestamp while (true) { ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { - double currentPts = GetTimestampAndUpdateIndex(_pDecodedFrame); + double currentPts = GetFrameTimestamp(_pDecodedFrame); - // Check from half frame earlier to account for floating point errors + // Check with half-frame margin to accommodate floating point rounding if (currentPts >= timeSeconds - (0.5 / FPS)) { - // Cache the first matching frame so it will be consumed by the next TryReadFrame call - ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame); + // Cache the first matching frame in system RAM for immediate consumption + if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) + { + ffmpeg.av_frame_unref(_pCachedFrame); + int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0); + if (transferRet < 0) + { + ffmpeg.av_frame_unref(_pDecodedFrame); + throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}"); + } + + ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame); + ffmpeg.av_frame_unref(_pDecodedFrame); + + _downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format; + } + else + { + ffmpeg.av_frame_unref(_pCachedFrame); + ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame); + } + _hasCachedFrame = true; break; } + // Discard non-matching intermediate frame without transferring to CPU + long ptsValue = _pDecodedFrame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE + ? _pDecodedFrame->best_effort_timestamp + : _pDecodedFrame->pts; + if (ptsValue == ffmpeg.AV_NOPTS_VALUE) + { + _currentFrameIndex++; + } + else + { + _currentFrameIndex = (long)Math.Round(currentPts * FPS); + } ffmpeg.av_frame_unref(_pDecodedFrame); continue; } @@ -778,23 +1384,34 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable /// /// Seeks to a specific frame index. /// - /// Target frame index to seek to. + /// Target frame index. public void SeekToFrame(long frameIndex) { SeekToTime(frameIndex / FPS); } + // ────────────────────────────────────────────────────────────── + // Error Handling & Disposal + // ────────────────────────────────────────────────────────────── + + /// + /// Translates an FFmpeg integer error code to a readable string using av_strerror. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html + /// private string GetFFmpegErrorString(int errnum) { const ulong bufferSize = 1024; byte* buffer = stackalloc byte[(int)bufferSize]; if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0) { - return System.Runtime.InteropServices.Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})"; + return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})"; } return $"Unknown error (Code: {errnum})"; } + /// + /// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures. + /// public void Dispose() { lock (_lock) @@ -802,12 +1419,14 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable if (_disposed) return; _disposed = true; + // 1. Release scaler context if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } + // 2. Release frame buffers if (_pDecodedFrame != null) { var pFrame = _pDecodedFrame; @@ -815,6 +1434,13 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _pDecodedFrame = null; } + if (_pHwTransferFrame != null) + { + var pFrame = _pHwTransferFrame; + ffmpeg.av_frame_free(&pFrame); + _pHwTransferFrame = null; + } + if (_pCachedFrame != null) { var pFrame = _pCachedFrame; @@ -829,6 +1455,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _pConvertedFrame = null; } + // 3. Release packet buffer if (_pPacket != null) { var pPacket = _pPacket; @@ -836,6 +1463,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _pPacket = null; } + // 4. Release codec context if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; @@ -843,6 +1471,15 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _pCodecContext = null; } + // 5. Release hardware device context + if (_hwDeviceCtx != null) + { + var pHw = _hwDeviceCtx; + ffmpeg.av_buffer_unref(&pHw); + _hwDeviceCtx = null; + } + + // 6. Close input file and release format context if (_pFormatContext != null) { var pFormatCtx = _pFormatContext; @@ -850,7 +1487,9 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable _pFormatContext = null; } + _getFormatDelegate = null; _isOpened = false; + _isHwDecoding = false; GC.SuppressFinalize(this); } }