using System; using System.Collections.Generic; using System.Runtime.InteropServices; using FFmpeg.AutoGen; using MelonLoader; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; namespace ADOFAIRenderer.Util.FFmpeg; /// /// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke. /// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and /// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback. /// /// ── Architecture & Lifecycle Flow ────────────────────────────────────────────── /// /// 1. Construction & Demuxing: /// a) avformat_open_input: Open container file and parse stream headers. /// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS. /// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO). /// /// 2. Codec & Hardware Device Discovery: /// a) avcodec_find_decoder: Locate codec implementation by codec ID. /// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder. /// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox). /// d) Attach hw_device_ctx and register custom get_format callback. /// e) If HW initialization fails or is unsupported, fall back to CPU software decoding /// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME). /// f) avcodec_open2: Initialize codec context. /// /// 3. Frame Demuxing & Decoding (TryRead*): /// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet. /// b) (Decoding) avcodec_receive_frame retrieves decoded frames. /// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data /// into system RAM as NV12 (av_frame_copy_props preserves timestamps). /// d) (Output Dispatch): /// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback. /// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback. /// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip). /// /// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame): /// a) av_seek_frame: Fast backward keyframe seek. /// b) avcodec_flush_buffers: Clear internal decoder state. /// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead; /// transfers and caches the first matching frame in _pCachedFrame for immediate consumption. /// /// 5. Resource Disposal (Dispose): /// a) Free scaler context (sws_freeContext). /// b) Free all allocated frames and packets (av_frame_free, av_packet_free). /// c) Free codec context (avcodec_free_context). /// d) Release hardware device context (av_buffer_unref). /// e) Close format context (avformat_close_input). /// /// ── Reference Documentation ──────────────────────────────────────────────────── /// • FFmpeg Demuxing & Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html /// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html /// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html /// • libswscale Scaling & Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// internal sealed unsafe class FFmpegNativeDecoder : IDisposable { // ────────────────────────────────────────────────────────────── // Hardware Acceleration Priorities & Candidate Structure // ────────────────────────────────────────────────────────────── /// /// Candidate hardware device configuration discovered via avcodec_get_hw_config. /// private record struct HwConfigCandidate { public AVHWDeviceType DeviceType; public AVPixelFormat PixelFormat; } /// /// Preferred hardware device types in priority order. /// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration, /// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA. /// Non-Windows systems fall back to VAAPI or VideoToolbox. /// private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[] { AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA, AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA, AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA, AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI, AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX }; // ────────────────────────────────────────────────────────────── // FFmpeg Core Struct Pointers // ────────────────────────────────────────────────────────────── /// /// Context managing container format demuxing (MP4, MKV, WebM, etc.). /// Used for packet demuxing and seeking. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html /// private AVFormatContext* _pFormatContext = null; /// /// Context managing video decoder state and parameters. /// Configured with codec parameters, hardware contexts, and threading options. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// private AVCodecContext* _pCodecContext = null; /// /// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.). /// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html /// private AVBufferRef* _hwDeviceCtx = null; /// /// Frame structure holding raw decoded frames received directly from avcodec_receive_frame. /// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView). /// In SW decoding mode, this contains pixel buffers in system memory. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pDecodedFrame = null; /// /// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces /// via av_hwframe_transfer_data. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html /// private AVFrame* _pHwTransferFrame = null; /// /// Frame structure holding converted pixel data produced by sws_scale (e.g., RGBA or fallback YUV). /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pConvertedFrame = null; /// /// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation. /// Consumed by the next TryRead* invocation. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pCachedFrame = null; private bool _hasCachedFrame; /// /// Packet structure temporarily holding compressed demuxed data read from the input container. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html /// private AVPacket* _pPacket = null; /// /// libswscale colorspace and format conversion context. /// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// private SwsContext* _pSwsContext = null; // ────────────────────────────────────────────────────────────── // Hardware Acceleration State & Delegates // ────────────────────────────────────────────────────────────── /// /// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code. /// Passed to _pCodecContext->get_format. /// private AVCodecContext_get_format? _getFormatDelegate; /// Whether hardware-accelerated decoding is currently active. private bool _isHwDecoding; /// Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV). private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; /// Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD). private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; /// /// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12). /// private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; // ────────────────────────────────────────────────────────────── // Playback, Stream & Scaler Synchronization Fields // ────────────────────────────────────────────────────────────── private int _videoStreamIndex = -1; private long _currentFrameIndex; private AVRational _timeBase; private bool _isOpened; private bool _disposed; private bool _isDraining; /// Number of video packets skipped because the decoder rejected them as invalid data. private int _skippedPacketCount; /// Number of skipped packets for which a warning is logged. private const int MaxLoggedSkippedPackets = 5; private readonly object _lock = new(); // Fields to detect SwsContext updates private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; private int _srcWidth; private int _srcHeight; // ────────────────────────────────────────────────────────────── // Public Properties // ────────────────────────────────────────────────────────────── /// Video width in pixels. public int Width { get; private set; } /// Video height in pixels. public int Height { get; private set; } /// Frames per second. public double FPS { get; private set; } /// Total duration in seconds. public double Duration { get; private set; } /// Estimated total number of frames. public long TotalFrames { get; private set; } /// Index of the currently read frame. public long CurrentFrameIndex { get { lock (_lock) { return _currentFrameIndex; } } } /// Current playback time in seconds. public double CurrentTime { get { lock (_lock) { return FPS > 0 ? _currentFrameIndex / FPS : 0; } } } /// /// Pixel format of the video frames produced by the decoder. /// When hardware decoding is active, returns the pixel format of the downloaded software frame /// in system RAM (typically ) rather than the GPU surface format /// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines. /// In software mode, returns the codec context's pixel format (e.g., YUV420P). /// public AVPixelFormat PixelFormat { get { lock (_lock) { if (_isHwDecoding) { return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE ? _downloadedPixFmt : AVPixelFormat.AV_PIX_FMT_NV12; } return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE; } } } /// /// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]). /// public AVColorRange ColorRange { get { lock (_lock) { if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED) { return _pCodecContext->color_range; } if (_pFormatContext != null && _videoStreamIndex >= 0) { return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range; } return AVColorRange.AVCOL_RANGE_UNSPECIFIED; } } } /// /// Gets a value indicating whether hardware-accelerated decoding is currently active. /// public bool IsHardwareAccelerated { get { lock (_lock) { return _isHwDecoding; } } } /// /// Gets the hardware device type in use for hardware acceleration, or null if software decoding. /// public AVHWDeviceType? HardwareDeviceType { get { lock (_lock) { return _isHwDecoding ? _hwDeviceType : null; } } } // ────────────────────────────────────────────────────────────── // Initialization & Setup // ────────────────────────────────────────────────────────────── /// /// Initializes a new instance of and opens the specified video file. /// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back /// to multi-threaded software decoding if unavailable. /// /// Absolute or relative file path to the video file. public FFmpegNativeDecoder(string videoPath) { Open(videoPath); } /// /// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers. /// /// Path to video file. private void Open(string videoPath) { lock (_lock) { if (_isOpened || _disposed) return; try { // 1. Open media container // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html fixed (AVFormatContext** ppFormatCtx = &_pFormatContext) { int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null); if (ret < 0) throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})"); } // 2. Retrieve stream information if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0) throw new Exception("Failed to retrieve stream info."); // 3. Find primary video stream _videoStreamIndex = -1; for (int i = 0; i < _pFormatContext->nb_streams; i++) { if (_pFormatContext->streams[i]->codecpar->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO) { _videoStreamIndex = i; break; } } if (_videoStreamIndex == -1) throw new Exception("Failed to find video stream."); // Only the video stream is used. The demuxer does not need to return the packets of the other streams // (audio, subtitles, ...) once they are marked as discarded. // Reference: AVStream.discard - "Selects which packets can be discarded at will and do not need to be demuxed." // https://ffmpeg.org/doxygen/trunk/structAVStream.html for (int i = 0; i < _pFormatContext->nb_streams; i++) { if (i != _videoStreamIndex) _pFormatContext->streams[i]->discard = AVDiscard.AVDISCARD_ALL; } var stream = _pFormatContext->streams[_videoStreamIndex]; _timeBase = stream->time_base; // 4. Calculate FPS if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->r_frame_rate); } else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->avg_frame_rate); } else { FPS = 30.0; // Fallback default } // 5. Find decoder by codec ID var codecId = stream->codecpar->codec_id; var decoder = ffmpeg.avcodec_find_decoder(codecId); if (decoder == null) throw new Exception($"Unsupported video codec: {codecId}"); // 6. Allocate codec context _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); if (_pCodecContext == null) throw new Exception("Failed to allocate codec context."); if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters."); // 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding bool hwSuccess = TryInitializeHwDecoder(decoder, stream); if (!hwSuccess) { Melon.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding."); _isHwDecoding = false; _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; // Multi-threading settings for improved software decoding performance (auto thread count) // Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html _pCodecContext->thread_count = 0; _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME; int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null); if (openRet < 0) throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}"); } Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width; Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height; // 8. Calculate total frames and duration if (stream->nb_frames > 0) { TotalFrames = stream->nb_frames; } else if (stream->duration > 0) { TotalFrames = (long)Math.Round(stream->duration * ffmpeg.av_q2d(stream->time_base) * FPS); } else if (_pFormatContext->duration > 0) { TotalFrames = (long)Math.Round((_pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE) * FPS); } else { TotalFrames = 0; } // Duration of the video stream (AVStream.duration, in the stream time base). // When the stream does not carry a duration, the container duration (AVFormatContext.duration, // in AV_TIME_BASE units, the longest of all streams) is used. // Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html, https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html Duration = stream->duration > 0 ? stream->duration * ffmpeg.av_q2d(stream->time_base) : _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0; // 9. Allocate frame and packet buffers // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html _pDecodedFrame = ffmpeg.av_frame_alloc(); _pHwTransferFrame = ffmpeg.av_frame_alloc(); _pConvertedFrame = ffmpeg.av_frame_alloc(); _pCachedFrame = ffmpeg.av_frame_alloc(); _pPacket = ffmpeg.av_packet_alloc(); _currentFrameIndex = 0; _isOpened = true; _isDraining = false; } catch (Exception) { Dispose(); throw; } } } /// /// Searches for supported hardware device configurations via avcodec_get_hw_config, /// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA), /// registers the get_format callback, and attempts to open the hardware decoder. /// /// Target decoder. /// Source video stream. /// True if hardware decoding was successfully configured and opened; false to fall back. private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream) { // ReSharper disable once InconsistentNaming // Collect all supported hardware configs from decoder const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01; var candidates = new List(); for (int i = 0; ; i++) { var config = ffmpeg.avcodec_get_hw_config(decoder, i); if (config == null) break; if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0) { candidates.Add(new HwConfigCandidate { DeviceType = config->device_type, PixelFormat = config->pix_fmt }); } } if (candidates.Count == 0) { return false; } // Order candidates according to preference var orderedCandidates = new List(); foreach (var preferred in PreferredHwDeviceTypes) { foreach (var candidate in candidates) { if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate)) { orderedCandidates.Add(candidate); } } } // Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes foreach (var candidate in candidates) { if (!orderedCandidates.Contains(candidate)) { orderedCandidates.Add(candidate); } } // Try candidate hardware configurations in priority order foreach (var candidate in orderedCandidates) { AVBufferRef* hwDeviceCtx = null; int err; // Device creation strategy // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV) { var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); string childDeviceName = isWindows ? "d3d11va" : "vaapi"; AVDictionary* opts = null; ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0); err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0); ffmpeg.av_dict_free(&opts); if (err < 0) { err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0); } } else { err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0); } if (err < 0 || hwDeviceCtx == null) { if (hwDeviceCtx != null) { ffmpeg.av_buffer_unref(&hwDeviceCtx); hwDeviceCtx = null; } Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate..."); continue; } // Assign hardware context and format selection callback _hwDeviceCtx = hwDeviceCtx; _hwPixFmt = candidate.PixelFormat; _hwDeviceType = candidate.DeviceType; _isHwDecoding = true; _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12; _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx); if (_pCodecContext->hw_device_ctx == null) { Melon.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate..."); var pHw = _hwDeviceCtx; ffmpeg.av_buffer_unref(&pHw); _hwDeviceCtx = null; _isHwDecoding = false; continue; } _getFormatDelegate = GetFormat; _pCodecContext->get_format = _getFormatDelegate; // Hardware decoding requires single-threaded decoding (thread_count = 1). // Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function // GPU decoders and causes surface pool allocation exhaustion or crashes in get_format. _pCodecContext->thread_count = 1; // Attempt opening codec context with hardware configuration int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null); if (openRet >= 0) { Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}"); return true; } // Failed to open codec with this hardware configuration, clean up and try next Melon.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate..."); if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; } if (_hwDeviceCtx != null) { var pHw = _hwDeviceCtx; ffmpeg.av_buffer_unref(&pHw); _hwDeviceCtx = null; } _isHwDecoding = false; _getFormatDelegate = null; // Reallocate fresh codec context for subsequent attempts _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); if (_pCodecContext == null) throw new Exception("Failed to re-allocate codec context during fallback."); if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters during fallback."); } return false; } /// /// Callback invoked by libavcodec during decoding to negotiate the output pixel format. /// Confirms the hardware surface format () if offered by the decoder, /// or falls back to software pixel formats via avcodec_default_get_format. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts) { for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++) { if (*p == _hwPixFmt) { return *p; } } Melon.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format."); _isHwDecoding = false; _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; return ffmpeg.avcodec_default_get_format(ctx, pixFmts); } // ────────────────────────────────────────────────────────────── // Demuxing & Decoding Engine // ────────────────────────────────────────────────────────────── /// /// Reads packets from the container until a packet belonging to the video stream is found, /// then sends it to the decoder via avcodec_send_packet. /// A packet that the decoder rejects as invalid data (AVERROR_INVALIDDATA, e.g. a damaged packet) /// is skipped and the next packet is read. Any other decoder error is thrown. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// /// True if a video packet was sent; false on EOF. private bool ReadAndSendPacket() { while (true) { ffmpeg.av_packet_unref(_pPacket); int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket); if (ret < 0) return false; // EOF or read error if (_pPacket->stream_index == _videoStreamIndex) { ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket); ffmpeg.av_packet_unref(_pPacket); if (ret == ffmpeg.AVERROR_INVALIDDATA) { // The warning is logged for the first few packets only (a damaged file can have many). _skippedPacketCount++; if (_skippedPacketCount <= MaxLoggedSkippedPackets) { Melon.Logger.Warning($"[FFmpegNativeDecoder] Skipped an invalid video packet ({_skippedPacketCount}): {GetFFmpegErrorString(ret)}"); } continue; } if (ret < 0) { throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}"); } return true; } } } /// /// Receives the next decoded video frame from the codec context. /// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV), /// it is automatically downloaded to system RAM via av_hwframe_transfer_data into _pHwTransferFrame, /// preserving presentation timestamps via av_frame_copy_props. /// /// The software frame containing pixel data in system RAM. /// True if a frame was successfully decoded and available; false on EOF or draining. private bool TryReceiveFrame(out AVFrame* outFrame) { outFrame = null; while (true) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { // Check if the received frame is on a hardware surface if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) { ffmpeg.av_frame_unref(_pHwTransferFrame); int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0); if (transferRet < 0) { ffmpeg.av_frame_unref(_pDecodedFrame); throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}"); } // Copy presentation timestamp and metadata properties ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); _downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format; outFrame = _pHwTransferFrame; return true; } // Frame is in system memory (CPU decoding or software fallback format) outFrame = _pDecodedFrame; return true; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return false; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { return false; } else { throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); } } } /// /// Lazily initializes or updates the when source/destination formats or dimensions change. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat) { if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat) { return; } if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } _srcPixFmt = srcFormat; _srcWidth = srcWidth; _srcHeight = srcHeight; _dstPixFmt = dstFormat; _pSwsContext = ffmpeg.sws_getContext( srcWidth, srcHeight, srcFormat, Width, Height, dstFormat, (int)SwsFlags.SWS_BILINEAR, null, null, null ); if (_pSwsContext == null) { throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height})."); } } /// /// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state. /// Used by seeking to inspect candidate frames without falsely incrementing frame indices. /// private double GetFrameTimestamp(AVFrame* frame) { long ptsValue = frame->best_effort_timestamp; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { ptsValue = frame->pts; } if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { return FPS > 0 ? _currentFrameIndex / FPS : 0; } return ptsValue * ffmpeg.av_q2d(_timeBase); } /// /// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter. /// private double GetTimestampAndUpdateIndex(AVFrame* frame) { double timestamp = GetFrameTimestamp(frame); long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE ? frame->best_effort_timestamp : frame->pts; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { _currentFrameIndex++; } else { _currentFrameIndex = (long)Math.Round(timestamp * FPS); } return timestamp; } /// /// Resets all data plane pointers and strides in to zero. /// Prevents stale or dangling pointers from previous conversions being retained across frames. /// private void ClearConvertedFramePointers() { if (_pConvertedFrame == null) return; for (uint i = 0; i < 8; i++) { _pConvertedFrame->data[i] = null; _pConvertedFrame->linesize[i] = 0; } } // ────────────────────────────────────────────────────────────── // Frame Reading APIs: RGBA (CPU Fallback) // ────────────────────────────────────────────────────────────── /// /// Reads the next video frame and copies it in 32-bit RGBA format using libswscale. /// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P). /// /// Pointer to destination memory storing RGBA data. /// Byte stride per row of destination memory (Width * 4). /// Whether to vertically flip the output to match Unity texture coordinates. /// Timestamp of the output frame in seconds. /// True if a frame was successfully retrieved; false on EOF. public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed || outBuffer == null) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA); ClearConvertedFramePointers(); if (vflip) { _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); _pConvertedFrame->linesize[0] = -outStride; } else { _pConvertedFrame->data[0] = outBuffer; _pConvertedFrame->linesize[0] = outStride; } ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ClearConvertedFramePointers(); ffmpeg.av_frame_unref(frame); return true; } } /// /// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer. /// public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp) { return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp); } /// /// Reads the next video frame and copies it in RGBA format to a Unity NativeArray. /// public bool TryReadFrame(NativeArray outBuffer, bool vflip, out double timestamp) { if (!outBuffer.IsCreated) { throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer)); } if (outBuffer.Length < (long)Width * Height * 4) { throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer)); } int stride = Width * 4; return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp); } // ────────────────────────────────────────────────────────────── // Frame Reading APIs: NV12 (Compute Shader Pipeline) // ────────────────────────────────────────────────────────────── /// /// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format. /// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed, /// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing. /// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale. /// /// Pointer to destination buffer for Y plane (Width * Height bytes). /// Row stride in bytes for Y plane. /// Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes). /// Row stride in bytes for UV plane. /// Timestamp of the output frame in seconds. /// True if a frame was successfully retrieved; false on EOF. // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( byte* yBuffer, int yStride, byte* uvBuffer, int uvStride, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12); ClearConvertedFramePointers(); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uvBuffer; _pConvertedFrame->linesize[0] = yStride; _pConvertedFrame->linesize[1] = uvStride; ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ClearConvertedFramePointers(); } else { int w = Width; int h = Height; int uvH = (h + 1) / 2; int uvRowBytes = (w / 2) * 2; // Copy Y plane (row by row to safely skip linesize GPU padding) for (int y = 0; y < h; y++) { Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, yStride, w ); } // Copy UV plane (interleaved U and V samples) for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uvBuffer + y * uvStride, uvStride, uvRowBytes ); } } ffmpeg.av_frame_unref(frame); return true; } } /// /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers. /// // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( IntPtr yBuffer, int yStride, IntPtr uvBuffer, int uvStride, out double timestamp) { return TryReadNV12Frame( (byte*)yBuffer.ToPointer(), yStride, (byte*)uvBuffer.ToPointer(), uvStride, out timestamp ); } /// /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays. /// // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( NativeArray yBuffer, NativeArray uvBuffer, out double timestamp) { if (!yBuffer.IsCreated || !uvBuffer.IsCreated) { throw new ArgumentException("Input NativeArray buffer is not created or already disposed."); } int w = Width; int h = Height; int uvH = (h + 1) / 2; int uvRowBytes = (w / 2) * 2; if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH) { throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution."); } return TryReadNV12Frame( (byte*)yBuffer.GetUnsafePtr(), w, (byte*)uvBuffer.GetUnsafePtr(), uvRowBytes, out timestamp ); } // ────────────────────────────────────────────────────────────── // Frame Reading APIs: YUV420P (Compute Shader Pipeline) // ────────────────────────────────────────────────────────────── /// /// Reads the next video frame and copies its planar Y, U, and V channel data separately. /// For native YUV420P sources, color conversion is bypassed and memory is copied directly. /// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale. /// // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( byte* yBuffer, int yStride, byte* uBuffer, int uStride, byte* vBuffer, int vStride, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P); ClearConvertedFramePointers(); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uBuffer; _pConvertedFrame->data[2] = vBuffer; _pConvertedFrame->linesize[0] = yStride; _pConvertedFrame->linesize[1] = uStride; _pConvertedFrame->linesize[2] = vStride; ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ClearConvertedFramePointers(); } else { int w = Width; int h = Height; int uvW = (w + 1) / 2; int uvH = (h + 1) / 2; // Copy Y plane for (int y = 0; y < h; y++) { Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, yStride, w ); } // Copy U plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uBuffer + y * uStride, uStride, uvW ); } // Copy V plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[2] + y * frame->linesize[2], vBuffer + y * vStride, vStride, uvW ); } } ffmpeg.av_frame_unref(frame); return true; } } /// /// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers. /// // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( IntPtr yBuffer, int yStride, IntPtr uBuffer, int uStride, IntPtr vBuffer, int vStride, out double timestamp) { return TryReadYUV420PFrame( (byte*)yBuffer.ToPointer(), yStride, (byte*)uBuffer.ToPointer(), uStride, (byte*)vBuffer.ToPointer(), vStride, out timestamp ); } /// /// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays. /// // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( NativeArray yBuffer, NativeArray uBuffer, NativeArray vBuffer, out double timestamp) { if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated) { throw new ArgumentException("Input NativeArray buffer is not created or already disposed."); } int w = Width; int h = Height; int uvW = (w + 1) / 2; int uvH = (h + 1) / 2; if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH) { throw new ArgumentException("Buffer size is smaller than video YUV plane resolution."); } return TryReadYUV420PFrame( (byte*)yBuffer.GetUnsafePtr(), w, (byte*)uBuffer.GetUnsafePtr(), uvW, (byte*)vBuffer.GetUnsafePtr(), uvW, out timestamp ); } // ────────────────────────────────────────────────────────────── // Seeking & Skipping // ────────────────────────────────────────────────────────────── /// /// Decodes and discards a specified number of frames immediately without GPU-to-CPU transfer overhead. /// /// Number of frames to discard. private void SkipFrames(int count) { lock (_lock) { if (!_isOpened || _disposed) return; if (count <= 0) return; int skipped = 0; if (_hasCachedFrame) { ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; skipped++; } while (skipped < count) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { GetTimestampAndUpdateIndex(_pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); skipped++; continue; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) break; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { break; } else { throw new Exception($"Error skipping frames: {GetFFmpegErrorString(ret)}"); } } } } /// /// Seeks to a specific playback time in seconds using backward keyframe seek followed by forward decode. /// Intermediate frames are discarded without GPU-to-host memory transfer to maximize seek speed; /// only the target frame is transferred and preserved in . /// /// Target time in seconds to seek to. public void SeekToTime(double timeSeconds) { lock (_lock) { if (!_isOpened || _disposed) return; if (timeSeconds < 0) timeSeconds = 0; double targetFrameIndex = timeSeconds * FPS; // If already at target index or very close future, skip frames instead of seek to avoid I-frame seek latency bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60; if (useSkip) { SkipFrames((int)(targetFrameIndex - _currentFrameIndex)); return; } if (_hasCachedFrame) { ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; } long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase)); int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { // Fallback to beginning of file on seek failure ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}"); } } // Flush decoder internal buffers // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html ffmpeg.avcodec_flush_buffers(_pCodecContext); _isDraining = false; // Decode frames until reaching the target presentation timestamp while (true) { ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { double currentPts = GetFrameTimestamp(_pDecodedFrame); // Check with half-frame margin to accommodate floating point rounding if (currentPts >= timeSeconds - (0.5 / FPS)) { // Cache the first matching frame in system RAM for immediate consumption if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) { ffmpeg.av_frame_unref(_pCachedFrame); int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0); if (transferRet < 0) { ffmpeg.av_frame_unref(_pDecodedFrame); throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}"); } ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); _downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format; } else { ffmpeg.av_frame_unref(_pCachedFrame); ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame); } _hasCachedFrame = true; break; } // Discard non-matching intermediate frame without transferring to CPU long ptsValue = _pDecodedFrame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE ? _pDecodedFrame->best_effort_timestamp : _pDecodedFrame->pts; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { _currentFrameIndex++; } else { _currentFrameIndex = (long)Math.Round(currentPts * FPS); } ffmpeg.av_frame_unref(_pDecodedFrame); continue; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) break; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { break; } else { throw new Exception($"Error during seek decoding: {GetFFmpegErrorString(ret)}"); } } } } /// /// Seeks to a specific frame index. /// /// Target frame index. public void SeekToFrame(long frameIndex) { SeekToTime(frameIndex / FPS); } // ────────────────────────────────────────────────────────────── // Error Handling & Disposal // ────────────────────────────────────────────────────────────── /// /// Translates an FFmpeg integer error code to a readable string using av_strerror. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html /// private string GetFFmpegErrorString(int errnum) { const ulong bufferSize = 1024; byte* buffer = stackalloc byte[(int)bufferSize]; if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0) { return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})"; } return $"Unknown error (Code: {errnum})"; } /// /// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures. /// public void Dispose() { lock (_lock) { if (_disposed) return; _disposed = true; // 1. Release scaler context if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } // 2. Release frame buffers if (_pDecodedFrame != null) { var pFrame = _pDecodedFrame; ffmpeg.av_frame_free(&pFrame); _pDecodedFrame = null; } if (_pHwTransferFrame != null) { var pFrame = _pHwTransferFrame; ffmpeg.av_frame_free(&pFrame); _pHwTransferFrame = null; } if (_pCachedFrame != null) { var pFrame = _pCachedFrame; ffmpeg.av_frame_free(&pFrame); _pCachedFrame = null; } if (_pConvertedFrame != null) { var pFrame = _pConvertedFrame; ffmpeg.av_frame_free(&pFrame); _pConvertedFrame = null; } // 3. Release packet buffer if (_pPacket != null) { var pPacket = _pPacket; ffmpeg.av_packet_free(&pPacket); _pPacket = null; } // 4. Release codec context if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; } // 5. Release hardware device context if (_hwDeviceCtx != null) { var pHw = _hwDeviceCtx; ffmpeg.av_buffer_unref(&pHw); _hwDeviceCtx = null; } // 6. Close input file and release format context if (_pFormatContext != null) { var pFormatCtx = _pFormatContext; ffmpeg.avformat_close_input(&pFormatCtx); _pFormatContext = null; } _getFormatDelegate = null; _isOpened = false; _isHwDecoding = false; GC.SuppressFinalize(this); } } ~FFmpegNativeDecoder() { Dispose(); } }