using System; using System.Collections.Generic; using System.Runtime.InteropServices; using FFmpeg.AutoGen; using MelonLoader; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; namespace ADOFAIRenderer.Util.FFmpeg; /// /// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke. /// Supports both GPU hardware-accelerated decoding (device types of : D3D11VA, DXVA2, /// CUDA, D3D12VA on Windows; VAAPI, CUDA or VideoToolbox on other platforms; others the decoder reports) and /// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback. /// /// Used by VideoBGPlayer to decode the background video of a level. Only the first video stream is decoded; /// audio and other streams are discarded by the demuxer. /// /// Threading: every public member takes _lock, so calls from different threads are serialized. /// Width, Height, FPS, Duration and TotalFrames are set once in the constructor and read without the lock. /// /// Timestamps returned by the TryRead* methods are the presentation times of the stream in seconds /// (PTS × stream time base), so a stream with a non-zero start time returns timestamps starting at that time. /// /// ── Architecture & Lifecycle Flow ────────────────────────────────────────────── /// /// 1. Construction & Demuxing: /// a) avformat_open_input: Open container file and parse stream headers. /// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS. /// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO). /// /// 2. Codec & Hardware Device Discovery: /// a) avcodec_find_decoder: Locate codec implementation by codec ID. /// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder. /// c) HW device creation for the reported configurations, in the order of PreferredHwDeviceTypes. /// d) Attach hw_device_ctx and register custom get_format callback. /// e) If HW initialization fails or is unsupported, fall back to CPU software decoding /// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME). /// f) avcodec_open2: Initialize codec context. /// /// 3. Frame Demuxing & Decoding (TryRead*): /// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet. /// b) (Decoding) avcodec_receive_frame retrieves decoded frames. /// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data /// into system RAM as NV12 (av_frame_copy_props preserves timestamps). /// d) (Output Dispatch): /// - TryReadNV12Frame: NV12 frames are copied row by row (Buffer.MemoryCopy) into the output buffers, /// other formats are converted with sws_scale. /// - TryReadYUV420PFrame: YUV420P/YUVJ420P planes are copied row by row into the output buffers, /// other formats are converted with sws_scale. /// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip). /// /// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame): /// (Short forward distances skip a) and b) and only decode forward; see SeekToTime.) /// a) av_seek_frame: Fast backward keyframe seek. /// b) avcodec_flush_buffers: Clear internal decoder state. /// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead; /// transfers and caches the first matching frame in _pCachedFrame for immediate consumption. /// /// 5. Resource Disposal (Dispose): /// a) Free scaler context (sws_freeContext). /// b) Free all allocated frames and packets (av_frame_free, av_packet_free). /// c) Free codec context (avcodec_free_context). /// d) Release hardware device context (av_buffer_unref). /// e) Close format context (avformat_close_input). /// /// ── Reference Documentation ──────────────────────────────────────────────────── /// • FFmpeg Demuxing & Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html /// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html /// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html /// • libswscale Scaling & Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// internal sealed unsafe class FFmpegNativeDecoder : IDisposable { // ────────────────────────────────────────────────────────────── // Hardware Acceleration Priorities & Candidate Structure // ────────────────────────────────────────────────────────────── /// /// Candidate hardware device configuration discovered via avcodec_get_hw_config. /// private record struct HwConfigCandidate { /// Hardware device type to create with av_hwdevice_ctx_create (AVCodecHWConfig.device_type). public AVHWDeviceType DeviceType; /// Hardware surface pixel format the decoder outputs with this device (AVCodecHWConfig.pix_fmt, e.g. AV_PIX_FMT_D3D11). public AVPixelFormat PixelFormat; } /// /// Preferred hardware device types in priority order. /// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config), /// sorted by this list; configurations not in the list follow in reported order. /// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Linux: CUDA (NVIDIA), then VAAPI. macOS: VideoToolbox. /// A device type that cannot be created on the machine (no driver, wrong platform) fails creation and is skipped. /// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by /// avcodec_find_decoder report hwaccel device types such as DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan (depending on the build); /// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name. /// private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[] { AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA, AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA, AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA, AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI, AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX }; // ────────────────────────────────────────────────────────────── // FFmpeg Core Struct Pointers // ────────────────────────────────────────────────────────────── /// /// Context managing container format demuxing (MP4, MKV, WebM, etc.). /// Used for packet demuxing and seeking. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html /// private AVFormatContext* _pFormatContext = null; /// /// Context managing video decoder state and parameters. /// Configured with codec parameters, hardware contexts, and threading options. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// private AVCodecContext* _pCodecContext = null; /// /// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.). /// This class owns one reference; the codec context holds a second one (hw_device_ctx). /// Both are released in Dispose (or when a hardware candidate fails). Null in software decoding. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html /// private AVBufferRef* _hwDeviceCtx = null; /// /// Frame structure holding raw decoded frames received directly from avcodec_receive_frame. /// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView). /// In SW decoding mode, this contains pixel buffers in system memory. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pDecodedFrame = null; /// /// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces /// via av_hwframe_transfer_data. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html /// private AVFrame* _pHwTransferFrame = null; /// /// Frame structure holding converted pixel data produced by sws_scale (e.g., RGBA or fallback YUV). /// Owns no pixel memory: during one conversion its data[]/linesize[] point to the caller's output buffers, /// and they are cleared before and after (ClearConvertedFramePointers). /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pConvertedFrame = null; /// /// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation. /// Consumed by the next TryRead* invocation. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html /// private AVFrame* _pCachedFrame = null; /// /// Whether holds a frame that the next TryRead* call returns /// before any new frame is decoded. Set by DecodeToTarget, cleared when the frame is consumed or dropped. /// private bool _hasCachedFrame; /// /// Packet structure temporarily holding compressed demuxed data read from the input container. /// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html /// private AVPacket* _pPacket = null; /// /// libswscale colorspace and format conversion context. /// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// private SwsContext* _pSwsContext = null; // ────────────────────────────────────────────────────────────── // Hardware Acceleration State & Delegates // ────────────────────────────────────────────────────────────── /// /// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code. /// Passed to _pCodecContext->get_format. /// private AVCodecContext_get_format? _getFormatDelegate; /// /// Whether hardware-accelerated decoding is currently active. /// Set when a hardware candidate is configured; cleared when it fails, or by GetFormat when the stream /// cannot be decoded on the hardware surface format (the decoder then outputs software frames). /// private bool _isHwDecoding; /// Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV). private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE; /// Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD). private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; /// /// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12). /// Assumed NV12 when hardware decoding is configured, then updated from every transferred frame /// (e.g. P010 for 10-bit streams). AV_PIX_FMT_NONE in software decoding. /// private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; // ────────────────────────────────────────────────────────────── // Playback, Stream & Scaler Synchronization Fields // ────────────────────────────────────────────────────────────── /// Index of the decoded video stream in the format context. -1 until a video stream is found. private int _videoStreamIndex = -1; /// /// Index of the frame most recently returned (or skipped while seeking): round(timestamp × FPS) for frames with a timestamp, /// the previous value + 1 for frames without one. 0 after opening. /// private long _currentFrameIndex; /// Time base of the video stream (seconds per PTS unit). Used to convert PTS to seconds and seek targets to PTS. private AVRational _timeBase; /// Whether Open completed. The read and seek methods do nothing until then and after Dispose. private bool _isOpened; /// Whether Dispose has run. Prevents duplicate cleanup. private bool _disposed; /// /// Whether the end of the input has been reached and the decoder was put into draining mode /// (avcodec_send_packet with null). Reset by a keyframe seek (after avcodec_flush_buffers). /// private bool _isDraining; /// Number of video packets skipped because the decoder rejected them as invalid data. private int _skippedPacketCount; /// Number of skipped packets for which a warning is logged. private const int MaxLoggedSkippedPackets = 5; /// Lock object serializing all public members (decoder state and native resources). private readonly object _lock = new(); // Fields to detect SwsContext updates // Parameters the current _pSwsContext was created with; a frame with different values recreates the context. /// Source pixel format of the current scaler context. private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; /// Destination pixel format of the current scaler context (RGBA, NV12 or YUV420P). private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; /// Source width of the current scaler context in pixels (the destination is always Width × Height). private int _srcWidth; /// Source height of the current scaler context in pixels. private int _srcHeight; /// YUV→RGB matrix id (SWS_CS_*) applied to the context. 0 when no matrix is set (non-RGBA output). private int _srcSwsColorspace; /// Whether the source range applied to the context is the full range. private bool _srcFullRange; // ────────────────────────────────────────────────────────────── // Public Properties // ────────────────────────────────────────────────────────────── /// /// Video width in pixels (codec context width after opening, or the stream parameters). /// Every output of the TryRead* methods has this width; frames of another size are scaled to it. /// public int Width { get; private set; } /// Video height in pixels. Every output of the TryRead* methods has this height. public int Height { get; private set; } /// /// Frames per second: r_frame_rate of the stream, else avg_frame_rate, else 30. /// For variable frame rate streams both values are approximations. /// public double FPS { get; private set; } /// Total duration in seconds (stream duration, else container duration, else 0). public double Duration { get; private set; } /// /// Estimated total number of frames: the frame count of the stream header (nb_frames) when present, /// otherwise round(duration × FPS); 0 when the duration is unknown. /// public long TotalFrames { get; private set; } /// Index of the currently read frame (the frame most recently returned or skipped; see _currentFrameIndex). public long CurrentFrameIndex { get { lock (_lock) { return _currentFrameIndex; } } } /// Current playback time in seconds: CurrentFrameIndex / FPS (0 when FPS is not positive). public double CurrentTime { get { lock (_lock) { return FPS > 0 ? _currentFrameIndex / FPS : 0; } } } /// /// Pixel format of the video frames produced by the decoder. /// When hardware decoding is active, returns the pixel format of the downloaded software frame /// in system RAM (typically ) rather than the GPU surface format /// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines. /// In software mode, returns the codec context's pixel format (e.g., YUV420P). /// Before the first hardware frame is transferred, NV12 is returned as the expected download format; /// the actual format (e.g. P010 for 10-bit video) is known after the first transfer. /// The TryRead* methods convert any decoded format to their output format, so this value only selects the fastest path. /// public AVPixelFormat PixelFormat { get { lock (_lock) { if (_isHwDecoding) { return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE ? _downloadedPixFmt : AVPixelFormat.AV_PIX_FMT_NV12; } return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE; } } } /// /// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]). /// Taken from the codec context, else from the stream parameters; AVCOL_RANGE_UNSPECIFIED when neither carries a tag. /// This property reads the color_range tags only; it does not inspect the pixel format (YUVJ* formats are full range). /// public AVColorRange ColorRange { get { lock (_lock) { if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED) { return _pCodecContext->color_range; } if (_pFormatContext != null && _videoStreamIndex >= 0) { return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range; } return AVColorRange.AVCOL_RANGE_UNSPECIFIED; } } } /// /// Color matrix id used to convert the YUV frames of this stream to RGB, selected from the colorspace tag of the stream. /// The ids are shared with the ColorMatrix uniform of ColorConversion.compute: /// 0 = BT.709 (also used when the stream has no usable tag), 1 = BT.601, 2 = BT.2020 (non-constant luminance). /// public int ColorMatrix { get { lock (_lock) { if (_pCodecContext != null && _pCodecContext->colorspace != AVColorSpace.AVCOL_SPC_UNSPECIFIED) { return GetColorMatrixId(_pCodecContext->colorspace); } if (_pFormatContext != null && _videoStreamIndex >= 0) { return GetColorMatrixId(_pFormatContext->streams[_videoStreamIndex]->codecpar->color_space); } return ColorMatrixBT709; } } } /// Color matrix id of BT.709. // ReSharper disable InconsistentNaming public const int ColorMatrixBT709 = 0; /// Color matrix id of BT.601 (SMPTE 170M / BT.470 BG). public const int ColorMatrixBT601 = 1; /// Color matrix id of BT.2020 (non-constant luminance). public const int ColorMatrixBT2020 = 2; // ReSharper restore InconsistentNaming /// /// Converts a colorspace tag to a color matrix id (see ). /// Tags that have no matrix of their own (unspecified, RGB, other standards) use BT.709. /// private static int GetColorMatrixId(AVColorSpace colorspace) { switch (colorspace) { case AVColorSpace.AVCOL_SPC_SMPTE170M: case AVColorSpace.AVCOL_SPC_BT470BG: return ColorMatrixBT601; case AVColorSpace.AVCOL_SPC_BT2020_NCL: return ColorMatrixBT2020; default: return ColorMatrixBT709; } } /// /// Gets a value indicating whether hardware-accelerated decoding is currently active. /// public bool IsHardwareAccelerated { get { lock (_lock) { return _isHwDecoding; } } } /// /// Gets the hardware device type in use for hardware acceleration, or null if software decoding. /// public AVHWDeviceType? HardwareDeviceType { get { lock (_lock) { return _isHwDecoding ? _hwDeviceType : null; } } } // ────────────────────────────────────────────────────────────── // Initialization & Setup // ────────────────────────────────────────────────────────────── /// /// Initializes a new instance of and opens the specified video file. /// Automatically attempts hardware acceleration (device types reported by the decoder, in the order of /// ), falling back to multi-threaded software decoding if unavailable. /// /// Absolute or relative file path to the video file. /// /// The file cannot be opened, has no video stream, or its codec has no decoder or cannot be opened. /// All native resources allocated so far are released before the exception propagates. /// public FFmpegNativeDecoder(string videoPath) { Open(videoPath); } /// /// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers. /// /// Path to video file. private void Open(string videoPath) { lock (_lock) { if (_isOpened || _disposed) return; try { // 1. Open media container // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html fixed (AVFormatContext** ppFormatCtx = &_pFormatContext) { int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null); if (ret < 0) throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})"); } // 2. Retrieve stream information // Reads (and buffers) the start of the file to fill in codec parameters, frame rates and durations. if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0) throw new Exception("Failed to retrieve stream info."); // 3. Find primary video stream (the first stream whose codec type is video) _videoStreamIndex = -1; for (int i = 0; i < _pFormatContext->nb_streams; i++) { if (_pFormatContext->streams[i]->codecpar->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO) { _videoStreamIndex = i; break; } } if (_videoStreamIndex == -1) throw new Exception("Failed to find video stream."); // Only the video stream is used. The demuxer does not need to return the packets of the other streams // (audio, subtitles, ...) once they are marked as discarded. // Reference: AVStream.discard - "Selects which packets can be discarded at will and do not need to be demuxed." // https://ffmpeg.org/doxygen/trunk/structAVStream.html for (int i = 0; i < _pFormatContext->nb_streams; i++) { if (i != _videoStreamIndex) _pFormatContext->streams[i]->discard = AVDiscard.AVDISCARD_ALL; } var stream = _pFormatContext->streams[_videoStreamIndex]; _timeBase = stream->time_base; // 4. Calculate FPS // r_frame_rate: lowest frame rate that represents all timestamps accurately (guessed by libavformat). // avg_frame_rate: average frame rate. Both are approximations for variable frame rate streams. // Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->r_frame_rate); } else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->avg_frame_rate); } else { FPS = 30.0; // Fallback default } // 5. Find decoder by codec ID // avcodec_find_decoder returns the default decoder registered for the ID (e.g. libdav1d for AV1 // when the build includes it, which reports no hardware configuration). var codecId = stream->codecpar->codec_id; var decoder = ffmpeg.avcodec_find_decoder(codecId); if (decoder == null) throw new Exception($"Unsupported video codec: {codecId}"); // 6. Allocate codec context _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); if (_pCodecContext == null) throw new Exception("Failed to allocate codec context."); // Copies the stream parameters (size, pixel format, extradata, color tags, ...) into the codec context. if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters."); // 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding bool hwSuccess = TryInitializeHwDecoder(decoder, stream); if (!hwSuccess) { Melon.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding."); _isHwDecoding = false; _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; // Multi-threading settings for improved software decoding performance (auto thread count) // thread_type = FF_THREAD_FRAME selects frame threading only (several frames decoded in parallel, // adds one frame of delay per thread); slice threading is not enabled. // TryInitializeHwDecoder leaves a fresh codec context when every hardware candidate fails. // Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html _pCodecContext->thread_count = 0; _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME; int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null); if (openRet < 0) throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}"); } Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width; Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height; // 8. Calculate total frames and duration // nb_frames is the frame count stored in the container (0 when unknown); // otherwise the count is estimated from the stream or container duration and FPS. if (stream->nb_frames > 0) { TotalFrames = stream->nb_frames; } else if (stream->duration > 0) { TotalFrames = (long)Math.Round(stream->duration * ffmpeg.av_q2d(stream->time_base) * FPS); } else if (_pFormatContext->duration > 0) { TotalFrames = (long)Math.Round((_pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE) * FPS); } else { TotalFrames = 0; } // Duration of the video stream (AVStream.duration, in the stream time base). // When the stream does not carry a duration, the container duration (AVFormatContext.duration, // in AV_TIME_BASE units, the longest of all streams) is used. // Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html, https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html Duration = stream->duration > 0 ? stream->duration * ffmpeg.av_q2d(stream->time_base) : _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0; // 9. Allocate frame and packet buffers // Structures only; pixel buffers are attached by the decoder, av_hwframe_transfer_data or av_frame_move_ref. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html _pDecodedFrame = ffmpeg.av_frame_alloc(); _pHwTransferFrame = ffmpeg.av_frame_alloc(); _pConvertedFrame = ffmpeg.av_frame_alloc(); _pCachedFrame = ffmpeg.av_frame_alloc(); _pPacket = ffmpeg.av_packet_alloc(); _currentFrameIndex = 0; _isOpened = true; _isDraining = false; } catch (Exception) { Dispose(); throw; } } } /// /// Searches for supported hardware device configurations via avcodec_get_hw_config, /// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA, VAAPI, ...), /// registers the get_format callback, and attempts to open the hardware decoder. /// /// Candidates are tried in the order of , then the remaining reported order. /// When a candidate fails to open, its codec context and device are freed and a fresh codec context /// (with the stream parameters) is allocated for the next candidate or for the software fallback. /// On success, , and describe the device /// and the codec context is open. /// /// Target decoder. /// Source video stream. /// True if hardware decoding was successfully configured and opened; false to fall back. private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream) { // ReSharper disable once InconsistentNaming // Collect all supported hardware configs from decoder // AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX (0x01): the decoder supports the hardware format through hw_device_ctx, // which has to be set on the codec context before avcodec_open2. Only these configurations are used here. // Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/codec.h const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01; var candidates = new List(); for (int i = 0; ; i++) { var config = ffmpeg.avcodec_get_hw_config(decoder, i); if (config == null) break; if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0) { candidates.Add(new HwConfigCandidate { DeviceType = config->device_type, PixelFormat = config->pix_fmt }); } } if (candidates.Count == 0) { return false; } // Order candidates according to preference var orderedCandidates = new List(); foreach (var preferred in PreferredHwDeviceTypes) { foreach (var candidate in candidates) { if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate)) { orderedCandidates.Add(candidate); } } } // Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes foreach (var candidate in candidates) { if (!orderedCandidates.Contains(candidate)) { orderedCandidates.Add(candidate); } } // Try candidate hardware configurations in priority order foreach (var candidate in orderedCandidates) { AVBufferRef* hwDeviceCtx = null; int err; // Device creation strategy // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV) { // QSV: first with an explicit child device (D3D11VA on Windows, VAAPI elsewhere), // then without options (FFmpeg selects the child device type). var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); string childDeviceName = isWindows ? "d3d11va" : "vaapi"; AVDictionary* opts = null; ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0); err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0); ffmpeg.av_dict_free(&opts); if (err < 0) { err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0); } } else { // Default device of the type (device = null: the first / default adapter chosen by FFmpeg). err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0); } if (err < 0 || hwDeviceCtx == null) { if (hwDeviceCtx != null) { ffmpeg.av_buffer_unref(&hwDeviceCtx); hwDeviceCtx = null; } Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate..."); continue; } // Assign hardware context and format selection callback // _hwDeviceCtx takes over the reference created above; the codec context receives its own reference. _hwDeviceCtx = hwDeviceCtx; _hwPixFmt = candidate.PixelFormat; _hwDeviceType = candidate.DeviceType; _isHwDecoding = true; _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12; _pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx); if (_pCodecContext->hw_device_ctx == null) { Melon.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate..."); var pHw = _hwDeviceCtx; ffmpeg.av_buffer_unref(&pHw); _hwDeviceCtx = null; _isHwDecoding = false; continue; } // The delegate is kept in a field so that it is not collected while native code can still call it // (FFmpeg.AutoGen marshals it to a function pointer). With thread_count = 1 it is called on the calling // thread inside avcodec_send_packet / avcodec_receive_frame, which run under _lock. _getFormatDelegate = GetFormat; _pCodecContext->get_format = _getFormatDelegate; // Hardware decoding runs with one decoder thread (thread_count = 1). // The GPU performs the decoding; the decoder thread only parses packets and submits them. // With frame threading, libavcodec adds one hardware surface per thread to the frame pool // (avcodec_get_hw_frames_parameters). The software fallback in GetFormat keeps this thread count. // Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/decode.c _pCodecContext->thread_count = 1; // Attempt opening codec context with hardware configuration int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null); if (openRet >= 0) { Melon.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}"); return true; } // Failed to open codec with this hardware configuration, clean up and try next. // Freeing the codec context releases its hw_device_ctx reference; _hwDeviceCtx holds the last one. Melon.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate..."); if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; } if (_hwDeviceCtx != null) { var pHw = _hwDeviceCtx; ffmpeg.av_buffer_unref(&pHw); _hwDeviceCtx = null; } _isHwDecoding = false; _getFormatDelegate = null; // Reallocate fresh codec context for subsequent attempts _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); if (_pCodecContext == null) throw new Exception("Failed to re-allocate codec context during fallback."); if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters during fallback."); } return false; } /// /// Callback invoked by libavcodec during decoding to negotiate the output pixel format. /// Confirms the hardware surface format () if offered by the decoder, /// or falls back to software pixel formats via avcodec_default_get_format. /// Called at the start of decoding and again when stream parameters change (e.g. resolution). /// The fallback clears ; the decoder then outputs frames in system memory /// with the thread count set for hardware decoding (1). /// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html /// /// Codec context negotiating the format. /// Formats the decoder can output for the stream, terminated by AV_PIX_FMT_NONE (hardware formats first). /// The selected pixel format. private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts) { for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++) { if (*p == _hwPixFmt) { return *p; } } Melon.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format."); _isHwDecoding = false; _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; return ffmpeg.avcodec_default_get_format(ctx, pixFmts); } // ────────────────────────────────────────────────────────────── // Demuxing & Decoding Engine // ────────────────────────────────────────────────────────────── /// /// Reads packets from the container until a packet belonging to the video stream is found, /// then sends it to the decoder via avcodec_send_packet. /// A packet that the decoder rejects as invalid data (AVERROR_INVALIDDATA, e.g. a damaged packet) /// is skipped and the next packet is read. Any other decoder error is thrown. /// Packets of other streams are discarded (they are normally not returned, see AVDISCARD_ALL in Open). /// Called only after avcodec_receive_frame returned EAGAIN, so avcodec_send_packet accepts the packet. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html /// /// True if a video packet was sent; false on EOF or on a read error (both end the stream). private bool ReadAndSendPacket() { while (true) { ffmpeg.av_packet_unref(_pPacket); int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket); if (ret < 0) return false; // EOF or read error if (_pPacket->stream_index == _videoStreamIndex) { // avcodec_send_packet takes its own reference to (or copy of) the packet data, // so the packet is unreferenced right away for the next av_read_frame. ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket); ffmpeg.av_packet_unref(_pPacket); if (ret == ffmpeg.AVERROR_INVALIDDATA) { // The warning is logged for the first few packets only (a damaged file can have many). _skippedPacketCount++; if (_skippedPacketCount <= MaxLoggedSkippedPackets) { Melon.Logger.Warning($"[FFmpegNativeDecoder] Skipped an invalid video packet ({_skippedPacketCount}): {GetFFmpegErrorString(ret)}"); } continue; } if (ret < 0) { throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}"); } return true; } } } /// /// Receives the next decoded video frame from the codec context. /// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV), /// it is automatically downloaded to system RAM via av_hwframe_transfer_data into _pHwTransferFrame, /// preserving presentation timestamps via av_frame_copy_props. /// /// Decode loop: avcodec_receive_frame → on EAGAIN read and send the next packet → retry. /// When the input ends, a null packet switches the decoder to draining mode; the remaining delayed frames /// are returned, then avcodec_receive_frame returns AVERROR_EOF. /// The returned frame ( or ) holds its data until /// the caller unreferences it (av_frame_unref) or the next frame is received. /// /// The software frame containing pixel data in system RAM. /// True if a frame was successfully decoded and available; false on EOF or draining. /// Decoding, packet sending or the GPU-to-host transfer fails. private bool TryReceiveFrame(out AVFrame* outFrame) { outFrame = null; while (true) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { // Check if the received frame is on a hardware surface if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) { // av_hwframe_transfer_data with an empty destination frame allocates a system memory buffer // in the first format the device supports for download (NV12 for 8-bit 4:2:0 on the common APIs). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html (av_hwframe_transfer_data) ffmpeg.av_frame_unref(_pHwTransferFrame); int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0); if (transferRet < 0) { ffmpeg.av_frame_unref(_pDecodedFrame); throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}"); } // Copy presentation timestamp and metadata properties // (pts, best_effort_timestamp, color tags; av_hwframe_transfer_data copies pixel data only). // The hardware surface is then released back to the decoder's pool. ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); _downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format; outFrame = _pHwTransferFrame; return true; } // Frame is in system memory (CPU decoding or software fallback format) outFrame = _pDecodedFrame; return true; } // EAGAIN: the decoder needs more input. Feed the next video packet, or start draining at the end of the input. // The return value of the flush call (null packet) is not checked; a failure shows up as an error or EOF // from the next avcodec_receive_frame. if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return false; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } // EOF: every delayed frame has been returned after draining. else if (ret == ffmpeg.AVERROR_EOF) { return false; } else { throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); } } } /// /// Selects the YUV→RGB conversion matrix (libswscale SWS_CS_*) from the colorspace tag of a frame. /// The matrix set is the same as that of the compute shader path (ColorConversion.compute, ColorMatrix uniform): /// BT.709 (also for frames without a usable tag), BT.601 and BT.2020, so both paths give the same colors. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_getCoefficients) /// private static int GetSwsColorspace(AVColorSpace colorspace) { switch (GetColorMatrixId(colorspace)) { case ColorMatrixBT601: return ffmpeg.SWS_CS_ITU601; case ColorMatrixBT2020: return ffmpeg.SWS_CS_BT2020; default: return ffmpeg.SWS_CS_ITU709; } } /// /// Returns true when the Y/U/V samples of the frame use the full range (0-255) instead of the limited range (16-235 / 16-240). /// The range is given by the frame's color_range tag or by a deprecated full range pixel format (YUVJ*). /// private static bool IsFullRange(AVFrame* frame) { if (frame->color_range == AVColorRange.AVCOL_RANGE_JPEG) return true; switch ((AVPixelFormat)frame->format) { case AVPixelFormat.AV_PIX_FMT_YUVJ420P: case AVPixelFormat.AV_PIX_FMT_YUVJ422P: case AVPixelFormat.AV_PIX_FMT_YUVJ444P: case AVPixelFormat.AV_PIX_FMT_YUVJ440P: return true; default: return false; } } /// /// Lazily initializes or updates the when source/destination formats, dimensions, /// or (for RGBA output) the colorspace / range of the source frame change. /// For RGBA output the YUV→RGB matrix and the source range are taken from . /// For NV12/YUV420P output no matrix is set; libswscale converts between the formats with its default range handling. /// The destination size is always × ; SWS_BILINEAR filters scaling and chroma resampling. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html /// /// sws_getContext cannot create a context for the conversion. /// Pixel format of the source frame. /// Width of the source frame in pixels. /// Height of the source frame in pixels. /// Pixel format of the output. /// Source frame whose colorspace / range tags are applied. Required for RGBA output. private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat, AVFrame* colorFrame = null) { var swsColorspace = 0; var fullRange = false; if (colorFrame != null && dstFormat == AVPixelFormat.AV_PIX_FMT_RGBA) { swsColorspace = GetSwsColorspace(colorFrame->colorspace); fullRange = IsFullRange(colorFrame); } if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat && _srcSwsColorspace == swsColorspace && _srcFullRange == fullRange) { return; } if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } _srcPixFmt = srcFormat; _srcWidth = srcWidth; _srcHeight = srcHeight; _dstPixFmt = dstFormat; _srcSwsColorspace = swsColorspace; _srcFullRange = fullRange; _pSwsContext = ffmpeg.sws_getContext( srcWidth, srcHeight, srcFormat, Width, Height, dstFormat, (int)SwsFlags.SWS_BILINEAR, null, null, null ); if (_pSwsContext == null) { throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height})."); } // A new context uses the BT.601 matrix and the range implied by the source pixel format (full for YUVJ*, // limited otherwise); set the frame's matrix and range for YUV → RGBA. if (swsColorspace != 0) { ApplySwsColorspace(swsColorspace, fullRange); } } /// /// Sets the YUV→RGB matrix and the source range of . /// The destination range, brightness, contrast and saturation keep the values of the context. /// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails) /// /// Matrix id (SWS_CS_*) of the source. /// True when the source uses the full range. private void ApplySwsColorspace(int swsColorspace, bool fullRange) { int* currentInvTable; int* currentTable; int srcRange, dstRange, brightness, contrast, saturation; // A negative return value means the details cannot be read; the context then keeps its default matrix and range. if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, ¤tInvTable, &srcRange, ¤tTable, &dstRange, &brightness, &contrast, &saturation) < 0) { return; } // inv_table: coefficients of the source YUV space (used for YUV → RGB); table: coefficients of the destination. // A failure of sws_setColorspaceDetails leaves the defaults of the context and is not reported. var coefficients = ffmpeg.sws_getCoefficients(swsColorspace); var invTable = new int_array4(); var table = new int_array4(); for (uint i = 0; i < 4; i++) { invTable[i] = coefficients[i]; table[i] = currentTable[i]; } ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, fullRange ? 1 : 0, table, dstRange, brightness, contrast, saturation); } /// /// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state. /// Used by seeking to inspect candidate frames without falsely incrementing frame indices. /// Uses best_effort_timestamp (libavcodec's estimate from PTS/DTS), else pts, converted with the stream time base. /// A frame without either timestamp gets / FPS. /// private double GetFrameTimestamp(AVFrame* frame) { long ptsValue = frame->best_effort_timestamp; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { ptsValue = frame->pts; } if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { return FPS > 0 ? _currentFrameIndex / FPS : 0; } return ptsValue * ffmpeg.av_q2d(_timeBase); } /// /// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter. /// With a timestamp, = round(timestamp × FPS); without one, it is incremented by 1. /// private double GetTimestampAndUpdateIndex(AVFrame* frame) { double timestamp = GetFrameTimestamp(frame); long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE ? frame->best_effort_timestamp : frame->pts; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { _currentFrameIndex++; } else { _currentFrameIndex = (long)Math.Round(timestamp * FPS); } return timestamp; } /// /// Resets all data plane pointers and strides in to zero. /// Prevents stale or dangling pointers from previous conversions being retained across frames. /// The frame never owns buffers (buf[] stays empty), so av_frame_free in Dispose does not touch caller memory. /// 8 = AV_NUM_DATA_POINTERS, the size of AVFrame.data / linesize. /// private void ClearConvertedFramePointers() { if (_pConvertedFrame == null) return; for (uint i = 0; i < 8; i++) { _pConvertedFrame->data[i] = null; _pConvertedFrame->linesize[i] = 0; } } // ────────────────────────────────────────────────────────────── // Frame Reading APIs: RGBA (CPU Fallback) // ────────────────────────────────────────────────────────────── /// /// Reads the next video frame and copies it in 32-bit RGBA format using libswscale. /// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P). /// The YUV→RGB matrix and range follow the frame's tags (see InitOrUpdateSwsContext). Frames of another size /// are scaled to Width × Height. A frame cached by a seek is returned first. /// /// Pointer to destination memory storing RGBA data (at least outStride × Height bytes). /// Byte stride per row of destination memory (Width * 4). /// Whether to vertically flip the output to match Unity texture coordinates. /// Timestamp of the output frame in seconds. /// True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer. /// Decoding, transfer or scaler creation fails. public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed || outBuffer == null) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame); // Destination: the caller's buffer. vflip uses a negative stride starting at the last row, // so sws_scale writes the rows bottom-up (Unity textures have their origin at the bottom-left). ClearConvertedFramePointers(); if (vflip) { _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); _pConvertedFrame->linesize[0] = -outStride; } else { _pConvertedFrame->data[0] = outBuffer; _pConvertedFrame->linesize[0] = outStride; } ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); // Release the decoded (or cached) frame's buffers; the output is now in the caller's buffer. ClearConvertedFramePointers(); ffmpeg.av_frame_unref(frame); return true; } } /// /// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer. /// Same as the byte* overload. /// public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp) { return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp); } /// /// Reads the next video frame and copies it in RGBA format to a Unity NativeArray. /// The rows are tightly packed (stride Width × 4). /// /// The array is not created or is smaller than Width × Height × 4 bytes. public bool TryReadFrame(NativeArray outBuffer, bool vflip, out double timestamp) { if (!outBuffer.IsCreated) { throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer)); } if (outBuffer.Length < (long)Width * Height * 4) { throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer)); } int stride = Width * 4; return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp); } // ────────────────────────────────────────────────────────────── // Frame Reading APIs: NV12 (Compute Shader Pipeline) // ────────────────────────────────────────────────────────────── /// /// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format. /// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed: /// the planes are copied row by row into the caller's buffers (e.g. texture data for Compute Shader processing). /// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale. /// The Y/UV values keep the range of the source (see ); no YUV→RGB conversion happens here. /// A frame cached by a seek is returned first. /// /// Pointer to destination buffer for Y plane (yStride × Height bytes; Width bytes per row are written). /// Row stride in bytes for Y plane. /// Pointer to destination buffer for interleaved UV plane (uvStride × ceil(Height / 2) bytes; ceil(Width / 2) × 2 bytes per row are written). /// Row stride in bytes for UV plane. /// Timestamp of the output frame in seconds. /// True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer. /// Decoding, transfer or scaler creation fails. // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( byte* yBuffer, int yStride, byte* uvBuffer, int uvStride, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); // Conversion path: any other format (e.g. YUV420P from software decoding, P010 from 10-bit hardware decoding) // or a frame size different from Width × Height. sws_scale writes directly into the caller's planes. if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12); ClearConvertedFramePointers(); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uvBuffer; _pConvertedFrame->linesize[0] = yStride; _pConvertedFrame->linesize[1] = uvStride; ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ClearConvertedFramePointers(); } else { // Direct copy path: NV12 frame of the output size. // The UV plane has ceil(Height / 2) rows of ceil(Width / 2) U/V byte pairs. int w = Width; int h = Height; int uvH = (h + 1) / 2; int uvRowBytes = ((w + 1) / 2) * 2; // Copy Y plane (row by row to safely skip linesize GPU padding) // Buffer.MemoryCopy(source, destination, destinationSizeInBytes, bytesToCopy): the destination size // argument is the stride, so a stride smaller than the row width throws instead of overrunning. for (int y = 0; y < h; y++) { Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, yStride, w ); } // Copy UV plane (interleaved U and V samples) for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uvBuffer + y * uvStride, uvStride, uvRowBytes ); } } ffmpeg.av_frame_unref(frame); return true; } } /// /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers. /// Same as the byte* overload. /// // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( IntPtr yBuffer, int yStride, IntPtr uvBuffer, int uvStride, out double timestamp) { return TryReadNV12Frame( (byte*)yBuffer.ToPointer(), yStride, (byte*)uvBuffer.ToPointer(), uvStride, out timestamp ); } /// /// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays. /// The planes are tightly packed: Y stride Width, UV stride ceil(Width / 2) × 2 with ceil(Height / 2) rows. /// /// An array is not created or is smaller than its plane. // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( NativeArray yBuffer, NativeArray uvBuffer, out double timestamp) { if (!yBuffer.IsCreated || !uvBuffer.IsCreated) { throw new ArgumentException("Input NativeArray buffer is not created or already disposed."); } int w = Width; int h = Height; int uvH = (h + 1) / 2; int uvRowBytes = ((w + 1) / 2) * 2; if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH) { throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution."); } return TryReadNV12Frame( (byte*)yBuffer.GetUnsafePtr(), w, (byte*)uvBuffer.GetUnsafePtr(), uvRowBytes, out timestamp ); } // ────────────────────────────────────────────────────────────── // Frame Reading APIs: YUV420P (Compute Shader Pipeline) // ────────────────────────────────────────────────────────────── /// /// Reads the next video frame and copies its planar Y, U, and V channel data separately. /// For native YUV420P sources, color conversion is bypassed and memory is copied directly. /// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale. /// YUVJ420P (full range YUV420P) is copied as well; the values keep the source range (see ). /// A frame cached by a seek is returned first. /// /// Destination of the Y plane (yStride × Height bytes; Width bytes per row are written). /// Row stride in bytes of the Y plane. /// Destination of the U plane (uStride × ceil(Height / 2) bytes; ceil(Width / 2) bytes per row). /// Row stride in bytes of the U plane. /// Destination of the V plane (same size as U). /// Row stride in bytes of the V plane. /// Timestamp of the output frame in seconds. /// True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer. /// Decoding, transfer or scaler creation fails. // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( byte* yBuffer, int yStride, byte* uBuffer, int uStride, byte* vBuffer, int vStride, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { if (!TryReceiveFrame(out frame)) return false; } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); // Conversion path: any format other than YUV420P/YUVJ420P, or a frame size different from Width × Height. if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P); ClearConvertedFramePointers(); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uBuffer; _pConvertedFrame->data[2] = vBuffer; _pConvertedFrame->linesize[0] = yStride; _pConvertedFrame->linesize[1] = uStride; _pConvertedFrame->linesize[2] = vStride; ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ClearConvertedFramePointers(); } else { // Direct copy path: each plane row by row (source rows can be padded, linesize ≥ row width). int w = Width; int h = Height; int uvW = (w + 1) / 2; int uvH = (h + 1) / 2; // Copy Y plane for (int y = 0; y < h; y++) { Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, yStride, w ); } // Copy U plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uBuffer + y * uStride, uStride, uvW ); } // Copy V plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[2] + y * frame->linesize[2], vBuffer + y * vStride, vStride, uvW ); } } ffmpeg.av_frame_unref(frame); return true; } } /// /// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers. /// Same as the byte* overload. /// // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( IntPtr yBuffer, int yStride, IntPtr uBuffer, int uStride, IntPtr vBuffer, int vStride, out double timestamp) { return TryReadYUV420PFrame( (byte*)yBuffer.ToPointer(), yStride, (byte*)uBuffer.ToPointer(), uStride, (byte*)vBuffer.ToPointer(), vStride, out timestamp ); } /// /// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays. /// The planes are tightly packed: Y stride Width, U/V stride ceil(Width / 2) with ceil(Height / 2) rows. /// /// An array is not created or is smaller than its plane. // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( NativeArray yBuffer, NativeArray uBuffer, NativeArray vBuffer, out double timestamp) { if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated) { throw new ArgumentException("Input NativeArray buffer is not created or already disposed."); } int w = Width; int h = Height; int uvW = (w + 1) / 2; int uvH = (h + 1) / 2; if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH) { throw new ArgumentException("Buffer size is smaller than video YUV plane resolution."); } return TryReadYUV420PFrame( (byte*)yBuffer.GetUnsafePtr(), w, (byte*)uBuffer.GetUnsafePtr(), uvW, (byte*)vBuffer.GetUnsafePtr(), uvW, out timestamp ); } // ────────────────────────────────────────────────────────────── // Seeking & Skipping // ────────────────────────────────────────────────────────────── /// /// Maximum distance in frames over which moves forward by decoding. /// Decoding forward needs no keyframe seek and no decoder flush, which is cheaper than seeking for short distances. /// private const int MaxForwardDecodeFrames = 60; /// /// Decodes forward from the current decoder position until the first frame whose presentation time is at least /// minus half a frame, and keeps that frame in /// for the next TryRead* call. /// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred. /// When the end of the stream is reached before the target, no frame is cached. /// Every discarded frame updates ; the cached target frame updates it when it is read. /// /// Target time in seconds. /// Decoding or the GPU-to-host transfer of the target frame fails. private void DecodeToTarget(double timeSeconds) { while (true) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { double currentPts = GetFrameTimestamp(_pDecodedFrame); // Check with half-frame margin to accommodate floating point rounding if (currentPts >= timeSeconds - (0.5 / FPS)) { // Cache the first matching frame in system RAM for immediate consumption if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt) { ffmpeg.av_frame_unref(_pCachedFrame); int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0); if (transferRet < 0) { ffmpeg.av_frame_unref(_pDecodedFrame); throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}"); } ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); _downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format; } else { // System memory frame: move the buffer references into the cache without copying // (_pDecodedFrame is left blank for the next receive). ffmpeg.av_frame_unref(_pCachedFrame); ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame); } _hasCachedFrame = true; return; } // Discard non-matching intermediate frame without transferring to CPU GetTimestampAndUpdateIndex(_pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); continue; } // Same input feeding and draining as TryReceiveFrame. if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { return; } else { throw new Exception($"Error during seek decoding: {GetFFmpegErrorString(ret)}"); } } } /// /// Moves the playback position to a time in seconds. /// The next TryRead* call returns the first frame whose presentation time is at least /// minus half a frame (the frame nearest to the target time). /// Both paths below return that same frame: /// - Forward decode: the target is ahead of the current position by less than frames. /// Frames are decoded and discarded up to the target. /// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read). /// A backward keyframe seek is followed by a forward decode up to the target. /// The target frame is transferred and preserved in ; when it is already there, nothing is decoded. /// Negative times are treated as 0. The time is in the timeline of the stream timestamps (a stream that starts at /// a non-zero time has no frame before its start; the first frame is returned for earlier targets). /// When the target is past the last frame, no frame is cached and the next TryRead* call returns false. /// /// Target time in seconds to seek to. /// Seeking (including the fallback seek to the start) or decoding fails. public void SeekToTime(double timeSeconds) { lock (_lock) { if (!_isOpened || _disposed) return; if (timeSeconds < 0) timeSeconds = 0; // Earliest presentation time that satisfies the target (target minus half a frame). double threshold = timeSeconds - (0.5 / FPS); // Index of the first frame that satisfies the target. The epsilon keeps a frame that lies exactly on the // margin from being excluded by floating point rounding. double targetFrameIndex = Math.Ceiling(timeSeconds * FPS - 0.5 - 1e-6); if (_hasCachedFrame) { // A frame is already positioned for the next read (e.g. by a previous SeekToTime call). double cachedPts = GetFrameTimestamp(_pCachedFrame); if (cachedPts >= threshold) { // The cached frame is the first frame that satisfies the target. // When the target lies before it, the frames in between are no longer available and a keyframe seek is required. if (cachedPts - threshold < 1.0 / FPS) return; } else if (targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames) { // The cached frame lies before the target: drop it and continue decoding forward. GetTimestampAndUpdateIndex(_pCachedFrame); ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; DecodeToTarget(timeSeconds); return; } } else if (targetFrameIndex > _currentFrameIndex && targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames) { // The target frame has not been read yet and is close: decode forward instead of seeking. DecodeToTarget(timeSeconds); return; } if (_hasCachedFrame) { ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; } // Keyframe seek. With a stream index, the timestamp is in the stream time base. // AVSEEK_FLAG_BACKWARD: seek to the nearest keyframe at or before the target, so that decoding forward // from there reaches the target frame. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_seek_frame) long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase)); int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { // Fallback to beginning of file on seek failure ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}"); } } // Flush decoder internal buffers // Drops the frames buffered from before the seek and ends draining mode, so new packets are accepted again. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html ffmpeg.avcodec_flush_buffers(_pCodecContext); _isDraining = false; // Decode frames until reaching the target presentation timestamp DecodeToTarget(timeSeconds); } } /// /// Seeks to a specific frame index: with frameIndex / FPS seconds. /// /// Target frame index (0-based, in units of 1 / FPS). public void SeekToFrame(long frameIndex) { SeekToTime(frameIndex / FPS); } // ────────────────────────────────────────────────────────────── // Error Handling & Disposal // ────────────────────────────────────────────────────────────── /// /// Translates an FFmpeg integer error code to a readable string using av_strerror /// (1024-byte stack buffer; "Unknown error (Code: n)" when there is no description). /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html /// private string GetFFmpegErrorString(int errnum) { const ulong bufferSize = 1024; byte* buffer = stackalloc byte[(int)bufferSize]; if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0) { return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})"; } return $"Unknown error (Code: {errnum})"; } /// /// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures. /// Safe to call more than once; also called by the constructor on failure and by the finalizer. /// Order: scaler → frames → packet → codec context (releases its hw_device_ctx reference) /// → this object's device reference → input file and format context. /// After Dispose, the read and seek methods return without doing anything. /// public void Dispose() { lock (_lock) { if (_disposed) return; _disposed = true; // 1. Release scaler context if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } // 2. Release frame buffers if (_pDecodedFrame != null) { var pFrame = _pDecodedFrame; ffmpeg.av_frame_free(&pFrame); _pDecodedFrame = null; } if (_pHwTransferFrame != null) { var pFrame = _pHwTransferFrame; ffmpeg.av_frame_free(&pFrame); _pHwTransferFrame = null; } if (_pCachedFrame != null) { var pFrame = _pCachedFrame; ffmpeg.av_frame_free(&pFrame); _pCachedFrame = null; } if (_pConvertedFrame != null) { var pFrame = _pConvertedFrame; ffmpeg.av_frame_free(&pFrame); _pConvertedFrame = null; } // 3. Release packet buffer if (_pPacket != null) { var pPacket = _pPacket; ffmpeg.av_packet_free(&pPacket); _pPacket = null; } // 4. Release codec context if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; } // 5. Release hardware device context if (_hwDeviceCtx != null) { var pHw = _hwDeviceCtx; ffmpeg.av_buffer_unref(&pHw); _hwDeviceCtx = null; } // 6. Close input file and release format context if (_pFormatContext != null) { var pFormatCtx = _pFormatContext; ffmpeg.avformat_close_input(&pFormatCtx); _pFormatContext = null; } _getFormatDelegate = null; _isOpened = false; _isHwDecoding = false; GC.SuppressFinalize(this); } } /// /// Finalizer. Releases the native resources when the decoder was not disposed (runs on the GC finalizer thread). /// ~FFmpegNativeDecoder() { Dispose(); } }