SHA256
1889 lines
88 KiB
C#
1889 lines
88 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.Runtime.InteropServices;
|
||
using FFmpeg.AutoGen;
|
||
using MelonLoader;
|
||
using Unity.Collections;
|
||
using Unity.Collections.LowLevel.Unsafe;
|
||
|
||
namespace ADOFAIRenderer.Util.FFmpeg;
|
||
|
||
/// <summary>
|
||
/// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke.
|
||
/// Supports both GPU hardware-accelerated decoding (device types of <see cref="PreferredHwDeviceTypes"/>: D3D11VA, DXVA2,
|
||
/// CUDA, D3D12VA on Windows; VAAPI, CUDA or VideoToolbox on other platforms; others the decoder reports) and
|
||
/// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback.
|
||
///
|
||
/// Used by VideoBGPlayer to decode the background video of a level. Only the first video stream is decoded;
|
||
/// audio and other streams are discarded by the demuxer.
|
||
///
|
||
/// Threading: every public member takes <c>_lock</c>, so calls from different threads are serialized.
|
||
/// Width, Height, FPS, Duration and TotalFrames are set once in the constructor and read without the lock.
|
||
///
|
||
/// Timestamps returned by the TryRead* methods are the presentation times of the stream in seconds
|
||
/// (PTS × stream time base), so a stream with a non-zero start time returns timestamps starting at that time.
|
||
///
|
||
/// ── Architecture & Lifecycle Flow ──────────────────────────────────────────────
|
||
///
|
||
/// 1. Construction & Demuxing:
|
||
/// a) avformat_open_input: Open container file and parse stream headers.
|
||
/// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS.
|
||
/// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO).
|
||
///
|
||
/// 2. Codec & Hardware Device Discovery:
|
||
/// a) avcodec_find_decoder: Locate codec implementation by codec ID.
|
||
/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder.
|
||
/// c) HW device creation for the reported configurations, in the order of PreferredHwDeviceTypes.
|
||
/// d) Attach hw_device_ctx and register custom get_format callback.
|
||
/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding
|
||
/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME).
|
||
/// f) avcodec_open2: Initialize codec context.
|
||
///
|
||
/// 3. Frame Demuxing & Decoding (TryRead*):
|
||
/// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet.
|
||
/// b) (Decoding) avcodec_receive_frame retrieves decoded frames.
|
||
/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data
|
||
/// into system RAM as NV12 (av_frame_copy_props preserves timestamps).
|
||
/// d) (Output Dispatch):
|
||
/// - TryReadNV12Frame: NV12 frames are copied row by row (Buffer.MemoryCopy) into the output buffers,
|
||
/// other formats are converted with sws_scale.
|
||
/// - TryReadYUV420PFrame: YUV420P/YUVJ420P planes are copied row by row into the output buffers,
|
||
/// other formats are converted with sws_scale.
|
||
/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
|
||
///
|
||
/// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame):
|
||
/// (Short forward distances skip a) and b) and only decode forward; see SeekToTime.)
|
||
/// a) av_seek_frame: Fast backward keyframe seek.
|
||
/// b) avcodec_flush_buffers: Clear internal decoder state.
|
||
/// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead;
|
||
/// transfers and caches the first matching frame in _pCachedFrame for immediate consumption.
|
||
///
|
||
/// 5. Resource Disposal (Dispose):
|
||
/// a) Free scaler context (sws_freeContext).
|
||
/// b) Free all allocated frames and packets (av_frame_free, av_packet_free).
|
||
/// c) Free codec context (avcodec_free_context).
|
||
/// d) Release hardware device context (av_buffer_unref).
|
||
/// e) Close format context (avformat_close_input).
|
||
///
|
||
/// ── Reference Documentation ────────────────────────────────────────────────────
|
||
/// • FFmpeg Demuxing & Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
|
||
/// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
||
/// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
/// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html
|
||
/// • libswscale Scaling & Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
||
/// </summary>
|
||
internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
||
{
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Hardware Acceleration Priorities & Candidate Structure
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Candidate hardware device configuration discovered via <c>avcodec_get_hw_config</c>.
|
||
/// </summary>
|
||
private record struct HwConfigCandidate
|
||
{
|
||
/// <summary>Hardware device type to create with av_hwdevice_ctx_create (AVCodecHWConfig.device_type).</summary>
|
||
public AVHWDeviceType DeviceType;
|
||
|
||
/// <summary>Hardware surface pixel format the decoder outputs with this device (AVCodecHWConfig.pix_fmt, e.g. AV_PIX_FMT_D3D11).</summary>
|
||
public AVPixelFormat PixelFormat;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Preferred hardware device types in priority order.
|
||
/// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config),
|
||
/// sorted by this list; configurations not in the list follow in reported order.
|
||
/// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Linux: CUDA (NVIDIA), then VAAPI. macOS: VideoToolbox.
|
||
/// A device type that cannot be created on the machine (no driver, wrong platform) fails creation and is skipped.
|
||
/// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by
|
||
/// avcodec_find_decoder report hwaccel device types such as DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan (depending on the build);
|
||
/// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name.
|
||
/// </summary>
|
||
private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
|
||
{
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA,
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2,
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_QSV,
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA,
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA,
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI,
|
||
AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX
|
||
};
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// FFmpeg Core Struct Pointers
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Context managing container format demuxing (MP4, MKV, WebM, etc.).
|
||
/// Used for packet demuxing and seeking.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
|
||
/// </summary>
|
||
private AVFormatContext* _pFormatContext = null;
|
||
|
||
/// <summary>
|
||
/// Context managing video decoder state and parameters.
|
||
/// Configured with codec parameters, hardware contexts, and threading options.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
||
/// </summary>
|
||
private AVCodecContext* _pCodecContext = null;
|
||
|
||
/// <summary>
|
||
/// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.).
|
||
/// This class owns one reference; the codec context holds a second one (hw_device_ctx).
|
||
/// Both are released in Dispose (or when a hardware candidate fails). Null in software decoding.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html
|
||
/// </summary>
|
||
private AVBufferRef* _hwDeviceCtx = null;
|
||
|
||
/// <summary>
|
||
/// Frame structure holding raw decoded frames received directly from <c>avcodec_receive_frame</c>.
|
||
/// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView).
|
||
/// In SW decoding mode, this contains pixel buffers in system memory.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
|
||
/// </summary>
|
||
private AVFrame* _pDecodedFrame = null;
|
||
|
||
/// <summary>
|
||
/// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces
|
||
/// via <c>av_hwframe_transfer_data</c>.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
/// </summary>
|
||
private AVFrame* _pHwTransferFrame = null;
|
||
|
||
/// <summary>
|
||
/// Frame structure holding converted pixel data produced by <c>sws_scale</c> (e.g., RGBA or fallback YUV).
|
||
/// Owns no pixel memory: during one conversion its data[]/linesize[] point to the caller's output buffers,
|
||
/// and they are cleared before and after (ClearConvertedFramePointers).
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
|
||
/// </summary>
|
||
private AVFrame* _pConvertedFrame = null;
|
||
|
||
/// <summary>
|
||
/// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation.
|
||
/// Consumed by the next <c>TryRead*</c> invocation.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
|
||
/// </summary>
|
||
private AVFrame* _pCachedFrame = null;
|
||
|
||
/// <summary>
|
||
/// Whether <see cref="_pCachedFrame"/> holds a frame that the next <c>TryRead*</c> call returns
|
||
/// before any new frame is decoded. Set by DecodeToTarget, cleared when the frame is consumed or dropped.
|
||
/// </summary>
|
||
private bool _hasCachedFrame;
|
||
|
||
/// <summary>
|
||
/// Packet structure temporarily holding compressed demuxed data read from the input container.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html
|
||
/// </summary>
|
||
private AVPacket* _pPacket = null;
|
||
|
||
/// <summary>
|
||
/// libswscale colorspace and format conversion context.
|
||
/// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
||
/// </summary>
|
||
private SwsContext* _pSwsContext = null;
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Hardware Acceleration State & Delegates
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code.
|
||
/// Passed to <c>_pCodecContext->get_format</c>.
|
||
/// </summary>
|
||
private AVCodecContext_get_format? _getFormatDelegate;
|
||
|
||
/// <summary>
|
||
/// Whether hardware-accelerated decoding is currently active.
|
||
/// Set when a hardware candidate is configured; cleared when it fails, or by GetFormat when the stream
|
||
/// cannot be decoded on the hardware surface format (the decoder then outputs software frames).
|
||
/// </summary>
|
||
private bool _isHwDecoding;
|
||
|
||
/// <summary>Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV).</summary>
|
||
private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
|
||
|
||
/// <summary>Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD).</summary>
|
||
private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
||
|
||
/// <summary>
|
||
/// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12).
|
||
/// Assumed NV12 when hardware decoding is configured, then updated from every transferred frame
|
||
/// (e.g. P010 for 10-bit streams). AV_PIX_FMT_NONE in software decoding.
|
||
/// </summary>
|
||
private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Playback, Stream & Scaler Synchronization Fields
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>Index of the decoded video stream in the format context. -1 until a video stream is found.</summary>
|
||
private int _videoStreamIndex = -1;
|
||
|
||
/// <summary>
|
||
/// Index of the frame most recently returned (or skipped while seeking): round(timestamp × FPS) for frames with a timestamp,
|
||
/// the previous value + 1 for frames without one. 0 after opening.
|
||
/// </summary>
|
||
private long _currentFrameIndex;
|
||
|
||
/// <summary>Time base of the video stream (seconds per PTS unit). Used to convert PTS to seconds and seek targets to PTS.</summary>
|
||
private AVRational _timeBase;
|
||
|
||
/// <summary>Whether Open completed. The read and seek methods do nothing until then and after Dispose.</summary>
|
||
private bool _isOpened;
|
||
|
||
/// <summary>Whether Dispose has run. Prevents duplicate cleanup.</summary>
|
||
private bool _disposed;
|
||
|
||
/// <summary>
|
||
/// Whether the end of the input has been reached and the decoder was put into draining mode
|
||
/// (avcodec_send_packet with null). Reset by a keyframe seek (after avcodec_flush_buffers).
|
||
/// </summary>
|
||
private bool _isDraining;
|
||
|
||
/// <summary>Number of video packets skipped because the decoder rejected them as invalid data.</summary>
|
||
private int _skippedPacketCount;
|
||
|
||
/// <summary>Number of skipped packets for which a warning is logged.</summary>
|
||
private const int MaxLoggedSkippedPackets = 5;
|
||
|
||
/// <summary>Lock object serializing all public members (decoder state and native resources).</summary>
|
||
private readonly object _lock = new();
|
||
|
||
// Fields to detect SwsContext updates
|
||
// Parameters the current _pSwsContext was created with; a frame with different values recreates the context.
|
||
/// <summary>Source pixel format of the current scaler context.</summary>
|
||
private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
||
|
||
/// <summary>Destination pixel format of the current scaler context (RGBA, NV12 or YUV420P).</summary>
|
||
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
||
|
||
/// <summary>Source width of the current scaler context in pixels (the destination is always Width × Height).</summary>
|
||
private int _srcWidth;
|
||
|
||
/// <summary>Source height of the current scaler context in pixels.</summary>
|
||
private int _srcHeight;
|
||
|
||
/// <summary>YUV→RGB matrix id (<c>SWS_CS_*</c>) applied to the context. 0 when no matrix is set (non-RGBA output).</summary>
|
||
private int _srcSwsColorspace;
|
||
|
||
/// <summary>Whether the source range applied to the context is the full range.</summary>
|
||
private bool _srcFullRange;
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Public Properties
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Video width in pixels (codec context width after opening, or the stream parameters).
|
||
/// Every output of the TryRead* methods has this width; frames of another size are scaled to it.
|
||
/// </summary>
|
||
public int Width { get; private set; }
|
||
|
||
/// <summary>Video height in pixels. Every output of the TryRead* methods has this height.</summary>
|
||
public int Height { get; private set; }
|
||
|
||
/// <summary>
|
||
/// Frames per second: r_frame_rate of the stream, else avg_frame_rate, else 30.
|
||
/// For variable frame rate streams both values are approximations.
|
||
/// </summary>
|
||
public double FPS { get; private set; }
|
||
|
||
/// <summary>Total duration in seconds (stream duration, else container duration, else 0).</summary>
|
||
public double Duration { get; private set; }
|
||
|
||
/// <summary>
|
||
/// Estimated total number of frames: the frame count of the stream header (nb_frames) when present,
|
||
/// otherwise round(duration × FPS); 0 when the duration is unknown.
|
||
/// </summary>
|
||
public long TotalFrames { get; private set; }
|
||
|
||
/// <summary>Index of the currently read frame (the frame most recently returned or skipped; see _currentFrameIndex).</summary>
|
||
public long CurrentFrameIndex
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
return _currentFrameIndex;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>Current playback time in seconds: CurrentFrameIndex / FPS (0 when FPS is not positive).</summary>
|
||
public double CurrentTime
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
return FPS > 0 ? _currentFrameIndex / FPS : 0;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Pixel format of the video frames produced by the decoder.
|
||
/// When hardware decoding is active, returns the pixel format of the downloaded software frame
|
||
/// in system RAM (typically <see cref="AVPixelFormat.AV_PIX_FMT_NV12"/>) rather than the GPU surface format
|
||
/// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines.
|
||
/// In software mode, returns the codec context's pixel format (e.g., YUV420P).
|
||
/// Before the first hardware frame is transferred, NV12 is returned as the expected download format;
|
||
/// the actual format (e.g. P010 for 10-bit video) is known after the first transfer.
|
||
/// The TryRead* methods convert any decoded format to their output format, so this value only selects the fastest path.
|
||
/// </summary>
|
||
public AVPixelFormat PixelFormat
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_isHwDecoding)
|
||
{
|
||
return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE
|
||
? _downloadedPixFmt
|
||
: AVPixelFormat.AV_PIX_FMT_NV12;
|
||
}
|
||
|
||
return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]).
|
||
/// Taken from the codec context, else from the stream parameters; AVCOL_RANGE_UNSPECIFIED when neither carries a tag.
|
||
/// This property reads the color_range tags only; it does not inspect the pixel format (YUVJ* formats are full range).
|
||
/// </summary>
|
||
public AVColorRange ColorRange
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED)
|
||
{
|
||
return _pCodecContext->color_range;
|
||
}
|
||
if (_pFormatContext != null && _videoStreamIndex >= 0)
|
||
{
|
||
return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range;
|
||
}
|
||
return AVColorRange.AVCOL_RANGE_UNSPECIFIED;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Color matrix id used to convert the YUV frames of this stream to RGB, selected from the colorspace tag of the stream.
|
||
/// The ids are shared with the <c>ColorMatrix</c> uniform of ColorConversion.compute:
|
||
/// 0 = BT.709 (also used when the stream has no usable tag), 1 = BT.601, 2 = BT.2020 (non-constant luminance).
|
||
/// </summary>
|
||
public int ColorMatrix
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_pCodecContext != null && _pCodecContext->colorspace != AVColorSpace.AVCOL_SPC_UNSPECIFIED)
|
||
{
|
||
return GetColorMatrixId(_pCodecContext->colorspace);
|
||
}
|
||
if (_pFormatContext != null && _videoStreamIndex >= 0)
|
||
{
|
||
return GetColorMatrixId(_pFormatContext->streams[_videoStreamIndex]->codecpar->color_space);
|
||
}
|
||
return ColorMatrixBT709;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>Color matrix id of BT.709.</summary>
|
||
// ReSharper disable InconsistentNaming
|
||
public const int ColorMatrixBT709 = 0;
|
||
|
||
/// <summary>Color matrix id of BT.601 (SMPTE 170M / BT.470 BG).</summary>
|
||
public const int ColorMatrixBT601 = 1;
|
||
|
||
/// <summary>Color matrix id of BT.2020 (non-constant luminance).</summary>
|
||
public const int ColorMatrixBT2020 = 2;
|
||
// ReSharper restore InconsistentNaming
|
||
|
||
/// <summary>
|
||
/// Converts a colorspace tag to a color matrix id (see <see cref="ColorMatrix"/>).
|
||
/// Tags that have no matrix of their own (unspecified, RGB, other standards) use BT.709.
|
||
/// </summary>
|
||
private static int GetColorMatrixId(AVColorSpace colorspace)
|
||
{
|
||
switch (colorspace)
|
||
{
|
||
case AVColorSpace.AVCOL_SPC_SMPTE170M:
|
||
case AVColorSpace.AVCOL_SPC_BT470BG:
|
||
return ColorMatrixBT601;
|
||
case AVColorSpace.AVCOL_SPC_BT2020_NCL:
|
||
return ColorMatrixBT2020;
|
||
default:
|
||
return ColorMatrixBT709;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gets a value indicating whether hardware-accelerated decoding is currently active.
|
||
/// </summary>
|
||
public bool IsHardwareAccelerated
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
return _isHwDecoding;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gets the hardware device type in use for hardware acceleration, or null if software decoding.
|
||
/// </summary>
|
||
public AVHWDeviceType? HardwareDeviceType
|
||
{
|
||
get
|
||
{
|
||
lock (_lock)
|
||
{
|
||
return _isHwDecoding ? _hwDeviceType : null;
|
||
}
|
||
}
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Initialization & Setup
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Initializes a new instance of <see cref="FFmpegNativeDecoder"/> and opens the specified video file.
|
||
/// Automatically attempts hardware acceleration (device types reported by the decoder, in the order of
|
||
/// <see cref="PreferredHwDeviceTypes"/>), falling back to multi-threaded software decoding if unavailable.
|
||
/// </summary>
|
||
/// <param name="videoPath">Absolute or relative file path to the video file.</param>
|
||
/// <exception cref="Exception">
|
||
/// The file cannot be opened, has no video stream, or its codec has no decoder or cannot be opened.
|
||
/// All native resources allocated so far are released before the exception propagates.
|
||
/// </exception>
|
||
public FFmpegNativeDecoder(string videoPath)
|
||
{
|
||
Open(videoPath);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers.
|
||
/// </summary>
|
||
/// <param name="videoPath">Path to video file.</param>
|
||
private void Open(string videoPath)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_isOpened || _disposed) return;
|
||
|
||
try
|
||
{
|
||
// 1. Open media container
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
|
||
fixed (AVFormatContext** ppFormatCtx = &_pFormatContext)
|
||
{
|
||
int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})");
|
||
}
|
||
|
||
// 2. Retrieve stream information
|
||
// Reads (and buffers) the start of the file to fill in codec parameters, frame rates and durations.
|
||
if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0)
|
||
throw new Exception("Failed to retrieve stream info.");
|
||
|
||
// 3. Find primary video stream (the first stream whose codec type is video)
|
||
_videoStreamIndex = -1;
|
||
for (int i = 0; i < _pFormatContext->nb_streams; i++)
|
||
{
|
||
if (_pFormatContext->streams[i]->codecpar->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO)
|
||
{
|
||
_videoStreamIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (_videoStreamIndex == -1)
|
||
throw new Exception("Failed to find video stream.");
|
||
|
||
// Only the video stream is used. The demuxer does not need to return the packets of the other streams
|
||
// (audio, subtitles, ...) once they are marked as discarded.
|
||
// Reference: AVStream.discard - "Selects which packets can be discarded at will and do not need to be demuxed."
|
||
// https://ffmpeg.org/doxygen/trunk/structAVStream.html
|
||
for (int i = 0; i < _pFormatContext->nb_streams; i++)
|
||
{
|
||
if (i != _videoStreamIndex)
|
||
_pFormatContext->streams[i]->discard = AVDiscard.AVDISCARD_ALL;
|
||
}
|
||
|
||
var stream = _pFormatContext->streams[_videoStreamIndex];
|
||
_timeBase = stream->time_base;
|
||
|
||
// 4. Calculate FPS
|
||
// r_frame_rate: lowest frame rate that represents all timestamps accurately (guessed by libavformat).
|
||
// avg_frame_rate: average frame rate. Both are approximations for variable frame rate streams.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html
|
||
if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0)
|
||
{
|
||
FPS = ffmpeg.av_q2d(stream->r_frame_rate);
|
||
}
|
||
else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0)
|
||
{
|
||
FPS = ffmpeg.av_q2d(stream->avg_frame_rate);
|
||
}
|
||
else
|
||
{
|
||
FPS = 30.0; // Fallback default
|
||
}
|
||
|
||
// 5. Find decoder by codec ID
|
||
// avcodec_find_decoder returns the default decoder registered for the ID (e.g. libdav1d for AV1
|
||
// when the build includes it, which reports no hardware configuration).
|
||
var codecId = stream->codecpar->codec_id;
|
||
var decoder = ffmpeg.avcodec_find_decoder(codecId);
|
||
if (decoder == null)
|
||
throw new Exception($"Unsupported video codec: {codecId}");
|
||
|
||
// 6. Allocate codec context
|
||
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
|
||
if (_pCodecContext == null)
|
||
throw new Exception("Failed to allocate codec context.");
|
||
|
||
// Copies the stream parameters (size, pixel format, extradata, color tags, ...) into the codec context.
|
||
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
|
||
throw new Exception("Failed to copy codec parameters.");
|
||
|
||
// 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding
|
||
bool hwSuccess = TryInitializeHwDecoder(decoder, stream);
|
||
if (!hwSuccess)
|
||
{
|
||
Melon<RenderMod>.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding.");
|
||
_isHwDecoding = false;
|
||
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
||
|
||
// Multi-threading settings for improved software decoding performance (auto thread count)
|
||
// thread_type = FF_THREAD_FRAME selects frame threading only (several frames decoded in parallel,
|
||
// adds one frame of delay per thread); slice threading is not enabled.
|
||
// TryInitializeHwDecoder leaves a fresh codec context when every hardware candidate fails.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
||
_pCodecContext->thread_count = 0;
|
||
_pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME;
|
||
|
||
int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
|
||
if (openRet < 0)
|
||
throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}");
|
||
}
|
||
|
||
Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width;
|
||
Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height;
|
||
|
||
// 8. Calculate total frames and duration
|
||
// nb_frames is the frame count stored in the container (0 when unknown);
|
||
// otherwise the count is estimated from the stream or container duration and FPS.
|
||
if (stream->nb_frames > 0)
|
||
{
|
||
TotalFrames = stream->nb_frames;
|
||
}
|
||
else if (stream->duration > 0)
|
||
{
|
||
TotalFrames = (long)Math.Round(stream->duration * ffmpeg.av_q2d(stream->time_base) * FPS);
|
||
}
|
||
else if (_pFormatContext->duration > 0)
|
||
{
|
||
TotalFrames = (long)Math.Round((_pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE) * FPS);
|
||
}
|
||
else
|
||
{
|
||
TotalFrames = 0;
|
||
}
|
||
|
||
// Duration of the video stream (AVStream.duration, in the stream time base).
|
||
// When the stream does not carry a duration, the container duration (AVFormatContext.duration,
|
||
// in AV_TIME_BASE units, the longest of all streams) is used.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html, https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
|
||
Duration = stream->duration > 0
|
||
? stream->duration * ffmpeg.av_q2d(stream->time_base)
|
||
: _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0;
|
||
|
||
// 9. Allocate frame and packet buffers
|
||
// Structures only; pixel buffers are attached by the decoder, av_hwframe_transfer_data or av_frame_move_ref.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
|
||
_pDecodedFrame = ffmpeg.av_frame_alloc();
|
||
_pHwTransferFrame = ffmpeg.av_frame_alloc();
|
||
_pConvertedFrame = ffmpeg.av_frame_alloc();
|
||
_pCachedFrame = ffmpeg.av_frame_alloc();
|
||
_pPacket = ffmpeg.av_packet_alloc();
|
||
|
||
_currentFrameIndex = 0;
|
||
_isOpened = true;
|
||
_isDraining = false;
|
||
}
|
||
catch (Exception)
|
||
{
|
||
Dispose();
|
||
throw;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Searches for supported hardware device configurations via <c>avcodec_get_hw_config</c>,
|
||
/// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA, VAAPI, ...),
|
||
/// registers the <c>get_format</c> callback, and attempts to open the hardware decoder.
|
||
///
|
||
/// Candidates are tried in the order of <see cref="PreferredHwDeviceTypes"/>, then the remaining reported order.
|
||
/// When a candidate fails to open, its codec context and device are freed and a fresh codec context
|
||
/// (with the stream parameters) is allocated for the next candidate or for the software fallback.
|
||
/// On success, <see cref="_hwDeviceCtx"/>, <see cref="_hwPixFmt"/> and <see cref="_hwDeviceType"/> describe the device
|
||
/// and the codec context is open.
|
||
/// </summary>
|
||
/// <param name="decoder">Target decoder.</param>
|
||
/// <param name="stream">Source video stream.</param>
|
||
/// <returns>True if hardware decoding was successfully configured and opened; false to fall back.</returns>
|
||
private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream)
|
||
{
|
||
// ReSharper disable once InconsistentNaming
|
||
// Collect all supported hardware configs from decoder
|
||
// AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX (0x01): the decoder supports the hardware format through hw_device_ctx,
|
||
// which has to be set on the codec context before avcodec_open2. Only these configurations are used here.
|
||
// Reference: https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/codec.h
|
||
const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01;
|
||
var candidates = new List<HwConfigCandidate>();
|
||
for (int i = 0; ; i++)
|
||
{
|
||
var config = ffmpeg.avcodec_get_hw_config(decoder, i);
|
||
if (config == null) break;
|
||
|
||
if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0)
|
||
{
|
||
candidates.Add(new HwConfigCandidate
|
||
{
|
||
DeviceType = config->device_type,
|
||
PixelFormat = config->pix_fmt
|
||
});
|
||
}
|
||
}
|
||
|
||
if (candidates.Count == 0)
|
||
{
|
||
return false;
|
||
}
|
||
|
||
// Order candidates according to preference
|
||
var orderedCandidates = new List<HwConfigCandidate>();
|
||
foreach (var preferred in PreferredHwDeviceTypes)
|
||
{
|
||
foreach (var candidate in candidates)
|
||
{
|
||
if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate))
|
||
{
|
||
orderedCandidates.Add(candidate);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes
|
||
foreach (var candidate in candidates)
|
||
{
|
||
if (!orderedCandidates.Contains(candidate))
|
||
{
|
||
orderedCandidates.Add(candidate);
|
||
}
|
||
}
|
||
|
||
// Try candidate hardware configurations in priority order
|
||
foreach (var candidate in orderedCandidates)
|
||
{
|
||
AVBufferRef* hwDeviceCtx = null;
|
||
int err;
|
||
|
||
// Device creation strategy
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV)
|
||
{
|
||
// QSV: first with an explicit child device (D3D11VA on Windows, VAAPI elsewhere),
|
||
// then without options (FFmpeg selects the child device type).
|
||
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
|
||
string childDeviceName = isWindows ? "d3d11va" : "vaapi";
|
||
AVDictionary* opts = null;
|
||
ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0);
|
||
|
||
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0);
|
||
ffmpeg.av_dict_free(&opts);
|
||
|
||
if (err < 0)
|
||
{
|
||
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// Default device of the type (device = null: the first / default adapter chosen by FFmpeg).
|
||
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0);
|
||
}
|
||
|
||
if (err < 0 || hwDeviceCtx == null)
|
||
{
|
||
if (hwDeviceCtx != null)
|
||
{
|
||
ffmpeg.av_buffer_unref(&hwDeviceCtx);
|
||
hwDeviceCtx = null;
|
||
}
|
||
Melon<RenderMod>.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate...");
|
||
continue;
|
||
}
|
||
|
||
// Assign hardware context and format selection callback
|
||
// _hwDeviceCtx takes over the reference created above; the codec context receives its own reference.
|
||
_hwDeviceCtx = hwDeviceCtx;
|
||
_hwPixFmt = candidate.PixelFormat;
|
||
_hwDeviceType = candidate.DeviceType;
|
||
_isHwDecoding = true;
|
||
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
||
|
||
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx);
|
||
if (_pCodecContext->hw_device_ctx == null)
|
||
{
|
||
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate...");
|
||
var pHw = _hwDeviceCtx;
|
||
ffmpeg.av_buffer_unref(&pHw);
|
||
_hwDeviceCtx = null;
|
||
_isHwDecoding = false;
|
||
continue;
|
||
}
|
||
|
||
// The delegate is kept in a field so that it is not collected while native code can still call it
|
||
// (FFmpeg.AutoGen marshals it to a function pointer). With thread_count = 1 it is called on the calling
|
||
// thread inside avcodec_send_packet / avcodec_receive_frame, which run under _lock.
|
||
_getFormatDelegate = GetFormat;
|
||
_pCodecContext->get_format = _getFormatDelegate;
|
||
|
||
// Hardware decoding runs with one decoder thread (thread_count = 1).
|
||
// The GPU performs the decoding; the decoder thread only parses packets and submits them.
|
||
// With frame threading, libavcodec adds one hardware surface per thread to the frame pool
|
||
// (avcodec_get_hw_frames_parameters). The software fallback in GetFormat keeps this thread count.
|
||
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/decode.c
|
||
_pCodecContext->thread_count = 1;
|
||
|
||
// Attempt opening codec context with hardware configuration
|
||
int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
|
||
if (openRet >= 0)
|
||
{
|
||
Melon<RenderMod>.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}");
|
||
return true;
|
||
}
|
||
|
||
// Failed to open codec with this hardware configuration, clean up and try next.
|
||
// Freeing the codec context releases its hw_device_ctx reference; _hwDeviceCtx holds the last one.
|
||
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate...");
|
||
|
||
if (_pCodecContext != null)
|
||
{
|
||
var pCodecCtx = _pCodecContext;
|
||
ffmpeg.avcodec_free_context(&pCodecCtx);
|
||
_pCodecContext = null;
|
||
}
|
||
|
||
if (_hwDeviceCtx != null)
|
||
{
|
||
var pHw = _hwDeviceCtx;
|
||
ffmpeg.av_buffer_unref(&pHw);
|
||
_hwDeviceCtx = null;
|
||
}
|
||
|
||
_isHwDecoding = false;
|
||
_getFormatDelegate = null;
|
||
|
||
// Reallocate fresh codec context for subsequent attempts
|
||
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
|
||
if (_pCodecContext == null)
|
||
throw new Exception("Failed to re-allocate codec context during fallback.");
|
||
|
||
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
|
||
throw new Exception("Failed to copy codec parameters during fallback.");
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Callback invoked by libavcodec during decoding to negotiate the output pixel format.
|
||
/// Confirms the hardware surface format (<see cref="_hwPixFmt"/>) if offered by the decoder,
|
||
/// or falls back to software pixel formats via <c>avcodec_default_get_format</c>.
|
||
/// Called at the start of decoding and again when stream parameters change (e.g. resolution).
|
||
/// The fallback clears <see cref="_isHwDecoding"/>; the decoder then outputs frames in system memory
|
||
/// with the thread count set for hardware decoding (1).
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
||
/// </summary>
|
||
/// <param name="ctx">Codec context negotiating the format.</param>
|
||
/// <param name="pixFmts">Formats the decoder can output for the stream, terminated by AV_PIX_FMT_NONE (hardware formats first).</param>
|
||
/// <returns>The selected pixel format.</returns>
|
||
private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts)
|
||
{
|
||
for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++)
|
||
{
|
||
if (*p == _hwPixFmt)
|
||
{
|
||
return *p;
|
||
}
|
||
}
|
||
|
||
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format.");
|
||
_isHwDecoding = false;
|
||
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
||
return ffmpeg.avcodec_default_get_format(ctx, pixFmts);
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Demuxing & Decoding Engine
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Reads packets from the container until a packet belonging to the video stream is found,
|
||
/// then sends it to the decoder via <c>avcodec_send_packet</c>.
|
||
/// A packet that the decoder rejects as invalid data (<c>AVERROR_INVALIDDATA</c>, e.g. a damaged packet)
|
||
/// is skipped and the next packet is read. Any other decoder error is thrown.
|
||
/// Packets of other streams are discarded (they are normally not returned, see AVDISCARD_ALL in Open).
|
||
/// Called only after avcodec_receive_frame returned EAGAIN, so avcodec_send_packet accepts the packet.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
||
/// </summary>
|
||
/// <returns>True if a video packet was sent; false on EOF or on a read error (both end the stream).</returns>
|
||
private bool ReadAndSendPacket()
|
||
{
|
||
while (true)
|
||
{
|
||
ffmpeg.av_packet_unref(_pPacket);
|
||
int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket);
|
||
if (ret < 0) return false; // EOF or read error
|
||
|
||
if (_pPacket->stream_index == _videoStreamIndex)
|
||
{
|
||
// avcodec_send_packet takes its own reference to (or copy of) the packet data,
|
||
// so the packet is unreferenced right away for the next av_read_frame.
|
||
ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket);
|
||
ffmpeg.av_packet_unref(_pPacket);
|
||
|
||
if (ret == ffmpeg.AVERROR_INVALIDDATA)
|
||
{
|
||
// The warning is logged for the first few packets only (a damaged file can have many).
|
||
_skippedPacketCount++;
|
||
if (_skippedPacketCount <= MaxLoggedSkippedPackets)
|
||
{
|
||
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Skipped an invalid video packet ({_skippedPacketCount}): {GetFFmpegErrorString(ret)}");
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (ret < 0)
|
||
{
|
||
throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}");
|
||
}
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Receives the next decoded video frame from the codec context.
|
||
/// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV),
|
||
/// it is automatically downloaded to system RAM via <c>av_hwframe_transfer_data</c> into <c>_pHwTransferFrame</c>,
|
||
/// preserving presentation timestamps via <c>av_frame_copy_props</c>.
|
||
///
|
||
/// Decode loop: avcodec_receive_frame → on EAGAIN read and send the next packet → retry.
|
||
/// When the input ends, a null packet switches the decoder to draining mode; the remaining delayed frames
|
||
/// are returned, then avcodec_receive_frame returns AVERROR_EOF.
|
||
/// The returned frame (<see cref="_pDecodedFrame"/> or <see cref="_pHwTransferFrame"/>) holds its data until
|
||
/// the caller unreferences it (av_frame_unref) or the next frame is received.
|
||
/// </summary>
|
||
/// <param name="outFrame">The software frame containing pixel data in system RAM.</param>
|
||
/// <returns>True if a frame was successfully decoded and available; false on EOF or draining.</returns>
|
||
/// <exception cref="Exception">Decoding, packet sending or the GPU-to-host transfer fails.</exception>
|
||
private bool TryReceiveFrame(out AVFrame* outFrame)
|
||
{
|
||
outFrame = null;
|
||
|
||
while (true)
|
||
{
|
||
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
|
||
if (ret == 0)
|
||
{
|
||
// Check if the received frame is on a hardware surface
|
||
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
|
||
{
|
||
// av_hwframe_transfer_data with an empty destination frame allocates a system memory buffer
|
||
// in the first format the device supports for download (NV12 for 8-bit 4:2:0 on the common APIs).
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html (av_hwframe_transfer_data)
|
||
ffmpeg.av_frame_unref(_pHwTransferFrame);
|
||
int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0);
|
||
if (transferRet < 0)
|
||
{
|
||
ffmpeg.av_frame_unref(_pDecodedFrame);
|
||
throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}");
|
||
}
|
||
|
||
// Copy presentation timestamp and metadata properties
|
||
// (pts, best_effort_timestamp, color tags; av_hwframe_transfer_data copies pixel data only).
|
||
// The hardware surface is then released back to the decoder's pool.
|
||
ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame);
|
||
ffmpeg.av_frame_unref(_pDecodedFrame);
|
||
|
||
_downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format;
|
||
outFrame = _pHwTransferFrame;
|
||
return true;
|
||
}
|
||
|
||
// Frame is in system memory (CPU decoding or software fallback format)
|
||
outFrame = _pDecodedFrame;
|
||
return true;
|
||
}
|
||
|
||
// EAGAIN: the decoder needs more input. Feed the next video packet, or start draining at the end of the input.
|
||
// The return value of the flush call (null packet) is not checked; a failure shows up as an error or EOF
|
||
// from the next avcodec_receive_frame.
|
||
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
|
||
{
|
||
if (_isDraining) return false;
|
||
|
||
if (!ReadAndSendPacket())
|
||
{
|
||
_isDraining = true;
|
||
ffmpeg.avcodec_send_packet(_pCodecContext, null);
|
||
}
|
||
}
|
||
// EOF: every delayed frame has been returned after draining.
|
||
else if (ret == ffmpeg.AVERROR_EOF)
|
||
{
|
||
return false;
|
||
}
|
||
else
|
||
{
|
||
throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Selects the YUV→RGB conversion matrix (libswscale <c>SWS_CS_*</c>) from the colorspace tag of a frame.
|
||
/// The matrix set is the same as that of the compute shader path (ColorConversion.compute, <c>ColorMatrix</c> uniform):
|
||
/// BT.709 (also for frames without a usable tag), BT.601 and BT.2020, so both paths give the same colors.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_getCoefficients)
|
||
/// </summary>
|
||
private static int GetSwsColorspace(AVColorSpace colorspace)
|
||
{
|
||
switch (GetColorMatrixId(colorspace))
|
||
{
|
||
case ColorMatrixBT601:
|
||
return ffmpeg.SWS_CS_ITU601;
|
||
case ColorMatrixBT2020:
|
||
return ffmpeg.SWS_CS_BT2020;
|
||
default:
|
||
return ffmpeg.SWS_CS_ITU709;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns true when the Y/U/V samples of the frame use the full range (0-255) instead of the limited range (16-235 / 16-240).
|
||
/// The range is given by the frame's color_range tag or by a deprecated full range pixel format (YUVJ*).
|
||
/// </summary>
|
||
private static bool IsFullRange(AVFrame* frame)
|
||
{
|
||
if (frame->color_range == AVColorRange.AVCOL_RANGE_JPEG) return true;
|
||
|
||
switch ((AVPixelFormat)frame->format)
|
||
{
|
||
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
|
||
case AVPixelFormat.AV_PIX_FMT_YUVJ422P:
|
||
case AVPixelFormat.AV_PIX_FMT_YUVJ444P:
|
||
case AVPixelFormat.AV_PIX_FMT_YUVJ440P:
|
||
return true;
|
||
default:
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats, dimensions,
|
||
/// or (for RGBA output) the colorspace / range of the source frame change.
|
||
/// For RGBA output the YUV→RGB matrix and the source range are taken from <paramref name="colorFrame"/>.
|
||
/// For NV12/YUV420P output no matrix is set; libswscale converts between the formats with its default range handling.
|
||
/// The destination size is always <see cref="Width"/> × <see cref="Height"/>; SWS_BILINEAR filters scaling and chroma resampling.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
||
/// </summary>
|
||
/// <exception cref="Exception">sws_getContext cannot create a context for the conversion.</exception>
|
||
/// <param name="srcFormat">Pixel format of the source frame.</param>
|
||
/// <param name="srcWidth">Width of the source frame in pixels.</param>
|
||
/// <param name="srcHeight">Height of the source frame in pixels.</param>
|
||
/// <param name="dstFormat">Pixel format of the output.</param>
|
||
/// <param name="colorFrame">Source frame whose colorspace / range tags are applied. Required for RGBA output.</param>
|
||
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat, AVFrame* colorFrame = null)
|
||
{
|
||
var swsColorspace = 0;
|
||
var fullRange = false;
|
||
if (colorFrame != null && dstFormat == AVPixelFormat.AV_PIX_FMT_RGBA)
|
||
{
|
||
swsColorspace = GetSwsColorspace(colorFrame->colorspace);
|
||
fullRange = IsFullRange(colorFrame);
|
||
}
|
||
|
||
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat
|
||
&& _srcSwsColorspace == swsColorspace && _srcFullRange == fullRange)
|
||
{
|
||
return;
|
||
}
|
||
|
||
if (_pSwsContext != null)
|
||
{
|
||
ffmpeg.sws_freeContext(_pSwsContext);
|
||
_pSwsContext = null;
|
||
}
|
||
|
||
_srcPixFmt = srcFormat;
|
||
_srcWidth = srcWidth;
|
||
_srcHeight = srcHeight;
|
||
_dstPixFmt = dstFormat;
|
||
_srcSwsColorspace = swsColorspace;
|
||
_srcFullRange = fullRange;
|
||
|
||
_pSwsContext = ffmpeg.sws_getContext(
|
||
srcWidth, srcHeight, srcFormat,
|
||
Width, Height, dstFormat,
|
||
(int)SwsFlags.SWS_BILINEAR, null, null, null
|
||
);
|
||
|
||
if (_pSwsContext == null)
|
||
{
|
||
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
|
||
}
|
||
|
||
// A new context uses the BT.601 matrix and the range implied by the source pixel format (full for YUVJ*,
|
||
// limited otherwise); set the frame's matrix and range for YUV → RGBA.
|
||
if (swsColorspace != 0)
|
||
{
|
||
ApplySwsColorspace(swsColorspace, fullRange);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Sets the YUV→RGB matrix and the source range of <see cref="_pSwsContext"/>.
|
||
/// The destination range, brightness, contrast and saturation keep the values of the context.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails)
|
||
/// </summary>
|
||
/// <param name="swsColorspace">Matrix id (<c>SWS_CS_*</c>) of the source.</param>
|
||
/// <param name="fullRange">True when the source uses the full range.</param>
|
||
private void ApplySwsColorspace(int swsColorspace, bool fullRange)
|
||
{
|
||
int* currentInvTable;
|
||
int* currentTable;
|
||
int srcRange, dstRange, brightness, contrast, saturation;
|
||
|
||
// A negative return value means the details cannot be read; the context then keeps its default matrix and range.
|
||
if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, ¤tInvTable, &srcRange, ¤tTable, &dstRange,
|
||
&brightness, &contrast, &saturation) < 0)
|
||
{
|
||
return;
|
||
}
|
||
|
||
// inv_table: coefficients of the source YUV space (used for YUV → RGB); table: coefficients of the destination.
|
||
// A failure of sws_setColorspaceDetails leaves the defaults of the context and is not reported.
|
||
var coefficients = ffmpeg.sws_getCoefficients(swsColorspace);
|
||
var invTable = new int_array4();
|
||
var table = new int_array4();
|
||
for (uint i = 0; i < 4; i++)
|
||
{
|
||
invTable[i] = coefficients[i];
|
||
table[i] = currentTable[i];
|
||
}
|
||
|
||
ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, fullRange ? 1 : 0, table, dstRange,
|
||
brightness, contrast, saturation);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state.
|
||
/// Used by seeking to inspect candidate frames without falsely incrementing frame indices.
|
||
/// Uses best_effort_timestamp (libavcodec's estimate from PTS/DTS), else pts, converted with the stream time base.
|
||
/// A frame without either timestamp gets <see cref="_currentFrameIndex"/> / FPS.
|
||
/// </summary>
|
||
private double GetFrameTimestamp(AVFrame* frame)
|
||
{
|
||
long ptsValue = frame->best_effort_timestamp;
|
||
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
|
||
{
|
||
ptsValue = frame->pts;
|
||
}
|
||
|
||
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
|
||
{
|
||
return FPS > 0 ? _currentFrameIndex / FPS : 0;
|
||
}
|
||
|
||
return ptsValue * ffmpeg.av_q2d(_timeBase);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter.
|
||
/// With a timestamp, <see cref="_currentFrameIndex"/> = round(timestamp × FPS); without one, it is incremented by 1.
|
||
/// </summary>
|
||
private double GetTimestampAndUpdateIndex(AVFrame* frame)
|
||
{
|
||
double timestamp = GetFrameTimestamp(frame);
|
||
long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE
|
||
? frame->best_effort_timestamp
|
||
: frame->pts;
|
||
|
||
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
|
||
{
|
||
_currentFrameIndex++;
|
||
}
|
||
else
|
||
{
|
||
_currentFrameIndex = (long)Math.Round(timestamp * FPS);
|
||
}
|
||
return timestamp;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Resets all data plane pointers and strides in <see cref="_pConvertedFrame"/> to zero.
|
||
/// Prevents stale or dangling pointers from previous conversions being retained across frames.
|
||
/// The frame never owns buffers (buf[] stays empty), so av_frame_free in Dispose does not touch caller memory.
|
||
/// 8 = AV_NUM_DATA_POINTERS, the size of AVFrame.data / linesize.
|
||
/// </summary>
|
||
private void ClearConvertedFramePointers()
|
||
{
|
||
if (_pConvertedFrame == null) return;
|
||
for (uint i = 0; i < 8; i++)
|
||
{
|
||
_pConvertedFrame->data[i] = null;
|
||
_pConvertedFrame->linesize[i] = 0;
|
||
}
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Frame Reading APIs: RGBA (CPU Fallback)
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies it in 32-bit RGBA format using libswscale.
|
||
/// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P).
|
||
/// The YUV→RGB matrix and range follow the frame's tags (see InitOrUpdateSwsContext). Frames of another size
|
||
/// are scaled to Width × Height. A frame cached by a seek is returned first.
|
||
/// </summary>
|
||
/// <param name="outBuffer">Pointer to destination memory storing RGBA data (at least outStride × Height bytes).</param>
|
||
/// <param name="outStride">Byte stride per row of destination memory (Width * 4).</param>
|
||
/// <param name="vflip">Whether to vertically flip the output to match Unity texture coordinates.</param>
|
||
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
|
||
/// <returns>True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer.</returns>
|
||
/// <exception cref="Exception">Decoding, transfer or scaler creation fails.</exception>
|
||
public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
|
||
{
|
||
timestamp = 0;
|
||
lock (_lock)
|
||
{
|
||
if (!_isOpened || _disposed || outBuffer == null) return false;
|
||
|
||
AVFrame* frame;
|
||
if (_hasCachedFrame)
|
||
{
|
||
frame = _pCachedFrame;
|
||
_hasCachedFrame = false;
|
||
}
|
||
else
|
||
{
|
||
if (!TryReceiveFrame(out frame)) return false;
|
||
}
|
||
|
||
if (frame == null) return false;
|
||
|
||
timestamp = GetTimestampAndUpdateIndex(frame);
|
||
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame);
|
||
|
||
// Destination: the caller's buffer. vflip uses a negative stride starting at the last row,
|
||
// so sws_scale writes the rows bottom-up (Unity textures have their origin at the bottom-left).
|
||
ClearConvertedFramePointers();
|
||
if (vflip)
|
||
{
|
||
_pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1);
|
||
_pConvertedFrame->linesize[0] = -outStride;
|
||
}
|
||
else
|
||
{
|
||
_pConvertedFrame->data[0] = outBuffer;
|
||
_pConvertedFrame->linesize[0] = outStride;
|
||
}
|
||
|
||
ffmpeg.sws_scale(
|
||
_pSwsContext,
|
||
frame->data, frame->linesize,
|
||
0, frame->height,
|
||
_pConvertedFrame->data, _pConvertedFrame->linesize
|
||
);
|
||
|
||
// Release the decoded (or cached) frame's buffers; the output is now in the caller's buffer.
|
||
ClearConvertedFramePointers();
|
||
ffmpeg.av_frame_unref(frame);
|
||
return true;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer.
|
||
/// Same as the byte* overload.
|
||
/// </summary>
|
||
public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp)
|
||
{
|
||
return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies it in RGBA format to a Unity NativeArray.
|
||
/// The rows are tightly packed (stride Width × 4).
|
||
/// </summary>
|
||
/// <exception cref="ArgumentException">The array is not created or is smaller than Width × Height × 4 bytes.</exception>
|
||
public bool TryReadFrame(NativeArray<byte> outBuffer, bool vflip, out double timestamp)
|
||
{
|
||
if (!outBuffer.IsCreated)
|
||
{
|
||
throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer));
|
||
}
|
||
if (outBuffer.Length < (long)Width * Height * 4)
|
||
{
|
||
throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer));
|
||
}
|
||
int stride = Width * 4;
|
||
return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp);
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Frame Reading APIs: NV12 (Compute Shader Pipeline)
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format.
|
||
/// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed:
|
||
/// the planes are copied row by row into the caller's buffers (e.g. texture data for Compute Shader processing).
|
||
/// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale.
|
||
/// The Y/UV values keep the range of the source (see <see cref="ColorRange"/>); no YUV→RGB conversion happens here.
|
||
/// A frame cached by a seek is returned first.
|
||
/// </summary>
|
||
/// <param name="yBuffer">Pointer to destination buffer for Y plane (yStride × Height bytes; Width bytes per row are written).</param>
|
||
/// <param name="yStride">Row stride in bytes for Y plane.</param>
|
||
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (uvStride × ceil(Height / 2) bytes; ceil(Width / 2) × 2 bytes per row are written).</param>
|
||
/// <param name="uvStride">Row stride in bytes for UV plane.</param>
|
||
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
|
||
/// <returns>True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer.</returns>
|
||
/// <exception cref="Exception">Decoding, transfer or scaler creation fails.</exception>
|
||
// ReSharper disable once InconsistentNaming
|
||
public bool TryReadNV12Frame(
|
||
byte* yBuffer, int yStride,
|
||
byte* uvBuffer, int uvStride,
|
||
out double timestamp)
|
||
{
|
||
timestamp = 0;
|
||
lock (_lock)
|
||
{
|
||
if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false;
|
||
|
||
AVFrame* frame;
|
||
if (_hasCachedFrame)
|
||
{
|
||
frame = _pCachedFrame;
|
||
_hasCachedFrame = false;
|
||
}
|
||
else
|
||
{
|
||
if (!TryReceiveFrame(out frame)) return false;
|
||
}
|
||
|
||
if (frame == null) return false;
|
||
|
||
timestamp = GetTimestampAndUpdateIndex(frame);
|
||
|
||
// Conversion path: any other format (e.g. YUV420P from software decoding, P010 from 10-bit hardware decoding)
|
||
// or a frame size different from Width × Height. sws_scale writes directly into the caller's planes.
|
||
if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height)
|
||
{
|
||
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12);
|
||
|
||
ClearConvertedFramePointers();
|
||
_pConvertedFrame->data[0] = yBuffer;
|
||
_pConvertedFrame->data[1] = uvBuffer;
|
||
_pConvertedFrame->linesize[0] = yStride;
|
||
_pConvertedFrame->linesize[1] = uvStride;
|
||
|
||
ffmpeg.sws_scale(
|
||
_pSwsContext,
|
||
frame->data, frame->linesize,
|
||
0, frame->height,
|
||
_pConvertedFrame->data, _pConvertedFrame->linesize
|
||
);
|
||
ClearConvertedFramePointers();
|
||
}
|
||
else
|
||
{
|
||
// Direct copy path: NV12 frame of the output size.
|
||
// The UV plane has ceil(Height / 2) rows of ceil(Width / 2) U/V byte pairs.
|
||
int w = Width;
|
||
int h = Height;
|
||
int uvH = (h + 1) / 2;
|
||
int uvRowBytes = ((w + 1) / 2) * 2;
|
||
|
||
// Copy Y plane (row by row to safely skip linesize GPU padding)
|
||
// Buffer.MemoryCopy(source, destination, destinationSizeInBytes, bytesToCopy): the destination size
|
||
// argument is the stride, so a stride smaller than the row width throws instead of overrunning.
|
||
for (int y = 0; y < h; y++)
|
||
{
|
||
Buffer.MemoryCopy(
|
||
frame->data[0] + y * frame->linesize[0],
|
||
yBuffer + y * yStride,
|
||
yStride, w
|
||
);
|
||
}
|
||
|
||
// Copy UV plane (interleaved U and V samples)
|
||
for (int y = 0; y < uvH; y++)
|
||
{
|
||
Buffer.MemoryCopy(
|
||
frame->data[1] + y * frame->linesize[1],
|
||
uvBuffer + y * uvStride,
|
||
uvStride, uvRowBytes
|
||
);
|
||
}
|
||
}
|
||
|
||
ffmpeg.av_frame_unref(frame);
|
||
return true;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers.
|
||
/// Same as the byte* overload.
|
||
/// </summary>
|
||
// ReSharper disable once InconsistentNaming
|
||
public bool TryReadNV12Frame(
|
||
IntPtr yBuffer, int yStride,
|
||
IntPtr uvBuffer, int uvStride,
|
||
out double timestamp)
|
||
{
|
||
return TryReadNV12Frame(
|
||
(byte*)yBuffer.ToPointer(), yStride,
|
||
(byte*)uvBuffer.ToPointer(), uvStride,
|
||
out timestamp
|
||
);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays.
|
||
/// The planes are tightly packed: Y stride Width, UV stride ceil(Width / 2) × 2 with ceil(Height / 2) rows.
|
||
/// </summary>
|
||
/// <exception cref="ArgumentException">An array is not created or is smaller than its plane.</exception>
|
||
// ReSharper disable once InconsistentNaming
|
||
public bool TryReadNV12Frame(
|
||
NativeArray<byte> yBuffer,
|
||
NativeArray<byte> uvBuffer,
|
||
out double timestamp)
|
||
{
|
||
if (!yBuffer.IsCreated || !uvBuffer.IsCreated)
|
||
{
|
||
throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
|
||
}
|
||
|
||
int w = Width;
|
||
int h = Height;
|
||
int uvH = (h + 1) / 2;
|
||
int uvRowBytes = ((w + 1) / 2) * 2;
|
||
|
||
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
|
||
{
|
||
throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution.");
|
||
}
|
||
|
||
return TryReadNV12Frame(
|
||
(byte*)yBuffer.GetUnsafePtr(), w,
|
||
(byte*)uvBuffer.GetUnsafePtr(), uvRowBytes,
|
||
out timestamp
|
||
);
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Frame Reading APIs: YUV420P (Compute Shader Pipeline)
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies its planar Y, U, and V channel data separately.
|
||
/// For native YUV420P sources, color conversion is bypassed and memory is copied directly.
|
||
/// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale.
|
||
/// YUVJ420P (full range YUV420P) is copied as well; the values keep the source range (see <see cref="ColorRange"/>).
|
||
/// A frame cached by a seek is returned first.
|
||
/// </summary>
|
||
/// <param name="yBuffer">Destination of the Y plane (yStride × Height bytes; Width bytes per row are written).</param>
|
||
/// <param name="yStride">Row stride in bytes of the Y plane.</param>
|
||
/// <param name="uBuffer">Destination of the U plane (uStride × ceil(Height / 2) bytes; ceil(Width / 2) bytes per row).</param>
|
||
/// <param name="uStride">Row stride in bytes of the U plane.</param>
|
||
/// <param name="vBuffer">Destination of the V plane (same size as U).</param>
|
||
/// <param name="vStride">Row stride in bytes of the V plane.</param>
|
||
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
|
||
/// <returns>True if a frame was successfully retrieved; false on EOF, before opening, after disposal or for a null buffer.</returns>
|
||
/// <exception cref="Exception">Decoding, transfer or scaler creation fails.</exception>
|
||
// ReSharper disable once InconsistentNaming
|
||
public bool TryReadYUV420PFrame(
|
||
byte* yBuffer, int yStride,
|
||
byte* uBuffer, int uStride,
|
||
byte* vBuffer, int vStride,
|
||
out double timestamp)
|
||
{
|
||
timestamp = 0;
|
||
lock (_lock)
|
||
{
|
||
if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false;
|
||
|
||
AVFrame* frame;
|
||
if (_hasCachedFrame)
|
||
{
|
||
frame = _pCachedFrame;
|
||
_hasCachedFrame = false;
|
||
}
|
||
else
|
||
{
|
||
if (!TryReceiveFrame(out frame)) return false;
|
||
}
|
||
|
||
if (frame == null) return false;
|
||
|
||
timestamp = GetTimestampAndUpdateIndex(frame);
|
||
|
||
// Conversion path: any format other than YUV420P/YUVJ420P, or a frame size different from Width × Height.
|
||
if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height)
|
||
{
|
||
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P);
|
||
|
||
ClearConvertedFramePointers();
|
||
_pConvertedFrame->data[0] = yBuffer;
|
||
_pConvertedFrame->data[1] = uBuffer;
|
||
_pConvertedFrame->data[2] = vBuffer;
|
||
_pConvertedFrame->linesize[0] = yStride;
|
||
_pConvertedFrame->linesize[1] = uStride;
|
||
_pConvertedFrame->linesize[2] = vStride;
|
||
|
||
ffmpeg.sws_scale(
|
||
_pSwsContext,
|
||
frame->data, frame->linesize,
|
||
0, frame->height,
|
||
_pConvertedFrame->data, _pConvertedFrame->linesize
|
||
);
|
||
ClearConvertedFramePointers();
|
||
}
|
||
else
|
||
{
|
||
// Direct copy path: each plane row by row (source rows can be padded, linesize ≥ row width).
|
||
int w = Width;
|
||
int h = Height;
|
||
int uvW = (w + 1) / 2;
|
||
int uvH = (h + 1) / 2;
|
||
|
||
// Copy Y plane
|
||
for (int y = 0; y < h; y++)
|
||
{
|
||
Buffer.MemoryCopy(
|
||
frame->data[0] + y * frame->linesize[0],
|
||
yBuffer + y * yStride,
|
||
yStride, w
|
||
);
|
||
}
|
||
|
||
// Copy U plane
|
||
for (int y = 0; y < uvH; y++)
|
||
{
|
||
Buffer.MemoryCopy(
|
||
frame->data[1] + y * frame->linesize[1],
|
||
uBuffer + y * uStride,
|
||
uStride, uvW
|
||
);
|
||
}
|
||
|
||
// Copy V plane
|
||
for (int y = 0; y < uvH; y++)
|
||
{
|
||
Buffer.MemoryCopy(
|
||
frame->data[2] + y * frame->linesize[2],
|
||
vBuffer + y * vStride,
|
||
vStride, uvW
|
||
);
|
||
}
|
||
}
|
||
|
||
ffmpeg.av_frame_unref(frame);
|
||
return true;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers.
|
||
/// Same as the byte* overload.
|
||
/// </summary>
|
||
// ReSharper disable once InconsistentNaming
|
||
public bool TryReadYUV420PFrame(
|
||
IntPtr yBuffer, int yStride,
|
||
IntPtr uBuffer, int uStride,
|
||
IntPtr vBuffer, int vStride,
|
||
out double timestamp)
|
||
{
|
||
return TryReadYUV420PFrame(
|
||
(byte*)yBuffer.ToPointer(), yStride,
|
||
(byte*)uBuffer.ToPointer(), uStride,
|
||
(byte*)vBuffer.ToPointer(), vStride,
|
||
out timestamp
|
||
);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays.
|
||
/// The planes are tightly packed: Y stride Width, U/V stride ceil(Width / 2) with ceil(Height / 2) rows.
|
||
/// </summary>
|
||
/// <exception cref="ArgumentException">An array is not created or is smaller than its plane.</exception>
|
||
// ReSharper disable once InconsistentNaming
|
||
public bool TryReadYUV420PFrame(
|
||
NativeArray<byte> yBuffer,
|
||
NativeArray<byte> uBuffer,
|
||
NativeArray<byte> vBuffer,
|
||
out double timestamp)
|
||
{
|
||
if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated)
|
||
{
|
||
throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
|
||
}
|
||
|
||
int w = Width;
|
||
int h = Height;
|
||
int uvW = (w + 1) / 2;
|
||
int uvH = (h + 1) / 2;
|
||
|
||
if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH)
|
||
{
|
||
throw new ArgumentException("Buffer size is smaller than video YUV plane resolution.");
|
||
}
|
||
|
||
return TryReadYUV420PFrame(
|
||
(byte*)yBuffer.GetUnsafePtr(), w,
|
||
(byte*)uBuffer.GetUnsafePtr(), uvW,
|
||
(byte*)vBuffer.GetUnsafePtr(), uvW,
|
||
out timestamp
|
||
);
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Seeking & Skipping
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Maximum distance in frames over which <see cref="SeekToTime"/> moves forward by decoding.
|
||
/// Decoding forward needs no keyframe seek and no decoder flush, which is cheaper than seeking for short distances.
|
||
/// </summary>
|
||
private const int MaxForwardDecodeFrames = 60;
|
||
|
||
/// <summary>
|
||
/// Decodes forward from the current decoder position until the first frame whose presentation time is at least
|
||
/// <paramref name="timeSeconds"/> minus half a frame, and keeps that frame in <see cref="_pCachedFrame"/>
|
||
/// for the next <c>TryRead*</c> call.
|
||
/// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred.
|
||
/// When the end of the stream is reached before the target, no frame is cached.
|
||
/// Every discarded frame updates <see cref="_currentFrameIndex"/>; the cached target frame updates it when it is read.
|
||
/// </summary>
|
||
/// <param name="timeSeconds">Target time in seconds.</param>
|
||
/// <exception cref="Exception">Decoding or the GPU-to-host transfer of the target frame fails.</exception>
|
||
private void DecodeToTarget(double timeSeconds)
|
||
{
|
||
while (true)
|
||
{
|
||
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
|
||
if (ret == 0)
|
||
{
|
||
double currentPts = GetFrameTimestamp(_pDecodedFrame);
|
||
|
||
// Check with half-frame margin to accommodate floating point rounding
|
||
if (currentPts >= timeSeconds - (0.5 / FPS))
|
||
{
|
||
// Cache the first matching frame in system RAM for immediate consumption
|
||
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
|
||
{
|
||
ffmpeg.av_frame_unref(_pCachedFrame);
|
||
int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0);
|
||
if (transferRet < 0)
|
||
{
|
||
ffmpeg.av_frame_unref(_pDecodedFrame);
|
||
throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}");
|
||
}
|
||
|
||
ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame);
|
||
ffmpeg.av_frame_unref(_pDecodedFrame);
|
||
|
||
_downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format;
|
||
}
|
||
else
|
||
{
|
||
// System memory frame: move the buffer references into the cache without copying
|
||
// (_pDecodedFrame is left blank for the next receive).
|
||
ffmpeg.av_frame_unref(_pCachedFrame);
|
||
ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame);
|
||
}
|
||
|
||
_hasCachedFrame = true;
|
||
return;
|
||
}
|
||
|
||
// Discard non-matching intermediate frame without transferring to CPU
|
||
GetTimestampAndUpdateIndex(_pDecodedFrame);
|
||
ffmpeg.av_frame_unref(_pDecodedFrame);
|
||
continue;
|
||
}
|
||
|
||
// Same input feeding and draining as TryReceiveFrame.
|
||
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
|
||
{
|
||
if (_isDraining) return;
|
||
|
||
if (!ReadAndSendPacket())
|
||
{
|
||
_isDraining = true;
|
||
ffmpeg.avcodec_send_packet(_pCodecContext, null);
|
||
}
|
||
}
|
||
else if (ret == ffmpeg.AVERROR_EOF)
|
||
{
|
||
return;
|
||
}
|
||
else
|
||
{
|
||
throw new Exception($"Error during seek decoding: {GetFFmpegErrorString(ret)}");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Moves the playback position to a time in seconds.
|
||
/// The next <c>TryRead*</c> call returns the first frame whose presentation time is at least
|
||
/// <paramref name="timeSeconds"/> minus half a frame (the frame nearest to the target time).
|
||
/// Both paths below return that same frame:
|
||
/// - Forward decode: the target is ahead of the current position by less than <see cref="MaxForwardDecodeFrames"/> frames.
|
||
/// Frames are decoded and discarded up to the target.
|
||
/// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read).
|
||
/// A backward keyframe seek is followed by a forward decode up to the target.
|
||
/// The target frame is transferred and preserved in <see cref="_pCachedFrame"/>; when it is already there, nothing is decoded.
|
||
/// Negative times are treated as 0. The time is in the timeline of the stream timestamps (a stream that starts at
|
||
/// a non-zero time has no frame before its start; the first frame is returned for earlier targets).
|
||
/// When the target is past the last frame, no frame is cached and the next TryRead* call returns false.
|
||
/// </summary>
|
||
/// <param name="timeSeconds">Target time in seconds to seek to.</param>
|
||
/// <exception cref="Exception">Seeking (including the fallback seek to the start) or decoding fails.</exception>
|
||
public void SeekToTime(double timeSeconds)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (!_isOpened || _disposed) return;
|
||
|
||
if (timeSeconds < 0) timeSeconds = 0;
|
||
|
||
// Earliest presentation time that satisfies the target (target minus half a frame).
|
||
double threshold = timeSeconds - (0.5 / FPS);
|
||
|
||
// Index of the first frame that satisfies the target. The epsilon keeps a frame that lies exactly on the
|
||
// margin from being excluded by floating point rounding.
|
||
double targetFrameIndex = Math.Ceiling(timeSeconds * FPS - 0.5 - 1e-6);
|
||
|
||
if (_hasCachedFrame)
|
||
{
|
||
// A frame is already positioned for the next read (e.g. by a previous SeekToTime call).
|
||
double cachedPts = GetFrameTimestamp(_pCachedFrame);
|
||
if (cachedPts >= threshold)
|
||
{
|
||
// The cached frame is the first frame that satisfies the target.
|
||
// When the target lies before it, the frames in between are no longer available and a keyframe seek is required.
|
||
if (cachedPts - threshold < 1.0 / FPS) return;
|
||
}
|
||
else if (targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
|
||
{
|
||
// The cached frame lies before the target: drop it and continue decoding forward.
|
||
GetTimestampAndUpdateIndex(_pCachedFrame);
|
||
ffmpeg.av_frame_unref(_pCachedFrame);
|
||
_hasCachedFrame = false;
|
||
DecodeToTarget(timeSeconds);
|
||
return;
|
||
}
|
||
}
|
||
else if (targetFrameIndex > _currentFrameIndex && targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
|
||
{
|
||
// The target frame has not been read yet and is close: decode forward instead of seeking.
|
||
DecodeToTarget(timeSeconds);
|
||
return;
|
||
}
|
||
|
||
if (_hasCachedFrame)
|
||
{
|
||
ffmpeg.av_frame_unref(_pCachedFrame);
|
||
_hasCachedFrame = false;
|
||
}
|
||
|
||
// Keyframe seek. With a stream index, the timestamp is in the stream time base.
|
||
// AVSEEK_FLAG_BACKWARD: seek to the nearest keyframe at or before the target, so that decoding forward
|
||
// from there reaches the target frame.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_seek_frame)
|
||
long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase));
|
||
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
|
||
if (ret < 0)
|
||
{
|
||
// Fallback to beginning of file on seek failure
|
||
ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD);
|
||
if (ret < 0)
|
||
{
|
||
throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}");
|
||
}
|
||
}
|
||
|
||
// Flush decoder internal buffers
|
||
// Drops the frames buffered from before the seek and ends draining mode, so new packets are accepted again.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
||
ffmpeg.avcodec_flush_buffers(_pCodecContext);
|
||
_isDraining = false;
|
||
|
||
// Decode frames until reaching the target presentation timestamp
|
||
DecodeToTarget(timeSeconds);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Seeks to a specific frame index: <see cref="SeekToTime"/> with frameIndex / FPS seconds.
|
||
/// </summary>
|
||
/// <param name="frameIndex">Target frame index (0-based, in units of 1 / FPS).</param>
|
||
public void SeekToFrame(long frameIndex)
|
||
{
|
||
SeekToTime(frameIndex / FPS);
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Error Handling & Disposal
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Translates an FFmpeg integer error code to a readable string using <c>av_strerror</c>
|
||
/// (1024-byte stack buffer; "Unknown error (Code: n)" when there is no description).
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
|
||
/// </summary>
|
||
private string GetFFmpegErrorString(int errnum)
|
||
{
|
||
const ulong bufferSize = 1024;
|
||
byte* buffer = stackalloc byte[(int)bufferSize];
|
||
if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0)
|
||
{
|
||
return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})";
|
||
}
|
||
return $"Unknown error (Code: {errnum})";
|
||
}
|
||
|
||
/// <summary>
|
||
/// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures.
|
||
/// Safe to call more than once; also called by the constructor on failure and by the finalizer.
|
||
/// Order: scaler → frames → packet → codec context (releases its hw_device_ctx reference)
|
||
/// → this object's device reference → input file and format context.
|
||
/// After Dispose, the read and seek methods return without doing anything.
|
||
/// </summary>
|
||
public void Dispose()
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed) return;
|
||
_disposed = true;
|
||
|
||
// 1. Release scaler context
|
||
if (_pSwsContext != null)
|
||
{
|
||
ffmpeg.sws_freeContext(_pSwsContext);
|
||
_pSwsContext = null;
|
||
}
|
||
|
||
// 2. Release frame buffers
|
||
if (_pDecodedFrame != null)
|
||
{
|
||
var pFrame = _pDecodedFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pDecodedFrame = null;
|
||
}
|
||
|
||
if (_pHwTransferFrame != null)
|
||
{
|
||
var pFrame = _pHwTransferFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pHwTransferFrame = null;
|
||
}
|
||
|
||
if (_pCachedFrame != null)
|
||
{
|
||
var pFrame = _pCachedFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pCachedFrame = null;
|
||
}
|
||
|
||
if (_pConvertedFrame != null)
|
||
{
|
||
var pFrame = _pConvertedFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pConvertedFrame = null;
|
||
}
|
||
|
||
// 3. Release packet buffer
|
||
if (_pPacket != null)
|
||
{
|
||
var pPacket = _pPacket;
|
||
ffmpeg.av_packet_free(&pPacket);
|
||
_pPacket = null;
|
||
}
|
||
|
||
// 4. Release codec context
|
||
if (_pCodecContext != null)
|
||
{
|
||
var pCodecCtx = _pCodecContext;
|
||
ffmpeg.avcodec_free_context(&pCodecCtx);
|
||
_pCodecContext = null;
|
||
}
|
||
|
||
// 5. Release hardware device context
|
||
if (_hwDeviceCtx != null)
|
||
{
|
||
var pHw = _hwDeviceCtx;
|
||
ffmpeg.av_buffer_unref(&pHw);
|
||
_hwDeviceCtx = null;
|
||
}
|
||
|
||
// 6. Close input file and release format context
|
||
if (_pFormatContext != null)
|
||
{
|
||
var pFormatCtx = _pFormatContext;
|
||
ffmpeg.avformat_close_input(&pFormatCtx);
|
||
_pFormatContext = null;
|
||
}
|
||
|
||
_getFormatDelegate = null;
|
||
_isOpened = false;
|
||
_isHwDecoding = false;
|
||
GC.SuppressFinalize(this);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Finalizer. Releases the native resources when the decoder was not disposed (runs on the GC finalizer thread).
|
||
/// </summary>
|
||
~FFmpegNativeDecoder()
|
||
{
|
||
Dispose();
|
||
}
|
||
} |