Files
ADOFAIRenderer/Util/FFmpeg/FFmpegNativeDecoder.cs
T

1688 lines
71 KiB
C#

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using FFmpeg.AutoGen;
using MelonLoader;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
namespace ADOFAIRenderer.Util.FFmpeg;
/// <summary>
/// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke.
/// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and
/// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback.
///
/// ── Architecture &amp; Lifecycle Flow ──────────────────────────────────────────────
///
/// 1. Construction &amp; Demuxing:
/// a) avformat_open_input: Open container file and parse stream headers.
/// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS.
/// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO).
///
/// 2. Codec &amp; Hardware Device Discovery:
/// a) avcodec_find_decoder: Locate codec implementation by codec ID.
/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder.
/// c) HW device creation for the reported configurations, in the order of PreferredHwDeviceTypes.
/// d) Attach hw_device_ctx and register custom get_format callback.
/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding
/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME).
/// f) avcodec_open2: Initialize codec context.
///
/// 3. Frame Demuxing &amp; Decoding (TryRead*):
/// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet.
/// b) (Decoding) avcodec_receive_frame retrieves decoded frames.
/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data
/// into system RAM as NV12 (av_frame_copy_props preserves timestamps).
/// d) (Output Dispatch):
/// - TryReadNV12Frame: NV12 frames are copied row by row (Buffer.MemoryCopy) into the output buffers,
/// other formats are converted with sws_scale.
/// - TryReadYUV420PFrame: YUV420P/YUVJ420P planes are copied row by row into the output buffers,
/// other formats are converted with sws_scale.
/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
///
/// 4. Seeking &amp; Scrubbing (SeekToTime / SeekToFrame):
/// a) av_seek_frame: Fast backward keyframe seek.
/// b) avcodec_flush_buffers: Clear internal decoder state.
/// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead;
/// transfers and caches the first matching frame in _pCachedFrame for immediate consumption.
///
/// 5. Resource Disposal (Dispose):
/// a) Free scaler context (sws_freeContext).
/// b) Free all allocated frames and packets (av_frame_free, av_packet_free).
/// c) Free codec context (avcodec_free_context).
/// d) Release hardware device context (av_buffer_unref).
/// e) Close format context (avformat_close_input).
///
/// ── Reference Documentation ────────────────────────────────────────────────────
/// • FFmpeg Demuxing &amp; Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
/// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
/// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
/// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html
/// • libswscale Scaling &amp; Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html
/// </summary>
internal sealed unsafe class FFmpegNativeDecoder : IDisposable
{
// ──────────────────────────────────────────────────────────────
// Hardware Acceleration Priorities &amp; Candidate Structure
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Candidate hardware device configuration discovered via <c>avcodec_get_hw_config</c>.
/// </summary>
private record struct HwConfigCandidate
{
public AVHWDeviceType DeviceType;
public AVPixelFormat PixelFormat;
}
/// <summary>
/// Preferred hardware device types in priority order.
/// The candidates are the hardware configurations the selected decoder reports (avcodec_get_hw_config),
/// sorted by this list; configurations not in the list follow in reported order.
/// Windows: D3D11VA, then DXVA2 (DirectX 9), CUDA and D3D12VA. Other platforms: VAAPI or VideoToolbox.
/// The QSV entry matches only decoders that report a QSV configuration. The native decoders returned by
/// avcodec_find_decoder report DXVA2/D3D11VA/D3D12VA/CUDA/VAAPI/Vulkan in the bundled FFmpeg 8 build;
/// the QSV decoders (h264_qsv, etc.) are separate decoders selected by name.
/// </summary>
private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
{
AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA,
AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2,
AVHWDeviceType.AV_HWDEVICE_TYPE_QSV,
AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA,
AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA,
AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI,
AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX
};
// ──────────────────────────────────────────────────────────────
// FFmpeg Core Struct Pointers
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Context managing container format demuxing (MP4, MKV, WebM, etc.).
/// Used for packet demuxing and seeking.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
/// </summary>
private AVFormatContext* _pFormatContext = null;
/// <summary>
/// Context managing video decoder state and parameters.
/// Configured with codec parameters, hardware contexts, and threading options.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
/// </summary>
private AVCodecContext* _pCodecContext = null;
/// <summary>
/// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.).
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html
/// </summary>
private AVBufferRef* _hwDeviceCtx = null;
/// <summary>
/// Frame structure holding raw decoded frames received directly from <c>avcodec_receive_frame</c>.
/// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView).
/// In SW decoding mode, this contains pixel buffers in system memory.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
/// </summary>
private AVFrame* _pDecodedFrame = null;
/// <summary>
/// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces
/// via <c>av_hwframe_transfer_data</c>.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
/// </summary>
private AVFrame* _pHwTransferFrame = null;
/// <summary>
/// Frame structure holding converted pixel data produced by <c>sws_scale</c> (e.g., RGBA or fallback YUV).
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
/// </summary>
private AVFrame* _pConvertedFrame = null;
/// <summary>
/// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation.
/// Consumed by the next <c>TryRead*</c> invocation.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
/// </summary>
private AVFrame* _pCachedFrame = null;
private bool _hasCachedFrame;
/// <summary>
/// Packet structure temporarily holding compressed demuxed data read from the input container.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html
/// </summary>
private AVPacket* _pPacket = null;
/// <summary>
/// libswscale colorspace and format conversion context.
/// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
/// </summary>
private SwsContext* _pSwsContext = null;
// ──────────────────────────────────────────────────────────────
// Hardware Acceleration State &amp; Delegates
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code.
/// Passed to <c>_pCodecContext-&gt;get_format</c>.
/// </summary>
private AVCodecContext_get_format? _getFormatDelegate;
/// <summary>Whether hardware-accelerated decoding is currently active.</summary>
private bool _isHwDecoding;
/// <summary>Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV).</summary>
private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
/// <summary>Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD).</summary>
private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
/// <summary>
/// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12).
/// </summary>
private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
// ──────────────────────────────────────────────────────────────
// Playback, Stream &amp; Scaler Synchronization Fields
// ──────────────────────────────────────────────────────────────
private int _videoStreamIndex = -1;
private long _currentFrameIndex;
private AVRational _timeBase;
private bool _isOpened;
private bool _disposed;
private bool _isDraining;
/// <summary>Number of video packets skipped because the decoder rejected them as invalid data.</summary>
private int _skippedPacketCount;
/// <summary>Number of skipped packets for which a warning is logged.</summary>
private const int MaxLoggedSkippedPackets = 5;
private readonly object _lock = new();
// Fields to detect SwsContext updates
private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
private int _srcWidth;
private int _srcHeight;
/// <summary>YUV→RGB matrix id (<c>SWS_CS_*</c>) applied to the context. 0 when no matrix is set (non-RGBA output).</summary>
private int _srcSwsColorspace;
/// <summary>Whether the source range applied to the context is the full range.</summary>
private bool _srcFullRange;
// ──────────────────────────────────────────────────────────────
// Public Properties
// ──────────────────────────────────────────────────────────────
/// <summary>Video width in pixels.</summary>
public int Width { get; private set; }
/// <summary>Video height in pixels.</summary>
public int Height { get; private set; }
/// <summary>Frames per second.</summary>
public double FPS { get; private set; }
/// <summary>Total duration in seconds.</summary>
public double Duration { get; private set; }
/// <summary>Estimated total number of frames.</summary>
public long TotalFrames { get; private set; }
/// <summary>Index of the currently read frame.</summary>
public long CurrentFrameIndex
{
get
{
lock (_lock)
{
return _currentFrameIndex;
}
}
}
/// <summary>Current playback time in seconds.</summary>
public double CurrentTime
{
get
{
lock (_lock)
{
return FPS > 0 ? _currentFrameIndex / FPS : 0;
}
}
}
/// <summary>
/// Pixel format of the video frames produced by the decoder.
/// When hardware decoding is active, returns the pixel format of the downloaded software frame
/// in system RAM (typically <see cref="AVPixelFormat.AV_PIX_FMT_NV12"/>) rather than the GPU surface format
/// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines.
/// In software mode, returns the codec context's pixel format (e.g., YUV420P).
/// </summary>
public AVPixelFormat PixelFormat
{
get
{
lock (_lock)
{
if (_isHwDecoding)
{
return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE
? _downloadedPixFmt
: AVPixelFormat.AV_PIX_FMT_NV12;
}
return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE;
}
}
}
/// <summary>
/// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]).
/// </summary>
public AVColorRange ColorRange
{
get
{
lock (_lock)
{
if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED)
{
return _pCodecContext->color_range;
}
if (_pFormatContext != null && _videoStreamIndex >= 0)
{
return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range;
}
return AVColorRange.AVCOL_RANGE_UNSPECIFIED;
}
}
}
/// <summary>
/// Color matrix id used to convert the YUV frames of this stream to RGB, selected from the colorspace tag of the stream.
/// The ids are shared with the <c>ColorMatrix</c> uniform of ColorConversion.compute:
/// 0 = BT.709 (also used when the stream has no usable tag), 1 = BT.601, 2 = BT.2020 (non-constant luminance).
/// </summary>
public int ColorMatrix
{
get
{
lock (_lock)
{
if (_pCodecContext != null && _pCodecContext->colorspace != AVColorSpace.AVCOL_SPC_UNSPECIFIED)
{
return GetColorMatrixId(_pCodecContext->colorspace);
}
if (_pFormatContext != null && _videoStreamIndex >= 0)
{
return GetColorMatrixId(_pFormatContext->streams[_videoStreamIndex]->codecpar->color_space);
}
return ColorMatrixBT709;
}
}
}
/// <summary>Color matrix id of BT.709.</summary>
// ReSharper disable InconsistentNaming
public const int ColorMatrixBT709 = 0;
/// <summary>Color matrix id of BT.601 (SMPTE 170M / BT.470 BG).</summary>
public const int ColorMatrixBT601 = 1;
/// <summary>Color matrix id of BT.2020 (non-constant luminance).</summary>
public const int ColorMatrixBT2020 = 2;
// ReSharper restore InconsistentNaming
/// <summary>
/// Converts a colorspace tag to a color matrix id (see <see cref="ColorMatrix"/>).
/// Tags that have no matrix of their own (unspecified, RGB, other standards) use BT.709.
/// </summary>
private static int GetColorMatrixId(AVColorSpace colorspace)
{
switch (colorspace)
{
case AVColorSpace.AVCOL_SPC_SMPTE170M:
case AVColorSpace.AVCOL_SPC_BT470BG:
return ColorMatrixBT601;
case AVColorSpace.AVCOL_SPC_BT2020_NCL:
return ColorMatrixBT2020;
default:
return ColorMatrixBT709;
}
}
/// <summary>
/// Gets a value indicating whether hardware-accelerated decoding is currently active.
/// </summary>
public bool IsHardwareAccelerated
{
get
{
lock (_lock)
{
return _isHwDecoding;
}
}
}
/// <summary>
/// Gets the hardware device type in use for hardware acceleration, or null if software decoding.
/// </summary>
public AVHWDeviceType? HardwareDeviceType
{
get
{
lock (_lock)
{
return _isHwDecoding ? _hwDeviceType : null;
}
}
}
// ──────────────────────────────────────────────────────────────
// Initialization &amp; Setup
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Initializes a new instance of <see cref="FFmpegNativeDecoder"/> and opens the specified video file.
/// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back
/// to multi-threaded software decoding if unavailable.
/// </summary>
/// <param name="videoPath">Absolute or relative file path to the video file.</param>
public FFmpegNativeDecoder(string videoPath)
{
Open(videoPath);
}
/// <summary>
/// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers.
/// </summary>
/// <param name="videoPath">Path to video file.</param>
private void Open(string videoPath)
{
lock (_lock)
{
if (_isOpened || _disposed) return;
try
{
// 1. Open media container
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
fixed (AVFormatContext** ppFormatCtx = &_pFormatContext)
{
int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null);
if (ret < 0)
throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})");
}
// 2. Retrieve stream information
if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0)
throw new Exception("Failed to retrieve stream info.");
// 3. Find primary video stream
_videoStreamIndex = -1;
for (int i = 0; i < _pFormatContext->nb_streams; i++)
{
if (_pFormatContext->streams[i]->codecpar->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO)
{
_videoStreamIndex = i;
break;
}
}
if (_videoStreamIndex == -1)
throw new Exception("Failed to find video stream.");
// Only the video stream is used. The demuxer does not need to return the packets of the other streams
// (audio, subtitles, ...) once they are marked as discarded.
// Reference: AVStream.discard - "Selects which packets can be discarded at will and do not need to be demuxed."
// https://ffmpeg.org/doxygen/trunk/structAVStream.html
for (int i = 0; i < _pFormatContext->nb_streams; i++)
{
if (i != _videoStreamIndex)
_pFormatContext->streams[i]->discard = AVDiscard.AVDISCARD_ALL;
}
var stream = _pFormatContext->streams[_videoStreamIndex];
_timeBase = stream->time_base;
// 4. Calculate FPS
if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0)
{
FPS = ffmpeg.av_q2d(stream->r_frame_rate);
}
else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0)
{
FPS = ffmpeg.av_q2d(stream->avg_frame_rate);
}
else
{
FPS = 30.0; // Fallback default
}
// 5. Find decoder by codec ID
var codecId = stream->codecpar->codec_id;
var decoder = ffmpeg.avcodec_find_decoder(codecId);
if (decoder == null)
throw new Exception($"Unsupported video codec: {codecId}");
// 6. Allocate codec context
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
if (_pCodecContext == null)
throw new Exception("Failed to allocate codec context.");
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
throw new Exception("Failed to copy codec parameters.");
// 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding
bool hwSuccess = TryInitializeHwDecoder(decoder, stream);
if (!hwSuccess)
{
Melon<RenderMod>.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding.");
_isHwDecoding = false;
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
// Multi-threading settings for improved software decoding performance (auto thread count)
// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
_pCodecContext->thread_count = 0;
_pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME;
int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
if (openRet < 0)
throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}");
}
Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width;
Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height;
// 8. Calculate total frames and duration
if (stream->nb_frames > 0)
{
TotalFrames = stream->nb_frames;
}
else if (stream->duration > 0)
{
TotalFrames = (long)Math.Round(stream->duration * ffmpeg.av_q2d(stream->time_base) * FPS);
}
else if (_pFormatContext->duration > 0)
{
TotalFrames = (long)Math.Round((_pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE) * FPS);
}
else
{
TotalFrames = 0;
}
// Duration of the video stream (AVStream.duration, in the stream time base).
// When the stream does not carry a duration, the container duration (AVFormatContext.duration,
// in AV_TIME_BASE units, the longest of all streams) is used.
// Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html, https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
Duration = stream->duration > 0
? stream->duration * ffmpeg.av_q2d(stream->time_base)
: _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0;
// 9. Allocate frame and packet buffers
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
_pDecodedFrame = ffmpeg.av_frame_alloc();
_pHwTransferFrame = ffmpeg.av_frame_alloc();
_pConvertedFrame = ffmpeg.av_frame_alloc();
_pCachedFrame = ffmpeg.av_frame_alloc();
_pPacket = ffmpeg.av_packet_alloc();
_currentFrameIndex = 0;
_isOpened = true;
_isDraining = false;
}
catch (Exception)
{
Dispose();
throw;
}
}
}
/// <summary>
/// Searches for supported hardware device configurations via <c>avcodec_get_hw_config</c>,
/// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA),
/// registers the <c>get_format</c> callback, and attempts to open the hardware decoder.
/// </summary>
/// <param name="decoder">Target decoder.</param>
/// <param name="stream">Source video stream.</param>
/// <returns>True if hardware decoding was successfully configured and opened; false to fall back.</returns>
private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream)
{
// ReSharper disable once InconsistentNaming
// Collect all supported hardware configs from decoder
const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01;
var candidates = new List<HwConfigCandidate>();
for (int i = 0; ; i++)
{
var config = ffmpeg.avcodec_get_hw_config(decoder, i);
if (config == null) break;
if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0)
{
candidates.Add(new HwConfigCandidate
{
DeviceType = config->device_type,
PixelFormat = config->pix_fmt
});
}
}
if (candidates.Count == 0)
{
return false;
}
// Order candidates according to preference
var orderedCandidates = new List<HwConfigCandidate>();
foreach (var preferred in PreferredHwDeviceTypes)
{
foreach (var candidate in candidates)
{
if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate))
{
orderedCandidates.Add(candidate);
}
}
}
// Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes
foreach (var candidate in candidates)
{
if (!orderedCandidates.Contains(candidate))
{
orderedCandidates.Add(candidate);
}
}
// Try candidate hardware configurations in priority order
foreach (var candidate in orderedCandidates)
{
AVBufferRef* hwDeviceCtx = null;
int err;
// Device creation strategy
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV)
{
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
string childDeviceName = isWindows ? "d3d11va" : "vaapi";
AVDictionary* opts = null;
ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0);
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0);
ffmpeg.av_dict_free(&opts);
if (err < 0)
{
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
}
}
else
{
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0);
}
if (err < 0 || hwDeviceCtx == null)
{
if (hwDeviceCtx != null)
{
ffmpeg.av_buffer_unref(&hwDeviceCtx);
hwDeviceCtx = null;
}
Melon<RenderMod>.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate...");
continue;
}
// Assign hardware context and format selection callback
_hwDeviceCtx = hwDeviceCtx;
_hwPixFmt = candidate.PixelFormat;
_hwDeviceType = candidate.DeviceType;
_isHwDecoding = true;
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12;
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx);
if (_pCodecContext->hw_device_ctx == null)
{
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate...");
var pHw = _hwDeviceCtx;
ffmpeg.av_buffer_unref(&pHw);
_hwDeviceCtx = null;
_isHwDecoding = false;
continue;
}
_getFormatDelegate = GetFormat;
_pCodecContext->get_format = _getFormatDelegate;
// Hardware decoding runs with one decoder thread (thread_count = 1).
// The GPU performs the decoding; the decoder thread only parses packets and submits them.
// With frame threading, libavcodec adds one hardware surface per thread to the frame pool
// (avcodec_get_hw_frames_parameters). The software fallback in GetFormat keeps this thread count.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/decode.c
_pCodecContext->thread_count = 1;
// Attempt opening codec context with hardware configuration
int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
if (openRet >= 0)
{
Melon<RenderMod>.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}");
return true;
}
// Failed to open codec with this hardware configuration, clean up and try next
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate...");
if (_pCodecContext != null)
{
var pCodecCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCodecCtx);
_pCodecContext = null;
}
if (_hwDeviceCtx != null)
{
var pHw = _hwDeviceCtx;
ffmpeg.av_buffer_unref(&pHw);
_hwDeviceCtx = null;
}
_isHwDecoding = false;
_getFormatDelegate = null;
// Reallocate fresh codec context for subsequent attempts
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
if (_pCodecContext == null)
throw new Exception("Failed to re-allocate codec context during fallback.");
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
throw new Exception("Failed to copy codec parameters during fallback.");
}
return false;
}
/// <summary>
/// Callback invoked by libavcodec during decoding to negotiate the output pixel format.
/// Confirms the hardware surface format (<see cref="_hwPixFmt"/>) if offered by the decoder,
/// or falls back to software pixel formats via <c>avcodec_default_get_format</c>.
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
/// </summary>
private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts)
{
for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++)
{
if (*p == _hwPixFmt)
{
return *p;
}
}
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format.");
_isHwDecoding = false;
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
return ffmpeg.avcodec_default_get_format(ctx, pixFmts);
}
// ──────────────────────────────────────────────────────────────
// Demuxing &amp; Decoding Engine
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Reads packets from the container until a packet belonging to the video stream is found,
/// then sends it to the decoder via <c>avcodec_send_packet</c>.
/// A packet that the decoder rejects as invalid data (<c>AVERROR_INVALIDDATA</c>, e.g. a damaged packet)
/// is skipped and the next packet is read. Any other decoder error is thrown.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
/// </summary>
/// <returns>True if a video packet was sent; false on EOF.</returns>
private bool ReadAndSendPacket()
{
while (true)
{
ffmpeg.av_packet_unref(_pPacket);
int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket);
if (ret < 0) return false; // EOF or read error
if (_pPacket->stream_index == _videoStreamIndex)
{
ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket);
ffmpeg.av_packet_unref(_pPacket);
if (ret == ffmpeg.AVERROR_INVALIDDATA)
{
// The warning is logged for the first few packets only (a damaged file can have many).
_skippedPacketCount++;
if (_skippedPacketCount <= MaxLoggedSkippedPackets)
{
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Skipped an invalid video packet ({_skippedPacketCount}): {GetFFmpegErrorString(ret)}");
}
continue;
}
if (ret < 0)
{
throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}");
}
return true;
}
}
}
/// <summary>
/// Receives the next decoded video frame from the codec context.
/// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV),
/// it is automatically downloaded to system RAM via <c>av_hwframe_transfer_data</c> into <c>_pHwTransferFrame</c>,
/// preserving presentation timestamps via <c>av_frame_copy_props</c>.
/// </summary>
/// <param name="outFrame">The software frame containing pixel data in system RAM.</param>
/// <returns>True if a frame was successfully decoded and available; false on EOF or draining.</returns>
private bool TryReceiveFrame(out AVFrame* outFrame)
{
outFrame = null;
while (true)
{
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
if (ret == 0)
{
// Check if the received frame is on a hardware surface
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
{
ffmpeg.av_frame_unref(_pHwTransferFrame);
int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0);
if (transferRet < 0)
{
ffmpeg.av_frame_unref(_pDecodedFrame);
throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}");
}
// Copy presentation timestamp and metadata properties
ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame);
ffmpeg.av_frame_unref(_pDecodedFrame);
_downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format;
outFrame = _pHwTransferFrame;
return true;
}
// Frame is in system memory (CPU decoding or software fallback format)
outFrame = _pDecodedFrame;
return true;
}
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
{
if (_isDraining) return false;
if (!ReadAndSendPacket())
{
_isDraining = true;
ffmpeg.avcodec_send_packet(_pCodecContext, null);
}
}
else if (ret == ffmpeg.AVERROR_EOF)
{
return false;
}
else
{
throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
}
}
}
/// <summary>
/// Selects the YUV→RGB conversion matrix (libswscale <c>SWS_CS_*</c>) from the colorspace tag of a frame.
/// The matrix set is the same as that of the compute shader path (ColorConversion.compute, <c>ColorMatrix</c> uniform):
/// BT.709 (also for frames without a usable tag), BT.601 and BT.2020, so both paths give the same colors.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_getCoefficients)
/// </summary>
private static int GetSwsColorspace(AVColorSpace colorspace)
{
switch (GetColorMatrixId(colorspace))
{
case ColorMatrixBT601:
return ffmpeg.SWS_CS_ITU601;
case ColorMatrixBT2020:
return ffmpeg.SWS_CS_BT2020;
default:
return ffmpeg.SWS_CS_ITU709;
}
}
/// <summary>
/// Returns true when the Y/U/V samples of the frame use the full range (0-255) instead of the limited range (16-235 / 16-240).
/// The range is given by the frame's color_range tag or by a deprecated full range pixel format (YUVJ*).
/// </summary>
private static bool IsFullRange(AVFrame* frame)
{
if (frame->color_range == AVColorRange.AVCOL_RANGE_JPEG) return true;
switch ((AVPixelFormat)frame->format)
{
case AVPixelFormat.AV_PIX_FMT_YUVJ420P:
case AVPixelFormat.AV_PIX_FMT_YUVJ422P:
case AVPixelFormat.AV_PIX_FMT_YUVJ444P:
case AVPixelFormat.AV_PIX_FMT_YUVJ440P:
return true;
default:
return false;
}
}
/// <summary>
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats, dimensions,
/// or (for RGBA output) the colorspace / range of the source frame change.
/// For RGBA output the YUV→RGB matrix and the source range are taken from <paramref name="colorFrame"/>.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
/// </summary>
/// <param name="srcFormat">Pixel format of the source frame.</param>
/// <param name="srcWidth">Width of the source frame in pixels.</param>
/// <param name="srcHeight">Height of the source frame in pixels.</param>
/// <param name="dstFormat">Pixel format of the output.</param>
/// <param name="colorFrame">Source frame whose colorspace / range tags are applied. Required for RGBA output.</param>
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat, AVFrame* colorFrame = null)
{
var swsColorspace = 0;
var fullRange = false;
if (colorFrame != null && dstFormat == AVPixelFormat.AV_PIX_FMT_RGBA)
{
swsColorspace = GetSwsColorspace(colorFrame->colorspace);
fullRange = IsFullRange(colorFrame);
}
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat
&& _srcSwsColorspace == swsColorspace && _srcFullRange == fullRange)
{
return;
}
if (_pSwsContext != null)
{
ffmpeg.sws_freeContext(_pSwsContext);
_pSwsContext = null;
}
_srcPixFmt = srcFormat;
_srcWidth = srcWidth;
_srcHeight = srcHeight;
_dstPixFmt = dstFormat;
_srcSwsColorspace = swsColorspace;
_srcFullRange = fullRange;
_pSwsContext = ffmpeg.sws_getContext(
srcWidth, srcHeight, srcFormat,
Width, Height, dstFormat,
(int)SwsFlags.SWS_BILINEAR, null, null, null
);
if (_pSwsContext == null)
{
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
}
if (swsColorspace != 0)
{
ApplySwsColorspace(swsColorspace, fullRange);
}
}
/// <summary>
/// Sets the YUV→RGB matrix and the source range of <see cref="_pSwsContext"/>.
/// The destination range, brightness, contrast and saturation keep the values of the context.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails)
/// </summary>
/// <param name="swsColorspace">Matrix id (<c>SWS_CS_*</c>) of the source.</param>
/// <param name="fullRange">True when the source uses the full range.</param>
private void ApplySwsColorspace(int swsColorspace, bool fullRange)
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
// Returns a negative value when the context cannot change its colorspace details (e.g. YUV to YUV conversion).
if (ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation) < 0)
{
return;
}
var coefficients = ffmpeg.sws_getCoefficients(swsColorspace);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = coefficients[i];
table[i] = currentTable[i];
}
ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, fullRange ? 1 : 0, table, dstRange,
brightness, contrast, saturation);
}
/// <summary>
/// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state.
/// Used by seeking to inspect candidate frames without falsely incrementing frame indices.
/// </summary>
private double GetFrameTimestamp(AVFrame* frame)
{
long ptsValue = frame->best_effort_timestamp;
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
{
ptsValue = frame->pts;
}
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
{
return FPS > 0 ? _currentFrameIndex / FPS : 0;
}
return ptsValue * ffmpeg.av_q2d(_timeBase);
}
/// <summary>
/// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter.
/// </summary>
private double GetTimestampAndUpdateIndex(AVFrame* frame)
{
double timestamp = GetFrameTimestamp(frame);
long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE
? frame->best_effort_timestamp
: frame->pts;
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
{
_currentFrameIndex++;
}
else
{
_currentFrameIndex = (long)Math.Round(timestamp * FPS);
}
return timestamp;
}
/// <summary>
/// Resets all data plane pointers and strides in <see cref="_pConvertedFrame"/> to zero.
/// Prevents stale or dangling pointers from previous conversions being retained across frames.
/// </summary>
private void ClearConvertedFramePointers()
{
if (_pConvertedFrame == null) return;
for (uint i = 0; i < 8; i++)
{
_pConvertedFrame->data[i] = null;
_pConvertedFrame->linesize[i] = 0;
}
}
// ──────────────────────────────────────────────────────────────
// Frame Reading APIs: RGBA (CPU Fallback)
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Reads the next video frame and copies it in 32-bit RGBA format using libswscale.
/// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P).
/// </summary>
/// <param name="outBuffer">Pointer to destination memory storing RGBA data.</param>
/// <param name="outStride">Byte stride per row of destination memory (Width * 4).</param>
/// <param name="vflip">Whether to vertically flip the output to match Unity texture coordinates.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
{
timestamp = 0;
lock (_lock)
{
if (!_isOpened || _disposed || outBuffer == null) return false;
AVFrame* frame;
if (_hasCachedFrame)
{
frame = _pCachedFrame;
_hasCachedFrame = false;
}
else
{
if (!TryReceiveFrame(out frame)) return false;
}
if (frame == null) return false;
timestamp = GetTimestampAndUpdateIndex(frame);
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA, frame);
ClearConvertedFramePointers();
if (vflip)
{
_pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1);
_pConvertedFrame->linesize[0] = -outStride;
}
else
{
_pConvertedFrame->data[0] = outBuffer;
_pConvertedFrame->linesize[0] = outStride;
}
ffmpeg.sws_scale(
_pSwsContext,
frame->data, frame->linesize,
0, frame->height,
_pConvertedFrame->data, _pConvertedFrame->linesize
);
ClearConvertedFramePointers();
ffmpeg.av_frame_unref(frame);
return true;
}
}
/// <summary>
/// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer.
/// </summary>
public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp)
{
return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp);
}
/// <summary>
/// Reads the next video frame and copies it in RGBA format to a Unity NativeArray.
/// </summary>
public bool TryReadFrame(NativeArray<byte> outBuffer, bool vflip, out double timestamp)
{
if (!outBuffer.IsCreated)
{
throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer));
}
if (outBuffer.Length < (long)Width * Height * 4)
{
throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer));
}
int stride = Width * 4;
return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp);
}
// ──────────────────────────────────────────────────────────────
// Frame Reading APIs: NV12 (Compute Shader Pipeline)
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format.
/// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed,
/// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing.
/// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale.
/// </summary>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
/// <param name="yStride">Row stride in bytes for Y plane.</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes).</param>
/// <param name="uvStride">Row stride in bytes for UV plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
// ReSharper disable once InconsistentNaming
public bool TryReadNV12Frame(
byte* yBuffer, int yStride,
byte* uvBuffer, int uvStride,
out double timestamp)
{
timestamp = 0;
lock (_lock)
{
if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false;
AVFrame* frame;
if (_hasCachedFrame)
{
frame = _pCachedFrame;
_hasCachedFrame = false;
}
else
{
if (!TryReceiveFrame(out frame)) return false;
}
if (frame == null) return false;
timestamp = GetTimestampAndUpdateIndex(frame);
if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height)
{
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12);
ClearConvertedFramePointers();
_pConvertedFrame->data[0] = yBuffer;
_pConvertedFrame->data[1] = uvBuffer;
_pConvertedFrame->linesize[0] = yStride;
_pConvertedFrame->linesize[1] = uvStride;
ffmpeg.sws_scale(
_pSwsContext,
frame->data, frame->linesize,
0, frame->height,
_pConvertedFrame->data, _pConvertedFrame->linesize
);
ClearConvertedFramePointers();
}
else
{
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = ((w + 1) / 2) * 2;
// Copy Y plane (row by row to safely skip linesize GPU padding)
for (int y = 0; y < h; y++)
{
Buffer.MemoryCopy(
frame->data[0] + y * frame->linesize[0],
yBuffer + y * yStride,
yStride, w
);
}
// Copy UV plane (interleaved U and V samples)
for (int y = 0; y < uvH; y++)
{
Buffer.MemoryCopy(
frame->data[1] + y * frame->linesize[1],
uvBuffer + y * uvStride,
uvStride, uvRowBytes
);
}
}
ffmpeg.av_frame_unref(frame);
return true;
}
}
/// <summary>
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadNV12Frame(
IntPtr yBuffer, int yStride,
IntPtr uvBuffer, int uvStride,
out double timestamp)
{
return TryReadNV12Frame(
(byte*)yBuffer.ToPointer(), yStride,
(byte*)uvBuffer.ToPointer(), uvStride,
out timestamp
);
}
/// <summary>
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadNV12Frame(
NativeArray<byte> yBuffer,
NativeArray<byte> uvBuffer,
out double timestamp)
{
if (!yBuffer.IsCreated || !uvBuffer.IsCreated)
{
throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
}
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = ((w + 1) / 2) * 2;
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
{
throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution.");
}
return TryReadNV12Frame(
(byte*)yBuffer.GetUnsafePtr(), w,
(byte*)uvBuffer.GetUnsafePtr(), uvRowBytes,
out timestamp
);
}
// ──────────────────────────────────────────────────────────────
// Frame Reading APIs: YUV420P (Compute Shader Pipeline)
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Reads the next video frame and copies its planar Y, U, and V channel data separately.
/// For native YUV420P sources, color conversion is bypassed and memory is copied directly.
/// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
byte* yBuffer, int yStride,
byte* uBuffer, int uStride,
byte* vBuffer, int vStride,
out double timestamp)
{
timestamp = 0;
lock (_lock)
{
if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false;
AVFrame* frame;
if (_hasCachedFrame)
{
frame = _pCachedFrame;
_hasCachedFrame = false;
}
else
{
if (!TryReceiveFrame(out frame)) return false;
}
if (frame == null) return false;
timestamp = GetTimestampAndUpdateIndex(frame);
if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height)
{
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P);
ClearConvertedFramePointers();
_pConvertedFrame->data[0] = yBuffer;
_pConvertedFrame->data[1] = uBuffer;
_pConvertedFrame->data[2] = vBuffer;
_pConvertedFrame->linesize[0] = yStride;
_pConvertedFrame->linesize[1] = uStride;
_pConvertedFrame->linesize[2] = vStride;
ffmpeg.sws_scale(
_pSwsContext,
frame->data, frame->linesize,
0, frame->height,
_pConvertedFrame->data, _pConvertedFrame->linesize
);
ClearConvertedFramePointers();
}
else
{
int w = Width;
int h = Height;
int uvW = (w + 1) / 2;
int uvH = (h + 1) / 2;
// Copy Y plane
for (int y = 0; y < h; y++)
{
Buffer.MemoryCopy(
frame->data[0] + y * frame->linesize[0],
yBuffer + y * yStride,
yStride, w
);
}
// Copy U plane
for (int y = 0; y < uvH; y++)
{
Buffer.MemoryCopy(
frame->data[1] + y * frame->linesize[1],
uBuffer + y * uStride,
uStride, uvW
);
}
// Copy V plane
for (int y = 0; y < uvH; y++)
{
Buffer.MemoryCopy(
frame->data[2] + y * frame->linesize[2],
vBuffer + y * vStride,
vStride, uvW
);
}
}
ffmpeg.av_frame_unref(frame);
return true;
}
}
/// <summary>
/// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
IntPtr yBuffer, int yStride,
IntPtr uBuffer, int uStride,
IntPtr vBuffer, int vStride,
out double timestamp)
{
return TryReadYUV420PFrame(
(byte*)yBuffer.ToPointer(), yStride,
(byte*)uBuffer.ToPointer(), uStride,
(byte*)vBuffer.ToPointer(), vStride,
out timestamp
);
}
/// <summary>
/// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays.
/// </summary>
// ReSharper disable once InconsistentNaming
public bool TryReadYUV420PFrame(
NativeArray<byte> yBuffer,
NativeArray<byte> uBuffer,
NativeArray<byte> vBuffer,
out double timestamp)
{
if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated)
{
throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
}
int w = Width;
int h = Height;
int uvW = (w + 1) / 2;
int uvH = (h + 1) / 2;
if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH)
{
throw new ArgumentException("Buffer size is smaller than video YUV plane resolution.");
}
return TryReadYUV420PFrame(
(byte*)yBuffer.GetUnsafePtr(), w,
(byte*)uBuffer.GetUnsafePtr(), uvW,
(byte*)vBuffer.GetUnsafePtr(), uvW,
out timestamp
);
}
// ──────────────────────────────────────────────────────────────
// Seeking &amp; Skipping
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Maximum distance in frames over which <see cref="SeekToTime"/> moves forward by decoding.
/// Decoding forward needs no keyframe seek and no decoder flush, which is cheaper than seeking for short distances.
/// </summary>
private const int MaxForwardDecodeFrames = 60;
/// <summary>
/// Decodes forward from the current decoder position until the first frame whose presentation time is at least
/// <paramref name="timeSeconds"/> minus half a frame, and keeps that frame in <see cref="_pCachedFrame"/>
/// for the next <c>TryRead*</c> call.
/// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred.
/// When the end of the stream is reached before the target, no frame is cached.
/// </summary>
/// <param name="timeSeconds">Target time in seconds.</param>
private void DecodeToTarget(double timeSeconds)
{
while (true)
{
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
if (ret == 0)
{
double currentPts = GetFrameTimestamp(_pDecodedFrame);
// Check with half-frame margin to accommodate floating point rounding
if (currentPts >= timeSeconds - (0.5 / FPS))
{
// Cache the first matching frame in system RAM for immediate consumption
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
{
ffmpeg.av_frame_unref(_pCachedFrame);
int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0);
if (transferRet < 0)
{
ffmpeg.av_frame_unref(_pDecodedFrame);
throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}");
}
ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame);
ffmpeg.av_frame_unref(_pDecodedFrame);
_downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format;
}
else
{
ffmpeg.av_frame_unref(_pCachedFrame);
ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame);
}
_hasCachedFrame = true;
return;
}
// Discard non-matching intermediate frame without transferring to CPU
GetTimestampAndUpdateIndex(_pDecodedFrame);
ffmpeg.av_frame_unref(_pDecodedFrame);
continue;
}
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
{
if (_isDraining) return;
if (!ReadAndSendPacket())
{
_isDraining = true;
ffmpeg.avcodec_send_packet(_pCodecContext, null);
}
}
else if (ret == ffmpeg.AVERROR_EOF)
{
return;
}
else
{
throw new Exception($"Error during seek decoding: {GetFFmpegErrorString(ret)}");
}
}
}
/// <summary>
/// Moves the playback position to a time in seconds.
/// The next <c>TryRead*</c> call returns the first frame whose presentation time is at least
/// <paramref name="timeSeconds"/> minus half a frame (the frame nearest to the target time).
/// Both paths below return that same frame:
/// - Forward decode: the target is ahead of the current position by less than <see cref="MaxForwardDecodeFrames"/> frames.
/// Frames are decoded and discarded up to the target.
/// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read).
/// A backward keyframe seek is followed by a forward decode up to the target.
/// The target frame is transferred and preserved in <see cref="_pCachedFrame"/>; when it is already there, nothing is decoded.
/// </summary>
/// <param name="timeSeconds">Target time in seconds to seek to.</param>
public void SeekToTime(double timeSeconds)
{
lock (_lock)
{
if (!_isOpened || _disposed) return;
if (timeSeconds < 0) timeSeconds = 0;
// Earliest presentation time that satisfies the target (target minus half a frame).
double threshold = timeSeconds - (0.5 / FPS);
// Index of the first frame that satisfies the target. The epsilon keeps a frame that lies exactly on the
// margin from being excluded by floating point rounding.
double targetFrameIndex = Math.Ceiling(timeSeconds * FPS - 0.5 - 1e-6);
if (_hasCachedFrame)
{
// A frame is already positioned for the next read (e.g. by a previous SeekToTime call).
double cachedPts = GetFrameTimestamp(_pCachedFrame);
if (cachedPts >= threshold)
{
// The cached frame is the first frame that satisfies the target.
// When the target lies before it, the frames in between are no longer available and a keyframe seek is required.
if (cachedPts - threshold < 1.0 / FPS) return;
}
else if (targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
{
// The cached frame lies before the target: drop it and continue decoding forward.
GetTimestampAndUpdateIndex(_pCachedFrame);
ffmpeg.av_frame_unref(_pCachedFrame);
_hasCachedFrame = false;
DecodeToTarget(timeSeconds);
return;
}
}
else if (targetFrameIndex > _currentFrameIndex && targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
{
// The target frame has not been read yet and is close: decode forward instead of seeking.
DecodeToTarget(timeSeconds);
return;
}
if (_hasCachedFrame)
{
ffmpeg.av_frame_unref(_pCachedFrame);
_hasCachedFrame = false;
}
long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase));
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
// Fallback to beginning of file on seek failure
ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
{
throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}");
}
}
// Flush decoder internal buffers
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
ffmpeg.avcodec_flush_buffers(_pCodecContext);
_isDraining = false;
// Decode frames until reaching the target presentation timestamp
DecodeToTarget(timeSeconds);
}
}
/// <summary>
/// Seeks to a specific frame index.
/// </summary>
/// <param name="frameIndex">Target frame index.</param>
public void SeekToFrame(long frameIndex)
{
SeekToTime(frameIndex / FPS);
}
// ──────────────────────────────────────────────────────────────
// Error Handling &amp; Disposal
// ──────────────────────────────────────────────────────────────
/// <summary>
/// Translates an FFmpeg integer error code to a readable string using <c>av_strerror</c>.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
/// </summary>
private string GetFFmpegErrorString(int errnum)
{
const ulong bufferSize = 1024;
byte* buffer = stackalloc byte[(int)bufferSize];
if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0)
{
return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})";
}
return $"Unknown error (Code: {errnum})";
}
/// <summary>
/// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures.
/// </summary>
public void Dispose()
{
lock (_lock)
{
if (_disposed) return;
_disposed = true;
// 1. Release scaler context
if (_pSwsContext != null)
{
ffmpeg.sws_freeContext(_pSwsContext);
_pSwsContext = null;
}
// 2. Release frame buffers
if (_pDecodedFrame != null)
{
var pFrame = _pDecodedFrame;
ffmpeg.av_frame_free(&pFrame);
_pDecodedFrame = null;
}
if (_pHwTransferFrame != null)
{
var pFrame = _pHwTransferFrame;
ffmpeg.av_frame_free(&pFrame);
_pHwTransferFrame = null;
}
if (_pCachedFrame != null)
{
var pFrame = _pCachedFrame;
ffmpeg.av_frame_free(&pFrame);
_pCachedFrame = null;
}
if (_pConvertedFrame != null)
{
var pFrame = _pConvertedFrame;
ffmpeg.av_frame_free(&pFrame);
_pConvertedFrame = null;
}
// 3. Release packet buffer
if (_pPacket != null)
{
var pPacket = _pPacket;
ffmpeg.av_packet_free(&pPacket);
_pPacket = null;
}
// 4. Release codec context
if (_pCodecContext != null)
{
var pCodecCtx = _pCodecContext;
ffmpeg.avcodec_free_context(&pCodecCtx);
_pCodecContext = null;
}
// 5. Release hardware device context
if (_hwDeviceCtx != null)
{
var pHw = _hwDeviceCtx;
ffmpeg.av_buffer_unref(&pHw);
_hwDeviceCtx = null;
}
// 6. Close input file and release format context
if (_pFormatContext != null)
{
var pFormatCtx = _pFormatContext;
ffmpeg.avformat_close_input(&pFormatCtx);
_pFormatContext = null;
}
_getFormatDelegate = null;
_isOpened = false;
_isHwDecoding = false;
GC.SuppressFinalize(this);
}
}
~FFmpegNativeDecoder()
{
Dispose();
}
}