SHA256
1527 lines
63 KiB
C#
1527 lines
63 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Runtime.InteropServices;
|
|
using FFmpeg.AutoGen;
|
|
using MelonLoader;
|
|
using Unity.Collections;
|
|
using Unity.Collections.LowLevel.Unsafe;
|
|
|
|
namespace ADOFAIRenderer.Util.FFmpeg;
|
|
|
|
/// <summary>
|
|
/// High-performance video decoder that directly invokes FFmpeg native C libraries via P/Invoke.
|
|
/// Supports both GPU hardware-accelerated decoding (D3D11VA, DXVA2, QSV, CUDA, D3D12VA) and
|
|
/// CPU multi-threaded software decoding (FF_THREAD_FRAME) with automatic fallback.
|
|
///
|
|
/// ── Architecture & Lifecycle Flow ──────────────────────────────────────────────
|
|
///
|
|
/// 1. Construction & Demuxing:
|
|
/// a) avformat_open_input: Open container file and parse stream headers.
|
|
/// b) avformat_find_stream_info: Inspect stream parameters, timebase, duration, FPS.
|
|
/// c) Identify the primary video stream (AVMEDIA_TYPE_VIDEO).
|
|
///
|
|
/// 2. Codec & Hardware Device Discovery:
|
|
/// a) avcodec_find_decoder: Locate codec implementation by codec ID.
|
|
/// b) avcodec_get_hw_config: Query supported hardware configurations for the decoder.
|
|
/// c) Prioritized HW device creation (D3D11VA → DXVA2 → QSV → CUDA → D3D12VA → VAAPI → VideoToolbox).
|
|
/// d) Attach hw_device_ctx and register custom get_format callback.
|
|
/// e) If HW initialization fails or is unsupported, fall back to CPU software decoding
|
|
/// with multi-threading enabled (thread_count = 0, FF_THREAD_FRAME).
|
|
/// f) avcodec_open2: Initialize codec context.
|
|
///
|
|
/// 3. Frame Demuxing & Decoding (TryRead*):
|
|
/// a) (Demuxing) av_read_frame reads packets and sends them to the decoder via avcodec_send_packet.
|
|
/// b) (Decoding) avcodec_receive_frame retrieves decoded frames.
|
|
/// c) (HW Transfer) For hardware frames, av_hwframe_transfer_data downloads GPU surface data
|
|
/// into system RAM as NV12 (av_frame_copy_props preserves timestamps).
|
|
/// d) (Output Dispatch):
|
|
/// - TryReadNV12Frame: Zero-copy direct memory transfer for NV12 frames, sws_scale fallback.
|
|
/// - TryReadYUV420PFrame: Zero-copy planar copy for YUV420P/YUVJ420P, sws_scale fallback.
|
|
/// - TryReadFrame: sws_scale conversion to RGBA32 with optional vertical flip (vflip).
|
|
///
|
|
/// 4. Seeking & Scrubbing (SeekToTime / SeekToFrame):
|
|
/// a) av_seek_frame: Fast backward keyframe seek.
|
|
/// b) avcodec_flush_buffers: Clear internal decoder state.
|
|
/// c) Forward decoding loop: Discards non-target frames without GPU transfer overhead;
|
|
/// transfers and caches the first matching frame in _pCachedFrame for immediate consumption.
|
|
///
|
|
/// 5. Resource Disposal (Dispose):
|
|
/// a) Free scaler context (sws_freeContext).
|
|
/// b) Free all allocated frames and packets (av_frame_free, av_packet_free).
|
|
/// c) Free codec context (avcodec_free_context).
|
|
/// d) Release hardware device context (av_buffer_unref).
|
|
/// e) Close format context (avformat_close_input).
|
|
///
|
|
/// ── Reference Documentation ────────────────────────────────────────────────────
|
|
/// • FFmpeg Demuxing & Decoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
|
|
/// • Send/Receive Codec API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
|
/// • Hardware Acceleration API: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
|
/// • FFmpeg HW Decoding Example: https://ffmpeg.org/doxygen/trunk/hw_decode_8c-example.html
|
|
/// • libswscale Scaling & Conversion: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
|
/// </summary>
|
|
internal sealed unsafe class FFmpegNativeDecoder : IDisposable
|
|
{
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Hardware Acceleration Priorities & Candidate Structure
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Candidate hardware device configuration discovered via <c>avcodec_get_hw_config</c>.
|
|
/// </summary>
|
|
private record struct HwConfigCandidate
|
|
{
|
|
public AVHWDeviceType DeviceType;
|
|
public AVPixelFormat PixelFormat;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Preferred hardware device types in priority order.
|
|
/// Modern Windows platforms prioritize D3D11VA for wide driver support and direct integration,
|
|
/// followed by DXVA2 (DirectX 9 / legacy), Intel QSV, NVIDIA CUDA, and DirectX 12 D3D12VA.
|
|
/// Non-Windows systems fall back to VAAPI or VideoToolbox.
|
|
/// </summary>
|
|
private static readonly AVHWDeviceType[] PreferredHwDeviceTypes = new[]
|
|
{
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA,
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_DXVA2,
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_QSV,
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_CUDA,
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_D3D12VA,
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI,
|
|
AVHWDeviceType.AV_HWDEVICE_TYPE_VIDEOTOOLBOX
|
|
};
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// FFmpeg Core Struct Pointers
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Context managing container format demuxing (MP4, MKV, WebM, etc.).
|
|
/// Used for packet demuxing and seeking.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
|
|
/// </summary>
|
|
private AVFormatContext* _pFormatContext = null;
|
|
|
|
/// <summary>
|
|
/// Context managing video decoder state and parameters.
|
|
/// Configured with codec parameters, hardware contexts, and threading options.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
|
/// </summary>
|
|
private AVCodecContext* _pCodecContext = null;
|
|
|
|
/// <summary>
|
|
/// Reference-counted context for the GPU hardware device (D3D11, DXVA2, QSV, CUDA, etc.).
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVHWDeviceContext.html
|
|
/// </summary>
|
|
private AVBufferRef* _hwDeviceCtx = null;
|
|
|
|
/// <summary>
|
|
/// Frame structure holding raw decoded frames received directly from <c>avcodec_receive_frame</c>.
|
|
/// In HW decoding mode, this contains GPU surface descriptors (e.g., ID3D11VideoDecoderOutputView).
|
|
/// In SW decoding mode, this contains pixel buffers in system memory.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
|
|
/// </summary>
|
|
private AVFrame* _pDecodedFrame = null;
|
|
|
|
/// <summary>
|
|
/// Software frame structure in system RAM used to receive pixel data transferred from GPU surfaces
|
|
/// via <c>av_hwframe_transfer_data</c>.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
|
/// </summary>
|
|
private AVFrame* _pHwTransferFrame = null;
|
|
|
|
/// <summary>
|
|
/// Frame structure holding converted pixel data produced by <c>sws_scale</c> (e.g., RGBA or fallback YUV).
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
|
|
/// </summary>
|
|
private AVFrame* _pConvertedFrame = null;
|
|
|
|
/// <summary>
|
|
/// Cached frame holding decoded system-memory pixel data after a seek or lookahead operation.
|
|
/// Consumed by the next <c>TryRead*</c> invocation.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVFrame.html
|
|
/// </summary>
|
|
private AVFrame* _pCachedFrame = null;
|
|
private bool _hasCachedFrame;
|
|
|
|
/// <summary>
|
|
/// Packet structure temporarily holding compressed demuxed data read from the input container.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVPacket.html
|
|
/// </summary>
|
|
private AVPacket* _pPacket = null;
|
|
|
|
/// <summary>
|
|
/// libswscale colorspace and format conversion context.
|
|
/// Handles conversions such as NV12/YUV420P → RGBA32 and format adaptation.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
|
/// </summary>
|
|
private SwsContext* _pSwsContext = null;
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Hardware Acceleration State & Delegates
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Delegate instance stored in managed memory to prevent GC collection while referenced by unmanaged code.
|
|
/// Passed to <c>_pCodecContext->get_format</c>.
|
|
/// </summary>
|
|
private AVCodecContext_get_format? _getFormatDelegate;
|
|
|
|
/// <summary>Whether hardware-accelerated decoding is currently active.</summary>
|
|
private bool _isHwDecoding;
|
|
|
|
/// <summary>Hardware device type in use (e.g. D3D11VA, DXVA2, CUDA, QSV).</summary>
|
|
private AVHWDeviceType _hwDeviceType = AVHWDeviceType.AV_HWDEVICE_TYPE_NONE;
|
|
|
|
/// <summary>Pixel format of the GPU surface (e.g. AV_PIX_FMT_D3D11, AV_PIX_FMT_DXVA2_VLD).</summary>
|
|
private AVPixelFormat _hwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
|
|
|
/// <summary>
|
|
/// Pixel format of the frame after downloading from GPU surface to system RAM (typically NV12).
|
|
/// </summary>
|
|
private AVPixelFormat _downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Playback, Stream & Scaler Synchronization Fields
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
private int _videoStreamIndex = -1;
|
|
private long _currentFrameIndex;
|
|
private AVRational _timeBase;
|
|
private bool _isOpened;
|
|
private bool _disposed;
|
|
private bool _isDraining;
|
|
|
|
/// <summary>Number of video packets skipped because the decoder rejected them as invalid data.</summary>
|
|
private int _skippedPacketCount;
|
|
|
|
/// <summary>Number of skipped packets for which a warning is logged.</summary>
|
|
private const int MaxLoggedSkippedPackets = 5;
|
|
|
|
private readonly object _lock = new();
|
|
|
|
// Fields to detect SwsContext updates
|
|
private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
|
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
|
private int _srcWidth;
|
|
private int _srcHeight;
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Public Properties
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>Video width in pixels.</summary>
|
|
public int Width { get; private set; }
|
|
|
|
/// <summary>Video height in pixels.</summary>
|
|
public int Height { get; private set; }
|
|
|
|
/// <summary>Frames per second.</summary>
|
|
public double FPS { get; private set; }
|
|
|
|
/// <summary>Total duration in seconds.</summary>
|
|
public double Duration { get; private set; }
|
|
|
|
/// <summary>Estimated total number of frames.</summary>
|
|
public long TotalFrames { get; private set; }
|
|
|
|
/// <summary>Index of the currently read frame.</summary>
|
|
public long CurrentFrameIndex
|
|
{
|
|
get
|
|
{
|
|
lock (_lock)
|
|
{
|
|
return _currentFrameIndex;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>Current playback time in seconds.</summary>
|
|
public double CurrentTime
|
|
{
|
|
get
|
|
{
|
|
lock (_lock)
|
|
{
|
|
return FPS > 0 ? _currentFrameIndex / FPS : 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Pixel format of the video frames produced by the decoder.
|
|
/// When hardware decoding is active, returns the pixel format of the downloaded software frame
|
|
/// in system RAM (typically <see cref="AVPixelFormat.AV_PIX_FMT_NV12"/>) rather than the GPU surface format
|
|
/// (e.g., D3D11VA, DXVA2, CUDA), allowing seamless binding to Compute Shader NV12 pipelines.
|
|
/// In software mode, returns the codec context's pixel format (e.g., YUV420P).
|
|
/// </summary>
|
|
public AVPixelFormat PixelFormat
|
|
{
|
|
get
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_isHwDecoding)
|
|
{
|
|
return _downloadedPixFmt != AVPixelFormat.AV_PIX_FMT_NONE
|
|
? _downloadedPixFmt
|
|
: AVPixelFormat.AV_PIX_FMT_NV12;
|
|
}
|
|
|
|
return _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Color range of the video stream (MPEG limited [16-235] or JPEG full [0-255]).
|
|
/// </summary>
|
|
public AVColorRange ColorRange
|
|
{
|
|
get
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_pCodecContext != null && _pCodecContext->color_range != AVColorRange.AVCOL_RANGE_UNSPECIFIED)
|
|
{
|
|
return _pCodecContext->color_range;
|
|
}
|
|
if (_pFormatContext != null && _videoStreamIndex >= 0)
|
|
{
|
|
return _pFormatContext->streams[_videoStreamIndex]->codecpar->color_range;
|
|
}
|
|
return AVColorRange.AVCOL_RANGE_UNSPECIFIED;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets a value indicating whether hardware-accelerated decoding is currently active.
|
|
/// </summary>
|
|
public bool IsHardwareAccelerated
|
|
{
|
|
get
|
|
{
|
|
lock (_lock)
|
|
{
|
|
return _isHwDecoding;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the hardware device type in use for hardware acceleration, or null if software decoding.
|
|
/// </summary>
|
|
public AVHWDeviceType? HardwareDeviceType
|
|
{
|
|
get
|
|
{
|
|
lock (_lock)
|
|
{
|
|
return _isHwDecoding ? _hwDeviceType : null;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Initialization & Setup
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of <see cref="FFmpegNativeDecoder"/> and opens the specified video file.
|
|
/// Automatically attempts hardware acceleration (D3D11VA, DXVA2, QSV, CUDA), falling back
|
|
/// to multi-threaded software decoding if unavailable.
|
|
/// </summary>
|
|
/// <param name="videoPath">Absolute or relative file path to the video file.</param>
|
|
public FFmpegNativeDecoder(string videoPath)
|
|
{
|
|
Open(videoPath);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Opens the video file, discovers streams, configures hardware/software decoding, and allocates frame buffers.
|
|
/// </summary>
|
|
/// <param name="videoPath">Path to video file.</param>
|
|
private void Open(string videoPath)
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_isOpened || _disposed) return;
|
|
|
|
try
|
|
{
|
|
// 1. Open media container
|
|
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html
|
|
fixed (AVFormatContext** ppFormatCtx = &_pFormatContext)
|
|
{
|
|
int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null);
|
|
if (ret < 0)
|
|
throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})");
|
|
}
|
|
|
|
// 2. Retrieve stream information
|
|
if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0)
|
|
throw new Exception("Failed to retrieve stream info.");
|
|
|
|
// 3. Find primary video stream
|
|
_videoStreamIndex = -1;
|
|
for (int i = 0; i < _pFormatContext->nb_streams; i++)
|
|
{
|
|
if (_pFormatContext->streams[i]->codecpar->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO)
|
|
{
|
|
_videoStreamIndex = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (_videoStreamIndex == -1)
|
|
throw new Exception("Failed to find video stream.");
|
|
|
|
// Only the video stream is used. The demuxer does not need to return the packets of the other streams
|
|
// (audio, subtitles, ...) once they are marked as discarded.
|
|
// Reference: AVStream.discard - "Selects which packets can be discarded at will and do not need to be demuxed."
|
|
// https://ffmpeg.org/doxygen/trunk/structAVStream.html
|
|
for (int i = 0; i < _pFormatContext->nb_streams; i++)
|
|
{
|
|
if (i != _videoStreamIndex)
|
|
_pFormatContext->streams[i]->discard = AVDiscard.AVDISCARD_ALL;
|
|
}
|
|
|
|
var stream = _pFormatContext->streams[_videoStreamIndex];
|
|
_timeBase = stream->time_base;
|
|
|
|
// 4. Calculate FPS
|
|
if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0)
|
|
{
|
|
FPS = ffmpeg.av_q2d(stream->r_frame_rate);
|
|
}
|
|
else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0)
|
|
{
|
|
FPS = ffmpeg.av_q2d(stream->avg_frame_rate);
|
|
}
|
|
else
|
|
{
|
|
FPS = 30.0; // Fallback default
|
|
}
|
|
|
|
// 5. Find decoder by codec ID
|
|
var codecId = stream->codecpar->codec_id;
|
|
var decoder = ffmpeg.avcodec_find_decoder(codecId);
|
|
if (decoder == null)
|
|
throw new Exception($"Unsupported video codec: {codecId}");
|
|
|
|
// 6. Allocate codec context
|
|
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
|
|
if (_pCodecContext == null)
|
|
throw new Exception("Failed to allocate codec context.");
|
|
|
|
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
|
|
throw new Exception("Failed to copy codec parameters.");
|
|
|
|
// 7. Attempt hardware decoding auto-discovery; fallback to multi-threaded CPU software decoding
|
|
bool hwSuccess = TryInitializeHwDecoder(decoder, stream);
|
|
if (!hwSuccess)
|
|
{
|
|
Melon<RenderMod>.Logger.Msg("[FFmpegNativeDecoder] Hardware decoding unavailable. Falling back to multi-threaded CPU software decoding.");
|
|
_isHwDecoding = false;
|
|
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
|
|
|
// Multi-threading settings for improved software decoding performance (auto thread count)
|
|
// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
|
_pCodecContext->thread_count = 0;
|
|
_pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME;
|
|
|
|
int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
|
|
if (openRet < 0)
|
|
throw new Exception($"Failed to open software codec: {GetFFmpegErrorString(openRet)}");
|
|
}
|
|
|
|
Width = _pCodecContext->width > 0 ? _pCodecContext->width : stream->codecpar->width;
|
|
Height = _pCodecContext->height > 0 ? _pCodecContext->height : stream->codecpar->height;
|
|
|
|
// 8. Calculate total frames and duration
|
|
if (stream->nb_frames > 0)
|
|
{
|
|
TotalFrames = stream->nb_frames;
|
|
}
|
|
else if (stream->duration > 0)
|
|
{
|
|
TotalFrames = (long)Math.Round(stream->duration * ffmpeg.av_q2d(stream->time_base) * FPS);
|
|
}
|
|
else if (_pFormatContext->duration > 0)
|
|
{
|
|
TotalFrames = (long)Math.Round((_pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE) * FPS);
|
|
}
|
|
else
|
|
{
|
|
TotalFrames = 0;
|
|
}
|
|
|
|
// Duration of the video stream (AVStream.duration, in the stream time base).
|
|
// When the stream does not carry a duration, the container duration (AVFormatContext.duration,
|
|
// in AV_TIME_BASE units, the longest of all streams) is used.
|
|
// Reference: https://ffmpeg.org/doxygen/trunk/structAVStream.html, https://ffmpeg.org/doxygen/trunk/structAVFormatContext.html
|
|
Duration = stream->duration > 0
|
|
? stream->duration * ffmpeg.av_q2d(stream->time_base)
|
|
: _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : 0;
|
|
|
|
// 9. Allocate frame and packet buffers
|
|
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
|
|
_pDecodedFrame = ffmpeg.av_frame_alloc();
|
|
_pHwTransferFrame = ffmpeg.av_frame_alloc();
|
|
_pConvertedFrame = ffmpeg.av_frame_alloc();
|
|
_pCachedFrame = ffmpeg.av_frame_alloc();
|
|
_pPacket = ffmpeg.av_packet_alloc();
|
|
|
|
_currentFrameIndex = 0;
|
|
_isOpened = true;
|
|
_isDraining = false;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
Dispose();
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Searches for supported hardware device configurations via <c>avcodec_get_hw_config</c>,
|
|
/// creates the appropriate hardware device context (D3D11VA, DXVA2, QSV, CUDA),
|
|
/// registers the <c>get_format</c> callback, and attempts to open the hardware decoder.
|
|
/// </summary>
|
|
/// <param name="decoder">Target decoder.</param>
|
|
/// <param name="stream">Source video stream.</param>
|
|
/// <returns>True if hardware decoding was successfully configured and opened; false to fall back.</returns>
|
|
private bool TryInitializeHwDecoder(AVCodec* decoder, AVStream* stream)
|
|
{
|
|
// ReSharper disable once InconsistentNaming
|
|
// Collect all supported hardware configs from decoder
|
|
const int AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX = 0x01;
|
|
var candidates = new List<HwConfigCandidate>();
|
|
for (int i = 0; ; i++)
|
|
{
|
|
var config = ffmpeg.avcodec_get_hw_config(decoder, i);
|
|
if (config == null) break;
|
|
|
|
if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) != 0)
|
|
{
|
|
candidates.Add(new HwConfigCandidate
|
|
{
|
|
DeviceType = config->device_type,
|
|
PixelFormat = config->pix_fmt
|
|
});
|
|
}
|
|
}
|
|
|
|
if (candidates.Count == 0)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Order candidates according to preference
|
|
var orderedCandidates = new List<HwConfigCandidate>();
|
|
foreach (var preferred in PreferredHwDeviceTypes)
|
|
{
|
|
foreach (var candidate in candidates)
|
|
{
|
|
if (candidate.DeviceType == preferred && !orderedCandidates.Contains(candidate))
|
|
{
|
|
orderedCandidates.Add(candidate);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add any remaining candidates not explicitly listed in PreferredHwDeviceTypes
|
|
foreach (var candidate in candidates)
|
|
{
|
|
if (!orderedCandidates.Contains(candidate))
|
|
{
|
|
orderedCandidates.Add(candidate);
|
|
}
|
|
}
|
|
|
|
// Try candidate hardware configurations in priority order
|
|
foreach (var candidate in orderedCandidates)
|
|
{
|
|
AVBufferRef* hwDeviceCtx = null;
|
|
int err;
|
|
|
|
// Device creation strategy
|
|
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
|
if (candidate.DeviceType == AVHWDeviceType.AV_HWDEVICE_TYPE_QSV)
|
|
{
|
|
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
|
|
string childDeviceName = isWindows ? "d3d11va" : "vaapi";
|
|
AVDictionary* opts = null;
|
|
ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0);
|
|
|
|
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0);
|
|
ffmpeg.av_dict_free(&opts);
|
|
|
|
if (err < 0)
|
|
{
|
|
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, candidate.DeviceType, null, null, 0);
|
|
}
|
|
|
|
if (err < 0 || hwDeviceCtx == null)
|
|
{
|
|
if (hwDeviceCtx != null)
|
|
{
|
|
ffmpeg.av_buffer_unref(&hwDeviceCtx);
|
|
hwDeviceCtx = null;
|
|
}
|
|
Melon<RenderMod>.Logger.Msg($"[FFmpegNativeDecoder] Hardware device creation failed for {candidate.DeviceType}: {GetFFmpegErrorString(err)}. Trying next candidate...");
|
|
continue;
|
|
}
|
|
|
|
// Assign hardware context and format selection callback
|
|
_hwDeviceCtx = hwDeviceCtx;
|
|
_hwPixFmt = candidate.PixelFormat;
|
|
_hwDeviceType = candidate.DeviceType;
|
|
_isHwDecoding = true;
|
|
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
|
|
|
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(_hwDeviceCtx);
|
|
if (_pCodecContext->hw_device_ctx == null)
|
|
{
|
|
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Failed to reference HW device context for {candidate.DeviceType}. Retrying next candidate...");
|
|
var pHw = _hwDeviceCtx;
|
|
ffmpeg.av_buffer_unref(&pHw);
|
|
_hwDeviceCtx = null;
|
|
_isHwDecoding = false;
|
|
continue;
|
|
}
|
|
|
|
_getFormatDelegate = GetFormat;
|
|
_pCodecContext->get_format = _getFormatDelegate;
|
|
|
|
// Hardware decoding requires single-threaded decoding (thread_count = 1).
|
|
// Multi-threading (thread_count = 0 / FF_THREAD_FRAME) is incompatible with fixed-function
|
|
// GPU decoders and causes surface pool allocation exhaustion or crashes in get_format.
|
|
_pCodecContext->thread_count = 1;
|
|
|
|
// Attempt opening codec context with hardware configuration
|
|
int openRet = ffmpeg.avcodec_open2(_pCodecContext, decoder, null);
|
|
if (openRet >= 0)
|
|
{
|
|
Melon<RenderMod>.Logger.Msg($"[FFmpegNativeDecoder] Hardware decoding enabled successfully: Device={candidate.DeviceType}, SurfaceFormat={candidate.PixelFormat}");
|
|
return true;
|
|
}
|
|
|
|
// Failed to open codec with this hardware configuration, clean up and try next
|
|
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] avcodec_open2 failed with HW device {candidate.DeviceType}: {GetFFmpegErrorString(openRet)}. Retrying next candidate...");
|
|
|
|
if (_pCodecContext != null)
|
|
{
|
|
var pCodecCtx = _pCodecContext;
|
|
ffmpeg.avcodec_free_context(&pCodecCtx);
|
|
_pCodecContext = null;
|
|
}
|
|
|
|
if (_hwDeviceCtx != null)
|
|
{
|
|
var pHw = _hwDeviceCtx;
|
|
ffmpeg.av_buffer_unref(&pHw);
|
|
_hwDeviceCtx = null;
|
|
}
|
|
|
|
_isHwDecoding = false;
|
|
_getFormatDelegate = null;
|
|
|
|
// Reallocate fresh codec context for subsequent attempts
|
|
_pCodecContext = ffmpeg.avcodec_alloc_context3(decoder);
|
|
if (_pCodecContext == null)
|
|
throw new Exception("Failed to re-allocate codec context during fallback.");
|
|
|
|
if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0)
|
|
throw new Exception("Failed to copy codec parameters during fallback.");
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Callback invoked by libavcodec during decoding to negotiate the output pixel format.
|
|
/// Confirms the hardware surface format (<see cref="_hwPixFmt"/>) if offered by the decoder,
|
|
/// or falls back to software pixel formats via <c>avcodec_default_get_format</c>.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
|
/// </summary>
|
|
private AVPixelFormat GetFormat(AVCodecContext* ctx, AVPixelFormat* pixFmts)
|
|
{
|
|
for (var p = pixFmts; *p != AVPixelFormat.AV_PIX_FMT_NONE; p++)
|
|
{
|
|
if (*p == _hwPixFmt)
|
|
{
|
|
return *p;
|
|
}
|
|
}
|
|
|
|
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Hardware surface format {_hwPixFmt} not offered by decoder for this stream. Falling back to software format.");
|
|
_isHwDecoding = false;
|
|
_downloadedPixFmt = AVPixelFormat.AV_PIX_FMT_NONE;
|
|
return ffmpeg.avcodec_default_get_format(ctx, pixFmts);
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Demuxing & Decoding Engine
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Reads packets from the container until a packet belonging to the video stream is found,
|
|
/// then sends it to the decoder via <c>avcodec_send_packet</c>.
|
|
/// A packet that the decoder rejects as invalid data (<c>AVERROR_INVALIDDATA</c>, e.g. a damaged packet)
|
|
/// is skipped and the next packet is read. Any other decoder error is thrown.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
|
/// </summary>
|
|
/// <returns>True if a video packet was sent; false on EOF.</returns>
|
|
private bool ReadAndSendPacket()
|
|
{
|
|
while (true)
|
|
{
|
|
ffmpeg.av_packet_unref(_pPacket);
|
|
int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket);
|
|
if (ret < 0) return false; // EOF or read error
|
|
|
|
if (_pPacket->stream_index == _videoStreamIndex)
|
|
{
|
|
ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket);
|
|
ffmpeg.av_packet_unref(_pPacket);
|
|
|
|
if (ret == ffmpeg.AVERROR_INVALIDDATA)
|
|
{
|
|
// The warning is logged for the first few packets only (a damaged file can have many).
|
|
_skippedPacketCount++;
|
|
if (_skippedPacketCount <= MaxLoggedSkippedPackets)
|
|
{
|
|
Melon<RenderMod>.Logger.Warning($"[FFmpegNativeDecoder] Skipped an invalid video packet ({_skippedPacketCount}): {GetFFmpegErrorString(ret)}");
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (ret < 0)
|
|
{
|
|
throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}");
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Receives the next decoded video frame from the codec context.
|
|
/// If the frame is decoded on a GPU hardware surface (D3D11VA, DXVA2, CUDA, QSV),
|
|
/// it is automatically downloaded to system RAM via <c>av_hwframe_transfer_data</c> into <c>_pHwTransferFrame</c>,
|
|
/// preserving presentation timestamps via <c>av_frame_copy_props</c>.
|
|
/// </summary>
|
|
/// <param name="outFrame">The software frame containing pixel data in system RAM.</param>
|
|
/// <returns>True if a frame was successfully decoded and available; false on EOF or draining.</returns>
|
|
private bool TryReceiveFrame(out AVFrame* outFrame)
|
|
{
|
|
outFrame = null;
|
|
|
|
while (true)
|
|
{
|
|
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
|
|
if (ret == 0)
|
|
{
|
|
// Check if the received frame is on a hardware surface
|
|
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
|
|
{
|
|
ffmpeg.av_frame_unref(_pHwTransferFrame);
|
|
int transferRet = ffmpeg.av_hwframe_transfer_data(_pHwTransferFrame, _pDecodedFrame, 0);
|
|
if (transferRet < 0)
|
|
{
|
|
ffmpeg.av_frame_unref(_pDecodedFrame);
|
|
throw new Exception($"Failed to transfer hardware frame to system memory: {GetFFmpegErrorString(transferRet)}");
|
|
}
|
|
|
|
// Copy presentation timestamp and metadata properties
|
|
ffmpeg.av_frame_copy_props(_pHwTransferFrame, _pDecodedFrame);
|
|
ffmpeg.av_frame_unref(_pDecodedFrame);
|
|
|
|
_downloadedPixFmt = (AVPixelFormat)_pHwTransferFrame->format;
|
|
outFrame = _pHwTransferFrame;
|
|
return true;
|
|
}
|
|
|
|
// Frame is in system memory (CPU decoding or software fallback format)
|
|
outFrame = _pDecodedFrame;
|
|
return true;
|
|
}
|
|
|
|
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
|
|
{
|
|
if (_isDraining) return false;
|
|
|
|
if (!ReadAndSendPacket())
|
|
{
|
|
_isDraining = true;
|
|
ffmpeg.avcodec_send_packet(_pCodecContext, null);
|
|
}
|
|
}
|
|
else if (ret == ffmpeg.AVERROR_EOF)
|
|
{
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Lazily initializes or updates the <see cref="SwsContext"/> when source/destination formats or dimensions change.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
|
/// </summary>
|
|
private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat)
|
|
{
|
|
if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (_pSwsContext != null)
|
|
{
|
|
ffmpeg.sws_freeContext(_pSwsContext);
|
|
_pSwsContext = null;
|
|
}
|
|
|
|
_srcPixFmt = srcFormat;
|
|
_srcWidth = srcWidth;
|
|
_srcHeight = srcHeight;
|
|
_dstPixFmt = dstFormat;
|
|
|
|
_pSwsContext = ffmpeg.sws_getContext(
|
|
srcWidth, srcHeight, srcFormat,
|
|
Width, Height, dstFormat,
|
|
(int)SwsFlags.SWS_BILINEAR, null, null, null
|
|
);
|
|
|
|
if (_pSwsContext == null)
|
|
{
|
|
throw new Exception($"Failed to initialize SwsContext (src: {srcFormat} {srcWidth}x{srcHeight} -> dst: {dstFormat} {Width}x{Height}).");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes presentation timestamp (PTS) in seconds from frame metadata without modifying decoder state.
|
|
/// Used by seeking to inspect candidate frames without falsely incrementing frame indices.
|
|
/// </summary>
|
|
private double GetFrameTimestamp(AVFrame* frame)
|
|
{
|
|
long ptsValue = frame->best_effort_timestamp;
|
|
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
|
|
{
|
|
ptsValue = frame->pts;
|
|
}
|
|
|
|
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
|
|
{
|
|
return FPS > 0 ? _currentFrameIndex / FPS : 0;
|
|
}
|
|
|
|
return ptsValue * ffmpeg.av_q2d(_timeBase);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Computes playback presentation timestamp (PTS) in seconds from frame metadata and updates the frame counter.
|
|
/// </summary>
|
|
private double GetTimestampAndUpdateIndex(AVFrame* frame)
|
|
{
|
|
double timestamp = GetFrameTimestamp(frame);
|
|
long ptsValue = frame->best_effort_timestamp != ffmpeg.AV_NOPTS_VALUE
|
|
? frame->best_effort_timestamp
|
|
: frame->pts;
|
|
|
|
if (ptsValue == ffmpeg.AV_NOPTS_VALUE)
|
|
{
|
|
_currentFrameIndex++;
|
|
}
|
|
else
|
|
{
|
|
_currentFrameIndex = (long)Math.Round(timestamp * FPS);
|
|
}
|
|
return timestamp;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Resets all data plane pointers and strides in <see cref="_pConvertedFrame"/> to zero.
|
|
/// Prevents stale or dangling pointers from previous conversions being retained across frames.
|
|
/// </summary>
|
|
private void ClearConvertedFramePointers()
|
|
{
|
|
if (_pConvertedFrame == null) return;
|
|
for (uint i = 0; i < 8; i++)
|
|
{
|
|
_pConvertedFrame->data[i] = null;
|
|
_pConvertedFrame->linesize[i] = 0;
|
|
}
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Frame Reading APIs: RGBA (CPU Fallback)
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies it in 32-bit RGBA format using libswscale.
|
|
/// Safely processes both hardware-downloaded frames (NV12) and software-decoded frames (YUV420P).
|
|
/// </summary>
|
|
/// <param name="outBuffer">Pointer to destination memory storing RGBA data.</param>
|
|
/// <param name="outStride">Byte stride per row of destination memory (Width * 4).</param>
|
|
/// <param name="vflip">Whether to vertically flip the output to match Unity texture coordinates.</param>
|
|
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
|
|
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
|
|
public bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp)
|
|
{
|
|
timestamp = 0;
|
|
lock (_lock)
|
|
{
|
|
if (!_isOpened || _disposed || outBuffer == null) return false;
|
|
|
|
AVFrame* frame;
|
|
if (_hasCachedFrame)
|
|
{
|
|
frame = _pCachedFrame;
|
|
_hasCachedFrame = false;
|
|
}
|
|
else
|
|
{
|
|
if (!TryReceiveFrame(out frame)) return false;
|
|
}
|
|
|
|
if (frame == null) return false;
|
|
|
|
timestamp = GetTimestampAndUpdateIndex(frame);
|
|
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_RGBA);
|
|
|
|
ClearConvertedFramePointers();
|
|
if (vflip)
|
|
{
|
|
_pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1);
|
|
_pConvertedFrame->linesize[0] = -outStride;
|
|
}
|
|
else
|
|
{
|
|
_pConvertedFrame->data[0] = outBuffer;
|
|
_pConvertedFrame->linesize[0] = outStride;
|
|
}
|
|
|
|
ffmpeg.sws_scale(
|
|
_pSwsContext,
|
|
frame->data, frame->linesize,
|
|
0, frame->height,
|
|
_pConvertedFrame->data, _pConvertedFrame->linesize
|
|
);
|
|
|
|
ClearConvertedFramePointers();
|
|
ffmpeg.av_frame_unref(frame);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies it in RGBA format to an unmanaged memory pointer.
|
|
/// </summary>
|
|
public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp)
|
|
{
|
|
return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies it in RGBA format to a Unity NativeArray.
|
|
/// </summary>
|
|
public bool TryReadFrame(NativeArray<byte> outBuffer, bool vflip, out double timestamp)
|
|
{
|
|
if (!outBuffer.IsCreated)
|
|
{
|
|
throw new ArgumentException("Output buffer is not created or already disposed.", nameof(outBuffer));
|
|
}
|
|
if (outBuffer.Length < (long)Width * Height * 4)
|
|
{
|
|
throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer));
|
|
}
|
|
int stride = Width * 4;
|
|
return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp);
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Frame Reading APIs: NV12 (Compute Shader Pipeline)
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies its Y and interleaved UV channel data separately in NV12 format.
|
|
/// When hardware decoding is active (producing NV12 in RAM), CPU color conversion is completely bypassed,
|
|
/// enabling maximum memory copy throughput directly into Unity textures for Compute Shader processing.
|
|
/// Other formats or mismatched dimensions are automatically converted and scaled to NV12 via libswscale.
|
|
/// </summary>
|
|
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
|
|
/// <param name="yStride">Row stride in bytes for Y plane.</param>
|
|
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes).</param>
|
|
/// <param name="uvStride">Row stride in bytes for UV plane.</param>
|
|
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
|
|
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
|
|
// ReSharper disable once InconsistentNaming
|
|
public bool TryReadNV12Frame(
|
|
byte* yBuffer, int yStride,
|
|
byte* uvBuffer, int uvStride,
|
|
out double timestamp)
|
|
{
|
|
timestamp = 0;
|
|
lock (_lock)
|
|
{
|
|
if (!_isOpened || _disposed || yBuffer == null || uvBuffer == null) return false;
|
|
|
|
AVFrame* frame;
|
|
if (_hasCachedFrame)
|
|
{
|
|
frame = _pCachedFrame;
|
|
_hasCachedFrame = false;
|
|
}
|
|
else
|
|
{
|
|
if (!TryReceiveFrame(out frame)) return false;
|
|
}
|
|
|
|
if (frame == null) return false;
|
|
|
|
timestamp = GetTimestampAndUpdateIndex(frame);
|
|
|
|
if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12 || frame->width != Width || frame->height != Height)
|
|
{
|
|
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12);
|
|
|
|
ClearConvertedFramePointers();
|
|
_pConvertedFrame->data[0] = yBuffer;
|
|
_pConvertedFrame->data[1] = uvBuffer;
|
|
_pConvertedFrame->linesize[0] = yStride;
|
|
_pConvertedFrame->linesize[1] = uvStride;
|
|
|
|
ffmpeg.sws_scale(
|
|
_pSwsContext,
|
|
frame->data, frame->linesize,
|
|
0, frame->height,
|
|
_pConvertedFrame->data, _pConvertedFrame->linesize
|
|
);
|
|
ClearConvertedFramePointers();
|
|
}
|
|
else
|
|
{
|
|
int w = Width;
|
|
int h = Height;
|
|
int uvH = (h + 1) / 2;
|
|
int uvRowBytes = (w / 2) * 2;
|
|
|
|
// Copy Y plane (row by row to safely skip linesize GPU padding)
|
|
for (int y = 0; y < h; y++)
|
|
{
|
|
Buffer.MemoryCopy(
|
|
frame->data[0] + y * frame->linesize[0],
|
|
yBuffer + y * yStride,
|
|
yStride, w
|
|
);
|
|
}
|
|
|
|
// Copy UV plane (interleaved U and V samples)
|
|
for (int y = 0; y < uvH; y++)
|
|
{
|
|
Buffer.MemoryCopy(
|
|
frame->data[1] + y * frame->linesize[1],
|
|
uvBuffer + y * uvStride,
|
|
uvStride, uvRowBytes
|
|
);
|
|
}
|
|
}
|
|
|
|
ffmpeg.av_frame_unref(frame);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data to unmanaged memory pointers.
|
|
/// </summary>
|
|
// ReSharper disable once InconsistentNaming
|
|
public bool TryReadNV12Frame(
|
|
IntPtr yBuffer, int yStride,
|
|
IntPtr uvBuffer, int uvStride,
|
|
out double timestamp)
|
|
{
|
|
return TryReadNV12Frame(
|
|
(byte*)yBuffer.ToPointer(), yStride,
|
|
(byte*)uvBuffer.ToPointer(), uvStride,
|
|
out timestamp
|
|
);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies its NV12 Y and interleaved UV channel data into Unity NativeArrays.
|
|
/// </summary>
|
|
// ReSharper disable once InconsistentNaming
|
|
public bool TryReadNV12Frame(
|
|
NativeArray<byte> yBuffer,
|
|
NativeArray<byte> uvBuffer,
|
|
out double timestamp)
|
|
{
|
|
if (!yBuffer.IsCreated || !uvBuffer.IsCreated)
|
|
{
|
|
throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
|
|
}
|
|
|
|
int w = Width;
|
|
int h = Height;
|
|
int uvH = (h + 1) / 2;
|
|
int uvRowBytes = (w / 2) * 2;
|
|
|
|
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
|
|
{
|
|
throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution.");
|
|
}
|
|
|
|
return TryReadNV12Frame(
|
|
(byte*)yBuffer.GetUnsafePtr(), w,
|
|
(byte*)uvBuffer.GetUnsafePtr(), uvRowBytes,
|
|
out timestamp
|
|
);
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Frame Reading APIs: YUV420P (Compute Shader Pipeline)
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies its planar Y, U, and V channel data separately.
|
|
/// For native YUV420P sources, color conversion is bypassed and memory is copied directly.
|
|
/// Non-YUV420P frames (including NV12 from HW decoding) or mismatched dimensions are converted via libswscale.
|
|
/// </summary>
|
|
// ReSharper disable once InconsistentNaming
|
|
public bool TryReadYUV420PFrame(
|
|
byte* yBuffer, int yStride,
|
|
byte* uBuffer, int uStride,
|
|
byte* vBuffer, int vStride,
|
|
out double timestamp)
|
|
{
|
|
timestamp = 0;
|
|
lock (_lock)
|
|
{
|
|
if (!_isOpened || _disposed || yBuffer == null || uBuffer == null || vBuffer == null) return false;
|
|
|
|
AVFrame* frame;
|
|
if (_hasCachedFrame)
|
|
{
|
|
frame = _pCachedFrame;
|
|
_hasCachedFrame = false;
|
|
}
|
|
else
|
|
{
|
|
if (!TryReceiveFrame(out frame)) return false;
|
|
}
|
|
|
|
if (frame == null) return false;
|
|
|
|
timestamp = GetTimestampAndUpdateIndex(frame);
|
|
|
|
if ((frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) || frame->width != Width || frame->height != Height)
|
|
{
|
|
InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P);
|
|
|
|
ClearConvertedFramePointers();
|
|
_pConvertedFrame->data[0] = yBuffer;
|
|
_pConvertedFrame->data[1] = uBuffer;
|
|
_pConvertedFrame->data[2] = vBuffer;
|
|
_pConvertedFrame->linesize[0] = yStride;
|
|
_pConvertedFrame->linesize[1] = uStride;
|
|
_pConvertedFrame->linesize[2] = vStride;
|
|
|
|
ffmpeg.sws_scale(
|
|
_pSwsContext,
|
|
frame->data, frame->linesize,
|
|
0, frame->height,
|
|
_pConvertedFrame->data, _pConvertedFrame->linesize
|
|
);
|
|
ClearConvertedFramePointers();
|
|
}
|
|
else
|
|
{
|
|
int w = Width;
|
|
int h = Height;
|
|
int uvW = (w + 1) / 2;
|
|
int uvH = (h + 1) / 2;
|
|
|
|
// Copy Y plane
|
|
for (int y = 0; y < h; y++)
|
|
{
|
|
Buffer.MemoryCopy(
|
|
frame->data[0] + y * frame->linesize[0],
|
|
yBuffer + y * yStride,
|
|
yStride, w
|
|
);
|
|
}
|
|
|
|
// Copy U plane
|
|
for (int y = 0; y < uvH; y++)
|
|
{
|
|
Buffer.MemoryCopy(
|
|
frame->data[1] + y * frame->linesize[1],
|
|
uBuffer + y * uStride,
|
|
uStride, uvW
|
|
);
|
|
}
|
|
|
|
// Copy V plane
|
|
for (int y = 0; y < uvH; y++)
|
|
{
|
|
Buffer.MemoryCopy(
|
|
frame->data[2] + y * frame->linesize[2],
|
|
vBuffer + y * vStride,
|
|
vStride, uvW
|
|
);
|
|
}
|
|
}
|
|
|
|
ffmpeg.av_frame_unref(frame);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies its planar Y, U, and V channel data to unmanaged memory pointers.
|
|
/// </summary>
|
|
// ReSharper disable once InconsistentNaming
|
|
public bool TryReadYUV420PFrame(
|
|
IntPtr yBuffer, int yStride,
|
|
IntPtr uBuffer, int uStride,
|
|
IntPtr vBuffer, int vStride,
|
|
out double timestamp)
|
|
{
|
|
return TryReadYUV420PFrame(
|
|
(byte*)yBuffer.ToPointer(), yStride,
|
|
(byte*)uBuffer.ToPointer(), uStride,
|
|
(byte*)vBuffer.ToPointer(), vStride,
|
|
out timestamp
|
|
);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Reads the next video frame and copies its planar Y, U, and V channel data into Unity NativeArrays.
|
|
/// </summary>
|
|
// ReSharper disable once InconsistentNaming
|
|
public bool TryReadYUV420PFrame(
|
|
NativeArray<byte> yBuffer,
|
|
NativeArray<byte> uBuffer,
|
|
NativeArray<byte> vBuffer,
|
|
out double timestamp)
|
|
{
|
|
if (!yBuffer.IsCreated || !uBuffer.IsCreated || !vBuffer.IsCreated)
|
|
{
|
|
throw new ArgumentException("Input NativeArray buffer is not created or already disposed.");
|
|
}
|
|
|
|
int w = Width;
|
|
int h = Height;
|
|
int uvW = (w + 1) / 2;
|
|
int uvH = (h + 1) / 2;
|
|
|
|
if (yBuffer.Length < (long)w * h || uBuffer.Length < (long)uvW * uvH || vBuffer.Length < (long)uvW * uvH)
|
|
{
|
|
throw new ArgumentException("Buffer size is smaller than video YUV plane resolution.");
|
|
}
|
|
|
|
return TryReadYUV420PFrame(
|
|
(byte*)yBuffer.GetUnsafePtr(), w,
|
|
(byte*)uBuffer.GetUnsafePtr(), uvW,
|
|
(byte*)vBuffer.GetUnsafePtr(), uvW,
|
|
out timestamp
|
|
);
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Seeking & Skipping
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Maximum distance in frames over which <see cref="SeekToTime"/> moves forward by decoding.
|
|
/// Decoding forward needs no keyframe seek and no decoder flush, which is cheaper than seeking for short distances.
|
|
/// </summary>
|
|
private const int MaxForwardDecodeFrames = 60;
|
|
|
|
/// <summary>
|
|
/// Decodes forward from the current decoder position until the first frame whose presentation time is at least
|
|
/// <paramref name="timeSeconds"/> minus half a frame, and keeps that frame in <see cref="_pCachedFrame"/>
|
|
/// for the next <c>TryRead*</c> call.
|
|
/// Frames before the target are discarded without GPU-to-host memory transfer; only the target frame is transferred.
|
|
/// When the end of the stream is reached before the target, no frame is cached.
|
|
/// </summary>
|
|
/// <param name="timeSeconds">Target time in seconds.</param>
|
|
private void DecodeToTarget(double timeSeconds)
|
|
{
|
|
while (true)
|
|
{
|
|
int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame);
|
|
if (ret == 0)
|
|
{
|
|
double currentPts = GetFrameTimestamp(_pDecodedFrame);
|
|
|
|
// Check with half-frame margin to accommodate floating point rounding
|
|
if (currentPts >= timeSeconds - (0.5 / FPS))
|
|
{
|
|
// Cache the first matching frame in system RAM for immediate consumption
|
|
if (_isHwDecoding && _pDecodedFrame->format == (int)_hwPixFmt)
|
|
{
|
|
ffmpeg.av_frame_unref(_pCachedFrame);
|
|
int transferRet = ffmpeg.av_hwframe_transfer_data(_pCachedFrame, _pDecodedFrame, 0);
|
|
if (transferRet < 0)
|
|
{
|
|
ffmpeg.av_frame_unref(_pDecodedFrame);
|
|
throw new Exception($"Failed to transfer cached hardware frame during seek: {GetFFmpegErrorString(transferRet)}");
|
|
}
|
|
|
|
ffmpeg.av_frame_copy_props(_pCachedFrame, _pDecodedFrame);
|
|
ffmpeg.av_frame_unref(_pDecodedFrame);
|
|
|
|
_downloadedPixFmt = (AVPixelFormat)_pCachedFrame->format;
|
|
}
|
|
else
|
|
{
|
|
ffmpeg.av_frame_unref(_pCachedFrame);
|
|
ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame);
|
|
}
|
|
|
|
_hasCachedFrame = true;
|
|
return;
|
|
}
|
|
|
|
// Discard non-matching intermediate frame without transferring to CPU
|
|
GetTimestampAndUpdateIndex(_pDecodedFrame);
|
|
ffmpeg.av_frame_unref(_pDecodedFrame);
|
|
continue;
|
|
}
|
|
|
|
if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN))
|
|
{
|
|
if (_isDraining) return;
|
|
|
|
if (!ReadAndSendPacket())
|
|
{
|
|
_isDraining = true;
|
|
ffmpeg.avcodec_send_packet(_pCodecContext, null);
|
|
}
|
|
}
|
|
else if (ret == ffmpeg.AVERROR_EOF)
|
|
{
|
|
return;
|
|
}
|
|
else
|
|
{
|
|
throw new Exception($"Error during seek decoding: {GetFFmpegErrorString(ret)}");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Moves the playback position to a time in seconds.
|
|
/// The next <c>TryRead*</c> call returns the first frame whose presentation time is at least
|
|
/// <paramref name="timeSeconds"/> minus half a frame (the frame nearest to the target time).
|
|
/// Both paths below return that same frame:
|
|
/// - Forward decode: the target is ahead of the current position by less than <see cref="MaxForwardDecodeFrames"/> frames.
|
|
/// Frames are decoded and discarded up to the target.
|
|
/// - Keyframe seek: all other cases (target before the current position, far ahead, or the target frame already read).
|
|
/// A backward keyframe seek is followed by a forward decode up to the target.
|
|
/// The target frame is transferred and preserved in <see cref="_pCachedFrame"/>; when it is already there, nothing is decoded.
|
|
/// </summary>
|
|
/// <param name="timeSeconds">Target time in seconds to seek to.</param>
|
|
public void SeekToTime(double timeSeconds)
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (!_isOpened || _disposed) return;
|
|
|
|
if (timeSeconds < 0) timeSeconds = 0;
|
|
|
|
// Earliest presentation time that satisfies the target (target minus half a frame).
|
|
double threshold = timeSeconds - (0.5 / FPS);
|
|
|
|
// Index of the first frame that satisfies the target. The epsilon keeps a frame that lies exactly on the
|
|
// margin from being excluded by floating point rounding.
|
|
double targetFrameIndex = Math.Ceiling(timeSeconds * FPS - 0.5 - 1e-6);
|
|
|
|
if (_hasCachedFrame)
|
|
{
|
|
// A frame is already positioned for the next read (e.g. by a previous SeekToTime call).
|
|
double cachedPts = GetFrameTimestamp(_pCachedFrame);
|
|
if (cachedPts >= threshold)
|
|
{
|
|
// The cached frame is the first frame that satisfies the target.
|
|
// When the target lies before it, the frames in between are no longer available and a keyframe seek is required.
|
|
if (cachedPts - threshold < 1.0 / FPS) return;
|
|
}
|
|
else if (targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
|
|
{
|
|
// The cached frame lies before the target: drop it and continue decoding forward.
|
|
GetTimestampAndUpdateIndex(_pCachedFrame);
|
|
ffmpeg.av_frame_unref(_pCachedFrame);
|
|
_hasCachedFrame = false;
|
|
DecodeToTarget(timeSeconds);
|
|
return;
|
|
}
|
|
}
|
|
else if (targetFrameIndex > _currentFrameIndex && targetFrameIndex - _currentFrameIndex < MaxForwardDecodeFrames)
|
|
{
|
|
// The target frame has not been read yet and is close: decode forward instead of seeking.
|
|
DecodeToTarget(timeSeconds);
|
|
return;
|
|
}
|
|
|
|
if (_hasCachedFrame)
|
|
{
|
|
ffmpeg.av_frame_unref(_pCachedFrame);
|
|
_hasCachedFrame = false;
|
|
}
|
|
|
|
long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase));
|
|
int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
|
|
if (ret < 0)
|
|
{
|
|
// Fallback to beginning of file on seek failure
|
|
ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD);
|
|
if (ret < 0)
|
|
{
|
|
throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}");
|
|
}
|
|
}
|
|
|
|
// Flush decoder internal buffers
|
|
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
|
ffmpeg.avcodec_flush_buffers(_pCodecContext);
|
|
_isDraining = false;
|
|
|
|
// Decode frames until reaching the target presentation timestamp
|
|
DecodeToTarget(timeSeconds);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Seeks to a specific frame index.
|
|
/// </summary>
|
|
/// <param name="frameIndex">Target frame index.</param>
|
|
public void SeekToFrame(long frameIndex)
|
|
{
|
|
SeekToTime(frameIndex / FPS);
|
|
}
|
|
|
|
// ──────────────────────────────────────────────────────────────
|
|
// Error Handling & Disposal
|
|
// ──────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Translates an FFmpeg integer error code to a readable string using <c>av_strerror</c>.
|
|
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
|
|
/// </summary>
|
|
private string GetFFmpegErrorString(int errnum)
|
|
{
|
|
const ulong bufferSize = 1024;
|
|
byte* buffer = stackalloc byte[(int)bufferSize];
|
|
if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0)
|
|
{
|
|
return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})";
|
|
}
|
|
return $"Unknown error (Code: {errnum})";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Releases all native FFmpeg decoder resources, hardware contexts, frame buffers, and scaler structures.
|
|
/// </summary>
|
|
public void Dispose()
|
|
{
|
|
lock (_lock)
|
|
{
|
|
if (_disposed) return;
|
|
_disposed = true;
|
|
|
|
// 1. Release scaler context
|
|
if (_pSwsContext != null)
|
|
{
|
|
ffmpeg.sws_freeContext(_pSwsContext);
|
|
_pSwsContext = null;
|
|
}
|
|
|
|
// 2. Release frame buffers
|
|
if (_pDecodedFrame != null)
|
|
{
|
|
var pFrame = _pDecodedFrame;
|
|
ffmpeg.av_frame_free(&pFrame);
|
|
_pDecodedFrame = null;
|
|
}
|
|
|
|
if (_pHwTransferFrame != null)
|
|
{
|
|
var pFrame = _pHwTransferFrame;
|
|
ffmpeg.av_frame_free(&pFrame);
|
|
_pHwTransferFrame = null;
|
|
}
|
|
|
|
if (_pCachedFrame != null)
|
|
{
|
|
var pFrame = _pCachedFrame;
|
|
ffmpeg.av_frame_free(&pFrame);
|
|
_pCachedFrame = null;
|
|
}
|
|
|
|
if (_pConvertedFrame != null)
|
|
{
|
|
var pFrame = _pConvertedFrame;
|
|
ffmpeg.av_frame_free(&pFrame);
|
|
_pConvertedFrame = null;
|
|
}
|
|
|
|
// 3. Release packet buffer
|
|
if (_pPacket != null)
|
|
{
|
|
var pPacket = _pPacket;
|
|
ffmpeg.av_packet_free(&pPacket);
|
|
_pPacket = null;
|
|
}
|
|
|
|
// 4. Release codec context
|
|
if (_pCodecContext != null)
|
|
{
|
|
var pCodecCtx = _pCodecContext;
|
|
ffmpeg.avcodec_free_context(&pCodecCtx);
|
|
_pCodecContext = null;
|
|
}
|
|
|
|
// 5. Release hardware device context
|
|
if (_hwDeviceCtx != null)
|
|
{
|
|
var pHw = _hwDeviceCtx;
|
|
ffmpeg.av_buffer_unref(&pHw);
|
|
_hwDeviceCtx = null;
|
|
}
|
|
|
|
// 6. Close input file and release format context
|
|
if (_pFormatContext != null)
|
|
{
|
|
var pFormatCtx = _pFormatContext;
|
|
ffmpeg.avformat_close_input(&pFormatCtx);
|
|
_pFormatContext = null;
|
|
}
|
|
|
|
_getFormatDelegate = null;
|
|
_isOpened = false;
|
|
_isHwDecoding = false;
|
|
GC.SuppressFinalize(this);
|
|
}
|
|
}
|
|
|
|
~FFmpegNativeDecoder()
|
|
{
|
|
Dispose();
|
|
}
|
|
} |