using System; using FFmpeg.AutoGen; using Unity.Collections; using Unity.Collections.LowLevel.Unsafe; namespace ADOFAIRenderer.Util.FFmpeg { /// /// Decodes video files in a streaming fashion using FFmpeg's native C API. /// Unlike a standalone player, it allows reading frames sequentially or seeking/skipping forward on demand. /// internal sealed unsafe class FFmpegNativeDecoder : IDisposable { private AVFormatContext* _pFormatContext = null; private AVCodecContext* _pCodecContext = null; private AVFrame* _pDecodedFrame = null; private AVFrame* _pConvertedFrame = null; private SwsContext* _pSwsContext = null; private AVPacket* _pPacket = null; private int _videoStreamIndex = -1; private long _currentFrameIndex; private AVRational _timeBase; private bool _isOpened; private bool _disposed; private bool _isDraining; private readonly object _lock = new(); // Cached frame used to hold matching frames after seeking private AVFrame* _pCachedFrame = null; private bool _hasCachedFrame; // Field to detect SwsContext updates private AVPixelFormat _srcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; private int _srcWidth; private int _srcHeight; /// Video width in pixels. public int Width { get; private set; } /// Video height in pixels. public int Height { get; private set; } /// Frames per second. public double FPS { get; private set; } /// Total duration in seconds. public double Duration { get; private set; } /// Estimated total number of frames. public long TotalFrames { get; private set; } /// Index of the currently read frame. public long CurrentFrameIndex => _currentFrameIndex; /// Current playback time in seconds. public double CurrentTime => _currentFrameIndex / FPS; /// Pixel format of the original video stream. public AVPixelFormat PixelFormat => _pCodecContext != null ? _pCodecContext->pix_fmt : AVPixelFormat.AV_PIX_FMT_NONE; /// Color range of the original video stream. public AVColorRange ColorRange => _pCodecContext != null ? _pCodecContext->color_range : AVColorRange.AVCOL_RANGE_UNSPECIFIED; public FFmpegNativeDecoder(string videoPath) { Open(videoPath); } private void Open(string videoPath) { lock (_lock) { if (_isOpened || _disposed) return; try { fixed (AVFormatContext** ppFormatCtx = &_pFormatContext) { int ret = ffmpeg.avformat_open_input(ppFormatCtx, videoPath, null, null); if (ret < 0) throw new Exception($"Failed to open video file: {videoPath} ({GetFFmpegErrorString(ret)})"); } if (ffmpeg.avformat_find_stream_info(_pFormatContext, null) < 0) throw new Exception("Failed to retrieve stream info."); _videoStreamIndex = -1; for (int i = 0; i < _pFormatContext->nb_streams; i++) { if (_pFormatContext->streams[i]->codecpar->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO) { _videoStreamIndex = i; break; } } if (_videoStreamIndex == -1) throw new Exception("Failed to find video stream."); var stream = _pFormatContext->streams[_videoStreamIndex]; _timeBase = stream->time_base; // Calculate FPS if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->r_frame_rate); } else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0) { FPS = ffmpeg.av_q2d(stream->avg_frame_rate); } else { FPS = 30.0; // Fallback default } var codecId = stream->codecpar->codec_id; var decoder = ffmpeg.avcodec_find_decoder(codecId); if (decoder == null) throw new Exception($"Unsupported video codec: {codecId}"); _pCodecContext = ffmpeg.avcodec_alloc_context3(decoder); if (_pCodecContext == null) throw new Exception("Failed to allocate codec context."); if (ffmpeg.avcodec_parameters_to_context(_pCodecContext, stream->codecpar) < 0) throw new Exception("Failed to copy codec parameters."); // Multi-threading settings for improved decoding performance (auto) _pCodecContext->thread_count = 0; _pCodecContext->thread_type = ffmpeg.FF_THREAD_FRAME; if (ffmpeg.avcodec_open2(_pCodecContext, decoder, null) < 0) throw new Exception("Failed to open codec."); Width = _pCodecContext->width; Height = _pCodecContext->height; // Calculate total frames and duration if (stream->nb_frames > 0) { TotalFrames = stream->nb_frames; } else if (stream->duration > 0) { TotalFrames = (long)Math.Round(stream->duration * ffmpeg.av_q2d(stream->time_base) * FPS); } else if (_pFormatContext->duration > 0) { TotalFrames = (long)Math.Round((_pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE) * FPS); } else { TotalFrames = 0; } Duration = _pFormatContext->duration > 0 ? _pFormatContext->duration / (double)ffmpeg.AV_TIME_BASE : (stream->duration > 0 ? stream->duration * ffmpeg.av_q2d(stream->time_base) : 0); _pDecodedFrame = ffmpeg.av_frame_alloc(); _pConvertedFrame = ffmpeg.av_frame_alloc(); _pCachedFrame = ffmpeg.av_frame_alloc(); _pPacket = ffmpeg.av_packet_alloc(); _currentFrameIndex = 0; _isOpened = true; _isDraining = false; } catch (Exception) { Dispose(); throw; } } } private bool ReadAndSendPacket() { while (true) { ffmpeg.av_packet_unref(_pPacket); int ret = ffmpeg.av_read_frame(_pFormatContext, _pPacket); if (ret < 0) return false; // EOF or error if (_pPacket->stream_index == _videoStreamIndex) { ret = ffmpeg.avcodec_send_packet(_pCodecContext, _pPacket); ffmpeg.av_packet_unref(_pPacket); if (ret < 0) { throw new Exception($"Error sending packet to decoder: {GetFFmpegErrorString(ret)}"); } return true; } } } private void InitOrUpdateSwsContext(AVPixelFormat srcFormat, int srcWidth, int srcHeight, AVPixelFormat dstFormat) { if (_pSwsContext != null && _srcPixFmt == srcFormat && _srcWidth == srcWidth && _srcHeight == srcHeight && _dstPixFmt == dstFormat) { return; } if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } _srcPixFmt = srcFormat; _srcWidth = srcWidth; _srcHeight = srcHeight; _dstPixFmt = dstFormat; _pSwsContext = ffmpeg.sws_getContext( srcWidth, srcHeight, srcFormat, Width, Height, dstFormat, (int)SwsFlags.SWS_BILINEAR, null, null, null ); if (_pSwsContext == null) { throw new Exception("Failed to initialize SwsContext."); } } private double GetTimestampAndUpdateIndex(AVFrame* frame) { long ptsValue = frame->best_effort_timestamp; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { ptsValue = frame->pts; } double timestamp; if (ptsValue == ffmpeg.AV_NOPTS_VALUE) { timestamp = _currentFrameIndex / FPS; _currentFrameIndex++; } else { timestamp = ptsValue * ffmpeg.av_q2d(_timeBase); _currentFrameIndex = (long)Math.Round(timestamp * FPS); } return timestamp; } /// /// Reads the next video frame and copies it in RGBA format. /// /// Pointer to the output buffer storing RGBA data. /// Byte stride per row of the output buffer (typically Width * 4). /// Whether to flip vertically to match Unity's texture coordinate system. /// Timestamp of the output frame in seconds. /// True if the frame was successfully decoded and copied; false on EOF or failure. private bool TryReadFrame(byte* outBuffer, int outStride, bool vflip, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed) return false; if (_hasCachedFrame) { timestamp = GetTimestampAndUpdateIndex(_pCachedFrame); InitOrUpdateSwsContext(_pCodecContext->pix_fmt, _pCachedFrame->width, _pCachedFrame->height, AVPixelFormat.AV_PIX_FMT_RGBA); if (vflip) { _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); _pConvertedFrame->linesize[0] = -outStride; } else { _pConvertedFrame->data[0] = outBuffer; _pConvertedFrame->linesize[0] = outStride; } ffmpeg.sws_scale( _pSwsContext, _pCachedFrame->data, _pCachedFrame->linesize, 0, _pCachedFrame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; return true; } while (true) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { timestamp = GetTimestampAndUpdateIndex(_pDecodedFrame); InitOrUpdateSwsContext(_pCodecContext->pix_fmt, _pDecodedFrame->width, _pDecodedFrame->height, AVPixelFormat.AV_PIX_FMT_RGBA); if (vflip) { _pConvertedFrame->data[0] = outBuffer + outStride * (Height - 1); _pConvertedFrame->linesize[0] = -outStride; } else { _pConvertedFrame->data[0] = outBuffer; _pConvertedFrame->linesize[0] = outStride; } ffmpeg.sws_scale( _pSwsContext, _pDecodedFrame->data, _pDecodedFrame->linesize, 0, _pDecodedFrame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); ffmpeg.av_frame_unref(_pDecodedFrame); return true; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return false; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { return false; } else { throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); } } } } public bool TryReadFrame(IntPtr outBuffer, int outStride, bool vflip, out double timestamp) { return TryReadFrame((byte*)outBuffer.ToPointer(), outStride, vflip, out timestamp); } public bool TryReadFrame(NativeArray outBuffer, bool vflip, out double timestamp) { if (outBuffer.Length < Width * Height * 4) { throw new ArgumentException("Output buffer size is smaller than video resolution.", nameof(outBuffer)); } int stride = Width * 4; return TryReadFrame((byte*)outBuffer.GetUnsafePtr(), stride, vflip, out timestamp); } /// /// Reads the next video frame and copies its Y, U, and V channel data separately. /// If the source format is YUV420P family (YUV420P, YUVJ420P), CPU color conversion (sws_scale) is bypassed /// and memory is copied directly for maximum performance. /// Other formats are automatically converted to YUV420P before copying. /// // ReSharper disable once InconsistentNaming private bool TryReadYUV420PFrame( byte* yBuffer, int yStride, byte* uBuffer, int uStride, byte* vBuffer, int vStride, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { while (true) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { frame = _pDecodedFrame; break; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return false; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { return false; } else { throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); } } } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUV420P && frame->format != (int)AVPixelFormat.AV_PIX_FMT_YUVJ420P) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_YUV420P); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uBuffer; _pConvertedFrame->data[2] = vBuffer; _pConvertedFrame->linesize[0] = yStride; _pConvertedFrame->linesize[1] = uStride; _pConvertedFrame->linesize[2] = vStride; ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); } else { int w = frame->width; int h = frame->height; int uvW = w / 2; int uvH = h / 2; // Copy Y plane for (int y = 0; y < h; y++) { Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, w, w ); } // Copy U plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uBuffer + y * uStride, uvW, uvW ); } // Copy V plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[2] + y * frame->linesize[2], vBuffer + y * vStride, uvW, uvW ); } } ffmpeg.av_frame_unref(frame); return true; } } // ReSharper disable once InconsistentNaming public bool TryReadYUV420PFrame( NativeArray yBuffer, NativeArray uBuffer, NativeArray vBuffer, out double timestamp) { int w = Width; int h = Height; int uvW = w / 2; int uvH = h / 2; if (yBuffer.Length < w * h || uBuffer.Length < uvW * uvH || vBuffer.Length < uvW * uvH) { throw new ArgumentException("Buffer size is smaller than video YUV plane resolution."); } return TryReadYUV420PFrame( (byte*)yBuffer.GetUnsafePtr(), w, (byte*)uBuffer.GetUnsafePtr(), uvW, (byte*)vBuffer.GetUnsafePtr(), uvW, out timestamp ); } /// /// Reads the next video frame and copies its NV12 format Y and UV channel data separately. /// If the source format is NV12 family, CPU color conversion (sws_scale) is bypassed /// and memory is copied directly for maximum performance. /// Other formats are automatically converted to NV12 before copying. /// // ReSharper disable once InconsistentNaming private bool TryReadNV12Frame( byte* yBuffer, int yStride, byte* uvBuffer, int uvStride, out double timestamp) { timestamp = 0; lock (_lock) { if (!_isOpened || _disposed) return false; AVFrame* frame; if (_hasCachedFrame) { frame = _pCachedFrame; _hasCachedFrame = false; } else { while (true) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { frame = _pDecodedFrame; break; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) return false; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { return false; } else { throw new Exception($"Error during decoding: {GetFFmpegErrorString(ret)}"); } } } if (frame == null) return false; timestamp = GetTimestampAndUpdateIndex(frame); if (frame->format != (int)AVPixelFormat.AV_PIX_FMT_NV12) { InitOrUpdateSwsContext((AVPixelFormat)frame->format, frame->width, frame->height, AVPixelFormat.AV_PIX_FMT_NV12); _pConvertedFrame->data[0] = yBuffer; _pConvertedFrame->data[1] = uvBuffer; _pConvertedFrame->linesize[0] = yStride; _pConvertedFrame->linesize[1] = uvStride; ffmpeg.sws_scale( _pSwsContext, frame->data, frame->linesize, 0, frame->height, _pConvertedFrame->data, _pConvertedFrame->linesize ); } else { int w = frame->width; int h = frame->height; int uvH = h / 2; // Copy Y plane for (int y = 0; y < h; y++) { Buffer.MemoryCopy( frame->data[0] + y * frame->linesize[0], yBuffer + y * yStride, w, w ); } // Copy UV plane for (int y = 0; y < uvH; y++) { Buffer.MemoryCopy( frame->data[1] + y * frame->linesize[1], uvBuffer + y * uvStride, w, w ); } } ffmpeg.av_frame_unref(frame); return true; } } // ReSharper disable once InconsistentNaming public bool TryReadNV12Frame( NativeArray yBuffer, NativeArray uvBuffer, out double timestamp) { int w = Width; int h = Height; int uvH = h / 2; if (yBuffer.Length < w * h || uvBuffer.Length < w * uvH) { throw new ArgumentException("Buffer size is smaller than video NV12 plane resolution."); } return TryReadNV12Frame( (byte*)yBuffer.GetUnsafePtr(), w, (byte*)uvBuffer.GetUnsafePtr(), w, out timestamp ); } /// /// Decodes and discards a specified number of frames immediately (for fast forward skipping). /// /// Number of frames to discard. private void SkipFrames(int count) { lock (_lock) { if (!_isOpened || _disposed) return; if (count <= 0) return; int skipped = 0; if (_hasCachedFrame) { ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; skipped++; } while (skipped < count) { int ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { GetTimestampAndUpdateIndex(_pDecodedFrame); ffmpeg.av_frame_unref(_pDecodedFrame); skipped++; continue; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) break; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { break; } else { throw new Exception($"Error skipping frames: {GetFFmpegErrorString(ret)}"); } } } } /// /// Seeks to a specific playback time in seconds. /// /// Target time in seconds to seek to. public void SeekToTime(double timeSeconds) { lock (_lock) { if (!_isOpened || _disposed) return; if (timeSeconds < 0) timeSeconds = 0; double targetFrameIndex = timeSeconds * FPS; // If already at target index or very close future, skip frames instead of seek to avoid I-frame overhead bool useSkip = targetFrameIndex >= _currentFrameIndex && targetFrameIndex - _currentFrameIndex < 60; if (useSkip) { SkipFrames((int)(targetFrameIndex - _currentFrameIndex)); return; } if (_hasCachedFrame) { ffmpeg.av_frame_unref(_pCachedFrame); _hasCachedFrame = false; } long targetPts = (long)Math.Round(timeSeconds / ffmpeg.av_q2d(_timeBase)); int ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, targetPts, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { // Fallback to start of file on seek failure ret = ffmpeg.av_seek_frame(_pFormatContext, _videoStreamIndex, 0, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) { throw new Exception($"Seek failed: {GetFFmpegErrorString(ret)}"); } } // Flush decoder internal cache ffmpeg.avcodec_flush_buffers(_pCodecContext); _isDraining = false; // Decode and discard frames until the first frame past the specified time is found while (true) { ret = ffmpeg.avcodec_receive_frame(_pCodecContext, _pDecodedFrame); if (ret == 0) { double currentPts = GetTimestampAndUpdateIndex(_pDecodedFrame); // Check from half frame earlier to account for floating point errors if (currentPts >= timeSeconds - (0.5 / FPS)) { // Cache the first matching frame so it will be consumed by the next TryReadFrame call ffmpeg.av_frame_move_ref(_pCachedFrame, _pDecodedFrame); _hasCachedFrame = true; break; } ffmpeg.av_frame_unref(_pDecodedFrame); continue; } if (ret == ffmpeg.AVERROR(ffmpeg.EAGAIN)) { if (_isDraining) break; if (!ReadAndSendPacket()) { _isDraining = true; ffmpeg.avcodec_send_packet(_pCodecContext, null); } } else if (ret == ffmpeg.AVERROR_EOF) { break; } else { throw new Exception($"Error during seek decoding: {GetFFmpegErrorString(ret)}"); } } } } /// /// Seeks to a specific frame index. /// /// Target frame index to seek to. public void SeekToFrame(long frameIndex) { SeekToTime(frameIndex / FPS); } private string GetFFmpegErrorString(int errnum) { const ulong bufferSize = 1024; byte* buffer = stackalloc byte[(int)bufferSize]; if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0) { return System.Runtime.InteropServices.Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})"; } return $"Unknown error (Code: {errnum})"; } public void Dispose() { lock (_lock) { if (_disposed) return; _disposed = true; if (_pSwsContext != null) { ffmpeg.sws_freeContext(_pSwsContext); _pSwsContext = null; } if (_pDecodedFrame != null) { var pFrame = _pDecodedFrame; ffmpeg.av_frame_free(&pFrame); _pDecodedFrame = null; } if (_pCachedFrame != null) { var pFrame = _pCachedFrame; ffmpeg.av_frame_free(&pFrame); _pCachedFrame = null; } if (_pConvertedFrame != null) { var pFrame = _pConvertedFrame; ffmpeg.av_frame_free(&pFrame); _pConvertedFrame = null; } if (_pPacket != null) { var pPacket = _pPacket; ffmpeg.av_packet_free(&pPacket); _pPacket = null; } if (_pCodecContext != null) { var pCodecCtx = _pCodecContext; ffmpeg.avcodec_free_context(&pCodecCtx); _pCodecContext = null; } if (_pFormatContext != null) { var pFormatCtx = _pFormatContext; ffmpeg.avformat_close_input(&pFormatCtx); _pFormatContext = null; } _isOpened = false; GC.SuppressFinalize(this); } } ~FFmpegNativeDecoder() { Dispose(); } } }