SHA256
forked from ADOFAIEditorForum/ADOFAIRenderer
1856 lines
94 KiB
C#
1856 lines
94 KiB
C#
using System;
|
||
using System.Runtime.InteropServices;
|
||
using FFmpeg.AutoGen;
|
||
using MelonLoader;
|
||
using Unity.Collections;
|
||
using Unity.Collections.LowLevel.Unsafe;
|
||
|
||
namespace ADOFAIRenderer.Util.FFmpeg
|
||
{
|
||
/// <summary>
|
||
/// Class that encodes video by directly invoking (P/Invoke) FFmpeg's C native API.
|
||
///
|
||
/// ── Lifecycle Flow ───────────────────────────────────────────────────
|
||
///
|
||
/// 1. Constructor: Stores encoding settings (resolution, FPS, codec, quality, etc.).
|
||
///
|
||
/// 2. Start(outputPath):
|
||
/// a) Create AVFormatContext for output container (mp4, etc.)
|
||
/// b) Add video stream (AVStream)
|
||
/// c) Try hardware encoders (NVENC → AMF → QSV) → software encoders (libx264, etc.)
|
||
/// in order to select an available encoder
|
||
/// d) Write file header and initialize frame/scaler buffers
|
||
///
|
||
/// 3. WriteFrame*(): Called for every frame
|
||
/// a) Convert Unity RGBA original frame to NV12/YUV420P using sws_scale
|
||
/// b) (In QSV hw_frames mode) Upload to GPU surface
|
||
/// c) avcodec_send_frame → avcodec_receive_packet → Write to file
|
||
///
|
||
/// 4. End(): Flush delayed frames remaining in the encoder and write trailer
|
||
///
|
||
/// 5. Dispose(): Release all FFmpeg resources (context, frames, scaler, I/O)
|
||
///
|
||
/// ── Reference Documentation ─────────────────────────────────────────
|
||
/// • FFmpeg Encoding Overview: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
|
||
/// • Send/Receive Encoding API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
||
/// • Hardware Acceleration Overview: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
/// • HW Acceleration Wiki: https://trac.ffmpeg.org/wiki/HWAccelIntro
|
||
/// </summary>
|
||
internal sealed unsafe class FFmpegNativeEncoder : IDisposable
|
||
{
|
||
/// <summary>
|
||
/// Struct containing configuration values to pass for audio encoding.
|
||
/// </summary>
|
||
public struct AudioSettings
|
||
{
|
||
/// <summary>Sample rate of input audio (Hz). e.g., 44100, 48000.</summary>
|
||
public int SampleRate;
|
||
|
||
/// <summary>Number of input audio channels. e.g., 1 (mono), 2 (stereo).</summary>
|
||
public int Channels;
|
||
|
||
/// <summary>
|
||
/// Sample format of input audio. Default is AV_SAMPLE_FMT_FLT (32-bit float).
|
||
/// Unity's AudioClip.GetData() returns float[], so FLT is suitable.
|
||
/// </summary>
|
||
public AVSampleFormat InputSampleFormat;
|
||
|
||
/// <summary>
|
||
/// Audio codec ID to use. If default (0), AAC is used.
|
||
/// e.g., AV_CODEC_ID_AAC, AV_CODEC_ID_FLAC, AV_CODEC_ID_OPUS, etc.
|
||
/// </summary>
|
||
public AVCodecID AudioCodecId;
|
||
|
||
/// <summary>
|
||
/// Audio bitrate (in bps). If default (0), 128000 bps is used.
|
||
/// For AAC, 128000–320000 bps is common.
|
||
/// </summary>
|
||
public int Bitrate;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Struct containing configuration values to pass to the encoder.
|
||
/// </summary>
|
||
public struct EncoderSettings
|
||
{
|
||
/// <summary>Width resolution of output video (pixels).</summary>
|
||
public int Width;
|
||
|
||
/// <summary>Height resolution of output video (pixels).</summary>
|
||
public int Height;
|
||
|
||
/// <summary>Frames per second (e.g., 60.0, 29.97, etc.).</summary>
|
||
public double FPS;
|
||
|
||
/// <summary>Video codec type to use (H264, H265, AV1, VP9).</summary>
|
||
public Codec Codec;
|
||
|
||
/// <summary>Bitrate control mode (CQP, CBR, VBR).</summary>
|
||
public RateControlMode Mode;
|
||
|
||
/// <summary>Quality value used in CQP mode. Lower values mean higher quality. (e.g., H.264 CRF 18 ≈ visually lossless)</summary>
|
||
public int QualityValue; // CRF
|
||
|
||
/// <summary>Target bitrate in VBR/CBR mode (in kbps). Converted to bps internally for FFmpeg.</summary>
|
||
public int TargetBitrate; // VBR
|
||
|
||
/// <summary>Maximum bitrate in VBR mode (in kbps). Determines peak quality limit.</summary>
|
||
public int MaxBitrate; // VBR
|
||
|
||
/// <summary>GOP (Group of Pictures) size (in seconds). Represents keyframe interval.</summary>
|
||
public double GopSize;
|
||
|
||
/// <summary>Whether to force the use of software CPU encoder.</summary>
|
||
public bool ForceSoftware;
|
||
|
||
/// <summary>
|
||
/// Audio encoding settings. If null, encodes video only without audio stream.
|
||
/// If configured, video and audio are muxed together.
|
||
/// </summary>
|
||
public AudioSettings? Audio;
|
||
}
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// FFmpeg Core Struct Pointers
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Context managing format information for the output container (mp4, mkv, etc.).
|
||
/// Used for writing file headers/trailers, recording packets, etc.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
|
||
/// </summary>
|
||
private AVFormatContext* _pFormatContext = null;
|
||
|
||
/// <summary>
|
||
/// Context managing codec (encoder) state.
|
||
/// All parameters required for encoding such as resolution, pixel format, framerate, and quality settings are configured here.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
||
/// </summary>
|
||
private AVCodecContext* _pCodecContext = null;
|
||
|
||
/// <summary>
|
||
/// Video stream within the output file.
|
||
/// A single container (format context) can contain multiple streams (video, audio, etc.),
|
||
/// but here only one video stream is used.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
|
||
/// </summary>
|
||
private AVStream* _pVideoStream = null;
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// Frame Buffers and Colorspace Converter (Scaler) Pointers
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Frame structure pointing to raw RGBA frame data captured from Unity.
|
||
/// Does not own pixel data; references external buffer via data[0] and linesize[0].
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
|
||
/// </summary>
|
||
private AVFrame* _pSrcFrame = null;
|
||
|
||
/// <summary>
|
||
/// Frame converted to the pixel format required by the encoder (YUV420P or NV12).
|
||
/// Actual pixel data memory is allocated via av_frame_get_buffer().
|
||
/// </summary>
|
||
private AVFrame* _pDstFrame = null;
|
||
|
||
/// <summary>
|
||
/// libswscale colorspace/resolution conversion context.
|
||
/// Performs RGBA (Unity) → YUV420P/NV12 (encoder) conversion.
|
||
/// sws_scale is used even when only converting colorspace without changing resolution.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
||
/// </summary>
|
||
private SwsContext* _pSwsContext = null;
|
||
|
||
/// <summary>Accumulated index of encoded frames. Used as the PTS (Presentation Time Stamp) value.</summary>
|
||
private long _frameIndex;
|
||
|
||
/// <summary>Whether Start() completed successfully and encoding is ready.</summary>
|
||
private bool _isInitialized;
|
||
|
||
/// <summary>Whether Dispose() has already been called. Prevents duplicate cleanup.</summary>
|
||
private bool _disposed;
|
||
|
||
/// <summary>Lock object for synchronizing encoder state and resource access in multithreaded environments.</summary>
|
||
private readonly object _lock = new();
|
||
|
||
/// <summary>
|
||
/// Whether hw_frames_ctx mode (GPU frame pool) is active in the QSV encoder.
|
||
/// If true, NV12 → QSV hardware surface upload is required every frame.
|
||
/// If false, passes NV12 frames in system memory directly to the encoder.
|
||
/// </summary>
|
||
private bool _isQsvEncoder;
|
||
|
||
/// <summary>
|
||
/// Target pixel format for the encoder.
|
||
/// - General encoders (NVENC, AMF, Software): YUV420P (most common 4:2:0 chroma subsampling)
|
||
/// - QSV encoder: NV12 (Y plane + UV interleaved plane, preferred format for Intel hardware)
|
||
///
|
||
/// Reference - Major pixel format relationships:
|
||
/// RGBA (Unity source, 4-channel 8-bit)
|
||
/// ↓ sws_scale conversion
|
||
/// YUV420P (3 planes Y/U/V, 1/4 chroma resolution) or
|
||
/// NV12 (1 Y plane + 1 UV interleaved plane, 1/4 chroma resolution)
|
||
///
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html
|
||
/// </summary>
|
||
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
|
||
|
||
/// <summary>Pixel format of the finally selected and running encoder.</summary>
|
||
public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
|
||
|
||
/// <summary>
|
||
/// Packet structure to temporarily hold encoded packet data.
|
||
/// Cached as a member variable to prevent allocation/deallocation overhead for each frame.
|
||
/// </summary>
|
||
private AVPacket* _pPacket = null;
|
||
|
||
/// <summary>
|
||
/// GPU frame structure used in QSV hardware acceleration mode.
|
||
/// Cached as a member variable to prevent allocation/deallocation overhead for each frame.
|
||
/// </summary>
|
||
private AVFrame* _pHwFrame = null;
|
||
private bool _loggedConversionMode;
|
||
|
||
// ──────────────────────────────────────────────────────────────
|
||
// FFmpeg Struct Pointers for Audio Encoding
|
||
// ──────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Context managing audio codec (encoder) state.
|
||
/// Audio encoding parameters such as sample rate, channel layout, and sample format are configured here.
|
||
/// </summary>
|
||
private AVCodecContext* _pAudioCodecContext = null;
|
||
|
||
/// <summary>Audio stream within the output file.</summary>
|
||
private AVStream* _pAudioStream = null;
|
||
|
||
/// <summary>Packet structure to temporarily hold encoded audio packet data.</summary>
|
||
private AVPacket* _pAudioPacket = null;
|
||
|
||
/// <summary>
|
||
/// Audio resampling context (SwrContext).
|
||
/// Converts input PCM format (e.g., float interleaved) → encoder required format (e.g., float planar).
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html
|
||
/// </summary>
|
||
private SwrContext* _pSwrContext = null;
|
||
|
||
/// <summary>
|
||
/// Audio FIFO buffer. Collects input samples of arbitrary sizes and passes them in frame_size units required by the encoder.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
|
||
/// </summary>
|
||
private AVAudioFifo* _pAudioFifo = null;
|
||
|
||
/// <summary>Frame structure to pass to the audio encoder.</summary>
|
||
private AVFrame* _pAudioFrame = null;
|
||
|
||
/// <summary>Accumulated index of encoded audio samples. Used as the PTS value.</summary>
|
||
private long _audioSampleIndex;
|
||
|
||
/// <summary>Whether audio encoding is enabled.</summary>
|
||
private bool _audioEnabled;
|
||
|
||
/// <summary>Encoding settings passed at construction time (immutable).</summary>
|
||
private readonly EncoderSettings _settings;
|
||
|
||
public FFmpegNativeEncoder(EncoderSettings settings)
|
||
{
|
||
_settings = settings;
|
||
}
|
||
|
||
// ================================================================
|
||
// Utility Methods
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg.
|
||
/// AVCodecID identifies "which codec it is (e.g., H.264, AV1)",
|
||
/// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately.
|
||
/// </summary>
|
||
// ReSharper disable once InconsistentNaming
|
||
// ReSharper disable once UnusedMember.Local
|
||
private AVCodecID GetAVCodecID()
|
||
{
|
||
return _settings.Codec switch
|
||
{
|
||
Codec.H264 => AVCodecID.AV_CODEC_ID_H264,
|
||
Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC,
|
||
Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1,
|
||
Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9,
|
||
Codec.VVC => AVCodecID.AV_CODEC_ID_VVC,
|
||
_ => throw new ArgumentOutOfRangeException()
|
||
};
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns a list of candidate encoders to try for the selected codec in priority order.
|
||
///
|
||
/// Priority: Hardware encoders (NVENC → AMF → QSV) → Software encoders (libx264, etc.)
|
||
///
|
||
/// - nvenc: Hardware encoder dedicated to NVIDIA GPUs (CUDA-based)
|
||
/// - amf: Hardware encoder dedicated to AMD GPUs (Advanced Media Framework)
|
||
/// - qsv: Hardware encoder dedicated to Intel GPUs/iGPUs (Quick Sync Video, Media SDK)
|
||
/// - lib*: CPU-based software encoders (always available, but slower)
|
||
///
|
||
/// Iterates through this list in Start() to use the first workable encoder.
|
||
/// </summary>
|
||
private VideoEncoder[] GetEncoderCandidates()
|
||
{
|
||
return GPUUtil.GetPrioritizedEncoders(_settings.Codec, _settings.ForceSoftware).ToArray();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Converts an FFmpeg error code (negative integer) to a human-readable string.
|
||
/// e.g., -1 → "Operation not permitted"
|
||
///
|
||
/// Calls av_strerror() internally to write the message to a stack-allocated buffer.
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__error.html
|
||
/// </summary>
|
||
private string GetFFmpegErrorString(int errnum)
|
||
{
|
||
const ulong bufferSize = 1024;
|
||
byte* buffer = stackalloc byte[(int)bufferSize];
|
||
if (ffmpeg.av_strerror(errnum, buffer, bufferSize) >= 0)
|
||
{
|
||
return Marshal.PtrToStringAnsi((IntPtr)buffer) ?? $"Unknown error (Code: {errnum})";
|
||
}
|
||
return $"Unknown error (Code: {errnum})";
|
||
}
|
||
|
||
// ================================================================
|
||
// Encoder Initialization (Start)
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Initializes the entire encoding pipeline.
|
||
///
|
||
/// Execution steps:
|
||
/// 1. Create container format context (AVFormatContext) for the output file
|
||
/// 2. Add video stream (AVStream)
|
||
/// 3. Iterate encoder candidates → configure codec context (AVCodecContext) → open encoder
|
||
/// 4. Copy codec parameters to stream
|
||
/// 5. Open output file and write header
|
||
/// 6. Initialize frame buffers (AVFrame) and colorspace converter (SwsContext)
|
||
///
|
||
/// On failure, calls Dispose() to clean up partially initialized resources before rethrowing the exception.
|
||
/// </summary>
|
||
/// <param name="outputPath">Output video file path (e.g., "C:\output\video.mp4")</param>
|
||
public void Start(string outputPath)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_isInitialized || _disposed) return;
|
||
|
||
try
|
||
{
|
||
// ── Step 1: Create output container format context ──
|
||
// avformat_alloc_output_context2 automatically selects an appropriate
|
||
// output format (mp4, mkv, etc.) based on the file extension (outputPath).
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
|
||
fixed (AVFormatContext** ppFormatCtx = &_pFormatContext)
|
||
{
|
||
ffmpeg.avformat_alloc_output_context2(ppFormatCtx, null, null, outputPath);
|
||
}
|
||
|
||
if (_pFormatContext == null) throw new Exception("Failed to create output format context.");
|
||
|
||
// ── Step 2: Add video stream ──
|
||
// Creates a single video track (stream) inside the container.
|
||
// Additional audio streams can be created here if needed.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
|
||
_pVideoStream = ffmpeg.avformat_new_stream(_pFormatContext, null);
|
||
if (_pVideoStream == null) throw new Exception("Failed to create video stream.");
|
||
|
||
// ── Step 3: Select and open encoder ──
|
||
// Iterates through the priority list returned by GetEncoderCandidates()
|
||
// to find the first available encoder.
|
||
var candidates = GetEncoderCandidates();
|
||
var openedSuccessfully = false;
|
||
|
||
foreach (var encoder in candidates)
|
||
{
|
||
// avcodec_find_encoder_by_name: Searches for an encoder by name (e.g., "h264_nvenc").
|
||
// Returns null if the encoder is not included in the build.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html
|
||
var pCodec = ffmpeg.avcodec_find_encoder_by_name(encoder.Name);
|
||
if (pCodec == null) continue;
|
||
|
||
// avcodec_alloc_context3: Allocates a codec context for the selected encoder.
|
||
// All encoding parameters such as resolution, framerate, quality, etc., are set on this context.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
|
||
_pCodecContext = ffmpeg.avcodec_alloc_context3(pCodec);
|
||
if (_pCodecContext == null) continue;
|
||
|
||
_pCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL;
|
||
|
||
// ── Basic video parameters configuration ──
|
||
_pCodecContext->width = _settings.Width;
|
||
_pCodecContext->height = _settings.Height;
|
||
_pCodecContext->color_range = AVColorRange.AVCOL_RANGE_JPEG;
|
||
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
|
||
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
|
||
// so setting 0 (auto) is required for parallel encoding utilizing all cores.
|
||
_pCodecContext->thread_count = 0;
|
||
|
||
// ── Pixel format configuration ──
|
||
// Video encoders generally expect input in YUV420P or NV12 format.
|
||
//
|
||
// • YUV420P: 3 independent planes Y/U/V. Used by most software encoders.
|
||
// • NV12: Y plane + UV interleaved plane. Preferred by hardware encoders (NVENC, AMF, QSV).
|
||
// • QSV Special Note: pix_fmt of codec context is AV_PIX_FMT_QSV (hardware surface identifier),
|
||
// while actual sws_scale output is set to NV12 (software format).
|
||
// FFmpeg internally handles mapping between NV12 ↔ QSV surfaces.
|
||
//
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/pixfmt_8h.html
|
||
var selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
|
||
if (encoder.Kind == EncoderKind.QSV)
|
||
{
|
||
selectedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
||
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_QSV;
|
||
}
|
||
else if (encoder.Kind is EncoderKind.NVENC or EncoderKind.AMF)
|
||
{
|
||
selectedPixFmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
||
_pCodecContext->pix_fmt = selectedPixFmt;
|
||
}
|
||
else if (encoder == VideoEncoder.LibVvenc)
|
||
{
|
||
// libvvenc: Only 10-bit YUV420P10LE supported (yuv420p verified unsupported in this build)
|
||
selectedPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P10LE;
|
||
_pCodecContext->pix_fmt = selectedPixFmt;
|
||
}
|
||
else
|
||
{
|
||
_pCodecContext->pix_fmt = selectedPixFmt;
|
||
}
|
||
|
||
// ── Framerate and timebase configuration ──
|
||
// time_base: Base unit of time interval between frames.
|
||
// e.g., 60fps → time_base = 1/60 sec (num=1, den=60)
|
||
// Multiplying PTS (Presentation Time Stamp) value by time_base yields actual time in seconds.
|
||
// PTS=150, time_base=1/60 → 150 * (1/60) = 2.5 seconds
|
||
//
|
||
// framerate: Framerate value giving hints to the encoder (reciprocal of time_base).
|
||
//
|
||
// RationalUtil: Converts FPS (double) into fractional form (numerator/denominator).
|
||
// e.g., 29.97 → 30000/1001 (NTSC standard)
|
||
//
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
||
var fpsRational = RationalUtil.Double2MaxPrecisionInt(_settings.FPS);
|
||
|
||
// NTSC
|
||
if (Math.Abs(_settings.FPS - 29.97) < 0.001) fpsRational = (30000, 1001);
|
||
if (Math.Abs(_settings.FPS - 59.94) < 0.001) fpsRational = (60000, 1001);
|
||
if (Math.Abs(_settings.FPS - 23.976) < 0.001 || Math.Abs(_settings.FPS - 23.98) < 0.001) fpsRational = (24000, 1001);
|
||
|
||
_pCodecContext->time_base = new AVRational
|
||
{ num = fpsRational.Denominator, den = fpsRational.Numerator };
|
||
_pCodecContext->framerate = new AVRational
|
||
{ num = fpsRational.Numerator, den = fpsRational.Denominator };
|
||
|
||
// ── GOP (Group of Pictures) size configuration ──
|
||
// GOP: Keyframe (I-frame) interval. e.g., gop_size=120, 60fps → keyframe every 2 seconds.
|
||
// Keyframes contain full images and serve as seek points.
|
||
// Too large reduces seek precision, too small increases file size.
|
||
var calculatedGop = Math.Max(1, (int)Math.Round(_settings.FPS * _settings.GopSize));
|
||
if (encoder == VideoEncoder.LibVvenc)
|
||
{
|
||
// libvvenc (VVenC) requires IntraPeriod (gop_size) to be a multiple of its GOPSize (typically 32).
|
||
_pCodecContext->gop_size = Math.Max(32, ((calculatedGop + 15) / 32) * 32);
|
||
}
|
||
else
|
||
{
|
||
_pCodecContext->gop_size = calculatedGop;
|
||
}
|
||
|
||
// ── Disable B-frames ──
|
||
// B-frames (Bidirectional predictive frames): Increase compression ratio by referencing before/after frames,
|
||
// but increase encoding latency and may cause compatibility issues on some encoders.
|
||
// In real-time recording scenarios, disable them to minimize latency.
|
||
//
|
||
// Reference: has_b_frames is a read-only field internally set by the encoder,
|
||
// so do not set it directly.
|
||
//
|
||
// SVT-AV1 Special Note: Setting max_b_frames directly on AVCodecContext
|
||
// causes a "bad parameter" error, so control via svtav1-params.
|
||
// pred-struct=1: Low Delay (IPPP) structure disabling B-frames.
|
||
if (encoder == VideoEncoder.LibSvtAv1)
|
||
{
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "svtav1-params", "pred-struct=1", 0);
|
||
}
|
||
else if (encoder == VideoEncoder.LibVvenc)
|
||
{
|
||
// libvvenc relies on hierarchical GOP structures; forcing max_b_frames = 0 causes initialization failure.
|
||
// We explicitly set max_b_frames to -1 to allow libvvenc to use its default GOP reference distance.
|
||
_pCodecContext->max_b_frames = -1;
|
||
}
|
||
else
|
||
{
|
||
_pCodecContext->max_b_frames = 0;
|
||
}
|
||
|
||
// ── Global header flag ──
|
||
// Some container formats (mp4, etc.) require codec configuration info (SPS/PPS, etc.)
|
||
// to be recorded once in the file header rather than in each packet.
|
||
// If format has AVFMT_GLOBALHEADER flag set, turn on AV_CODEC_FLAG_GLOBAL_HEADER
|
||
// so the encoder stores configuration info in extradata.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/avformat_8h.html
|
||
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0)
|
||
{
|
||
_pCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER;
|
||
}
|
||
|
||
// ── Apply quality/bitrate settings ──
|
||
ApplyQualitySettings(encoder);
|
||
|
||
// ── QSV Encoder Hardware Device Initialization ──
|
||
// QSV (Quick Sync Video) uses fixed-function encoding hardware on Intel GPUs.
|
||
// Using QSV in FFmpeg requires creating a hardware device context.
|
||
//
|
||
// 3 strategies are attempted sequentially below:
|
||
// Strategy 1) Create QSV device directly with child_device_type=d3d11va option
|
||
// → FFmpeg internally creates D3D11 device as child and connects to QSV
|
||
// Strategy 2) Create D3D11VA device first, then derive QSV device
|
||
// → Effective when Strategy 1 fails on some drivers
|
||
// Strategy 3) Create QSV device directly (legacy method)
|
||
// → Fallback on older drivers/systems
|
||
//
|
||
// After obtaining device, attempt using GPU frame pool:
|
||
// hw_frames_ctx: Pre-allocate frame pool in GPU VRAM (zero-copy, highest performance)
|
||
// hw_device_ctx: Encoder reads frames directly from system memory (better compatibility)
|
||
//
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
if (encoder.Kind == EncoderKind.QSV)
|
||
{
|
||
AVBufferRef* hwDeviceCtx = null;
|
||
var qsvReady = false;
|
||
int err;
|
||
|
||
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
|
||
string childDeviceName = isWindows ? "d3d11va" : "vaapi";
|
||
AVHWDeviceType childDeviceType = isWindows ? AVHWDeviceType.AV_HWDEVICE_TYPE_D3D11VA : AVHWDeviceType.AV_HWDEVICE_TYPE_VAAPI;
|
||
|
||
// ── Strategy 1: child_device_type option ──
|
||
// Passing "child_device_type" dictionary to av_hwdevice_ctx_create
|
||
// lets FFmpeg internally create an appropriate lower device (Windows: D3D11, Linux: VAAPI) and connect to QSV session.
|
||
{
|
||
AVDictionary* opts = null;
|
||
ffmpeg.av_dict_set(&opts, "child_device_type", childDeviceName, 0);
|
||
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, opts, 0);
|
||
ffmpeg.av_dict_free(&opts);
|
||
if (err >= 0)
|
||
{
|
||
qsvReady = true;
|
||
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Successfully created QSV device using child_device_type={childDeviceName}");
|
||
}
|
||
else
|
||
{
|
||
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: child_device_type failed — {GetFFmpegErrorString(err)} (Code: {err})");
|
||
}
|
||
}
|
||
|
||
// ── Strategy 2: Lower device → QSV derive ──
|
||
// Create lower device (D3D11VA / VAAPI) first,
|
||
// then derive QSV device using av_hwdevice_ctx_create_derived.
|
||
if (!qsvReady)
|
||
{
|
||
AVBufferRef* childDeviceCtx = null;
|
||
err = ffmpeg.av_hwdevice_ctx_create(&childDeviceCtx, childDeviceType, null, null, 0);
|
||
if (err >= 0)
|
||
{
|
||
err = ffmpeg.av_hwdevice_ctx_create_derived(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, childDeviceCtx, 0);
|
||
if (err >= 0)
|
||
{
|
||
qsvReady = true;
|
||
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: Successfully created device by deriving {childDeviceType} → QSV");
|
||
}
|
||
ffmpeg.av_buffer_unref(&childDeviceCtx);
|
||
}
|
||
}
|
||
|
||
// ── Strategy 3: Direct QSV creation (legacy) ──
|
||
if (!qsvReady)
|
||
{
|
||
err = ffmpeg.av_hwdevice_ctx_create(&hwDeviceCtx, AVHWDeviceType.AV_HWDEVICE_TYPE_QSV, null, null, 0);
|
||
if (err >= 0) qsvReady = true;
|
||
}
|
||
|
||
if (!qsvReady)
|
||
{
|
||
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: All QSV device creation strategies failed. Testing next candidate codec.");
|
||
{
|
||
var pCtx = _pCodecContext;
|
||
ffmpeg.avcodec_free_context(&pCtx);
|
||
}
|
||
_pCodecContext = null;
|
||
continue;
|
||
}
|
||
|
||
// ── Attempt hw_frames_ctx: Pre-create GPU frame pool ──
|
||
// AVHWFramesContext pre-allocates frame pool in GPU VRAM,
|
||
// eliminating memory allocation/deallocation overhead for each frame.
|
||
//
|
||
// Pros: Zero-copy capability, highest performance
|
||
// Cons: May fail depending on GPU/driver resolution/format constraints
|
||
// (e.g. specific texture size constraints on Intel Arc → E_INVALIDARG)
|
||
//
|
||
// initial_pool_size: Number of frames to pre-allocate. Matches encoder pipeline depth.
|
||
// Align resolution to multiples of 32: Texture alignment condition required by QSV hardware.
|
||
//
|
||
// Reference:
|
||
// AVHWFramesContext: https://ffmpeg.org/doxygen/trunk/structAVHWFramesContext.html
|
||
// av_hwframe_ctx_alloc: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
bool framesCtxReady = false;
|
||
AVBufferRef* hwFramesRef = ffmpeg.av_hwframe_ctx_alloc(hwDeviceCtx);
|
||
if (hwFramesRef != null)
|
||
{
|
||
var framesCtx = (AVHWFramesContext*)hwFramesRef->data;
|
||
framesCtx->format = AVPixelFormat.AV_PIX_FMT_QSV; // Hardware surface format
|
||
framesCtx->sw_format = AVPixelFormat.AV_PIX_FMT_NV12; // Software-side format
|
||
framesCtx->width = (_settings.Width + 31) & ~31; // 32-multiple alignment (bitmask)
|
||
framesCtx->height = (_settings.Height + 31) & ~31; // 32-multiple alignment (bitmask)
|
||
framesCtx->initial_pool_size = 4; // Frame pool size
|
||
|
||
err = ffmpeg.av_hwframe_ctx_init(hwFramesRef);
|
||
if (err >= 0)
|
||
{
|
||
// Success: Connect hw_frames_ctx to codec context.
|
||
// Increment ref count with av_buffer_ref to keep valid after hwFramesRef is released.
|
||
_pCodecContext->hw_frames_ctx = ffmpeg.av_buffer_ref(hwFramesRef);
|
||
_isQsvEncoder = true;
|
||
framesCtxReady = true;
|
||
Melon<RenderMod>.Logger.Msg($"{encoder.Name}: hw_frames_ctx initialization succeeded (GPU frame pool mode)");
|
||
|
||
_pHwFrame = ffmpeg.av_frame_alloc();
|
||
if (_pHwFrame == null)
|
||
throw new Exception("Failed to allocate QSV hardware frame structure.");
|
||
}
|
||
else
|
||
{
|
||
Melon<RenderMod>.Logger.Warning($"{encoder.Name}: hw_frames_ctx failed — {GetFFmpegErrorString(err)} (Code: {err}). Falling back to hw_device_ctx mode.");
|
||
}
|
||
ffmpeg.av_buffer_unref(&hwFramesRef);
|
||
}
|
||
|
||
// ── When hw_frames_ctx fails: Fallback to hw_device_ctx + NV12 system memory ──
|
||
// In hw_device_ctx mode, the encoder receives NV12 frames from system memory (RAM)
|
||
// directly and uploads to GPU internally.
|
||
// Slightly slower than hw_frames_ctx, but compatibility is much better.
|
||
//
|
||
// Change pix_fmt to NV12: Use software format directly instead of hardware surface (QSV).
|
||
// _isQsvEncoder = false: Skip manual upload in WriteFrame().
|
||
if (!framesCtxReady)
|
||
{
|
||
_pCodecContext->pix_fmt = AVPixelFormat.AV_PIX_FMT_NV12;
|
||
_pCodecContext->hw_device_ctx = ffmpeg.av_buffer_ref(hwDeviceCtx);
|
||
_isQsvEncoder = false; // System memory mode, no manual frame upload required
|
||
}
|
||
|
||
// Transfer ownership of hwDeviceCtx to codec context,
|
||
// so release local reference (reference count management).
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__buffer.html
|
||
ffmpeg.av_buffer_unref(&hwDeviceCtx);
|
||
}
|
||
|
||
// ── Specify per-encoder speed preset ──
|
||
// Presets determine the tradeoff between encoding speed ↔ compression efficiency.
|
||
// Fast preset: Fast encoding, but larger file size
|
||
// Slow preset: Slow encoding, but smaller file size at same quality
|
||
//
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__opt__set__funcs.html
|
||
if (encoder.Kind == EncoderKind.NVENC)
|
||
{
|
||
// NVENC preset range: p1 (fastest) ~ p7 (highest quality). p4 = medium (default)
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "p4", 0);
|
||
}
|
||
else if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265)
|
||
{
|
||
// x264/x265 presets: ultrafast, superfast, veryfast, faster, fast, medium, slow, ...
|
||
// CPU encoding easily becomes a bottleneck, so use fast presets
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "veryfast", 0);
|
||
}
|
||
else if (encoder == VideoEncoder.LibSvtAv1)
|
||
{
|
||
// SVT-AV1 presets: 0 (highest quality/lowest speed) ~ 13 (highest speed/lowest quality)
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "10", 0);
|
||
}
|
||
else if (encoder == VideoEncoder.LibAomAv1)
|
||
{
|
||
// libaom cpu-used: 0 (highest quality) ~ 8 (highest speed)
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "cpu-used", "6", 0);
|
||
// "realtime" usage: Single pass, low latency encoding mode
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "usage", "realtime", 0);
|
||
}
|
||
else if (encoder == VideoEncoder.LibVvenc)
|
||
{
|
||
// libvvenc preset: slower, slow, medium, fast, faster. Default is medium, change to faster to gain performance
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "preset", "faster", 0);
|
||
}
|
||
// Other encoders (AMF, QSV, etc.) use default presets.
|
||
// Setting unsupported options can cause errors, so skip.
|
||
|
||
// ── Open encoder ──
|
||
// avcodec_open2: Initializes encoder based on all parameters configured above.
|
||
// Returns >= 0 on success, enabling frame encoding via avcodec_send_frame/avcodec_receive_packet afterwards.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
|
||
Melon<RenderMod>.Logger.Msg($"[Encoder Diagnostic] {encoder.Name}: {_pCodecContext->width}x{_pCodecContext->height}, pix_fmt={_pCodecContext->pix_fmt}, gop={_pCodecContext->gop_size}, max_b_frames={_pCodecContext->max_b_frames}, framerate={_pCodecContext->framerate.num}/{_pCodecContext->framerate.den}, thread_count={_pCodecContext->thread_count}, bit_rate={_pCodecContext->bit_rate}, rc_max_rate={_pCodecContext->rc_max_rate}, rc_buffer_size={_pCodecContext->rc_buffer_size}, mode={_settings.Mode}");
|
||
var openResult = ffmpeg.avcodec_open2(_pCodecContext, pCodec, null);
|
||
if (openResult >= 0)
|
||
{
|
||
var isHardware = encoder.IsHardware;
|
||
var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage";
|
||
Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})");
|
||
_dstPixFmt = selectedPixFmt;
|
||
openedSuccessfully = true;
|
||
break;
|
||
}
|
||
|
||
// If open fails, free codec context and try next encoder.
|
||
Melon<RenderMod>.Logger.Warning($"Failed to initialize hardware encoder {encoder.Name}: {GetFFmpegErrorString(openResult)} (Code: {openResult}). Testing next candidate codec.");
|
||
var pCodecCtx = _pCodecContext;
|
||
ffmpeg.avcodec_free_context(&pCodecCtx);
|
||
_pCodecContext = null;
|
||
}
|
||
|
||
if (!openedSuccessfully || _pCodecContext == null)
|
||
{
|
||
throw new Exception("Failed to initialize any available video encoder (both hardware accelerated and CPU software).");
|
||
}
|
||
|
||
// ── Step 4: Copy codec parameters to stream ──
|
||
// Copy parameters set in codec context (resolution, codec ID, profile, etc.)
|
||
// to stream's codecpar. Information to be written to container header.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
|
||
if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0)
|
||
throw new Exception("Failed to copy codec parameters to stream.");
|
||
|
||
// ── Step 4.5 (Optional): Initialize audio stream ──
|
||
// If EncoderSettings.Audio is set, add audio stream.
|
||
if (_settings.Audio.HasValue)
|
||
{
|
||
InitializeAudioStream(_settings.Audio.Value);
|
||
}
|
||
|
||
// ── Step 5: Open output file I/O ──
|
||
// Formats without AVFMT_NOFILE flag require actual file I/O.
|
||
// Open file in write mode using avio_open and connect to pb (I/O context) of format context.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
|
||
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
|
||
{
|
||
if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0)
|
||
throw new Exception($"Cannot open output file {outputPath}.");
|
||
}
|
||
|
||
// Write file header (ftyp, moov, container metadata).
|
||
// Packets can be written with av_interleaved_write_frame after this point.
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
|
||
if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0)
|
||
throw new Exception("Failed to write file header.");
|
||
|
||
// ── Step 6: Initialize frame buffers and colorspace converter ──
|
||
// av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet).
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
|
||
_pSrcFrame = ffmpeg.av_frame_alloc();
|
||
_pDstFrame = ffmpeg.av_frame_alloc();
|
||
_pPacket = ffmpeg.av_packet_alloc();
|
||
if (_pSrcFrame == null || _pDstFrame == null || _pPacket == null)
|
||
throw new Exception("Failed to allocate frame or packet structure.");
|
||
|
||
// Set format and size of destination frame, then allocate actual pixel buffer.
|
||
// av_frame_get_buffer: Allocates memory matching format, width, height and
|
||
// sets data[]/linesize[] pointers. Second argument 0 means default alignment.
|
||
_pDstFrame->format = (int)_dstPixFmt;
|
||
_pDstFrame->width = _settings.Width;
|
||
_pDstFrame->height = _settings.Height;
|
||
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
|
||
|
||
if (ffmpeg.av_frame_get_buffer(_pDstFrame, 0) < 0)
|
||
throw new Exception($"Failed to allocate {_dstPixFmt} frame buffer.");
|
||
|
||
// sws_getContext: Initialize software scaler (colorspace converter).
|
||
// Input: RGBA (Unity capture format, 4 bytes/pixel)
|
||
// Output: YUV420P or NV12 (encoder required format)
|
||
//
|
||
// Colorspace is converted without resolution change here, but sws_scale
|
||
// can also perform resizing simultaneously.
|
||
//
|
||
// SWS_POINT: Nearest neighbor interpolation. Since only colorspace is converted without resolution change,
|
||
// interpolation algorithm does not affect output quality, and SWS_POINT is fastest.
|
||
//
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html
|
||
_pSwsContext = ffmpeg.sws_getContext(
|
||
_settings.Width, _settings.Height, AVPixelFormat.AV_PIX_FMT_RGBA,
|
||
_settings.Width, _settings.Height, _dstPixFmt,
|
||
(int)SwsFlags.SWS_POINT, null, null, null);
|
||
|
||
if (_pSwsContext == null)
|
||
throw new Exception("Failed to initialize software scaler.");
|
||
|
||
_isInitialized = true;
|
||
_loggedConversionMode = false;
|
||
}
|
||
catch (NotSupportedException ex)
|
||
{
|
||
Dispose();
|
||
throw new Exception("Failed to load FFmpeg native library (DLL). Please check if FFmpeg DLL files (avcodec, avformat, etc.) in UserLibs folder are missing or corrupted, delete the DLL files in UserLibs, and restart the game.", ex);
|
||
}
|
||
catch (Exception)
|
||
{
|
||
// Clean up all partially allocated resources if failure occurs during initialization.
|
||
Dispose();
|
||
throw;
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
// ================================================================
|
||
// Quality / Bitrate Settings
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Sets quality parameters on the codec context according to the rate control mode (CBR, VBR, CQP) selected by the user.
|
||
///
|
||
/// ── Bitrate Control Mode Description ──
|
||
///
|
||
/// • CBR (Constant Bitrate): Maintains a constant bitrate. Suitable for streaming.
|
||
/// → bit_rate = rc_max_rate = rc_min_rate (all identical)
|
||
///
|
||
/// • VBR (Variable Bitrate): Bitrate varies according to scene complexity.
|
||
/// → bit_rate = target value, rc_max_rate = maximum allowed value
|
||
///
|
||
/// • CQP (Constant Quantization Parameter): Uses a fixed quantization value.
|
||
/// → Lower value = higher quality/larger size, Higher value = lower quality/smaller size
|
||
/// → Configuration method varies per encoder (see below)
|
||
///
|
||
/// Reference - CQP options per encoder:
|
||
/// NVENC: "qp" (single value applied to I/P/B frames)
|
||
/// AMF: "qp_i" + "qp_p" (I-frame and P-frame configured separately)
|
||
/// QSV: "global_quality" (Intel unique option)
|
||
/// Software: "crf" (Constant Rate Factor, smarter quality control)
|
||
///
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/structAVCodecContext.html
|
||
/// </summary>
|
||
private void ApplyQualitySettings(VideoEncoder encoder)
|
||
{
|
||
switch (_settings.Mode)
|
||
{
|
||
case RateControlMode.CBR:
|
||
// kbps → bps conversion (FFmpeg bit_rate field is in bps)
|
||
_pCodecContext->bit_rate = _settings.TargetBitrate * 1000L;
|
||
|
||
// libvvenc does not support rc_min_rate/rc_max_rate/rc_buffer_size,
|
||
// so set bit_rate only and delegate the rest to internal rate control.
|
||
if (encoder != VideoEncoder.LibVvenc)
|
||
{
|
||
// In CBR, set target/max/min bitrates all equal.
|
||
// rc_buffer_size determines the bitrate buffering size.
|
||
_pCodecContext->rc_max_rate = _pCodecContext->rc_min_rate = _settings.TargetBitrate * 1000L;
|
||
_pCodecContext->rc_buffer_size = _settings.TargetBitrate * 1000;
|
||
|
||
// nal-hrd=cbr: x264/x265 dedicated option. Generates CBR bitstream matching
|
||
// NAL HRD (Hypothetical Reference Decoder) specification. May be required for streaming compatibility.
|
||
if (encoder == VideoEncoder.Libx264 || encoder == VideoEncoder.Libx265)
|
||
{
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "nal-hrd", "cbr", 0);
|
||
}
|
||
}
|
||
break;
|
||
|
||
case RateControlMode.VBR:
|
||
// kbps → bps conversion
|
||
_pCodecContext->bit_rate = _settings.TargetBitrate * 1000L;
|
||
|
||
// Skip libvvenc as it does not support rc_max_rate/rc_buffer_size.
|
||
if (encoder != VideoEncoder.LibVvenc)
|
||
{
|
||
// If MaxBitrate is larger than TargetBitrate, use as is;
|
||
// otherwise set 1.5x TargetBitrate as max rate.
|
||
var maxRate = _settings.MaxBitrate > _settings.TargetBitrate
|
||
? _settings.MaxBitrate * 1000L : _settings.TargetBitrate * 1500L;
|
||
|
||
_pCodecContext->rc_max_rate = maxRate;
|
||
// Set buffer size to 2x max bitrate to accommodate temporary bitrate spikes.
|
||
_pCodecContext->rc_buffer_size = (int)(maxRate * 2);
|
||
}
|
||
break;
|
||
|
||
case RateControlMode.CQP:
|
||
// Default value 18: Quality close to "visually lossless" in H.264.
|
||
// Provides similar level of quality in other codecs as well.
|
||
var qpValue = _settings.QualityValue is >= 0 and <= 51 ? _settings.QualityValue : 18;
|
||
|
||
if (encoder.Kind == EncoderKind.NVENC)
|
||
{
|
||
// NVENC: Applies same QP to I/P/B frames with single "qp" option
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp", qpValue.ToString(), 0);
|
||
}
|
||
else if (encoder.Kind == EncoderKind.AMF)
|
||
{
|
||
// AMF: QP for I-frame (qp_i) and P-frame (qp_p) must be set separately.
|
||
// Since B-frames are disabled, qp_b is not set.
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp_i", qpValue.ToString(), 0);
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp_p", qpValue.ToString(), 0);
|
||
}
|
||
else if (encoder.Kind == EncoderKind.QSV)
|
||
{
|
||
// QSV: Uses ICQ (Intelligent Constant Quality) mode with "global_quality" option
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "global_quality", qpValue.ToString(), 0);
|
||
}
|
||
else if (encoder == VideoEncoder.LibVvenc)
|
||
{
|
||
// libvvenc: Specifies Constant QP mode with "qp" option
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "qp", qpValue.ToString(), 0);
|
||
}
|
||
else // Software encoder (libx264, libx265, libvpx-vp9, etc.)
|
||
{
|
||
// CRF (Constant Rate Factor): Smarter quality control than QP.
|
||
// Automatically adjusts bitrate according to scene complexity to maintain consistent visual quality.
|
||
// Range: 0 (lossless) ~ 51 (lowest quality). Typically between 18 and 28.
|
||
ffmpeg.av_opt_set(_pCodecContext->priv_data, "crf", qpValue.ToString(), 0);
|
||
}
|
||
break;
|
||
default:
|
||
throw new ArgumentOutOfRangeException();
|
||
}
|
||
}
|
||
|
||
// ================================================================
|
||
// Write Frame
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Encodes a single frame receiving RGBA pixel data (byte* pointer) captured from Unity.
|
||
///
|
||
/// Unity textures use Bottom-Left origin ((0,0) at bottom-left of screen),
|
||
/// while video standards use Top-Left origin ((0,0) at top-left of screen).
|
||
/// To resolve this, a "negative stride trick" is used:
|
||
///
|
||
/// ── Negative stride (linesize) trick ──
|
||
/// • Set data[0] to the start address of the last row (bottom-most line) of the buffer
|
||
/// • Set linesize[0] to -stride (negative)
|
||
/// → sws_scale reads rows in reverse, naturally performing vertical flip (vflip)
|
||
/// → Highly efficient as no extra memory copy or separate flip operation is required
|
||
///
|
||
/// stride = Width × 4 (RGBA has 4 bytes per pixel)
|
||
/// </summary>
|
||
/// <param name="buffer">Starting address of Unity RGBA pixel data</param>
|
||
public void WriteFrameFromBuffer(byte* buffer)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized) return;
|
||
|
||
var stride = _settings.Width * 4; // RGBA = 4 bytes per pixel
|
||
_pSrcFrame->data[0] = buffer + stride * (_settings.Height - 1); // Starting address of last row
|
||
_pSrcFrame->linesize[0] = -stride; // Vertical flip with negative stride
|
||
_pSrcFrame->width = _settings.Width;
|
||
_pSrcFrame->height = _settings.Height;
|
||
_pSrcFrame->format = (int)AVPixelFormat.AV_PIX_FMT_RGBA;
|
||
|
||
WriteFrame();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Encodes a frame by obtaining a read-only pointer from Unity NativeArray<byte>.
|
||
/// NativeArray is managed by Unity's native memory allocator,
|
||
/// allowing processing of large amounts of data without GC (Garbage Collector) pressure.
|
||
/// </summary>
|
||
public void WriteFrameFromNativeBuffer(NativeArray<byte> buffer)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized) return;
|
||
WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr());
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Encodes a frame from IntPtr (native pointer).
|
||
/// Can be used from external native plugins or custom capture sources.
|
||
/// </summary>
|
||
public void SendDataFromNativeBuffer(IntPtr pointer, int length)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized) return;
|
||
var preConvertedSize = _settings.Width * _settings.Height * 3 / 2;
|
||
|
||
if (!_loggedConversionMode)
|
||
{
|
||
_loggedConversionMode = true;
|
||
Melon<RenderMod>.Logger.Msg(length == preConvertedSize
|
||
? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
|
||
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
|
||
}
|
||
|
||
if (length == preConvertedSize)
|
||
{
|
||
WritePreConvertedFrame((byte*)pointer.ToPointer());
|
||
}
|
||
else
|
||
{
|
||
WriteFrameFromBuffer((byte*)pointer.ToPointer());
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Encodes a frame from a managed byte[].
|
||
/// Pins the array using fixed keyword to prevent GC from moving it before passing the pointer.
|
||
/// </summary>
|
||
public void SendDataFromBuffer(byte[] buffer, int length)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized) return;
|
||
fixed (byte* pBuffer = buffer)
|
||
{
|
||
var preConvertedSize = _settings.Width * _settings.Height * 3 / 2;
|
||
|
||
if (!_loggedConversionMode)
|
||
{
|
||
_loggedConversionMode = true;
|
||
Melon<RenderMod>.Logger.Msg(length == preConvertedSize
|
||
? "[Encoder] High-speed GPU color conversion enabled (NV12/YUV420P Direct Write, bypassing sws_scale)"
|
||
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
|
||
}
|
||
|
||
if (length == preConvertedSize)
|
||
{
|
||
WritePreConvertedFrame(pBuffer);
|
||
}
|
||
else
|
||
{
|
||
WriteFrameFromBuffer(pBuffer);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Encodes a frame by directly copying already pre-converted NV12 or YUV420P frame data.
|
||
/// </summary>
|
||
private void WritePreConvertedFrame(byte* buffer)
|
||
{
|
||
ffmpeg.av_frame_make_writable(_pDstFrame);
|
||
|
||
var width = _settings.Width;
|
||
var height = _settings.Height;
|
||
|
||
switch (_dstPixFmt)
|
||
{
|
||
case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV:
|
||
{
|
||
// Copy Y Plane
|
||
var pSrcY = buffer;
|
||
var pDstY = _pDstFrame->data[0];
|
||
var dstStrideY = _pDstFrame->linesize[0];
|
||
|
||
if (width == dstStrideY)
|
||
{
|
||
Buffer.MemoryCopy(pSrcY, pDstY, dstStrideY * height, width * height);
|
||
}
|
||
else
|
||
{
|
||
for (var y = 0; y < height; y++)
|
||
{
|
||
Buffer.MemoryCopy(pSrcY + y * width, pDstY + y * dstStrideY, width, width);
|
||
}
|
||
}
|
||
|
||
// Copy UV Plane (positioned after Y plane size)
|
||
var pSrcUV = buffer + width * height;
|
||
var pDstUV = _pDstFrame->data[1];
|
||
var dstStrideUV = _pDstFrame->linesize[1];
|
||
var uvHeight = height / 2;
|
||
|
||
if (width == dstStrideUV)
|
||
{
|
||
Buffer.MemoryCopy(pSrcUV, pDstUV, dstStrideUV * uvHeight, width * uvHeight);
|
||
}
|
||
else
|
||
{
|
||
for (var y = 0; y < uvHeight; y++)
|
||
{
|
||
Buffer.MemoryCopy(pSrcUV + y * width, pDstUV + y * dstStrideUV, width, width);
|
||
}
|
||
}
|
||
|
||
break;
|
||
}
|
||
case AVPixelFormat.AV_PIX_FMT_YUV420P:
|
||
{
|
||
// Copy Y Plane
|
||
var pSrcY = buffer;
|
||
var pDstY = _pDstFrame->data[0];
|
||
var dstStrideY = _pDstFrame->linesize[0];
|
||
|
||
if (width == dstStrideY)
|
||
{
|
||
Buffer.MemoryCopy(pSrcY, pDstY, dstStrideY * height, width * height);
|
||
}
|
||
else
|
||
{
|
||
for (var y = 0; y < height; y++)
|
||
{
|
||
Buffer.MemoryCopy(pSrcY + y * width, pDstY + y * dstStrideY, width, width);
|
||
}
|
||
}
|
||
|
||
// Copy U Plane
|
||
var pSrcU = buffer + width * height;
|
||
var pDstU = _pDstFrame->data[1];
|
||
var dstStrideU = _pDstFrame->linesize[1];
|
||
var chromaWidth = width / 2;
|
||
var chromaHeight = height / 2;
|
||
|
||
if (chromaWidth == dstStrideU)
|
||
{
|
||
Buffer.MemoryCopy(pSrcU, pDstU, dstStrideU * chromaHeight, chromaWidth * chromaHeight);
|
||
}
|
||
else
|
||
{
|
||
for (var y = 0; y < chromaHeight; y++)
|
||
{
|
||
Buffer.MemoryCopy(pSrcU + y * chromaWidth, pDstU + y * dstStrideU, chromaWidth, chromaWidth);
|
||
}
|
||
}
|
||
|
||
// Copy V Plane
|
||
var pSrcV = pSrcU + chromaWidth * chromaHeight;
|
||
var pDstV = _pDstFrame->data[2];
|
||
var dstStrideV = _pDstFrame->linesize[2];
|
||
|
||
if (chromaWidth == dstStrideV)
|
||
{
|
||
Buffer.MemoryCopy(pSrcV, pDstV, dstStrideV * chromaHeight, chromaWidth * chromaHeight);
|
||
}
|
||
else
|
||
{
|
||
for (var y = 0; y < chromaHeight; y++)
|
||
{
|
||
Buffer.MemoryCopy(pSrcV + y * chromaWidth, pDstV + y * dstStrideV, chromaWidth, chromaWidth);
|
||
}
|
||
}
|
||
|
||
break;
|
||
}
|
||
default:
|
||
throw new NotSupportedException($"This pixel format ({_dstPixFmt}) does not support direct copying of pre-converted buffers.");
|
||
}
|
||
|
||
// QSV acceleration handling or encoder send
|
||
if (_isQsvEncoder)
|
||
{
|
||
ffmpeg.av_frame_unref(_pHwFrame);
|
||
try
|
||
{
|
||
var ret = ffmpeg.av_hwframe_get_buffer(_pCodecContext->hw_frames_ctx, _pHwFrame, 0);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to allocate QSV hardware frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
ret = ffmpeg.av_hwframe_transfer_data(_pHwFrame, _pDstFrame, 0);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to upload QSV frame — {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
_pHwFrame->pts = _frameIndex++;
|
||
SendFrameAndWritePackets(_pHwFrame);
|
||
}
|
||
finally
|
||
{
|
||
ffmpeg.av_frame_unref(_pHwFrame);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
_pDstFrame->pts = _frameIndex++;
|
||
SendFrameAndWritePackets(_pDstFrame);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Core logic for actual frame encoding.
|
||
///
|
||
/// Processing flow:
|
||
/// 1. av_frame_make_writable: Check if destination frame buffer is writable
|
||
/// (Allocates a new buffer if encoder is still referencing previous frame's buffer)
|
||
/// 2. sws_scale: RGBA → YUV420P/NV12 colorspace conversion
|
||
/// 3. (QSV hw_frames mode) av_hwframe_transfer_data: Upload NV12 → QSV hardware surface
|
||
/// 4. Pass to encoder after setting PTS
|
||
///
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_scale)
|
||
/// </summary>
|
||
private void WriteFrame()
|
||
{
|
||
// Ensure writable state since encoder might still be referencing previous frame's buffer.
|
||
// Internally allocates a new buffer if referenced.
|
||
ffmpeg.av_frame_make_writable(_pDstFrame);
|
||
|
||
// ── Colorspace conversion: RGBA → YUV420P or NV12 ──
|
||
// sws_scale parameters explanation:
|
||
// _pSwsContext: Pre-initialized conversion context
|
||
// _pSrcFrame->data: Input data plane array (RGBA uses data[0] only)
|
||
// _pSrcFrame->linesize: Input stride per plane (bytes/row)
|
||
// 0: First source row index to process
|
||
// _settings.Height: Number of rows to process (= full height)
|
||
// _pDstFrame->data: Output data plane array
|
||
// _pDstFrame->linesize: Output stride per plane
|
||
ffmpeg.sws_scale(
|
||
_pSwsContext,
|
||
_pSrcFrame->data, _pSrcFrame->linesize,
|
||
0, _settings.Height,
|
||
_pDstFrame->data, _pDstFrame->linesize
|
||
);
|
||
|
||
if (_isQsvEncoder)
|
||
{
|
||
// ── QSV hw_frames_ctx mode: System memory → GPU surface upload ──
|
||
//
|
||
// av_hwframe_get_buffer: Get an empty hardware frame from hw_frames_ctx frame pool.
|
||
// av_hwframe_transfer_data: Copy (upload) NV12 software frame data to QSV surface.
|
||
//
|
||
// Through this process, the encoder can read frames directly from GPU memory,
|
||
// minimizing PCIe bandwidth usage.
|
||
//
|
||
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__hwcontext.html
|
||
|
||
ffmpeg.av_frame_unref(_pHwFrame);
|
||
try
|
||
{
|
||
var ret = ffmpeg.av_hwframe_get_buffer(_pCodecContext->hw_frames_ctx, _pHwFrame, 0);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to allocate QSV hardware frame buffer — {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
ret = ffmpeg.av_hwframe_transfer_data(_pHwFrame, _pDstFrame, 0);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to upload QSV frame — {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
// PTS (Presentation Time Stamp): Point in time when this frame should be displayed.
|
||
// Increments _frameIndex by 1, actual time calculated as PTS × time_base.
|
||
_pHwFrame->pts = _frameIndex++;
|
||
SendFrameAndWritePackets(_pHwFrame);
|
||
}
|
||
finally
|
||
{
|
||
// Release reference immediately after use so it can be returned to pool.
|
||
ffmpeg.av_frame_unref(_pHwFrame);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// ── General Encoders (Software or hw_device_ctx mode) ──
|
||
// Pass software frame directly to encoder.
|
||
// In hw_device_ctx mode, encoder handles GPU upload internally.
|
||
_pDstFrame->pts = _frameIndex++;
|
||
SendFrameAndWritePackets(_pDstFrame);
|
||
}
|
||
}
|
||
|
||
// ================================================================
|
||
// Encoder Send/Receive and Packet Writing
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Encodes frames using FFmpeg's Send/Receive pattern and writes result packets to file.
|
||
///
|
||
/// ── Send/Receive API Description ──
|
||
/// Asynchronous encoding API introduced in FFmpeg 3.x:
|
||
///
|
||
/// avcodec_send_frame(ctx, frame)
|
||
/// → Pushes frame into encoder's internal queue.
|
||
/// → Starts encoding once enough frames are gathered.
|
||
///
|
||
/// avcodec_receive_packet(ctx, packet)
|
||
/// → Pops encoded packet (compressed data).
|
||
/// → EAGAIN: Output not ready yet (more frames needed)
|
||
/// → EOF: Encoding complete (on flush)
|
||
///
|
||
/// Since zero or more packets can result from a single send_frame call,
|
||
/// receive_packet is repeated until EAGAIN/EOF is returned.
|
||
///
|
||
/// ── Packet Timestamp Handling ──
|
||
/// av_packet_rescale_ts: Converts PTS/DTS from codec's time_base to stream's time_base.
|
||
/// Mandatory because codec and container may use different time_bases.
|
||
/// e.g., codec time_base=1/60, stream time_base=1/90000 → PTS 150 → PTS 225000
|
||
///
|
||
/// Reference:
|
||
/// Send/Receive API: https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html
|
||
/// Packet: https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html
|
||
/// Interleaved Write: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
|
||
/// </summary>
|
||
/// <param name="pFrame">Frame to encode. Pass null to flush encoder (output delayed frames).</param>
|
||
private void SendFrameAndWritePackets(AVFrame* pFrame)
|
||
{
|
||
// Push frame to encoder internal queue
|
||
var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame);
|
||
if (response < 0)
|
||
{
|
||
throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})");
|
||
}
|
||
|
||
// Pop all encoded packets and write to file.
|
||
try
|
||
{
|
||
while (true)
|
||
{
|
||
response = ffmpeg.avcodec_receive_packet(_pCodecContext, _pPacket);
|
||
|
||
// EAGAIN: Encoder needs more input frames → exit loop
|
||
// EOF: Encoder flush complete (all delayed frames outputted) → exit loop
|
||
if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
|
||
if (response < 0)
|
||
throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})");
|
||
|
||
// Convert PTS/DTS from codec timebase → stream timebase
|
||
ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base);
|
||
_pPacket->stream_index = _pVideoStream->index;
|
||
|
||
// av_interleaved_write_frame: Write packet to container.
|
||
// "Interleaved" means packets from multiple streams (video+audio) are interleaved
|
||
// and written in time order. This function is used even when only video is present.
|
||
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
|
||
if (response < 0)
|
||
throw new Exception($"Error occurred while writing packet to file. (Code: {response})");
|
||
|
||
// Unref packet data reference for reuse in next receive_packet.
|
||
ffmpeg.av_packet_unref(_pPacket);
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
// Safely unref packet reference even on abnormal exit (exception)
|
||
ffmpeg.av_packet_unref(_pPacket);
|
||
}
|
||
}
|
||
|
||
// ================================================================
|
||
// Encoding Termination (End)
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Finalizes encoding.
|
||
///
|
||
/// Most encoders queue frames in internal buffers (delayed frames)
|
||
/// and perform encoding when enough frames have accumulated. In End():
|
||
///
|
||
/// 1. Send a null frame to switch encoder to "flush mode".
|
||
/// → Encoder encodes and outputs all remaining frames in internal buffer.
|
||
/// → Pops packets until receive_packet returns EOF.
|
||
///
|
||
/// 2. av_write_trailer: Writes container trailer (end-of-file metadata).
|
||
/// For MP4, the moov atom (index info) is completed here.
|
||
/// ⚠️ If trailer is not written, the file becomes corrupted and unplayable!
|
||
///
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
|
||
/// </summary>
|
||
public void End()
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized) return;
|
||
|
||
// 1. Flush delayed frames from video encoder (send null frame as "end of input" signal)
|
||
try
|
||
{
|
||
SendFrameAndWritePackets(null);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
// Log error occurring during flush process
|
||
Melon<RenderMod>.Logger.Error($"Error during video encoder flush: {ex.Message}");
|
||
}
|
||
|
||
// 1.5. Flush remaining samples in audio FIFO + flush audio encoder
|
||
if (_audioEnabled)
|
||
{
|
||
try
|
||
{
|
||
FlushAudioEncoder();
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Melon<RenderMod>.Logger.Error($"Error during audio encoder flush: {ex.Message}");
|
||
}
|
||
}
|
||
|
||
// 2. Write container trailer (moov atom and file termination metadata)
|
||
if (_pFormatContext != null)
|
||
{
|
||
ffmpeg.av_write_trailer(_pFormatContext);
|
||
}
|
||
|
||
_isInitialized = false;
|
||
}
|
||
}
|
||
|
||
// ================================================================
|
||
// Resource Cleanup (Dispose)
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Actual resource cleanup logic. Cleanup order is important:
|
||
///
|
||
/// 1. Call End() (if not called yet) → flush delayed frames + write trailer
|
||
/// 2. SwsContext (scaler) → independent, can be freed first
|
||
/// 3. AVFrame (src/dst) → freed before codec as codec may reference them
|
||
/// 4. AVCodecContext (codec) → hw_frames_ctx/hw_device_ctx freed together
|
||
/// 5. AVFormatContext (format/IO) → includes closing file handle
|
||
///
|
||
/// Reference:
|
||
/// av_frame_free: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html
|
||
/// avcodec_free_context: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
|
||
/// avformat_free_context: https://ffmpeg.org/doxygen/trunk/group__lavf__core.html
|
||
/// avio_closep: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
|
||
/// </summary>
|
||
public void Dispose()
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed) return;
|
||
_disposed = true;
|
||
|
||
// Trigger End() safely in case Dispose is called without calling End() first
|
||
if (_isInitialized)
|
||
{
|
||
try { End(); } catch { /* Ignore */ }
|
||
}
|
||
|
||
// Release Sws scaler
|
||
if (_pSwsContext != null)
|
||
{
|
||
ffmpeg.sws_freeContext(_pSwsContext);
|
||
_pSwsContext = null;
|
||
}
|
||
|
||
// Release source frame memory (frame referencing Unity RGBA data)
|
||
if (_pSrcFrame != null)
|
||
{
|
||
var pFrame = _pSrcFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pSrcFrame = null;
|
||
}
|
||
|
||
// Release destination frame memory (YUV420P/NV12 converted frame)
|
||
if (_pDstFrame != null)
|
||
{
|
||
var pFrame = _pDstFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pDstFrame = null;
|
||
}
|
||
|
||
// Release QSV hardware frame memory
|
||
if (_pHwFrame != null)
|
||
{
|
||
var pFrame = _pHwFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pHwFrame = null;
|
||
}
|
||
|
||
// Release packet memory
|
||
if (_pPacket != null)
|
||
{
|
||
var pPacket = _pPacket;
|
||
ffmpeg.av_packet_free(&pPacket);
|
||
_pPacket = null;
|
||
}
|
||
|
||
// Release codec context
|
||
// This function internally frees hw_frames_ctx and hw_device_ctx as well.
|
||
if (_pCodecContext != null)
|
||
{
|
||
var pCodecCtx = _pCodecContext;
|
||
ffmpeg.avcodec_free_context(&pCodecCtx);
|
||
_pCodecContext = null;
|
||
}
|
||
|
||
// ── Release audio-related resources ──
|
||
DisposeAudioResources();
|
||
|
||
// Close format context and I/O
|
||
if (_pFormatContext != null)
|
||
{
|
||
// Must manually close avio when not AVFMT_NOFILE.
|
||
// (Opened with avio_open, so close with avio_closep)
|
||
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
|
||
{
|
||
var pPb = _pFormatContext->pb;
|
||
if (pPb != null)
|
||
{
|
||
ffmpeg.avio_closep(&pPb);
|
||
_pFormatContext->pb = null;
|
||
}
|
||
}
|
||
// Release format context (internal streams, etc. freed as well)
|
||
ffmpeg.avformat_free_context(_pFormatContext);
|
||
_pFormatContext = null;
|
||
}
|
||
|
||
GC.SuppressFinalize(this);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Finalizer (destructor). Executed when collected by GC without Dispose() being called.
|
||
/// Safety net to prevent native resource leaks.
|
||
/// ⚠️ Finalizer execution timing is not guaranteed, so always manage resources
|
||
/// using a using statement or explicit Dispose() call.
|
||
/// </summary>
|
||
~FFmpegNativeEncoder() { Dispose(); }
|
||
|
||
// ================================================================
|
||
// Audio Encoding
|
||
// ================================================================
|
||
|
||
/// <summary>
|
||
/// Initializes the audio stream.
|
||
///
|
||
/// Execution steps:
|
||
/// 1. Search audio encoder (codec) and allocate codec context
|
||
/// 2. Set sample rate, channel layout, sample format, bitrate
|
||
/// 3. Add audio stream (AVStream) and copy codec parameters
|
||
/// 4. Initialize resampler (SwrContext) (input format → encoder format conversion)
|
||
/// 5. Initialize FIFO buffer (arbitrary input size → collect in frame_size units)
|
||
/// 6. Allocate audio frame and packet structures
|
||
///
|
||
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__encoding.html
|
||
/// </summary>
|
||
private void InitializeAudioStream(AudioSettings audioSettings)
|
||
{
|
||
// ── 1. Determine and search audio codec ──
|
||
var audioCodecId = audioSettings.AudioCodecId != 0
|
||
? audioSettings.AudioCodecId
|
||
: AVCodecID.AV_CODEC_ID_AAC;
|
||
|
||
var pAudioCodec = ffmpeg.avcodec_find_encoder(audioCodecId);
|
||
if (pAudioCodec == null)
|
||
throw new Exception($"Audio encoder not found: {audioCodecId}");
|
||
|
||
// ── 2. Allocate codec context and configure parameters ──
|
||
_pAudioCodecContext = ffmpeg.avcodec_alloc_context3(pAudioCodec);
|
||
if (_pAudioCodecContext == null)
|
||
throw new Exception("Failed to allocate audio codec context.");
|
||
|
||
var sampleRate = audioSettings.SampleRate > 0 ? audioSettings.SampleRate : 44100;
|
||
var channels = audioSettings.Channels > 0 ? audioSettings.Channels : 2;
|
||
var bitrate = audioSettings.Bitrate > 0 ? audioSettings.Bitrate : 128000;
|
||
|
||
// Select the first sample format supported by the encoder.
|
||
// Most encoders prefer AV_SAMPLE_FMT_FLTP (float planar).
|
||
// AVCodec.sample_fmts is deprecated but still functional; avcodec_get_supported_config() is the replacement
|
||
#pragma warning disable CS0618
|
||
var encoderSampleFmt = pAudioCodec->sample_fmts != null ? pAudioCodec->sample_fmts[0] : AVSampleFormat.AV_SAMPLE_FMT_FLTP;
|
||
#pragma warning restore CS0618
|
||
|
||
_pAudioCodecContext->sample_rate = sampleRate;
|
||
_pAudioCodecContext->sample_fmt = encoderSampleFmt;
|
||
_pAudioCodecContext->bit_rate = bitrate;
|
||
_pAudioCodecContext->strict_std_compliance = ffmpeg.FF_COMPLIANCE_EXPERIMENTAL;
|
||
|
||
// Configure channel layout (FFmpeg 7.x+: use AVChannelLayout)
|
||
ffmpeg.av_channel_layout_default(&_pAudioCodecContext->ch_layout, channels);
|
||
|
||
// time_base: Audio typically uses 1/sample_rate.
|
||
_pAudioCodecContext->time_base = new AVRational { num = 1, den = sampleRate };
|
||
|
||
// Global header flag
|
||
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_GLOBALHEADER) != 0)
|
||
{
|
||
_pAudioCodecContext->flags |= ffmpeg.AV_CODEC_FLAG_GLOBAL_HEADER;
|
||
}
|
||
|
||
// Open encoder
|
||
var ret = ffmpeg.avcodec_open2(_pAudioCodecContext, pAudioCodec, null);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to initialize audio encoder: {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
Melon<RenderMod>.Logger.Msg($"Audio encoder successfully initialized: {Marshal.PtrToStringAnsi((IntPtr)pAudioCodec->name)} (sample_rate={sampleRate}, channels={channels}, fmt={encoderSampleFmt})");
|
||
|
||
// ── 3. Add audio stream ──
|
||
_pAudioStream = ffmpeg.avformat_new_stream(_pFormatContext, null);
|
||
if (_pAudioStream == null)
|
||
throw new Exception("Failed to create audio stream.");
|
||
|
||
if (ffmpeg.avcodec_parameters_from_context(_pAudioStream->codecpar, _pAudioCodecContext) < 0)
|
||
throw new Exception("Failed to copy audio codec parameters to stream.");
|
||
|
||
_pAudioStream->time_base = _pAudioCodecContext->time_base;
|
||
|
||
// ── 4. Initialize resampler (SwrContext) ──
|
||
// Input: interleaved float (Unity AudioClip.GetData default) → Output: Encoder required format
|
||
var inputSampleFmt = audioSettings.InputSampleFormat != 0
|
||
? audioSettings.InputSampleFormat
|
||
: AVSampleFormat.AV_SAMPLE_FMT_FLT;
|
||
|
||
// Allocate SwrContext
|
||
fixed (SwrContext** ppSwr = &_pSwrContext)
|
||
{
|
||
// Configure input channel layout
|
||
AVChannelLayout inLayout;
|
||
ffmpeg.av_channel_layout_default(&inLayout, channels);
|
||
|
||
ret = ffmpeg.swr_alloc_set_opts2(
|
||
ppSwr,
|
||
&_pAudioCodecContext->ch_layout, encoderSampleFmt, sampleRate,
|
||
&inLayout, inputSampleFmt, sampleRate,
|
||
0, null);
|
||
|
||
if (ret < 0 || _pSwrContext == null)
|
||
throw new Exception($"Failed to configure audio resampler options: {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
}
|
||
|
||
ret = ffmpeg.swr_init(_pSwrContext);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to initialize audio resampler: {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
// ── 5. Initialize FIFO buffer ──
|
||
_pAudioFifo = ffmpeg.av_audio_fifo_alloc(encoderSampleFmt, channels, 1);
|
||
if (_pAudioFifo == null)
|
||
throw new Exception("Failed to allocate audio FIFO buffer.");
|
||
|
||
// ── 6. Allocate audio frame and packet ──
|
||
_pAudioFrame = ffmpeg.av_frame_alloc();
|
||
_pAudioPacket = ffmpeg.av_packet_alloc();
|
||
if (_pAudioFrame == null || _pAudioPacket == null)
|
||
throw new Exception("Failed to allocate audio frame or packet structure.");
|
||
|
||
// Determine frame size: Some codecs (e.g. AAC) have a fixed frame_size.
|
||
// Use default 1024 if frame_size is 0 (e.g. PCM family).
|
||
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
|
||
_pAudioFrame->nb_samples = frameSize;
|
||
_pAudioFrame->format = (int)encoderSampleFmt;
|
||
ffmpeg.av_channel_layout_copy(&_pAudioFrame->ch_layout, &_pAudioCodecContext->ch_layout);
|
||
_pAudioFrame->sample_rate = sampleRate;
|
||
|
||
ret = ffmpeg.av_frame_get_buffer(_pAudioFrame, 0);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to allocate audio frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
_audioSampleIndex = 0;
|
||
_audioEnabled = true;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Encodes raw PCM audio sample data into the audio stream.
|
||
///
|
||
/// Can pass float[] array returned by Unity's AudioClip.GetData() or other PCM sources.
|
||
/// Internally executes resampling → FIFO buffering → frame_size unit encoding pipeline.
|
||
///
|
||
/// Thread safety: Safe to call from multiple threads as it uses _lock internally.
|
||
/// </summary>
|
||
/// <param name="samples">PCM audio sample data. Must be in interleaved format (LRLRLR...).</param>
|
||
/// <param name="numSamples">Number of samples per channel (= total float count / channel count).</param>
|
||
public void WriteAudioSamples(float[] samples, int numSamples)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized || !_audioEnabled) return;
|
||
|
||
fixed (float* pSamples = samples)
|
||
{
|
||
WriteAudioSamplesInternal((byte*)pSamples, numSamples);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Encodes raw PCM audio sample data from pointer into the audio stream.
|
||
/// </summary>
|
||
/// <param name="samples">Pointer to PCM audio sample data.</param>
|
||
/// <param name="numSamples">Number of samples per channel.</param>
|
||
public void WriteAudioSamples(IntPtr samples, int numSamples)
|
||
{
|
||
lock (_lock)
|
||
{
|
||
if (_disposed || !_isInitialized || !_audioEnabled) return;
|
||
WriteAudioSamplesInternal((byte*)samples.ToPointer(), numSamples);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Internal implementation of audio sample encoding.
|
||
///
|
||
/// Processing flow:
|
||
/// 1. Resample input PCM → encoder format via swr_convert
|
||
/// 2. Add resampled data to FIFO buffer
|
||
/// 3. Pop from FIFO when frame_size or more is accumulated and pass to encoder
|
||
///
|
||
/// Reference:
|
||
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
|
||
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
|
||
/// </summary>
|
||
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
|
||
{
|
||
// ── 1. Resampling: Input format → Encoder format ──
|
||
// Allocate frame for output buffer (temporary frame to hold resampling result)
|
||
var pConvertedFrame = ffmpeg.av_frame_alloc();
|
||
if (pConvertedFrame == null) return;
|
||
|
||
try
|
||
{
|
||
pConvertedFrame->nb_samples = numSamples;
|
||
pConvertedFrame->format = (int)_pAudioCodecContext->sample_fmt;
|
||
ffmpeg.av_channel_layout_copy(&pConvertedFrame->ch_layout, &_pAudioCodecContext->ch_layout);
|
||
pConvertedFrame->sample_rate = _pAudioCodecContext->sample_rate;
|
||
|
||
var ret = ffmpeg.av_frame_get_buffer(pConvertedFrame, 0);
|
||
if (ret < 0)
|
||
throw new Exception($"Failed to allocate resampling output frame buffer: {GetFFmpegErrorString(ret)} (Code: {ret})");
|
||
|
||
// swr_convert: Converts input data to format required by encoder.
|
||
var convertedSamples = ffmpeg.swr_convert(
|
||
_pSwrContext,
|
||
(byte**)&pConvertedFrame->data, numSamples,
|
||
&inputBuffer, numSamples);
|
||
|
||
switch (convertedSamples)
|
||
{
|
||
case < 0: throw new Exception($"Audio resampling failed: {GetFFmpegErrorString(convertedSamples)} (Code: {convertedSamples})");
|
||
case 0: return;
|
||
}
|
||
|
||
// ── 2. Add to FIFO buffer ──
|
||
var written = ffmpeg.av_audio_fifo_write(_pAudioFifo, (void**)&pConvertedFrame->data, convertedSamples);
|
||
if (written < convertedSamples) throw new Exception("Failed to write to audio FIFO");
|
||
|
||
// ── 3. Pop from FIFO in frame_size units and encode ──
|
||
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
|
||
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
|
||
{
|
||
ffmpeg.av_frame_make_writable(_pAudioFrame);
|
||
_pAudioFrame->nb_samples = frameSize;
|
||
|
||
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize);
|
||
if (read < frameSize) throw new Exception("Failed to read from audio FIFO");
|
||
|
||
_pAudioFrame->pts = _audioSampleIndex;
|
||
_audioSampleIndex += frameSize;
|
||
|
||
SendAudioFrameAndWritePackets(_pAudioFrame);
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
ffmpeg.av_frame_free(&pConvertedFrame);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Sends audio frame to encoder and writes result packets to file.
|
||
/// Uses same Send/Receive pattern as video SendFrameAndWritePackets.
|
||
/// </summary>
|
||
private void SendAudioFrameAndWritePackets(AVFrame* pFrame)
|
||
{
|
||
var response = ffmpeg.avcodec_send_frame(_pAudioCodecContext, pFrame);
|
||
if (response < 0)
|
||
{
|
||
throw new Exception($"Error occurred while sending frame to audio encoder: {GetFFmpegErrorString(response)} (Code: {response})");
|
||
}
|
||
|
||
try
|
||
{
|
||
while (true)
|
||
{
|
||
response = ffmpeg.avcodec_receive_packet(_pAudioCodecContext, _pAudioPacket);
|
||
if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
|
||
|
||
if (response < 0)
|
||
{
|
||
throw new Exception($"Error occurred while receiving packet from audio encoder: {GetFFmpegErrorString(response)} (Code: {response})");
|
||
}
|
||
|
||
// Convert codec timebase → stream timebase
|
||
ffmpeg.av_packet_rescale_ts(_pAudioPacket, _pAudioCodecContext->time_base, _pAudioStream->time_base);
|
||
_pAudioPacket->stream_index = _pAudioStream->index;
|
||
|
||
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pAudioPacket);
|
||
if (response < 0)
|
||
{
|
||
throw new Exception($"Error occurred while writing audio packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
|
||
}
|
||
|
||
ffmpeg.av_packet_unref(_pAudioPacket);
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
ffmpeg.av_packet_unref(_pAudioPacket);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Flushes the audio encoder.
|
||
/// Pads and encodes remaining samples (less than frame_size) in FIFO,
|
||
/// then sends null frame to output all delayed frames in encoder.
|
||
/// </summary>
|
||
private void FlushAudioEncoder()
|
||
{
|
||
if (_pAudioFifo == null || _pAudioCodecContext == null) return;
|
||
|
||
// Encode remaining samples in FIFO (last chunk smaller than frame_size)
|
||
var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo);
|
||
if (remaining > 0)
|
||
{
|
||
ffmpeg.av_frame_make_writable(_pAudioFrame);
|
||
_pAudioFrame->nb_samples = remaining;
|
||
|
||
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);
|
||
if (read > 0)
|
||
{
|
||
_pAudioFrame->pts = _audioSampleIndex;
|
||
_audioSampleIndex += read;
|
||
SendAudioFrameAndWritePackets(_pAudioFrame);
|
||
}
|
||
}
|
||
|
||
// Flush encoder by sending null frame
|
||
SendAudioFrameAndWritePackets(null);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Releases audio-related FFmpeg resources.
|
||
/// Called inside Dispose().
|
||
/// </summary>
|
||
private void DisposeAudioResources()
|
||
{
|
||
if (_pSwrContext != null)
|
||
{
|
||
fixed (SwrContext** ppSwr = &_pSwrContext)
|
||
{
|
||
ffmpeg.swr_free(ppSwr);
|
||
}
|
||
_pSwrContext = null;
|
||
}
|
||
|
||
if (_pAudioFifo != null)
|
||
{
|
||
ffmpeg.av_audio_fifo_free(_pAudioFifo);
|
||
_pAudioFifo = null;
|
||
}
|
||
|
||
if (_pAudioFrame != null)
|
||
{
|
||
var pFrame = _pAudioFrame;
|
||
ffmpeg.av_frame_free(&pFrame);
|
||
_pAudioFrame = null;
|
||
}
|
||
|
||
if (_pAudioPacket != null)
|
||
{
|
||
var pPacket = _pAudioPacket;
|
||
ffmpeg.av_packet_free(&pPacket);
|
||
_pAudioPacket = null;
|
||
}
|
||
|
||
if (_pAudioCodecContext != null)
|
||
{
|
||
var pCtx = _pAudioCodecContext;
|
||
ffmpeg.avcodec_free_context(&pCtx);
|
||
_pAudioCodecContext = null;
|
||
}
|
||
|
||
_audioEnabled = false;
|
||
}
|
||
}
|
||
} |