From 5080e7ea1d3bb60ecc239f098f12156af32ddf2e924515c6e55aaec1bec51e79 Mon Sep 17 00:00:00 2001 From: copecone Date: Mon, 12 Oct 2026 21:15:38 +0900 Subject: [PATCH] Harden Encoder Error Handling --- Util/FFmpeg/FFmpegNativeEncoder.cs | 110 ++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 32 deletions(-) diff --git a/Util/FFmpeg/FFmpegNativeEncoder.cs b/Util/FFmpeg/FFmpegNativeEncoder.cs index 5d3a773..6e7e762 100644 --- a/Util/FFmpeg/FFmpegNativeEncoder.cs +++ b/Util/FFmpeg/FFmpegNativeEncoder.cs @@ -255,6 +255,9 @@ namespace ADOFAIRenderer.Util.FFmpeg /// Whether audio encoding is enabled. private bool _audioEnabled; + /// Bytes of one input sample of every channel (interleaved input format × channel count). + private int _audioInputBytesPerSample; + /// Encoding settings passed at construction time (immutable). private readonly EncoderSettings _settings; @@ -267,26 +270,6 @@ namespace ADOFAIRenderer.Util.FFmpeg // Utility Methods // ================================================================ - /// - /// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg. - /// AVCodecID identifies "which codec it is (e.g., H.264, AV1)", - /// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately. - /// - // ReSharper disable once InconsistentNaming - // ReSharper disable once UnusedMember.Local - private AVCodecID GetAVCodecID() - { - return _settings.Codec switch - { - Codec.H264 => AVCodecID.AV_CODEC_ID_H264, - Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC, - Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1, - Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9, - Codec.VVC => AVCodecID.AV_CODEC_ID_VVC, - _ => throw new ArgumentOutOfRangeException() - }; - } - /// /// Returns a list of candidate encoders to try for the selected codec in priority order. /// @@ -690,7 +673,11 @@ namespace ADOFAIRenderer.Util.FFmpeg _pHwFrame = ffmpeg.av_frame_alloc(); if (_pHwFrame == null) + { + // The codec context (and the frames context it holds) is released by Dispose in the catch block of Start. + ffmpeg.av_buffer_unref(&hwDeviceCtx); throw new Exception("Failed to allocate QSV hardware frame structure."); + } break; } @@ -797,8 +784,9 @@ namespace ADOFAIRenderer.Util.FFmpeg // Copy parameters set in codec context (resolution, codec ID, profile, etc.) // to stream's codecpar. Information to be written to container header. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html - if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0) - throw new Exception("Failed to copy codec parameters to stream."); + var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext); + if (paramResult < 0) + throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})"); // Stream time base: unit of the timestamps stored in the container. // Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks. @@ -823,15 +811,17 @@ namespace ADOFAIRenderer.Util.FFmpeg // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0) { - if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0) - throw new Exception($"Cannot open output file {outputPath}."); + var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE); + if (avioResult < 0) + throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})"); } // Write file header (ftyp, moov, container metadata). // Packets can be written with av_interleaved_write_frame after this point. // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html - if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0) - throw new Exception("Failed to write file header."); + var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null); + if (headerResult < 0) + throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})"); // ── Step 6: Initialize frame buffers and colorspace converter ── // av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet). @@ -1160,10 +1150,42 @@ namespace ADOFAIRenderer.Util.FFmpeg lock (_lock) { if (_disposed || !_isInitialized) return; + ValidateInputLength(buffer.Length, isColorConverted: false); WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr()); } } + /// + /// Number of bytes read from the input buffer of one frame. + /// RGBA input (isColorConverted = false): Width × Height × 4 + /// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV) + /// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V) + /// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it). + /// + private long GetRequiredInputLength(bool isColorConverted) + { + long w = _settings.Width, h = _settings.Height; + if (!isColorConverted) return w * h * 4; + + return _dstPixFmt switch + { + AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2), + AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2), + _ => 0 + }; + } + + /// + /// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it + /// (GetRequiredInputLength), so that the copy does not read outside the buffer. + /// + private void ValidateInputLength(long length, bool isColorConverted) + { + var required = GetRequiredInputLength(isColorConverted); + if (length < required) + throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input."); + } + /// /// Encodes a frame from IntPtr (native pointer). /// Can be used from external native plugins or custom capture sources. @@ -1182,6 +1204,7 @@ namespace ADOFAIRenderer.Util.FFmpeg : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)"); } + ValidateInputLength(length, isColorConverted); if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer()); else WriteFrameFromBuffer((byte*)pointer.ToPointer()); } @@ -1196,6 +1219,10 @@ namespace ADOFAIRenderer.Util.FFmpeg lock (_lock) { if (_disposed || !_isInitialized) return; + if (length > buffer.Length) + throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes)."); + ValidateInputLength(length, isColorConverted); + fixed (byte* pBuffer = buffer) { if (!_loggedConversionMode) @@ -1262,7 +1289,7 @@ namespace ADOFAIRenderer.Util.FFmpeg switch (_dstPixFmt) { - case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV: + case AVPixelFormat.AV_PIX_FMT_NV12: { // Copy Y Plane var pSrcY = buffer; @@ -1485,7 +1512,7 @@ namespace ADOFAIRenderer.Util.FFmpeg var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame); if (response < 0) { - throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})"); + throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})"); } // Pop all encoded packets and write to file. @@ -1499,7 +1526,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // EOF: Encoder flush complete (all delayed frames outputted) → exit loop if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break; if (response < 0) - throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})"); + throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})"); // Convert PTS/DTS from codec timebase → stream timebase ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base); @@ -1510,7 +1537,7 @@ namespace ADOFAIRenderer.Util.FFmpeg // and written in time order. This function is used even when only video is present. response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket); if (response < 0) - throw new Exception($"Error occurred while writing packet to file. (Code: {response})"); + throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})"); // Unref packet data reference for reuse in next receive_packet. ffmpeg.av_packet_unref(_pPacket); @@ -1775,6 +1802,10 @@ namespace ADOFAIRenderer.Util.FFmpeg ? audioSettings.InputSampleFormat : AVSampleFormat.AV_SAMPLE_FMT_FLT; + // Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes). + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample) + _audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels; + // Allocate SwrContext fixed (SwrContext** ppSwr = &_pSwrContext) { @@ -1838,6 +1869,8 @@ namespace ADOFAIRenderer.Util.FFmpeg lock (_lock) { if (_disposed || !_isInitialized || !_audioEnabled) return; + if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float)) + throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats)."); fixed (float* pSamples = samples) { @@ -1872,6 +1905,19 @@ namespace ADOFAIRenderer.Util.FFmpeg /// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html /// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html /// + /// + /// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it. + /// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer. + /// Throws an exception when the buffer cannot be allocated. + /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable) + /// + private void MakeAudioFrameWritable() + { + var ret = ffmpeg.av_frame_make_writable(_pAudioFrame); + if (ret < 0) + throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})"); + } + private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples) { // ── 1. Resampling: Input format → Encoder format ── @@ -1910,7 +1956,7 @@ namespace ADOFAIRenderer.Util.FFmpeg var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize) { - ffmpeg.av_frame_make_writable(_pAudioFrame); + MakeAudioFrameWritable(); _pAudioFrame->nb_samples = frameSize; var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize); @@ -1989,7 +2035,7 @@ namespace ADOFAIRenderer.Util.FFmpeg var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo); if (remaining > 0) { - ffmpeg.av_frame_make_writable(_pAudioFrame); + MakeAudioFrameWritable(); _pAudioFrame->nb_samples = remaining; var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);