Harden Encoder Error Handling

Util/FFmpeg/FFmpegNativeEncoder.cs

@@ -255,6 +255,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
         /// <summary>Whether audio encoding is enabled.</summary>
         private bool _audioEnabled;
 
+        /// <summary>Bytes of one input sample of every channel (interleaved input format × channel count).</summary>
+        private int _audioInputBytesPerSample;
+
         /// <summary>Encoding settings passed at construction time (immutable).</summary>
         private readonly EncoderSettings _settings;
 
@@ -267,26 +270,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
         // Utility Methods
         // ================================================================
 
-        /// <summary>
-        /// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg.
-        /// AVCodecID identifies "which codec it is (e.g., H.264, AV1)",
-        /// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately.
-        /// </summary>
-        // ReSharper disable once InconsistentNaming
-        // ReSharper disable once UnusedMember.Local
-        private AVCodecID GetAVCodecID()
-        {
-            return _settings.Codec switch
-            {
-                Codec.H264 => AVCodecID.AV_CODEC_ID_H264,
-                Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC,
-                Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1,
-                Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9,
-                Codec.VVC => AVCodecID.AV_CODEC_ID_VVC,
-                _ => throw new ArgumentOutOfRangeException()
-            };
-        }
-
         /// <summary>
         /// Returns a list of candidate encoders to try for the selected codec in priority order.
         ///
@@ -690,7 +673,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
 
                                         _pHwFrame = ffmpeg.av_frame_alloc();
                                         if (_pHwFrame == null)
+                                        {
+                                            // The codec context (and the frames context it holds) is released by Dispose in the catch block of Start.
+                                            ffmpeg.av_buffer_unref(&hwDeviceCtx);
                                             throw new Exception("Failed to allocate QSV hardware frame structure.");
+                                        }
                                         break;
                                     }
 
@@ -797,8 +784,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
                     // Copy parameters set in codec context (resolution, codec ID, profile, etc.)
                     // to stream's codecpar. Information to be written to container header.
                     // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
-                    if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0)
+                    var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext);
-                        throw new Exception("Failed to copy codec parameters to stream.");
+                    if (paramResult < 0)
+                        throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})");
 
                     // Stream time base: unit of the timestamps stored in the container.
                     // Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks.
@@ -823,15 +811,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
                     // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
                     if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
                     {
-                        if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0)
+                        var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE);
-                            throw new Exception($"Cannot open output file {outputPath}.");
+                        if (avioResult < 0)
+                            throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
                     }
 
                     // Write file header (ftyp, moov, container metadata).
                     // Packets can be written with av_interleaved_write_frame after this point.
                     // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
-                    if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0)
+                    var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
-                        throw new Exception("Failed to write file header.");
+                    if (headerResult < 0)
+                        throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})");
 
                     // ── Step 6: Initialize frame buffers and colorspace converter ──
                     // av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet).
@@ -1160,10 +1150,42 @@ namespace ADOFAIRenderer.Util.FFmpeg
             lock (_lock)
             {
                 if (_disposed || !_isInitialized) return;
+                ValidateInputLength(buffer.Length, isColorConverted: false);
                 WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr());
             }
         }
 
+        /// <summary>
+        /// Number of bytes read from the input buffer of one frame.
+        ///   RGBA input (isColorConverted = false):  Width × Height × 4
+        ///   Pre-converted NV12:                      Width × Height (Y) + Width × (Height / 2) (interleaved UV)
+        ///   Pre-converted YUV420P:                   Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
+        /// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
+        /// </summary>
+        private long GetRequiredInputLength(bool isColorConverted)
+        {
+            long w = _settings.Width, h = _settings.Height;
+            if (!isColorConverted) return w * h * 4;
+
+            return _dstPixFmt switch
+            {
+                AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2),
+                AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2),
+                _ => 0
+            };
+        }
+
+        /// <summary>
+        /// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it
+        /// (GetRequiredInputLength), so that the copy does not read outside the buffer.
+        /// </summary>
+        private void ValidateInputLength(long length, bool isColorConverted)
+        {
+            var required = GetRequiredInputLength(isColorConverted);
+            if (length < required)
+                throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input.");
+        }
+
         /// <summary>
         /// Encodes a frame from IntPtr (native pointer).
         /// Can be used from external native plugins or custom capture sources.
@@ -1182,6 +1204,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                         : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
                 }
 
+                ValidateInputLength(length, isColorConverted);
                 if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
                 else WriteFrameFromBuffer((byte*)pointer.ToPointer());
             }
@@ -1196,6 +1219,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
             lock (_lock)
             {
                 if (_disposed || !_isInitialized) return;
+                if (length > buffer.Length)
+                    throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes).");
+                ValidateInputLength(length, isColorConverted);
+
                 fixed (byte* pBuffer = buffer)
                 {
                     if (!_loggedConversionMode)
@@ -1262,7 +1289,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
 
             switch (_dstPixFmt)
             {
-                case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV:
+                case AVPixelFormat.AV_PIX_FMT_NV12:
                 {
                     // Copy Y Plane
                     var pSrcY = buffer;
@@ -1485,7 +1512,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame);
             if (response < 0)
             {
-                throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})");
+                throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})");
             }
 
             // Pop all encoded packets and write to file.
@@ -1499,7 +1526,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                     // EOF:    Encoder flush complete (all delayed frames outputted) → exit loop
                     if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
                     if (response < 0)
-                        throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})");
+                        throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})");
 
                     // Convert PTS/DTS from codec timebase → stream timebase
                     ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base);
@@ -1510,7 +1537,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                     // and written in time order. This function is used even when only video is present.
                     response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
                     if (response < 0)
-                        throw new Exception($"Error occurred while writing packet to file. (Code: {response})");
+                        throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
 
                     // Unref packet data reference for reuse in next receive_packet.
                     ffmpeg.av_packet_unref(_pPacket);
@@ -1775,6 +1802,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
                 ? audioSettings.InputSampleFormat
                 : AVSampleFormat.AV_SAMPLE_FMT_FLT;
 
+            // Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
+            // Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
+            _audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
+
             // Allocate SwrContext
             fixed (SwrContext** ppSwr = &_pSwrContext)
             {
@@ -1838,6 +1869,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
             lock (_lock)
             {
                 if (_disposed || !_isInitialized || !_audioEnabled) return;
+                if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float))
+                    throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats).");
 
                 fixed (float* pSamples = samples)
                 {
@@ -1872,6 +1905,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
         ///   swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
         ///   av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
         /// </summary>
+        /// <summary>
+        /// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
+        /// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
+        /// Throws an exception when the buffer cannot be allocated.
+        /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
+        /// </summary>
+        private void MakeAudioFrameWritable()
+        {
+            var ret = ffmpeg.av_frame_make_writable(_pAudioFrame);
+            if (ret < 0)
+                throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
+        }
+
         private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
         {
             // ── 1. Resampling: Input format → Encoder format ──
@@ -1910,7 +1956,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
                 var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
                 while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
                 {
-                    ffmpeg.av_frame_make_writable(_pAudioFrame);
+                    MakeAudioFrameWritable();
                     _pAudioFrame->nb_samples = frameSize;
 
                     var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize);
@@ -1989,7 +2035,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
             var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo);
             if (remaining > 0)
             {
-                ffmpeg.av_frame_make_writable(_pAudioFrame);
+                MakeAudioFrameWritable();
                 _pAudioFrame->nb_samples = remaining;
 
                 var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);