Match Game MP3 Decoding Timing

Util/Audio/Clip/FFmpegPCMReader.cs

@@ -15,6 +15,11 @@ namespace ADOFAIRenderer.Util.Audio.Clip;
 ///
 /// <see cref="TotalFrames"/> comes from the stream duration (or the container duration) and may differ from the decoded length;
 /// reads stop at whichever ends first.
+///
+/// MP3: decoded with the game's timing (MP3Sharp, see <see cref="MP3HeaderFrame"/>): encoder delay and padding are kept
+/// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed
+/// before it when the file starts with a Xing/Info/VBRI header frame. <see cref="TotalFrames"/> is the sum of all packet
+/// durations plus that silence, counted once when the file is opened.
 /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API)
 /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame)
 /// </summary>
@@ -33,7 +38,8 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
     private AVFrame* _frame;
     private readonly int _streamIndex;
     private readonly AVRational _timeBase;
-    private readonly long _startPts;
+    private readonly long _ptsOrigin;  // Stream timestamp of output frame _leadFrames
+    private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary)
 
     // Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels].
     private float[] _pending = Array.Empty<float>();
@@ -53,6 +59,9 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
     public bool IsForwardOnly => false;
     public long Position { get; private set; }
 
+    /// <summary>True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary).</summary>
+    public bool IsMP3 { get; }
+
     /// <summary>Number of demuxer seeks performed (for diagnostics).</summary>
     public int SeekCount { get; private set; }
 
@@ -80,13 +89,20 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
 
             var stream = _format->streams[_streamIndex];
             _timeBase = stream->time_base;
-            _startPts = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0;
+            _ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0;
 
             _codec = ffmpeg.avcodec_alloc_context3(decoder);
             if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed.");
             ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar);
             if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}");
             _codec->pkt_timebase = _timeBase;
+            IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3;
+            if (IsMP3)
+            {
+                // Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead).
+                // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL)
+                _codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL;
+            }
             ret = ffmpeg.avcodec_open2(_codec, decoder, null);
             if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}");
 
@@ -94,7 +110,16 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
             SampleRate = _codec->sample_rate;
             if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz.");
 
-            TotalFrames = EstimateTotalFrames(stream);
+            if (IsMP3)
+            {
+                _leadFrames = MP3HeaderFrame.SilentFrames(path);
+                ScanPackets(out _ptsOrigin, out var decodedFrames);
+                TotalFrames = _leadFrames + decodedFrames;
+            }
+            else
+            {
+                TotalFrames = EstimateTotalFrames(stream);
+            }
 
             SwrContext* swr = null;
             ret = ffmpeg.swr_alloc_set_opts2(&swr,
@@ -118,6 +143,44 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
         }
     }
 
+    /// <summary>
+    /// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed
+    /// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes
+    /// to a full frame when trimming is off.
+    /// </summary>
+    private void ScanPackets(out long firstPts, out long frames)
+    {
+        firstPts = ffmpeg.AV_NOPTS_VALUE;
+        long durationTicks = 0;
+        var packet = ffmpeg.av_packet_alloc();
+        try
+        {
+            int ret;
+            while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0)
+            {
+                if (packet->stream_index == _streamIndex)
+                {
+                    if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts;
+                    durationTicks += packet->duration;
+                }
+                ffmpeg.av_packet_unref(packet);
+            }
+            if (ret != ffmpeg.AVERROR_EOF)
+                throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}");
+        }
+        finally
+        {
+            ffmpeg.av_packet_free(&packet);
+        }
+
+        if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0;
+        frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate });
+
+        var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
+        if (seek < 0)
+            throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}");
+    }
+
     private long EstimateTotalFrames(AVStream* stream)
     {
         if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0)
@@ -142,10 +205,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
             return;
         }
 
-        var targetTs = _startPts + ffmpeg.av_rescale_q(Math.Max(0, frame - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase);
+        var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase);
         var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD);
         if (ret < 0)
-            ret = ffmpeg.av_seek_frame(_format, _streamIndex, _startPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
+            ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD);
         if (ret < 0)
             throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}");
 
@@ -261,7 +324,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
         {
             var pts = _frame->best_effort_timestamp;
             _nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE
-                ? ffmpeg.av_rescale_q(pts - _startPts, _timeBase, new AVRational { num = 1, den = SampleRate })
+                ? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate })
                 : Position;
             _syncToTimestamp = false;
         }