Match Game MP3 Decoding Timing

Util/Audio/Clip/FFmpegPCMReader.cs

@@ -15,6 +15,11 @@ namespace ADOFAIRenderer.Util.Audio.Clip;
 ///
 /// <see cref="TotalFrames"/> comes from the stream duration (or the container duration) and may differ from the decoded length;
 /// reads stop at whichever ends first.
+///
+/// MP3: decoded with the game's timing (MP3Sharp, see <see cref="MP3HeaderFrame"/>): encoder delay and padding are kept
+/// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed
+/// before it when the file starts with a Xing/Info/VBRI header frame. <see cref="TotalFrames"/> is the sum of all packet
+/// durations plus that silence, counted once when the file is opened.
 /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API)
 /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame)
 /// </summary>
@@ -33,7 +38,8 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
     private AVFrame* _frame;
     private readonly int _streamIndex;
     private readonly AVRational _timeBase;
-    private readonly long _startPts;
+    private readonly long _ptsOrigin;  // Stream timestamp of output frame _leadFrames
+    private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary)
 
     // Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels].
     private float[] _pending = Array.Empty<float>();
@@ -53,6 +59,9 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
     public bool IsForwardOnly => false;
     public long Position { get; private set; }
 
+    /// <summary>True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary).</summary>
+    public bool IsMP3 { get; }
+
     /// <summary>Number of demuxer seeks performed (for diagnostics).</summary>
     public int SeekCount { get; private set; }
 
@@ -80,13 +89,20 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
 
             var stream = _format->streams[_streamIndex];
             _timeBase = stream->time_base;
-            _startPts = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0;
+            _ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0;
 
             _codec = ffmpeg.avcodec_alloc_context3(decoder);
             if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed.");
             ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar);
             if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}");
             _codec->pkt_timebase = _timeBase;
+            IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3;
+            if (IsMP3)
+            {
+                // Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead).
+                // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL)
+                _codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL;
+            }
             ret = ffmpeg.avcodec_open2(_codec, decoder, null);
             if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}");
 
@@ -94,7 +110,16 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
             SampleRate = _codec->sample_rate;
             if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz.");
 
+            if (IsMP3)
+            {
+                _leadFrames = MP3HeaderFrame.SilentFrames(path);
+                ScanPackets(out _ptsOrigin, out var decodedFrames);
+                TotalFrames = _leadFrames + decodedFrames;
+            }
+            else
+            {
                 TotalFrames = EstimateTotalFrames(stream);
+            }
 
             SwrContext* swr = null;
             ret = ffmpeg.swr_alloc_set_opts2(&swr,
@@ -118,6 +143,44 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
         }
     }
 
+    /// <summary>
+    /// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed
+    /// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes
+    /// to a full frame when trimming is off.
+    /// </summary>
+    private void ScanPackets(out long firstPts, out long frames)
+    {
+        firstPts = ffmpeg.AV_NOPTS_VALUE;
+        long durationTicks = 0;
+        var packet = ffmpeg.av_packet_alloc();
+        try
+        {
+            int ret;
+            while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0)
+            {
+                if (packet->stream_index == _streamIndex)
+                {
+                    if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts;
+                    durationTicks += packet->duration;
+                }
+                ffmpeg.av_packet_unref(packet);
+            }
+            if (ret != ffmpeg.AVERROR_EOF)
+                throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}");
+        }
+        finally
+        {
+            ffmpeg.av_packet_free(&packet);
+        }
+
+        if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0;
+        frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate });
+
+        var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
+        if (seek < 0)
+            throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}");
+    }
+
     private long EstimateTotalFrames(AVStream* stream)
     {
         if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0)
@@ -142,10 +205,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
             return;
         }
 
-        var targetTs = _startPts + ffmpeg.av_rescale_q(Math.Max(0, frame - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase);
+        var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase);
         var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD);
         if (ret < 0)
-            ret = ffmpeg.av_seek_frame(_format, _streamIndex, _startPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
+            ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD);
         if (ret < 0)
             throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}");
 
@@ -261,7 +324,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader
         {
             var pts = _frame->best_effort_timestamp;
             _nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE
-                ? ffmpeg.av_rescale_q(pts - _startPts, _timeBase, new AVRational { num = 1, den = SampleRate })
+                ? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate })
                 : Position;
             _syncToTimestamp = false;
         }

Util/Audio/Clip/MP3HeaderFrame.cs

@@ -0,0 +1,107 @@
+using System;
+using System.IO;
+
+namespace ADOFAIRenderer.Util.Audio.Clip;
+
+/// <summary>
+/// Detects the VBR/info header frame ("Xing", "Info" or "VBRI") at the start of an MP3 file.
+///
+/// The game decodes MP3 files with MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.MP3Stream), which has no header-frame handling:
+/// it decodes that frame as one frame of silence and keeps the encoder delay. FFmpeg's MP3 demuxer consumes the header frame
+/// without output. Measured with the clip harness (LAME 192k CBR, LAME VBR, 44.1/48 kHz, with and without ID3v2):
+/// MP3Sharp output is FFmpeg's untrimmed output preceded by one frame of silence when the header frame exists,
+/// and identical in timing when it does not.
+///
+/// Frame layout used here (ISO/IEC 11172-3 and 13818-3 frame header; Xing/Info tag after the Layer III side information,
+/// VBRI tag 32 bytes after the frame header):
+/// - Header: 11 sync bits, version (2 bits: 3 = MPEG-1, 2 = MPEG-2, 0 = MPEG-2.5), layer (2 bits: 1 = III, 2 = II, 3 = I),
+///   protection bit (0 = 16-bit CRC follows), bitrate index, sample rate index, padding, private, channel mode (3 = mono).
+/// - Layer III side information: MPEG-1 32 bytes (17 mono), MPEG-2/2.5 17 bytes (9 mono).
+/// - Samples per frame: Layer I 384, Layer II 1152, Layer III 1152 (MPEG-1) or 576 (MPEG-2/2.5).
+/// - ID3v2 tags before the first frame: "ID3", version (2 bytes), flags (bit 0x10 = 10-byte footer), size (4 bytes, 7 bits each).
+/// </summary>
+internal static class MP3HeaderFrame
+{
+    /// <summary>Bytes searched for the first frame sync after the ID3v2 tags.</summary>
+    private const int SyncSearchBytes = 64 * 1024;
+
+    /// <summary>
+    /// Frames of silence MP3Sharp outputs for the header frame of <paramref name="path"/>:
+    /// the samples per frame of the first frame when it is a Xing/Info/VBRI header frame, otherwise 0.
+    /// </summary>
+    public static int SilentFrames(string path)
+    {
+        using var file = File.OpenRead(path);
+        return SilentFrames(file);
+    }
+
+    public static int SilentFrames(Stream stream)
+    {
+        var tag = new byte[10];
+        long offset = 0;
+        while (true)
+        {
+            stream.Position = offset;
+            if (ReadFully(stream, tag, 10) < 10 || tag[0] != 'I' || tag[1] != 'D' || tag[2] != '3') break;
+            var size = (tag[6] & 0x7F) << 21 | (tag[7] & 0x7F) << 14 | (tag[8] & 0x7F) << 7 | (tag[9] & 0x7F);
+            offset += 10 + size + ((tag[5] & 0x10) != 0 ? 10 : 0);
+        }
+
+        stream.Position = offset;
+        var data = new byte[SyncSearchBytes];
+        var length = ReadFully(stream, data, data.Length);
+        for (var i = 0; i + 4 <= length; i++)
+        {
+            if (!TryParseHeader(data, i, out var samplesPerFrame, out var tagOffset)) continue;
+
+            // Xing/Info after the side information; VBRI at a fixed offset of 32 bytes after the 4-byte header.
+            if (HasTag(data, length, i + tagOffset, "Xing") || HasTag(data, length, i + tagOffset, "Info")
+                || HasTag(data, length, i + 4 + 32, "VBRI"))
+                return samplesPerFrame;
+            return 0;
+        }
+        return 0;
+    }
+
+    private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset)
+    {
+        samplesPerFrame = 0;
+        tagOffset = 0;
+        if (data[i] != 0xFF || (data[i + 1] & 0xE0) != 0xE0) return false;
+
+        var version = (data[i + 1] >> 3) & 0x3;
+        var layer = (data[i + 1] >> 1) & 0x3;
+        var hasCrc = (data[i + 1] & 0x1) == 0;
+        var bitrateIndex = (data[i + 2] >> 4) & 0xF;
+        var sampleRateIndex = (data[i + 2] >> 2) & 0x3;
+        var mono = ((data[i + 3] >> 6) & 0x3) == 3;
+        if (version == 1 || layer == 0 || bitrateIndex == 0xF || sampleRateIndex == 3) return false;
+
+        var mpeg1 = version == 3;
+        samplesPerFrame = layer switch
+        {
+            3 => 384,
+            2 => 1152,
+            _ => mpeg1 ? 1152 : 576
+        };
+        var sideInfo = layer == 1 ? (mpeg1 ? (mono ? 17 : 32) : (mono ? 9 : 17)) : 0;
+        tagOffset = 4 + (hasCrc ? 2 : 0) + sideInfo;
+        return true;
+    }
+
+    private static bool HasTag(byte[] data, int length, int at, string tag)
+    {
+        if (at < 0 || at + tag.Length > length) return false;
+        for (var k = 0; k < tag.Length; k++)
+            if (data[at + k] != tag[k]) return false;
+        return true;
+    }
+
+    private static int ReadFully(Stream stream, byte[] buffer, int count)
+    {
+        var total = 0;
+        int read;
+        while (total < count && (read = stream.Read(buffer, total, count - total)) > 0) total += read;
+        return total;
+    }
+}

Util/Audio/Clip/MP3MonoLayoutReader.cs

@@ -0,0 +1,73 @@
+using System;
+
+namespace ADOFAIRenderer.Util.Audio.Clip;
+
+/// <summary>
+/// <see cref="IPCMReader"/> that reproduces the game's output for mono MP3 files.
+///
+/// MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.Buffer16BitStereo) writes every frame in stereo layout and fills only the
+/// channel 0 slot for mono streams, while MP3Stream.ChannelCount reports 1. The game creates a 1-channel clip from that data,
+/// so a mono MP3 plays as [s0, 0, s1, 0, ...]: twice as long, at half speed and one octave lower
+/// (confirmed in game with TestMaps/mp3_mono, 2026-10-13).
+///
+/// Output: 1 channel at the inner reader's sample rate. Frame 2k is inner frame k, frame 2k + 1 is 0.
+/// <see cref="TotalFrames"/> = 2 * inner TotalFrames.
+/// </summary>
+internal sealed class MP3MonoLayoutReader : IPCMReader
+{
+    private readonly IPCMReader _inner;
+    private float[] _innerBuffer = Array.Empty<float>();
+
+    /// <param name="inner">Mono reader of the MP3 file with the game's MP3 timing (<see cref="FFmpegPCMReader.IsMP3"/>). Owned by this reader.</param>
+    public MP3MonoLayoutReader(IPCMReader inner)
+    {
+        if (inner.Channels != 1) throw new ArgumentException("Inner reader must be mono.", nameof(inner));
+        _inner = inner;
+        Position = inner.Position * 2;
+    }
+
+    public int Channels => 1;
+    public int SampleRate => _inner.SampleRate;
+    public long TotalFrames => _inner.TotalFrames * 2;
+    public bool IsForwardOnly => _inner.IsForwardOnly;
+    public long Position { get; private set; }
+
+    /// <summary>Moves to <paramref name="frame"/>; the inner reader moves to the inner frame of the next even position.</summary>
+    public void Seek(long frame)
+    {
+        _inner.Seek((frame + 1) / 2);
+        Position = frame;
+    }
+
+    public int Read(float[] buffer, int frameCount)
+    {
+        var limit = (int)Math.Min(frameCount, TotalFrames - Position);
+        var written = 0;
+        while (written < limit)
+        {
+            if ((Position & 1) == 1)
+            {
+                buffer[written++] = 0f;
+                Position++;
+                continue;
+            }
+
+            var want = (limit - written + 1) / 2;
+            if (_innerBuffer.Length < want) _innerBuffer = new float[want];
+            var got = _inner.Read(_innerBuffer, want);
+            if (got == 0) break;
+
+            for (var i = 0; i < got && written < limit; i++)
+            {
+                buffer[written++] = _innerBuffer[i];
+                Position++;
+                if (written >= limit) break;
+                buffer[written++] = 0f;
+                Position++;
+            }
+        }
+        return written;
+    }
+
+    public void Dispose() => _inner.Dispose();
+}