From 8addbfb79d759541bcad7bb79cac62a229203bf6913b2cf1092469976011ce90 Mon Sep 17 00:00:00 2001 From: copecone Date: Tue, 13 Oct 2026 01:59:53 +0900 Subject: [PATCH] Match Game MP3 Decoding Timing --- Util/Audio/Clip/FFmpegPCMReader.cs | 75 +++++++++++++++-- Util/Audio/Clip/MP3HeaderFrame.cs | 107 +++++++++++++++++++++++++ Util/Audio/Clip/MP3MonoLayoutReader.cs | 73 +++++++++++++++++ 3 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 Util/Audio/Clip/MP3HeaderFrame.cs create mode 100644 Util/Audio/Clip/MP3MonoLayoutReader.cs diff --git a/Util/Audio/Clip/FFmpegPCMReader.cs b/Util/Audio/Clip/FFmpegPCMReader.cs index c9896a3..3690b6a 100644 --- a/Util/Audio/Clip/FFmpegPCMReader.cs +++ b/Util/Audio/Clip/FFmpegPCMReader.cs @@ -15,6 +15,11 @@ namespace ADOFAIRenderer.Util.Audio.Clip; /// /// comes from the stream duration (or the container duration) and may differ from the decoded length; /// reads stop at whichever ends first. +/// +/// MP3: decoded with the game's timing (MP3Sharp, see ): encoder delay and padding are kept +/// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed +/// before it when the file starts with a Xing/Info/VBRI header frame. is the sum of all packet +/// durations plus that silence, counted once when the file is opened. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API) /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame) /// @@ -33,7 +38,8 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader private AVFrame* _frame; private readonly int _streamIndex; private readonly AVRational _timeBase; - private readonly long _startPts; + private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames + private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary) // Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels]. private float[] _pending = Array.Empty(); @@ -53,6 +59,9 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader public bool IsForwardOnly => false; public long Position { get; private set; } + /// True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary). + public bool IsMP3 { get; } + /// Number of demuxer seeks performed (for diagnostics). public int SeekCount { get; private set; } @@ -80,13 +89,20 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader var stream = _format->streams[_streamIndex]; _timeBase = stream->time_base; - _startPts = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0; + _ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0; _codec = ffmpeg.avcodec_alloc_context3(decoder); if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed."); ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar); if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}"); _codec->pkt_timebase = _timeBase; + IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3; + if (IsMP3) + { + // Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead). + // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL) + _codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL; + } ret = ffmpeg.avcodec_open2(_codec, decoder, null); if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}"); @@ -94,7 +110,16 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader SampleRate = _codec->sample_rate; if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz."); - TotalFrames = EstimateTotalFrames(stream); + if (IsMP3) + { + _leadFrames = MP3HeaderFrame.SilentFrames(path); + ScanPackets(out _ptsOrigin, out var decodedFrames); + TotalFrames = _leadFrames + decodedFrames; + } + else + { + TotalFrames = EstimateTotalFrames(stream); + } SwrContext* swr = null; ret = ffmpeg.swr_alloc_set_opts2(&swr, @@ -118,6 +143,44 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader } } + /// + /// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed + /// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes + /// to a full frame when trimming is off. + /// + private void ScanPackets(out long firstPts, out long frames) + { + firstPts = ffmpeg.AV_NOPTS_VALUE; + long durationTicks = 0; + var packet = ffmpeg.av_packet_alloc(); + try + { + int ret; + while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0) + { + if (packet->stream_index == _streamIndex) + { + if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts; + durationTicks += packet->duration; + } + ffmpeg.av_packet_unref(packet); + } + if (ret != ffmpeg.AVERROR_EOF) + throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}"); + } + finally + { + ffmpeg.av_packet_free(&packet); + } + + if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0; + frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate }); + + var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD); + if (seek < 0) + throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}"); + } + private long EstimateTotalFrames(AVStream* stream) { if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0) @@ -142,10 +205,10 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader return; } - var targetTs = _startPts + ffmpeg.av_rescale_q(Math.Max(0, frame - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); + var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) - ret = ffmpeg.av_seek_frame(_format, _streamIndex, _startPts, ffmpeg.AVSEEK_FLAG_BACKWARD); + ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}"); @@ -261,7 +324,7 @@ internal sealed unsafe class FFmpegPCMReader : IPCMReader { var pts = _frame->best_effort_timestamp; _nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE - ? ffmpeg.av_rescale_q(pts - _startPts, _timeBase, new AVRational { num = 1, den = SampleRate }) + ? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate }) : Position; _syncToTimestamp = false; } diff --git a/Util/Audio/Clip/MP3HeaderFrame.cs b/Util/Audio/Clip/MP3HeaderFrame.cs new file mode 100644 index 0000000..c74e03b --- /dev/null +++ b/Util/Audio/Clip/MP3HeaderFrame.cs @@ -0,0 +1,107 @@ +using System; +using System.IO; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// Detects the VBR/info header frame ("Xing", "Info" or "VBRI") at the start of an MP3 file. +/// +/// The game decodes MP3 files with MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.MP3Stream), which has no header-frame handling: +/// it decodes that frame as one frame of silence and keeps the encoder delay. FFmpeg's MP3 demuxer consumes the header frame +/// without output. Measured with the clip harness (LAME 192k CBR, LAME VBR, 44.1/48 kHz, with and without ID3v2): +/// MP3Sharp output is FFmpeg's untrimmed output preceded by one frame of silence when the header frame exists, +/// and identical in timing when it does not. +/// +/// Frame layout used here (ISO/IEC 11172-3 and 13818-3 frame header; Xing/Info tag after the Layer III side information, +/// VBRI tag 32 bytes after the frame header): +/// - Header: 11 sync bits, version (2 bits: 3 = MPEG-1, 2 = MPEG-2, 0 = MPEG-2.5), layer (2 bits: 1 = III, 2 = II, 3 = I), +/// protection bit (0 = 16-bit CRC follows), bitrate index, sample rate index, padding, private, channel mode (3 = mono). +/// - Layer III side information: MPEG-1 32 bytes (17 mono), MPEG-2/2.5 17 bytes (9 mono). +/// - Samples per frame: Layer I 384, Layer II 1152, Layer III 1152 (MPEG-1) or 576 (MPEG-2/2.5). +/// - ID3v2 tags before the first frame: "ID3", version (2 bytes), flags (bit 0x10 = 10-byte footer), size (4 bytes, 7 bits each). +/// +internal static class MP3HeaderFrame +{ + /// Bytes searched for the first frame sync after the ID3v2 tags. + private const int SyncSearchBytes = 64 * 1024; + + /// + /// Frames of silence MP3Sharp outputs for the header frame of : + /// the samples per frame of the first frame when it is a Xing/Info/VBRI header frame, otherwise 0. + /// + public static int SilentFrames(string path) + { + using var file = File.OpenRead(path); + return SilentFrames(file); + } + + public static int SilentFrames(Stream stream) + { + var tag = new byte[10]; + long offset = 0; + while (true) + { + stream.Position = offset; + if (ReadFully(stream, tag, 10) < 10 || tag[0] != 'I' || tag[1] != 'D' || tag[2] != '3') break; + var size = (tag[6] & 0x7F) << 21 | (tag[7] & 0x7F) << 14 | (tag[8] & 0x7F) << 7 | (tag[9] & 0x7F); + offset += 10 + size + ((tag[5] & 0x10) != 0 ? 10 : 0); + } + + stream.Position = offset; + var data = new byte[SyncSearchBytes]; + var length = ReadFully(stream, data, data.Length); + for (var i = 0; i + 4 <= length; i++) + { + if (!TryParseHeader(data, i, out var samplesPerFrame, out var tagOffset)) continue; + + // Xing/Info after the side information; VBRI at a fixed offset of 32 bytes after the 4-byte header. + if (HasTag(data, length, i + tagOffset, "Xing") || HasTag(data, length, i + tagOffset, "Info") + || HasTag(data, length, i + 4 + 32, "VBRI")) + return samplesPerFrame; + return 0; + } + return 0; + } + + private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset) + { + samplesPerFrame = 0; + tagOffset = 0; + if (data[i] != 0xFF || (data[i + 1] & 0xE0) != 0xE0) return false; + + var version = (data[i + 1] >> 3) & 0x3; + var layer = (data[i + 1] >> 1) & 0x3; + var hasCrc = (data[i + 1] & 0x1) == 0; + var bitrateIndex = (data[i + 2] >> 4) & 0xF; + var sampleRateIndex = (data[i + 2] >> 2) & 0x3; + var mono = ((data[i + 3] >> 6) & 0x3) == 3; + if (version == 1 || layer == 0 || bitrateIndex == 0xF || sampleRateIndex == 3) return false; + + var mpeg1 = version == 3; + samplesPerFrame = layer switch + { + 3 => 384, + 2 => 1152, + _ => mpeg1 ? 1152 : 576 + }; + var sideInfo = layer == 1 ? (mpeg1 ? (mono ? 17 : 32) : (mono ? 9 : 17)) : 0; + tagOffset = 4 + (hasCrc ? 2 : 0) + sideInfo; + return true; + } + + private static bool HasTag(byte[] data, int length, int at, string tag) + { + if (at < 0 || at + tag.Length > length) return false; + for (var k = 0; k < tag.Length; k++) + if (data[at + k] != tag[k]) return false; + return true; + } + + private static int ReadFully(Stream stream, byte[] buffer, int count) + { + var total = 0; + int read; + while (total < count && (read = stream.Read(buffer, total, count - total)) > 0) total += read; + return total; + } +} diff --git a/Util/Audio/Clip/MP3MonoLayoutReader.cs b/Util/Audio/Clip/MP3MonoLayoutReader.cs new file mode 100644 index 0000000..b1e35ab --- /dev/null +++ b/Util/Audio/Clip/MP3MonoLayoutReader.cs @@ -0,0 +1,73 @@ +using System; + +namespace ADOFAIRenderer.Util.Audio.Clip; + +/// +/// that reproduces the game's output for mono MP3 files. +/// +/// MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.Buffer16BitStereo) writes every frame in stereo layout and fills only the +/// channel 0 slot for mono streams, while MP3Stream.ChannelCount reports 1. The game creates a 1-channel clip from that data, +/// so a mono MP3 plays as [s0, 0, s1, 0, ...]: twice as long, at half speed and one octave lower +/// (confirmed in game with TestMaps/mp3_mono, 2026-10-13). +/// +/// Output: 1 channel at the inner reader's sample rate. Frame 2k is inner frame k, frame 2k + 1 is 0. +/// = 2 * inner TotalFrames. +/// +internal sealed class MP3MonoLayoutReader : IPCMReader +{ + private readonly IPCMReader _inner; + private float[] _innerBuffer = Array.Empty(); + + /// Mono reader of the MP3 file with the game's MP3 timing (). Owned by this reader. + public MP3MonoLayoutReader(IPCMReader inner) + { + if (inner.Channels != 1) throw new ArgumentException("Inner reader must be mono.", nameof(inner)); + _inner = inner; + Position = inner.Position * 2; + } + + public int Channels => 1; + public int SampleRate => _inner.SampleRate; + public long TotalFrames => _inner.TotalFrames * 2; + public bool IsForwardOnly => _inner.IsForwardOnly; + public long Position { get; private set; } + + /// Moves to ; the inner reader moves to the inner frame of the next even position. + public void Seek(long frame) + { + _inner.Seek((frame + 1) / 2); + Position = frame; + } + + public int Read(float[] buffer, int frameCount) + { + var limit = (int)Math.Min(frameCount, TotalFrames - Position); + var written = 0; + while (written < limit) + { + if ((Position & 1) == 1) + { + buffer[written++] = 0f; + Position++; + continue; + } + + var want = (limit - written + 1) / 2; + if (_innerBuffer.Length < want) _innerBuffer = new float[want]; + var got = _inner.Read(_innerBuffer, want); + if (got == 0) break; + + for (var i = 0; i < got && written < limit; i++) + { + buffer[written++] = _innerBuffer[i]; + Position++; + if (written >= limit) break; + buffer[written++] = 0f; + Position++; + } + } + return written; + } + + public void Dispose() => _inner.Dispose(); +}