using System; using System.Runtime.InteropServices; using FFmpeg.AutoGen; namespace ADOFAIRenderer.Util.Audio.Clip; /// /// that decodes the best audio stream of a file with libavformat/libavcodec. /// Output keeps the stream's sample rate and channel count; libswresample converts only the sample format to interleaved float. /// /// Frame positions: frame 0 is the first sample the decoder outputs for the stream's start time. /// The position of the first frame decoded after opening or seeking is taken from its timestamp /// (best_effort_timestamp - stream start_time, in samples); following frames are counted sequentially. /// Seek moves the demuxer to before the target and discards decoded samples before the target. /// /// comes from the stream duration (or the container duration) and may differ from the decoded length; /// reads stop at whichever ends first. /// /// MP3: decoded with the game's timing (MP3Sharp, see ): encoder delay and padding are kept /// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed /// before it when the file starts with a Xing/Info/VBRI header frame. is the sum of all packet /// durations plus that silence, counted once when the file is opened. /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API) /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame) /// internal sealed unsafe class FFmpegPCMReader : IPCMReader { /// /// Time the demuxer is moved before a seek target, in seconds. Decoders that depend on previous packets /// (for example the MP3 bit reservoir) produce settled output after this margin; the samples in it are discarded. /// private const double SeekMarginSeconds = 0.25; private AVFormatContext* _format; private AVCodecContext* _codec; private SwrContext* _swr; private AVPacket* _packet; private AVFrame* _frame; private readonly int _streamIndex; private readonly AVRational _timeBase; private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary) // Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels]. private float[] _pending = Array.Empty(); private int _pendingOffset; private int _pendingCount; private long _pendingStart; private long _nextDecodedFrame; // Frame index of the next decoded frame when counting sequentially private bool _syncToTimestamp; // True until the first frame after open/seek sets _nextDecodedFrame from its timestamp private bool _flushSent; private bool _endOfStream; public string Path { get; } public int Channels { get; } public int SampleRate { get; } public long TotalFrames { get; } public bool IsForwardOnly => false; public long Position { get; private set; } /// True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary). public bool IsMP3 { get; } /// Number of demuxer seeks performed (for diagnostics). public int SeekCount { get; private set; } public FFmpegPCMReader(string path) { Path = path; try { AVFormatContext* format = null; var ret = ffmpeg.avformat_open_input(&format, path, null, null); if (ret < 0) throw new InvalidOperationException($"Failed to open audio file '{path}': {ErrorString(ret)}"); _format = format; ret = ffmpeg.avformat_find_stream_info(_format, null); if (ret < 0) throw new InvalidOperationException($"Failed to read stream info of '{path}': {ErrorString(ret)}"); AVCodec* decoder = null; _streamIndex = ffmpeg.av_find_best_stream(_format, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &decoder, 0); if (_streamIndex < 0 || decoder == null) throw new InvalidOperationException($"No decodable audio stream in '{path}': {ErrorString(_streamIndex)}"); for (var i = 0; i < (int)_format->nb_streams; i++) { if (i != _streamIndex) _format->streams[i]->discard = AVDiscard.AVDISCARD_ALL; } var stream = _format->streams[_streamIndex]; _timeBase = stream->time_base; _ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0; _codec = ffmpeg.avcodec_alloc_context3(decoder); if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed."); ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar); if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}"); _codec->pkt_timebase = _timeBase; IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3; if (IsMP3) { // Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead). // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL) _codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL; } ret = ffmpeg.avcodec_open2(_codec, decoder, null); if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}"); Channels = _codec->ch_layout.nb_channels; SampleRate = _codec->sample_rate; if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz."); if (IsMP3) { _leadFrames = MP3HeaderFrame.SilentFrames(path); ScanPackets(out _ptsOrigin, out var decodedFrames); TotalFrames = _leadFrames + decodedFrames; } else { TotalFrames = EstimateTotalFrames(stream); } SwrContext* swr = null; ret = ffmpeg.swr_alloc_set_opts2(&swr, &_codec->ch_layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, SampleRate, &_codec->ch_layout, _codec->sample_fmt, SampleRate, 0, null); if (ret < 0 || swr == null) throw new InvalidOperationException($"swr_alloc_set_opts2 failed: {ErrorString(ret)}"); _swr = swr; ret = ffmpeg.swr_init(_swr); if (ret < 0) throw new InvalidOperationException($"swr_init failed: {ErrorString(ret)}"); _packet = ffmpeg.av_packet_alloc(); _frame = ffmpeg.av_frame_alloc(); if (_packet == null || _frame == null) throw new InvalidOperationException("Failed to allocate packet/frame."); _syncToTimestamp = true; } catch { Dispose(); throw; } } /// /// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed /// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes /// to a full frame when trimming is off. /// private void ScanPackets(out long firstPts, out long frames) { firstPts = ffmpeg.AV_NOPTS_VALUE; long durationTicks = 0; var packet = ffmpeg.av_packet_alloc(); try { int ret; while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0) { if (packet->stream_index == _streamIndex) { if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts; durationTicks += packet->duration; } ffmpeg.av_packet_unref(packet); } if (ret != ffmpeg.AVERROR_EOF) throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}"); } finally { ffmpeg.av_packet_free(&packet); } if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0; frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate }); var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD); if (seek < 0) throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}"); } private long EstimateTotalFrames(AVStream* stream) { if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0) return ffmpeg.av_rescale_q(stream->duration, _timeBase, new AVRational { num = 1, den = SampleRate }); if (_format->duration != ffmpeg.AV_NOPTS_VALUE && _format->duration > 0) return ffmpeg.av_rescale(_format->duration, SampleRate, ffmpeg.AV_TIME_BASE); throw new InvalidOperationException($"Audio file '{Path}' has no duration."); } public void Seek(long frame) { if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame)); if (frame == Position) return; var marginFrames = (long)(SeekMarginSeconds * SampleRate); // Target inside the pending block or less than the seek margin after it: decode forward and discard. if (!_syncToTimestamp && frame >= _pendingStart + _pendingOffset && frame <= _pendingStart + _pendingOffset + _pendingCount + marginFrames) { Position = frame; return; } var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase); var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD); if (ret < 0) throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}"); ffmpeg.avcodec_flush_buffers(_codec); SeekCount++; _pendingCount = 0; _pendingOffset = 0; _syncToTimestamp = true; _flushSent = false; _endOfStream = false; Position = frame; } public int Read(float[] buffer, int frameCount) { var limit = (int)Math.Min(frameCount, TotalFrames - Position); var written = 0; while (written < limit) { if (_pendingCount == 0 || Position >= _pendingStart + _pendingOffset + _pendingCount) { if (!DecodeNextFrame()) break; continue; } var pendingPosition = _pendingStart + _pendingOffset; if (Position > pendingPosition) { // Discard decoded samples before the read position (after a seek). var skip = (int)Math.Min(Position - pendingPosition, _pendingCount); _pendingOffset += skip; _pendingCount -= skip; continue; } if (Position < pendingPosition) { // A gap in the decoded timestamps: fill with silence up to the next decoded sample. var gap = (int)Math.Min(pendingPosition - Position, limit - written); Array.Clear(buffer, written * Channels, gap * Channels); written += gap; Position += gap; continue; } var count = Math.Min(_pendingCount, limit - written); Array.Copy(_pending, _pendingOffset * Channels, buffer, written * Channels, count * Channels); _pendingOffset += count; _pendingCount -= count; written += count; Position += count; } return written; } /// Decodes the next audio frame into the pending buffer. Returns false at the end of the stream. private bool DecodeNextFrame() { while (!_endOfStream) { var ret = ffmpeg.avcodec_receive_frame(_codec, _frame); if (ret == 0) { StoreFrame(); ffmpeg.av_frame_unref(_frame); return true; } if (ret == ffmpeg.AVERROR_EOF) { _endOfStream = true; break; } if (ret != ffmpeg.AVERROR(ffmpeg.EAGAIN)) throw new InvalidOperationException($"Audio decoding failed in '{Path}': {ErrorString(ret)}"); if (_flushSent) { _endOfStream = true; break; } ret = ffmpeg.av_read_frame(_format, _packet); if (ret < 0) { if (ret != ffmpeg.AVERROR_EOF) throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}"); ffmpeg.avcodec_send_packet(_codec, null); _flushSent = true; continue; } try { if (_packet->stream_index != _streamIndex) continue; ret = ffmpeg.avcodec_send_packet(_codec, _packet); // A corrupt packet is skipped; decoding continues with the next one. if (ret < 0 && ret != ffmpeg.AVERROR_INVALIDDATA && ret != ffmpeg.AVERROR(ffmpeg.EAGAIN)) throw new InvalidOperationException($"Failed to send audio packet of '{Path}': {ErrorString(ret)}"); } finally { ffmpeg.av_packet_unref(_packet); } } return false; } private void StoreFrame() { var samples = _frame->nb_samples; if (samples <= 0) return; if (_syncToTimestamp) { var pts = _frame->best_effort_timestamp; _nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE ? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate }) : Position; _syncToTimestamp = false; } if (_pending.Length < samples * Channels) _pending = new float[samples * Channels]; fixed (float* pPending = _pending) { var outPlane = (byte*)pPending; var converted = ffmpeg.swr_convert(_swr, &outPlane, samples, _frame->extended_data, samples); if (converted < 0) throw new InvalidOperationException($"Sample format conversion failed in '{Path}': {ErrorString(converted)}"); samples = converted; } _pendingStart = _nextDecodedFrame; _pendingOffset = 0; _pendingCount = samples; _nextDecodedFrame += samples; } public void Dispose() { if (_frame != null) { var frame = _frame; ffmpeg.av_frame_free(&frame); _frame = null; } if (_packet != null) { var packet = _packet; ffmpeg.av_packet_free(&packet); _packet = null; } if (_swr != null) { var swr = _swr; ffmpeg.swr_free(&swr); _swr = null; } if (_codec != null) { var codec = _codec; ffmpeg.avcodec_free_context(&codec); _codec = null; } if (_format != null) { var format = _format; ffmpeg.avformat_close_input(&format); _format = null; } } private static string ErrorString(int error) { var buffer = stackalloc byte[1024]; ffmpeg.av_strerror(error, buffer, 1024); return $"{Marshal.PtrToStringAnsi((IntPtr)buffer)} ({error})"; } }