5 Commits
Author SHA256 Message Date
copecone 6ef1538c3f Add Voice Mixer With Timed Changes 2026-10-13 09:43:07 +09:00
copecone 61373f8271 Resolve Audio Clips From Level Files 2026-10-13 02:00:18 +09:00
copecone 8addbfb79d Match Game MP3 Decoding Timing 2026-10-13 01:59:53 +09:00
copecone e9b23b981f Add FFmpeg Audio File Reader 2026-10-13 00:30:07 +09:00
copecone a472c6f72e Add Chunked Audio Clip Store 2026-10-13 00:30:02 +09:00
24 changed files with 2696 additions and 346 deletions

No files matched your search

+1 -3
View File
@@ -520,9 +520,7 @@ public class RenderSession
}
RenderManager.Instance.levelPath = levelPath;
// The render uses its own copy of the configuration: values adjusted during the render
// (e.g. the audio sample rate in SetupPlay) stay in this copy and do not change the caller's object.
RenderManager.Instance.Configuration = renderConfig.Clone();
RenderManager.Instance.Configuration = renderConfig;
RenderManager.Instance.stage = RenderStage.Load;
+10 -86
View File
@@ -5,6 +5,7 @@ using System.IO;
using System.Threading;
using System.Buffers;
using System.Diagnostics;
using System.Linq;
using ADOFAIRenderer.API;
using ADOFAIRenderer.Core.Data;
using ADOFAIRenderer.Data;
@@ -80,28 +81,8 @@ internal class RenderManager : MonoBehaviour
public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene;
public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render;
/// <summary>
/// Maximum number of frames in ImageQueue. The main thread waits (throttles) while the queue holds more frames.
/// </summary>
public const int MaxQueuedFrames = 64;
/// <summary>
/// Upper bound of the readback buffer bytes of the frames in ImageQueue (4 GiB).
/// The frame limit becomes min(MaxQueuedFrames, MaxQueuedBytes / bytes per frame) (see ComputeQueueFrameLimit),
/// so it only lowers the frame count at resolutions where MaxQueuedFrames frames exceed this size.
/// </summary>
public const long MaxQueuedBytes = 4L * 1024 * 1024 * 1024;
/// <summary>
/// Wait time of the throttle loop (OnPostFrame) for the render thread to take a frame from ImageQueue.
/// When it elapses without a dequeue, AsyncGPUReadback.WaitAllRequests() is called once so the readbacks
/// still in flight complete (the render thread may be waiting for one of them to fill a gap in ImageQueue).
/// </summary>
private const int ThrottleWaitMilliseconds = 10;
/// <summary>Frame limit of ImageQueue applied by the last throttle check (MaxQueuedFrames or lower).</summary>
public int QueueFrameLimit { get; private set; } = MaxQueuedFrames;
// Statistics
public long framesSent;
private readonly Stopwatch _renderStopwatch = new();
@@ -232,8 +213,6 @@ internal class RenderManager : MonoBehaviour
// Thread and Queue State
internal SortedList<long, CapturedFrame> ImageQueue = new();
internal AutoResetEvent FrameAvailableEvent = new(false);
// Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it.
internal AutoResetEvent FrameDequeuedEvent = new(false);
public Thread? RenderThread;
public long passCount;
public volatile bool endRequested;
@@ -417,73 +396,26 @@ internal class RenderManager : MonoBehaviour
frameCount++;
}
// Floors that own a hold renderer, collected from scrLevelMaker.listFloors (see RefreshHoldFloors).
private readonly List<scrFloor> _holdFloors = new();
private List<scrFloor>? _holdFloorsSource;
private int _holdFloorsSourceCount = -1;
public void OnLastUpdate()
{
if (Instance.stage < RenderStage.Load) return;
if (Configuration.AdvancedConfig.NonTile)
if (!Configuration.AdvancedConfig.NonTile) return;
foreach (var floor in scrLevelMaker.instance.listFloors
.Where(floor => floor.holdRenderer && floor.holdRenderer.m_meshRenderer.enabled))
{
// scrHoldRenderer.UpdateColor sets m_meshRenderer.enabled = true every time it runs
// (scrHoldRenderer.cs, called from scrPlanet/scrPlayer), so the hold meshes are disabled again every frame.
RefreshHoldFloors();
foreach (var floor in _holdFloors)
{
if (!floor || !floor.holdRenderer) continue;
var meshRenderer = floor.holdRenderer.m_meshRenderer;
if (meshRenderer && meshRenderer.enabled) meshRenderer.enabled = false;
}
floor.holdRenderer.m_meshRenderer.enabled = false;
}
TriggerOnLastUpdate();
}
/// <summary>
/// Rebuilds the hold floor list when scrLevelMaker.listFloors is a different list or has a different count
/// (hold renderers are created together with the floors in scrLevelMaker).
/// </summary>
private void RefreshHoldFloors()
{
var floors = scrLevelMaker.instance.listFloors;
if (ReferenceEquals(floors, _holdFloorsSource) && floors.Count == _holdFloorsSourceCount) return;
_holdFloors.Clear();
foreach (var floor in floors)
{
if (floor && floor.holdRenderer) _holdFloors.Add(floor);
}
_holdFloorsSource = floors;
_holdFloorsSourceCount = floors.Count;
}
/// <summary>
/// Frame limit of ImageQueue for the given capture texture: MaxQueuedBytes divided by the readback bytes of
/// one frame (YUV 4:2:0 when the GPU conversion is active, RGBA otherwise), within 4 to MaxQueuedFrames.
/// </summary>
private int ComputeQueueFrameLimit(RenderTexture texture)
{
var frameBytes = _frameReader?.GetFrameBufferBytes(texture.width, texture.height)
?? (long)texture.width * texture.height * 4;
if (frameBytes <= 0) return MaxQueuedFrames;
return (int)Math.Min(MaxQueuedFrames, Math.Max(4, MaxQueuedBytes / frameBytes));
}
// ReSharper disable once Unity.IncorrectMethodSignature
public void OnPostFrame(RenderTexture? texture)
{
if (Instance.stage < RenderStage.Render || !texture) return;
// [Backpressure Throttling]
// The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread
// removes frames, and it signals FrameDequeuedEvent for each one. The wait has a timeout so that the loop
// re-checks the render thread state and pumps pending readback completions (see ThrottleWaitMilliseconds).
isThrottling = false;
var frameLimit = QueueFrameLimit = ComputeQueueFrameLimit(texture!);
while (Instance.stage >= RenderStage.Render)
{
@@ -499,7 +431,7 @@ internal class RenderManager : MonoBehaviour
count = ImageQueue.Count;
}
if (count <= frameLimit) break;
if (count <= MaxQueuedFrames) break;
if (!isThrottling)
{
@@ -507,10 +439,7 @@ internal class RenderManager : MonoBehaviour
throttleCount++;
}
if (!FrameDequeuedEvent.WaitOne(ThrottleWaitMilliseconds))
{
AsyncGPUReadback.WaitAllRequests();
}
AsyncGPUReadback.WaitAllRequests();
}
isThrottling = false;
@@ -690,7 +619,6 @@ internal class RenderManager : MonoBehaviour
endRequested = false;
endSuccessful = false;
FrameAvailableEvent.Reset();
FrameDequeuedEvent.Reset();
var renderFPS = Configuration.RenderFPS;
var videoFPS = Configuration.VideoFPS;
@@ -754,7 +682,7 @@ internal class RenderManager : MonoBehaviour
throw;
}
var encoder = videoFFmpeg;
var encoder = videoFFmpeg!;
RenderThread = new Thread(() => RunRenderThread(encoder))
{
IsBackground = true,
@@ -806,18 +734,14 @@ internal class RenderManager : MonoBehaviour
/// <summary>
/// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio
/// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready.
/// Returns when the render is aborted (checked before every frame, frames left in ImageQueue are released
/// by the cleanup after the thread ends) or all requested frames have been sent after endRequested.
/// Returns when the render is aborted or all requested frames have been sent after endRequested.
/// </summary>
private void EncodeQueuedFrames(FFmpegNativeEncoder videoFFmpeg)
{
while (true)
{
if (IsRenderAborted) return;
if (TryDequeueNextFrame(out var frame))
{
FrameDequeuedEvent.Set();
SendVideoFrame(videoFFmpeg, frame);
WriteAudioForFrame(videoFFmpeg);
continue;
@@ -960,7 +884,7 @@ internal class RenderManager : MonoBehaviour
var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED";
Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources.");
// 1. Wait (up to 2s) for the render background thread to see the abort flag and terminate
// 1. Wait for render background thread to finish processing remaining queued frames and terminate
if (RenderThread is { IsAlive: true })
{
RenderThread.Join(2000);
+20
View File
@@ -1,11 +1,31 @@
using System;
using ADOFAIRenderer.Core;
using ADOFAIRenderer.Util.Audio.Clip;
using HarmonyLib;
using UnityEngine;
using UnityEngine.Audio;
namespace ADOFAIRenderer.Patch.Engine;
/// <summary>
/// Records the file path of each external clip the game loads (always active, also outside rendering,
/// because levels load their song and PlaySound files before a render starts).
/// FindOrLoadAudioClipExternal returns the clip already in audioLib when its key exists, so the path is recorded
/// only when the key is missing and the file is about to be loaded.
/// </summary>
[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.FindOrLoadAudioClipExternal))]
public static class AudioManagerFindOrLoadAudioClipExternalPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static void Prefix(AudioManager __instance, string path)
{
if (string.IsNullOrEmpty(path)) return;
if (__instance.audioLib.ContainsKey(AudioClipPaths.KeyOf(path))) return;
AudioClipPaths.Record(path);
}
}
[HarmonyPatch(typeof(AudioManager), "_Play")]
public static class AudioManagerPlayPatch
{
+1 -1
View File
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine;
using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.4.0-BETA", "ADOFAI Editor Forum")]
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer;
+1 -1
View File
@@ -164,7 +164,7 @@ internal class DebugMonitorUI : MonoBehaviour
}
}
sb.AppendLine($"• Queue: <b>{queueCount}</b> / {(RenderManager.Instance ? RenderManager.Instance.QueueFrameLimit : RenderManager.MaxQueuedFrames)} frames");
sb.AppendLine($"• Queue: <b>{queueCount}</b> / {RenderManager.MaxQueuedFrames} frames");
sb.AppendLine($"• Rendered Frames: <b>{(RenderManager.Instance ? RenderManager.Instance.frameCount : 0)}</b>");
sb.AppendLine($"• Frames Sent: <b>{(RenderManager.Instance ? Interlocked.Read(ref RenderManager.Instance.framesSent) : 0)}</b>");
+16 -60
View File
@@ -9,7 +9,6 @@ using ADOFAIRenderer.Data;
using ADOFAIRenderer.Localization;
using ADOFAIRenderer.Util.OS;
using MelonLoader;
using Newtonsoft.Json;
using O5Kit.Control;
using O5Kit.Core;
using O5Kit.Factory;
@@ -33,14 +32,6 @@ internal static class RenderConfigWindow
private static Coroutine? _deleteResetCoroutine;
private static string? _lastLoadedPresetName;
/// <summary>
/// Configuration edited by this window. Kept for the whole game session; a render started from the window
/// receives a copy of it (RenderSession.Start), so values adjusted during the render are not written back here.
/// </summary>
private static RenderConfig? _uiConfig;
private static RenderConfig UiConfig => _uiConfig ??= new RenderConfig();
// Controls cache for validation & sync
private static O5Slider? _renderFpsSlider;
private static O5Slider? _videoFPSSlider;
@@ -120,7 +111,8 @@ internal static class RenderConfigWindow
ReloadPresetDropdown(GlobalSettings.Current.LastPresetName);
}
var config = UiConfig;
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
ApplyConfigToUI(config);
SetWindowActive(true);
@@ -133,7 +125,8 @@ internal static class RenderConfigWindow
var (_, scrollContent, _) =
O5Factory.ScrollView(ctx, content, spacing: 10f, padding: 8f);
var config = UiConfig;
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
EnsureDefaultConfig(config);
double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144,
@@ -751,14 +744,11 @@ internal static class RenderConfigWindow
{
EnsureDefaultConfig(config);
// Sliders show the config values as they are (noFilter: true).
// O5Slider.Set applies the slider filter (snapping to the nearest snap value) unless noFilter is set;
// the filter is meant for dragging, and typed values are not snapped either (O5Slider.SetFromInput).
_renderFpsSlider?.Set(config.RenderFPS, false, noFilter: true);
_videoFPSSlider?.Set(config.VideoFPS, false, noFilter: true);
_renderWidthSlider?.Set(config.RenderWidth, false, noFilter: true);
_renderHeightSlider?.Set(config.RenderHeight, false, noFilter: true);
_delayAfterEndSlider?.Set(config.DelayAfterEnd, false, noFilter: true);
_renderFpsSlider?.Set((float) config.RenderFPS, false);
_videoFPSSlider?.Set((float) config.VideoFPS, false);
_renderWidthSlider?.Set(config.RenderWidth, false);
_renderHeightSlider?.Set(config.RenderHeight, false);
_delayAfterEndSlider?.Set((float) config.DelayAfterEnd, false);
_showJudgementToggle?.Set(config.ShowJudgementText, false);
_codecDropdown?.Set(config.VideoConfig.Codec, false);
@@ -766,13 +756,13 @@ internal static class RenderConfigWindow
_bitrateInput?.Set(config.VideoConfig.Bitrate.ToString(), false);
_maxBitrateInput?.Set(config.VideoConfig.MaxBitrate.ToString(), false);
_qpInput?.Set(config.VideoConfig.QP.ToString(), false);
_keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false, noFilter: true);
_keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false);
_forceSoftwareToggle?.Set(config.VideoConfig.ForceSoftwareEncoder, false);
_renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false);
_sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false);
_channelsDropdown?.Set(config.AudioConfig.AudioChannels, false);
_audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false, noFilter: true);
_audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false);
_outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false);
@@ -799,51 +789,17 @@ internal static class RenderConfigWindow
UpdateVideoCodecUI(config.VideoConfig.Codec, config);
UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true);
UpdatePresetLabel(config);
}
/// <summary>
/// Sets the preset dropdown from the config values.
/// Every value equals the last loaded preset (_lastLoadedPresetName): the preset name is shown.
/// Any value differs (edited, or the preset file changed): the Custom label is shown (MarkConfigModified),
/// the same state as after editing a value in the window.
/// </summary>
private static void UpdatePresetLabel(RenderConfig config)
{
if (_presetDropdown == null) return;
var presetName = _lastLoadedPresetName;
var preset = string.IsNullOrEmpty(presetName)
? null
: PresetManager.GetPresets().FirstOrDefault(p => string.Equals(p.Name, presetName, StringComparison.OrdinalIgnoreCase));
if (preset == null || !ConfigValuesEqual(config, preset.Config))
if (!string.IsNullOrEmpty(GlobalSettings.Current.LastPresetName))
{
MarkConfigModified();
return;
_presetDropdown?.Set(GlobalSettings.Current.LastPresetName, false);
_lastLoadedPresetName = GlobalSettings.Current.LastPresetName;
UpdateDeleteButtonState();
}
if (_presetDropdown.Values.Contains(CustomPresetLabel))
{
_presetDropdown.SetValues(_presetDropdown.Values.Where(v => v != CustomPresetLabel).ToList());
}
_presetDropdown.Set(preset.Name, false);
UpdateDeleteButtonState();
}
/// <summary>
/// Compares two configs by their serialized JSON (the same form as the preset files), so every saved value is compared.
/// </summary>
private static bool ConfigValuesEqual(RenderConfig a, RenderConfig b)
{
var settings = new JsonSerializerSettings();
settings.Converters.Add(new ColorJsonConverter());
return JsonConvert.SerializeObject(a, settings) == JsonConvert.SerializeObject(b, settings);
}
private static void TryGoRender()
{
var config = UiConfig;
var config = RenderManager.Instance.Configuration;
SyncUItoConfig(config);
// Sync Output Directory
+46
View File
@@ -0,0 +1,46 @@
using System.Collections.Generic;
using System.IO;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// File path of every external audio clip the game loads, keyed by the game's audioLib key.
///
/// The game loads external clips with AudioManager.FindOrLoadAudioClipExternal(path, ...) and stores them in audioLib under
/// Path.GetFileName(path) + "*external" (the same string becomes AudioClip.name). Recorded paths:
/// - Song: Path.Combine(Path.GetDirectoryName(levelPath), levelData.songFilename) (scnGame song loading).
/// - PlaySound event sound: Path.Combine(Path.GetDirectoryName(levelPath), hitsound) (scnGame.LoadSoundCo);
/// ffxPlaySound plays it as hitsound + "*external".
/// <see cref="Record"/> is called only when the game actually loads the file (the key is not in audioLib yet),
/// so the recorded path is the file of the clip audioLib holds under that key.
/// Thread-safe.
/// </summary>
internal static class AudioClipPaths
{
/// <summary>Suffix of the game's audioLib keys for external clips.</summary>
public const string ExternalSuffix = "*external";
private static readonly Dictionary<string, string> Paths = new();
/// <summary>audioLib key of an external clip loaded from <paramref name="path"/>.</summary>
public static string KeyOf(string path) => Path.GetFileName(path) + ExternalSuffix;
/// <summary>Records <paramref name="path"/> under <see cref="KeyOf"/>(path), replacing an earlier path of the same key.</summary>
public static void Record(string path)
{
if (string.IsNullOrEmpty(path)) return;
lock (Paths)
{
Paths[KeyOf(path)] = path;
}
}
/// <summary>Recorded path for an audioLib key (or AudioClip.name) of an external clip.</summary>
public static bool TryGetPath(string key, out string path)
{
lock (Paths)
{
return Paths.TryGetValue(key, out path!);
}
}
}
+60
View File
@@ -0,0 +1,60 @@
using System;
using System.IO;
using MelonLoader;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Opens the <see cref="AudioClipSource"/> for a clip the game plays, keyed by the game's audioLib key (also AudioClip.name):
/// - Key recorded in <see cref="AudioClipPaths"/> and the file exists: decoded from the file with <see cref="FFmpegPCMReader"/>
/// (mono MP3 through <see cref="MP3MonoLayoutReader"/>)
/// (song files and PlaySound files of custom levels).
/// - Otherwise, or when FFmpeg cannot open the file: the Unity clip is copied with <see cref="UnityClipLoader"/>
/// (built-in hit sounds from Resources, official/DLC level songs from Addressables).
/// Main thread only (the Unity copy calls AudioClip.GetData).
/// </summary>
internal static class AudioClipResolver
{
/// <param name="store">Store of the current render.</param>
/// <param name="key">audioLib key of the clip (for example "song.ogg*external" or "sndKick").</param>
/// <param name="clip">Loaded Unity clip for the key, used when the clip has no file; may be null when the key has a file.</param>
/// <returns>The source, or null when neither the file nor the clip can be read.</returns>
public static AudioClipSource? GetOrAdd(AudioClipStore store, string key, AudioClip? clip)
{
if (store.TryGet(key, out var existing)) return existing;
if (AudioClipPaths.TryGetPath(key, out var path) && File.Exists(path))
{
IPCMReader? fileReader = null;
try
{
fileReader = OpenFile(path);
if (fileReader is MP3MonoLayoutReader)
Melon<RenderMod>.Logger.Warning($"[AudioClipResolver] Mono MP3 '{path}' is rendered like the game: half speed, one octave lower.");
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Warning($"[AudioClipResolver] FFmpeg could not open '{path}' for '{key}', using the Unity clip: {e.Message}");
}
if (fileReader != null)
{
var source = store.GetOrAdd(key, () => fileReader, () => OpenFile(path));
if (source == null) fileReader.Dispose();
return source;
}
}
if (!clip) return store.GetOrAdd(key, () => null);
return store.GetOrAdd(key, () => UnityClipLoader.Load(clip!, store));
}
/// <summary>Opens <paramref name="path"/> with FFmpeg; mono MP3 files are wrapped in <see cref="MP3MonoLayoutReader"/> (game layout).</summary>
private static IPCMReader OpenFile(string path)
{
var reader = new FFmpegPCMReader(path);
if (!reader.IsMP3 || reader.Channels != 1) return reader;
return new MP3MonoLayoutReader(reader);
}
}
+613
View File
@@ -0,0 +1,613 @@
using System;
using System.Collections.Generic;
using System.Threading;
using MelonLoader;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Read position registered on an <see cref="AudioClipSource"/> by one consumer (a playing voice or the spectrum generator).
/// For a windowed source, the chunk containing <see cref="Position"/> and the following chunks
/// (<see cref="AudioClipSource.SlotCount"/> chunks in total) are kept loaded.
/// </summary>
internal sealed class AudioCursor
{
private readonly AudioClipSource _source;
private long _position;
private volatile bool _loop;
internal AudioCursor(AudioClipSource source, long position, bool loop)
{
_source = source;
_position = position;
_loop = loop;
}
/// <summary>
/// Frame in the source at the output sample rate (0 ~ TotalFrames - 1).
/// Written by the consumer that reads the source; read by the decoder thread.
/// Moving into another chunk wakes the decoder thread.
/// </summary>
public long Position
{
get => Interlocked.Read(ref _position);
set
{
var previous = Interlocked.Exchange(ref _position, value);
if (previous / _source.FramesPerChunk != value / _source.FramesPerChunk)
_source.OnCursorMoved();
}
}
/// <summary>When true, the chunk after the last chunk is chunk 0.</summary>
public bool Loop
{
get => _loop;
set
{
_loop = value;
_source.OnCursorMoved();
}
}
}
/// <summary>
/// One loaded chunk of an <see cref="AudioClipSource"/>, held by a consumer between
/// <see cref="AudioClipSource.TryAcquire"/> and <see cref="AudioClipSource.Release"/>.
/// The chunk is not freed while held.
/// </summary>
internal readonly struct ChunkView
{
/// <summary>Interleaved samples. Frame f of the source is at (f - StartFrame) * Channels.</summary>
public readonly float[] Data;
/// <summary>Source frame of the first frame in <see cref="Data"/>.</summary>
public readonly long StartFrame;
/// <summary>Valid frames in this chunk. Frames past the end of the source are not included.</summary>
public readonly int FrameCount;
/// <summary>Channel count of the stored samples (same as the source clip).</summary>
public readonly int Channels;
public readonly int ChunkIndex;
internal ChunkView(float[] data, long startFrame, int frameCount, int channels, int chunkIndex)
{
Data = data;
StartFrame = startFrame;
FrameCount = frameCount;
Channels = channels;
ChunkIndex = chunkIndex;
}
public bool IsValid => Data != null;
public long EndFrame => StartFrame + FrameCount;
public bool Contains(long frame) => frame >= StartFrame && frame < StartFrame + FrameCount;
/// <summary>
/// Sample of <paramref name="frame"/> mapped to output channel <paramref name="outChannel"/> of <paramref name="outChannels"/>.
/// Mapping: mono output averages all source channels; a mono source is copied to every output channel;
/// otherwise output channel c takes source channel c, and output channels beyond the source take source channel 0.
/// <paramref name="frame"/> must be inside this chunk.
/// </summary>
public float GetSample(long frame, int outChannel, int outChannels)
{
var baseIndex = (int)(frame - StartFrame) * Channels;
if (Channels == 1)
return Data[baseIndex];
if (outChannels == 1)
{
var sum = 0f;
for (var c = 0; c < Channels; c++)
sum += Data[baseIndex + c];
return sum / Channels;
}
return Data[baseIndex + (outChannel < Channels ? outChannel : 0)];
}
}
/// <summary>
/// How an <see cref="AudioClipSource"/> keeps its chunks.
/// </summary>
internal enum ChunkPolicy
{
/// <summary>
/// Only the chunks required by registered cursors (current chunk and the next SlotCount - 1 chunks per cursor),
/// chunks requested by a waiting consumer, and chunks currently held are kept. Other chunks are freed and reloaded on demand.
/// Used for readers that can seek (files decoded with FFmpeg) when the clip is longer than one chunk.
/// </summary>
Windowed,
/// <summary>
/// All chunks are loaded once, in order from chunk 0, starting when the source is created, and kept until the store is disposed.
/// Used for forward-only readers (clip data copied from Unity, released as it is consumed) and for clips of one chunk or less.
/// </summary>
Resident
}
/// <summary>
/// Decoded audio of one clip, resampled to the output sample rate and stored in fixed-size chunks.
/// Channel count stays as in the clip.
///
/// Chunks: chunk k covers output frames [k * FramesPerChunk, (k + 1) * FramesPerChunk), where
/// <see cref="FramesPerChunk"/> = <see cref="ChunkFloats"/> / Channels (rounded down; a frame never spans two chunks).
/// Windowed chunks use pooled arrays of ChunkFloats floats; resident chunks use arrays of their exact length.
/// See <see cref="ChunkPolicy"/> for which chunks are kept.
///
/// Threads: cursors and TryAcquire/Release may be used from any thread.
/// Loading and freeing run only on the store's decoder thread (<see cref="AudioClipStore"/>), through
/// <see cref="FreeUnused"/>, <see cref="TryGetNextLoad"/> and <see cref="LoadChunk"/>.
/// </summary>
internal sealed class AudioClipSource
{
/// <summary>Floats per chunk array: 2^22 floats = 16 MiB.</summary>
public const int ChunkFloats = 1 << 22;
/// <summary>Chunks kept per cursor of a windowed source: the chunk containing the cursor and the next SlotCount - 1 chunks.</summary>
public const int SlotCount = 3;
/// <summary>Load priority of a chunk a consumer is waiting for (lowest value is loaded first).</summary>
internal const int PriorityDemanded = 0;
/// <summary>Load priority of slot s of a cursor window is PriorityWindow + s.</summary>
internal const int PriorityWindow = 1;
/// <summary>Load priority of resident chunks nobody is waiting for.</summary>
internal const int PriorityResident = PriorityWindow + SlotCount;
/// <summary>
/// Most decode streams (reader + resampler) a windowed source opens. A load continues the stream positioned at the chunk start,
/// otherwise opens another stream up to this count, otherwise restarts the least recently used stream.
/// Each stream holds one open reader (file handle and decoder state, measured 0.3 ~ 2.7 MiB for ogg/flac/mp3 with the clip harness).
/// </summary>
public const int MaxDecodeStreams = 4;
/// <summary>Decode stream limit of this source (<see cref="MaxDecodeStreams"/> unless changed for tests).</summary>
internal int DecodeStreamLimit { get; set; } = MaxDecodeStreams;
private readonly AudioClipStore _store;
private readonly IPCMReader _firstReader;
private readonly Func<IPCMReader?>? _openReader;
// Guarded by _sync: chunk table, hold counts, cursors, demand set, loaded set.
private readonly object _sync = new();
private readonly float[]?[] _chunks;
private readonly int[] _holdCounts;
private readonly List<AudioCursor> _cursors = new();
private readonly HashSet<int> _demanded = new();
private readonly HashSet<int> _loaded = new();
private int _residentLoadedCount; // Resident: chunks [0, _residentLoadedCount) are loaded
// Guarded by _loadLock: decode streams.
private readonly object _loadLock = new();
private readonly List<DecodeStream> _streams = new();
private long _streamUseCounter;
private bool _openFailed;
private volatile bool _isFailed;
public string Key { get; }
/// <summary>Channel count of the stored samples (same as the clip).</summary>
public int Channels { get; }
/// <summary>Sample rate of the stored samples in Hz (the output sample rate).</summary>
public int Frequency { get; }
/// <summary>Frame count at <see cref="Frequency"/>: ceil(clip frames * Frequency / clip rate).</summary>
public long TotalFrames { get; }
/// <summary>Clip length in seconds: clip frames / clip rate.</summary>
public double Duration { get; }
public int FramesPerChunk { get; }
public int ChunkCount { get; }
public ChunkPolicy Policy { get; }
public bool IsResident => Policy == ChunkPolicy.Resident;
/// <summary>True after loading failed. A failed source returns no chunks.</summary>
public bool IsFailed => _isFailed;
/// <summary>True when every chunk of a resident source is loaded.</summary>
public bool IsFullyLoaded
{
get
{
lock (_sync) return _loaded.Count == ChunkCount;
}
}
/// <summary>Number of chunk loads performed (for diagnostics).</summary>
public int LoadCount { get; private set; }
/// <summary>Decode streams opened so far (for diagnostics).</summary>
public int StreamCount
{
get
{
lock (_loadLock) return _streams.Count;
}
}
/// <param name="store">Store that owns this source.</param>
/// <param name="key">Clip name or path the source is registered under.</param>
/// <param name="reader">First reader; also defines the clip format.</param>
/// <param name="openReader">Opens another reader of the same clip for an additional decode stream, or null when not possible.</param>
/// <param name="outputRate">Output sample rate in Hz.</param>
internal AudioClipSource(AudioClipStore store, string key, IPCMReader reader, Func<IPCMReader?>? openReader, int outputRate)
{
if (reader.Channels <= 0) throw new ArgumentException("Reader channel count must be positive.", nameof(reader));
if (reader.SampleRate <= 0) throw new ArgumentException("Reader sample rate must be positive.", nameof(reader));
if (reader.TotalFrames < 0) throw new ArgumentException("Reader frame count must not be negative.", nameof(reader));
_store = store;
_firstReader = reader;
_openReader = openReader;
Key = key;
Channels = reader.Channels;
Frequency = outputRate;
Duration = (double)reader.TotalFrames / reader.SampleRate;
TotalFrames = CeilDiv(reader.TotalFrames * outputRate, reader.SampleRate);
FramesPerChunk = ChunkFloats / Channels;
ChunkCount = (int)CeilDiv(TotalFrames, FramesPerChunk);
Policy = reader.IsForwardOnly || ChunkCount <= 1 ? ChunkPolicy.Resident : ChunkPolicy.Windowed;
_chunks = new float[ChunkCount][];
_holdCounts = new int[ChunkCount];
_streams.Add(new DecodeStream(reader, outputRate));
}
// ── Cursors ──
/// <summary>Registers a read position. The decoder thread loads the chunks it requires.</summary>
public AudioCursor AddCursor(long position, bool loop)
{
var cursor = new AudioCursor(this, position, loop);
lock (_sync)
{
_cursors.Add(cursor);
}
_store.RequestService();
return cursor;
}
public void RemoveCursor(AudioCursor cursor)
{
lock (_sync)
{
_cursors.Remove(cursor);
}
_store.RequestService();
}
internal void OnCursorMoved()
{
if (!IsResident) _store.RequestService();
}
// ── Consumer access ──
/// <summary>Chunk index containing <paramref name="frame"/> (0 ~ TotalFrames - 1).</summary>
public int ChunkIndexOf(long frame) => (int)(frame / FramesPerChunk);
/// <summary>
/// Holds the chunk containing <paramref name="frame"/> when it is loaded.
/// When it is not loaded, records a load request (kept until the chunk is acquired), wakes the decoder thread and returns false.
/// Every successful call must be paired with <see cref="Release"/>.
/// </summary>
public bool TryAcquire(long frame, out ChunkView view)
{
view = default;
if (_isFailed || frame < 0 || frame >= TotalFrames) return false;
var index = ChunkIndexOf(frame);
lock (_sync)
{
var data = _chunks[index];
if (data != null)
{
_demanded.Remove(index);
_holdCounts[index]++;
view = CreateView(index, data);
return true;
}
_demanded.Add(index);
}
_store.RequestService();
return false;
}
public void Release(in ChunkView view)
{
if (!view.IsValid) return;
lock (_sync)
{
if (_holdCounts[view.ChunkIndex] > 0)
_holdCounts[view.ChunkIndex]--;
}
}
private ChunkView CreateView(int index, float[] data)
{
var start = (long)index * FramesPerChunk;
var count = (int)Math.Min(FramesPerChunk, TotalFrames - start);
return new ChunkView(data, start, count, Channels, index);
}
// ── Decoder thread ──
/// <summary>
/// Windowed: frees loaded chunks that are not required (see <see cref="ChunkPolicy.Windowed"/>) and not held.
/// Resident: does nothing. Decoder thread only.
/// </summary>
internal void FreeUnused()
{
if (IsResident) return;
List<float[]>? freed = null;
lock (_sync)
{
if (_loaded.Count == 0) return;
List<int>? remove = null;
foreach (var index in _loaded)
{
if (_holdCounts[index] > 0 || IsRequired(index)) continue;
(remove ??= new List<int>()).Add(index);
}
if (remove == null) return;
foreach (var index in remove)
{
(freed ??= new List<float[]>()).Add(_chunks[index]!);
_chunks[index] = null;
_loaded.Remove(index);
}
}
foreach (var data in freed!)
_store.ReturnChunkArray(data);
}
/// <summary>
/// Next chunk to load and its priority (lower loads first), or false when nothing is missing.
/// Windowed: demanded chunks, then each cursor's window by slot.
/// Resident: the next chunk in order; priority is <see cref="PriorityDemanded"/> while a consumer is waiting on this source.
/// Decoder thread only.
/// </summary>
internal bool TryGetNextLoad(out int index, out int priority)
{
index = -1;
priority = int.MaxValue;
if (_isFailed) return false;
lock (_sync)
{
if (IsResident)
{
if (_residentLoadedCount >= ChunkCount) return false;
index = _residentLoadedCount;
priority = _demanded.Count > 0 ? PriorityDemanded : PriorityResident;
return true;
}
foreach (var demanded in _demanded)
{
if (_chunks[demanded] != null) continue;
index = demanded;
priority = PriorityDemanded;
return true;
}
foreach (var cursor in _cursors)
{
if (!TryGetWindowStart(cursor, out var first, out var loop)) continue;
for (var slot = 0; slot < SlotCount; slot++)
{
if (!TryWindowIndex(first, slot, loop, out var candidate)) break;
if (_chunks[candidate] != null) continue;
if (PriorityWindow + slot < priority)
{
index = candidate;
priority = PriorityWindow + slot;
}
break;
}
}
return index >= 0;
}
}
/// <summary>True when a demand or a cursor window covers <paramref name="index"/>. Caller holds _sync.</summary>
private bool IsRequired(int index)
{
if (_demanded.Contains(index)) return true;
foreach (var cursor in _cursors)
{
if (!TryGetWindowStart(cursor, out var first, out var loop)) continue;
for (var slot = 0; slot < SlotCount; slot++)
{
if (!TryWindowIndex(first, slot, loop, out var candidate)) break;
if (candidate == index) return true;
}
}
return false;
}
private bool TryGetWindowStart(AudioCursor cursor, out int first, out bool loop)
{
var position = cursor.Position;
loop = cursor.Loop;
first = 0;
if (position < 0) position = 0;
if (position >= TotalFrames)
{
if (!loop || TotalFrames == 0) return false;
position %= TotalFrames;
}
first = ChunkIndexOf(position);
return true;
}
private bool TryWindowIndex(int first, int slot, bool loop, out int index)
{
index = first + slot;
if (index < ChunkCount) return true;
if (!loop) return false;
index %= ChunkCount;
return true;
}
/// <summary>
/// Loads chunk <paramref name="index"/> if it is not loaded. Resident sources load only the next chunk in order.
/// Decoder thread only.
/// </summary>
/// <returns>True when this call loaded the chunk.</returns>
internal bool LoadChunk(int index)
{
if (_isFailed || index < 0 || index >= ChunkCount) return false;
lock (_loadLock)
{
lock (_sync)
{
if (_chunks[index] != null) return false;
if (IsResident && index != _residentLoadedCount)
throw new InvalidOperationException($"Resident clip '{Key}' loads chunks in order: expected {_residentLoadedCount}, got {index}.");
}
float[]? data = null;
try
{
var start = (long)index * FramesPerChunk;
var frames = (int)Math.Min(FramesPerChunk, TotalFrames - start);
data = IsResident ? _store.AllocateExactArray(frames * Channels) : _store.RentChunkArray();
var stream = SelectStream(start);
stream.LastUse = ++_streamUseCounter;
stream.Fill(start, frames, data);
}
catch (OutOfMemoryException e)
{
ReturnArray(data);
Fail();
_store.ReportFatal(this, e);
return false;
}
catch (Exception e)
{
ReturnArray(data);
Fail();
_store.ReportSourceFailure(this, e);
return false;
}
lock (_sync)
{
_chunks[index] = data;
_loaded.Add(index);
if (IsResident) _residentLoadedCount++;
}
LoadCount++;
// A resident source never reads again after its last chunk; the reader releases its data now.
if (IsResident && _residentLoadedCount == ChunkCount)
DisposeStreams();
}
_store.NotifyChunkLoaded();
return true;
}
/// <summary>
/// Decode stream for a load starting at output frame <paramref name="start"/>:
/// the stream positioned at <paramref name="start"/>; otherwise a new stream while fewer than <see cref="DecodeStreamLimit"/>
/// are open (windowed sources only); otherwise the least recently used stream. Caller holds _loadLock.
/// </summary>
private DecodeStream SelectStream(long start)
{
DecodeStream? oldest = null;
foreach (var stream in _streams)
{
if (stream.NextOutFrame == start) return stream;
if (oldest == null || stream.LastUse < oldest.LastUse) oldest = stream;
}
if (!IsResident && !_openFailed && _openReader != null && _streams.Count < DecodeStreamLimit)
{
IPCMReader? reader = null;
try
{
reader = _openReader();
if (reader is { IsForwardOnly: true })
{
reader.Dispose();
reader = null;
}
if (reader != null)
{
var stream = new DecodeStream(reader, Frequency);
_streams.Add(stream);
return stream;
}
}
catch (Exception e)
{
reader?.Dispose();
Melon<RenderMod>.Logger.Warning($"[AudioClipSource] Could not open another decode stream for '{Key}', reusing open streams: {e.Message}");
}
_openFailed = true;
}
return oldest!;
}
private void Fail()
{
_isFailed = true;
if (IsResident) DisposeStreams();
}
private void DisposeStreams()
{
foreach (var stream in _streams)
stream.Dispose();
_streams.Clear();
}
private void ReturnArray(float[]? data)
{
if (data == null) return;
if (IsResident) _store.FreeExactArray(data);
else _store.ReturnChunkArray(data);
}
/// <summary>Releases all chunk arrays and readers. Called by the store after the decoder thread stopped.</summary>
internal void DisposeInternal()
{
lock (_loadLock)
{
lock (_sync)
{
foreach (var index in _loaded)
{
ReturnArray(_chunks[index]);
_chunks[index] = null;
}
_loaded.Clear();
_cursors.Clear();
_demanded.Clear();
}
DisposeStreams();
_firstReader.Dispose();
}
}
private static long CeilDiv(long a, long b) => (a + b - 1) / b;
}
+348
View File
@@ -0,0 +1,348 @@
using System;
using System.Collections.Generic;
using System.Threading;
using MelonLoader;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Clip sources of one render, keyed by clip name or path, with a shared pool of chunk arrays and one decoder thread.
///
/// Threads:
/// - Decoder thread (<see cref="Start"/>): the only thread that decodes, resamples, loads and frees chunks.
/// Each step loads one chunk, the one with the lowest priority value over all sources
/// (a chunk a consumer waits for, then cursor windows by slot, then resident chunks in order).
/// - Consumers (render thread, main thread for the spectrum): <see cref="Acquire"/> waits for the decoder thread and never loads.
/// - Main thread: <see cref="GetOrAdd"/> creates readers (for Unity clips this copies the clip data).
///
/// Memory has no limit; current and peak usage are tracked and logged by <see cref="LogMemoryUsage"/>.
/// An <see cref="OutOfMemoryException"/> while loading is kept in <see cref="FatalError"/> and rethrown by <see cref="Acquire"/>.
/// </summary>
internal sealed class AudioClipStore : IDisposable
{
/// <summary>Longest wait of the decoder thread between checks when nothing signals it, in milliseconds.</summary>
private const int DecoderIdleWaitMs = 50;
/// <summary>Longest wait of a consumer between checks when no chunk is loaded, in milliseconds.</summary>
private const int ConsumerWaitMs = 50;
/// <summary>Longest wait for the decoder thread to stop in <see cref="Dispose"/>, in milliseconds.</summary>
private const int DecoderStopTimeoutMs = 3000;
private readonly Dictionary<string, AudioClipSource?> _sources = new();
private readonly List<AudioClipSource> _sourceList = new();
private readonly Stack<float[]> _chunkPool = new();
private readonly object _poolLock = new();
private readonly object _signal = new();
private readonly AutoResetEvent _serviceRequested = new(false);
private Thread? _decoderThread;
private volatile bool _stopRequested;
private long _allocatedBytes;
private long _peakBytes;
private volatile Exception? _fatalError;
private volatile bool _disposed;
/// <summary>Sample rate every source is resampled to, in Hz.</summary>
public int OutputRate { get; }
/// <summary>Bytes of sample arrays currently allocated (loaded chunks, pooled chunk arrays and copied Unity clip data).</summary>
public long AllocatedBytes => Interlocked.Read(ref _allocatedBytes);
/// <summary>Highest value of <see cref="AllocatedBytes"/> since creation.</summary>
public long PeakBytes => Interlocked.Read(ref _peakBytes);
/// <summary>Out-of-memory error raised while loading, or null.</summary>
public Exception? FatalError => _fatalError;
public AudioClipStore(int outputRate)
{
if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate));
OutputRate = outputRate;
}
// ── Decoder thread ──
/// <summary>Starts the decoder thread. Called once.</summary>
public void Start()
{
if (_decoderThread != null) throw new InvalidOperationException("Decoder thread already started.");
_decoderThread = new Thread(DecoderLoop)
{
Name = "Audio Clip Decoder",
IsBackground = true
};
_decoderThread.Start();
}
private void DecoderLoop()
{
try
{
while (!_stopRequested)
{
if (!LoadNext())
_serviceRequested.WaitOne(DecoderIdleWaitMs);
}
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Decoder thread stopped: {e}");
_fatalError ??= e;
NotifyChunkLoaded();
}
}
/// <summary>
/// Frees unused windowed chunks, then loads the single chunk with the lowest priority value over all sources.
/// Runs on the decoder thread (tests may call it directly while the thread is not started).
/// </summary>
/// <returns>True when a chunk was loaded or a load was attempted.</returns>
internal bool LoadNext()
{
AudioClipSource[] sources;
lock (_sources)
{
if (_disposed) return false;
sources = _sourceList.ToArray();
}
AudioClipSource? best = null;
var bestIndex = -1;
var bestPriority = int.MaxValue;
foreach (var source in sources)
{
source.FreeUnused();
if (!source.TryGetNextLoad(out var index, out var priority) || priority >= bestPriority) continue;
best = source;
bestIndex = index;
bestPriority = priority;
}
if (best == null) return false;
best.LoadChunk(bestIndex);
return true;
}
internal void RequestService() => _serviceRequested.Set();
// ── Sources ──
/// <summary>
/// Returns the source registered under <paramref name="key"/>, creating it with <paramref name="createReader"/> on first use.
/// A key whose reader could not be created is remembered as null and not retried.
/// For readers that can seek, additional decode streams (see <see cref="AudioClipSource.MaxDecodeStreams"/>) are opened on the
/// decoder thread with <paramref name="openAdditionalReader"/>, or with <paramref name="createReader"/> when it is null.
/// </summary>
public AudioClipSource? GetOrAdd(string key, Func<IPCMReader?> createReader, Func<IPCMReader?>? openAdditionalReader = null)
{
lock (_sources)
{
if (_disposed) return null;
if (_sources.TryGetValue(key, out var existing)) return existing;
}
AudioClipSource? source = null;
try
{
var reader = createReader();
if (reader != null)
{
try
{
source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : openAdditionalReader ?? createReader, OutputRate);
}
catch
{
reader.Dispose();
throw;
}
}
}
catch (OutOfMemoryException e)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Out of memory while opening clip '{key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {e}");
_fatalError ??= e;
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Failed to open clip '{key}': {e}");
}
lock (_sources)
{
if (_disposed || _sources.TryGetValue(key, out var raced))
{
source?.DisposeInternal();
return _disposed ? null : _sources[key];
}
_sources[key] = source;
if (source != null) _sourceList.Add(source);
}
if (source != null)
{
Melon<RenderMod>.Logger.Msg($"[AudioClipStore] Opened clip '{key}': {source.Channels}ch, {source.Duration:F3}s, {source.TotalFrames} frames at {OutputRate}Hz, {source.ChunkCount} chunk(s), {source.Policy}");
RequestService();
}
return source;
}
public bool TryGet(string key, out AudioClipSource? source)
{
lock (_sources)
{
return _sources.TryGetValue(key, out source);
}
}
// ── Consumers ──
/// <summary>
/// Holds the chunk containing <paramref name="frame"/>, waiting for the decoder thread until it is loaded.
/// Returns an invalid view when the source failed, the frame is outside the source, or the store was disposed.
/// </summary>
/// <exception cref="InvalidOperationException">Loading ran out of memory or the decoder thread stopped.</exception>
/// <exception cref="OperationCanceledException"><paramref name="cancellationToken"/> was canceled while waiting.</exception>
public ChunkView Acquire(AudioClipSource source, long frame, CancellationToken cancellationToken = default)
{
if (frame < 0 || frame >= source.TotalFrames) return default;
while (true)
{
ThrowIfFatal();
if (source.IsFailed || _disposed) return default;
if (source.TryAcquire(frame, out var view)) return view;
lock (_signal)
{
if (source.TryAcquire(frame, out view)) return view;
if (source.IsFailed || _fatalError != null || _disposed) continue;
Monitor.Wait(_signal, ConsumerWaitMs);
}
cancellationToken.ThrowIfCancellationRequested();
}
}
private void ThrowIfFatal()
{
var error = _fatalError;
if (error != null)
throw new InvalidOperationException("Audio clip loading failed: " + error.Message, error);
}
internal void NotifyChunkLoaded()
{
lock (_signal)
{
Monitor.PulseAll(_signal);
}
}
internal void ReportSourceFailure(AudioClipSource source, Exception error)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Failed to load clip '{source.Key}', it will be silent: {error}");
NotifyChunkLoaded();
}
internal void ReportFatal(AudioClipSource source, OutOfMemoryException error)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Out of memory while loading clip '{source.Key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {error}");
_fatalError ??= error;
NotifyChunkLoaded();
}
// ── Arrays ──
internal float[] RentChunkArray()
{
lock (_poolLock)
{
if (_chunkPool.Count > 0) return _chunkPool.Pop();
}
var data = new float[AudioClipSource.ChunkFloats];
AddAllocated(data.LongLength * sizeof(float));
return data;
}
internal void ReturnChunkArray(float[] data)
{
if (_disposed)
{
AddAllocated(-data.LongLength * sizeof(float));
return;
}
lock (_poolLock)
{
_chunkPool.Push(data);
}
}
internal float[] AllocateExactArray(int floats)
{
var data = new float[floats];
AddAllocated(data.LongLength * sizeof(float));
return data;
}
internal void FreeExactArray(float[] data) => AddAllocated(-data.LongLength * sizeof(float));
private void AddAllocated(long bytes)
{
var current = Interlocked.Add(ref _allocatedBytes, bytes);
long peak;
while (current > (peak = Interlocked.Read(ref _peakBytes)))
{
if (Interlocked.CompareExchange(ref _peakBytes, current, peak) == peak) break;
}
}
public void LogMemoryUsage()
{
int sourceCount;
lock (_sources) sourceCount = _sourceList.Count;
Melon<RenderMod>.Logger.Msg($"[AudioClipStore] Clips: {sourceCount}, sample memory now {AllocatedBytes / 1048576.0:F1} MiB, peak {PeakBytes / 1048576.0:F1} MiB");
}
/// <summary>Stops the decoder thread, then releases every source and pooled array.</summary>
public void Dispose()
{
AudioClipSource[] sources;
lock (_sources)
{
if (_disposed) return;
_disposed = true;
sources = _sourceList.ToArray();
}
_stopRequested = true;
_serviceRequested.Set();
NotifyChunkLoaded();
// A decoder thread blocked inside a read (for example a file on an unresponsive drive) is left running:
// it is a background thread, and its sources keep their readers so the blocked call does not touch freed FFmpeg contexts.
var decoderStopped = _decoderThread == null || _decoderThread.Join(DecoderStopTimeoutMs);
if (!decoderStopped)
Melon<RenderMod>.Logger.Warning($"[AudioClipStore] Decoder thread did not stop within {DecoderStopTimeoutMs} ms; its readers are left open.");
LogMemoryUsage();
lock (_sources)
{
_sourceList.Clear();
_sources.Clear();
}
if (decoderStopped)
{
foreach (var source in sources)
source.DisposeInternal();
}
lock (_poolLock)
{
while (_chunkPool.Count > 0)
AddAllocated(-_chunkPool.Pop().LongLength * sizeof(float));
}
}
}
+181
View File
@@ -0,0 +1,181 @@
using System;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// One reader plus its resampler, producing output-rate frames of a clip as a continuous stream.
/// Consecutive <see cref="Fill"/> calls whose ranges follow each other continue the stream (reader position and resampler state);
/// any other range starts a new stream near the requested position.
///
/// Restart: the input position is moved back by a preroll and down to a multiple of in_rate / gcd(in_rate, out_rate),
/// where input frame k * (in_rate / gcd) maps exactly to output frame k * (out_rate / gcd);
/// the resampler is reset and output before the requested frame is discarded.
/// Measured with the clip harness: output is bit-identical to one continuous conversion of the whole clip.
/// Used by one thread at a time (the store's decoder thread).
/// </summary>
internal sealed class DecodeStream : IDisposable
{
/// <summary>Input frames read from the reader per resampler call.</summary>
private const int ReadBlockFrames = 4096;
private readonly PCMResampler? _resampler;
private readonly int _channels;
private readonly int _outputRate;
private readonly float[] _readBuffer;
private float[]? _discardBuffer;
private bool _flushing;
private bool _ended;
private readonly long _inputStep;
private readonly long _outputStep;
private readonly long _prerollInputFrames;
public IPCMReader Reader { get; }
/// <summary>Output frame the stream continues at, or -1 when the stream is not positioned.</summary>
public long NextOutFrame { get; private set; } = -1;
/// <summary>Value of the owner's use counter at the last <see cref="Fill"/> (for least-recently-used selection).</summary>
public long LastUse { get; set; }
/// <summary>Number of times the stream started anew (seek + resampler reset).</summary>
public int RestartCount { get; private set; }
public DecodeStream(IPCMReader reader, int outputRate)
{
Reader = reader;
_channels = reader.Channels;
_outputRate = outputRate;
_readBuffer = new float[ReadBlockFrames * _channels];
if (reader.SampleRate != outputRate)
{
_resampler = new PCMResampler(_channels, reader.SampleRate, outputRate);
var gcd = GCD(reader.SampleRate, outputRate);
_inputStep = reader.SampleRate / gcd;
_outputStep = outputRate / gcd;
// History fed before the first kept output frame when a stream starts mid-clip.
// The filter spans about filter_size input frames per output frame at ratios up to 1:1
// and filter_size * (input rate / output rate) input frames when downsampling.
var ratio = Math.Max(1L, (reader.SampleRate + outputRate - 1L) / outputRate);
_prerollInputFrames = 4L * _resampler.FilterSize * ratio + _inputStep;
}
else
{
_inputStep = 1;
_outputStep = 1;
}
}
/// <summary>
/// Writes output frames [start, start + frames) into <paramref name="data"/> and zeros the rest of the array.
/// Frames past the end of the decoded input are zero.
/// </summary>
public unsafe void Fill(long start, int frames, float[] data)
{
var channels = _channels;
var filled = 0;
if (_resampler == null)
{
if (Reader.Position != start)
{
Reader.Seek(start);
RestartCount++;
}
while (filled < frames)
{
var request = Math.Min(ReadBlockFrames, frames - filled);
var read = Reader.Read(_readBuffer, request);
if (read <= 0) break;
Array.Copy(_readBuffer, 0, data, filled * channels, read * channels);
filled += read;
}
Array.Clear(data, filled * channels, data.Length - filled * channels);
NextOutFrame = start + frames;
return;
}
long discard = 0;
if (NextOutFrame != start)
{
// Input frame of the requested output frame, moved back by the preroll and down to a multiple of _inputStep.
var inputFrame = start * Reader.SampleRate / _outputRate - _prerollInputFrames;
var step = inputFrame > 0 ? inputFrame / _inputStep : 0;
_resampler.Reset();
Reader.Seek(step * _inputStep);
_flushing = false;
_ended = false;
discard = start - step * _outputStep;
RestartCount++;
}
fixed (float* pIn = _readBuffer)
fixed (float* pOut = data)
{
while (discard > 0 && !_ended)
{
_discardBuffer ??= new float[ReadBlockFrames * channels];
fixed (float* pDiscard = _discardBuffer)
{
var capacity = (int)Math.Min(ReadBlockFrames, discard);
discard -= Pull(pIn, pDiscard, capacity);
}
}
while (filled < frames && !_ended)
filled += Pull(pIn, pOut + (long)filled * channels, frames - filled);
}
Array.Clear(data, filled * channels, data.Length - filled * channels);
NextOutFrame = _ended ? -1 : start + frames;
}
/// <summary>
/// Writes 1 ~ <paramref name="capacity"/> resampled frames to <paramref name="output"/>, reading input as needed.
/// Returns 0 and sets _ended once the input and the resampler's flushed tail are exhausted.
/// </summary>
private unsafe int Pull(float* input, float* output, int capacity)
{
var resampler = _resampler!;
if (!_flushing)
{
while (true)
{
var produced = resampler.Convert(input, 0, output, capacity);
if (produced > 0) return produced;
var read = Reader.Read(_readBuffer, ReadBlockFrames);
if (read <= 0)
{
_flushing = true;
break;
}
produced = resampler.Convert(input, read, output, capacity);
if (produced > 0) return produced;
}
}
// Input is exhausted: take the resampler's remaining tail until it returns 0.
var flushed = resampler.Flush(output, capacity);
if (flushed > 0) return flushed;
_ended = true;
return 0;
}
public void Dispose()
{
_resampler?.Dispose();
Reader.Dispose();
}
private static int GCD(int a, int b)
{
while (b != 0) (a, b) = (b, a % b);
return a;
}
}
+390
View File
@@ -0,0 +1,390 @@
using System;
using System.Runtime.InteropServices;
using FFmpeg.AutoGen;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// <see cref="IPCMReader"/> that decodes the best audio stream of a file with libavformat/libavcodec.
/// Output keeps the stream's sample rate and channel count; libswresample converts only the sample format to interleaved float.
///
/// Frame positions: frame 0 is the first sample the decoder outputs for the stream's start time.
/// The position of the first frame decoded after opening or seeking is taken from its timestamp
/// (best_effort_timestamp - stream start_time, in samples); following frames are counted sequentially.
/// Seek moves the demuxer to <see cref="SeekMarginSeconds"/> before the target and discards decoded samples before the target.
///
/// <see cref="TotalFrames"/> comes from the stream duration (or the container duration) and may differ from the decoded length;
/// reads stop at whichever ends first.
///
/// MP3: decoded with the game's timing (MP3Sharp, see <see cref="MP3HeaderFrame"/>): encoder delay and padding are kept
/// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed
/// before it when the file starts with a Xing/Info/VBRI header frame. <see cref="TotalFrames"/> is the sum of all packet
/// durations plus that silence, counted once when the file is opened.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API)
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame)
/// </summary>
internal sealed unsafe class FFmpegPCMReader : IPCMReader
{
/// <summary>
/// Time the demuxer is moved before a seek target, in seconds. Decoders that depend on previous packets
/// (for example the MP3 bit reservoir) produce settled output after this margin; the samples in it are discarded.
/// </summary>
private const double SeekMarginSeconds = 0.25;
private AVFormatContext* _format;
private AVCodecContext* _codec;
private SwrContext* _swr;
private AVPacket* _packet;
private AVFrame* _frame;
private readonly int _streamIndex;
private readonly AVRational _timeBase;
private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames
private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary)
// Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels].
private float[] _pending = Array.Empty<float>();
private int _pendingOffset;
private int _pendingCount;
private long _pendingStart;
private long _nextDecodedFrame; // Frame index of the next decoded frame when counting sequentially
private bool _syncToTimestamp; // True until the first frame after open/seek sets _nextDecodedFrame from its timestamp
private bool _flushSent;
private bool _endOfStream;
public string Path { get; }
public int Channels { get; }
public int SampleRate { get; }
public long TotalFrames { get; }
public bool IsForwardOnly => false;
public long Position { get; private set; }
/// <summary>True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary).</summary>
public bool IsMP3 { get; }
/// <summary>Number of demuxer seeks performed (for diagnostics).</summary>
public int SeekCount { get; private set; }
public FFmpegPCMReader(string path)
{
Path = path;
try
{
AVFormatContext* format = null;
var ret = ffmpeg.avformat_open_input(&format, path, null, null);
if (ret < 0) throw new InvalidOperationException($"Failed to open audio file '{path}': {ErrorString(ret)}");
_format = format;
ret = ffmpeg.avformat_find_stream_info(_format, null);
if (ret < 0) throw new InvalidOperationException($"Failed to read stream info of '{path}': {ErrorString(ret)}");
AVCodec* decoder = null;
_streamIndex = ffmpeg.av_find_best_stream(_format, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &decoder, 0);
if (_streamIndex < 0 || decoder == null) throw new InvalidOperationException($"No decodable audio stream in '{path}': {ErrorString(_streamIndex)}");
for (var i = 0; i < (int)_format->nb_streams; i++)
{
if (i != _streamIndex) _format->streams[i]->discard = AVDiscard.AVDISCARD_ALL;
}
var stream = _format->streams[_streamIndex];
_timeBase = stream->time_base;
_ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0;
_codec = ffmpeg.avcodec_alloc_context3(decoder);
if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed.");
ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar);
if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}");
_codec->pkt_timebase = _timeBase;
IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3;
if (IsMP3)
{
// Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL)
_codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL;
}
ret = ffmpeg.avcodec_open2(_codec, decoder, null);
if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}");
Channels = _codec->ch_layout.nb_channels;
SampleRate = _codec->sample_rate;
if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz.");
if (IsMP3)
{
_leadFrames = MP3HeaderFrame.SilentFrames(path);
ScanPackets(out _ptsOrigin, out var decodedFrames);
TotalFrames = _leadFrames + decodedFrames;
}
else
{
TotalFrames = EstimateTotalFrames(stream);
}
SwrContext* swr = null;
ret = ffmpeg.swr_alloc_set_opts2(&swr,
&_codec->ch_layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, SampleRate,
&_codec->ch_layout, _codec->sample_fmt, SampleRate, 0, null);
if (ret < 0 || swr == null) throw new InvalidOperationException($"swr_alloc_set_opts2 failed: {ErrorString(ret)}");
_swr = swr;
ret = ffmpeg.swr_init(_swr);
if (ret < 0) throw new InvalidOperationException($"swr_init failed: {ErrorString(ret)}");
_packet = ffmpeg.av_packet_alloc();
_frame = ffmpeg.av_frame_alloc();
if (_packet == null || _frame == null) throw new InvalidOperationException("Failed to allocate packet/frame.");
_syncToTimestamp = true;
}
catch
{
Dispose();
throw;
}
}
/// <summary>
/// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed
/// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes
/// to a full frame when trimming is off.
/// </summary>
private void ScanPackets(out long firstPts, out long frames)
{
firstPts = ffmpeg.AV_NOPTS_VALUE;
long durationTicks = 0;
var packet = ffmpeg.av_packet_alloc();
try
{
int ret;
while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0)
{
if (packet->stream_index == _streamIndex)
{
if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts;
durationTicks += packet->duration;
}
ffmpeg.av_packet_unref(packet);
}
if (ret != ffmpeg.AVERROR_EOF)
throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}");
}
finally
{
ffmpeg.av_packet_free(&packet);
}
if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0;
frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate });
var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (seek < 0)
throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}");
}
private long EstimateTotalFrames(AVStream* stream)
{
if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0)
return ffmpeg.av_rescale_q(stream->duration, _timeBase, new AVRational { num = 1, den = SampleRate });
if (_format->duration != ffmpeg.AV_NOPTS_VALUE && _format->duration > 0)
return ffmpeg.av_rescale(_format->duration, SampleRate, ffmpeg.AV_TIME_BASE);
throw new InvalidOperationException($"Audio file '{Path}' has no duration.");
}
public void Seek(long frame)
{
if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame));
if (frame == Position) return;
var marginFrames = (long)(SeekMarginSeconds * SampleRate);
// Target inside the pending block or less than the seek margin after it: decode forward and discard.
if (!_syncToTimestamp && frame >= _pendingStart + _pendingOffset
&& frame <= _pendingStart + _pendingOffset + _pendingCount + marginFrames)
{
Position = frame;
return;
}
var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase);
var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}");
ffmpeg.avcodec_flush_buffers(_codec);
SeekCount++;
_pendingCount = 0;
_pendingOffset = 0;
_syncToTimestamp = true;
_flushSent = false;
_endOfStream = false;
Position = frame;
}
public int Read(float[] buffer, int frameCount)
{
var limit = (int)Math.Min(frameCount, TotalFrames - Position);
var written = 0;
while (written < limit)
{
if (_pendingCount == 0 || Position >= _pendingStart + _pendingOffset + _pendingCount)
{
if (!DecodeNextFrame()) break;
continue;
}
var pendingPosition = _pendingStart + _pendingOffset;
if (Position > pendingPosition)
{
// Discard decoded samples before the read position (after a seek).
var skip = (int)Math.Min(Position - pendingPosition, _pendingCount);
_pendingOffset += skip;
_pendingCount -= skip;
continue;
}
if (Position < pendingPosition)
{
// A gap in the decoded timestamps: fill with silence up to the next decoded sample.
var gap = (int)Math.Min(pendingPosition - Position, limit - written);
Array.Clear(buffer, written * Channels, gap * Channels);
written += gap;
Position += gap;
continue;
}
var count = Math.Min(_pendingCount, limit - written);
Array.Copy(_pending, _pendingOffset * Channels, buffer, written * Channels, count * Channels);
_pendingOffset += count;
_pendingCount -= count;
written += count;
Position += count;
}
return written;
}
/// <summary>Decodes the next audio frame into the pending buffer. Returns false at the end of the stream.</summary>
private bool DecodeNextFrame()
{
while (!_endOfStream)
{
var ret = ffmpeg.avcodec_receive_frame(_codec, _frame);
if (ret == 0)
{
StoreFrame();
ffmpeg.av_frame_unref(_frame);
return true;
}
if (ret == ffmpeg.AVERROR_EOF)
{
_endOfStream = true;
break;
}
if (ret != ffmpeg.AVERROR(ffmpeg.EAGAIN))
throw new InvalidOperationException($"Audio decoding failed in '{Path}': {ErrorString(ret)}");
if (_flushSent)
{
_endOfStream = true;
break;
}
ret = ffmpeg.av_read_frame(_format, _packet);
if (ret < 0)
{
if (ret != ffmpeg.AVERROR_EOF)
throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}");
ffmpeg.avcodec_send_packet(_codec, null);
_flushSent = true;
continue;
}
try
{
if (_packet->stream_index != _streamIndex) continue;
ret = ffmpeg.avcodec_send_packet(_codec, _packet);
// A corrupt packet is skipped; decoding continues with the next one.
if (ret < 0 && ret != ffmpeg.AVERROR_INVALIDDATA && ret != ffmpeg.AVERROR(ffmpeg.EAGAIN))
throw new InvalidOperationException($"Failed to send audio packet of '{Path}': {ErrorString(ret)}");
}
finally
{
ffmpeg.av_packet_unref(_packet);
}
}
return false;
}
private void StoreFrame()
{
var samples = _frame->nb_samples;
if (samples <= 0) return;
if (_syncToTimestamp)
{
var pts = _frame->best_effort_timestamp;
_nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE
? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate })
: Position;
_syncToTimestamp = false;
}
if (_pending.Length < samples * Channels)
_pending = new float[samples * Channels];
fixed (float* pPending = _pending)
{
var outPlane = (byte*)pPending;
var converted = ffmpeg.swr_convert(_swr, &outPlane, samples, _frame->extended_data, samples);
if (converted < 0)
throw new InvalidOperationException($"Sample format conversion failed in '{Path}': {ErrorString(converted)}");
samples = converted;
}
_pendingStart = _nextDecodedFrame;
_pendingOffset = 0;
_pendingCount = samples;
_nextDecodedFrame += samples;
}
public void Dispose()
{
if (_frame != null)
{
var frame = _frame;
ffmpeg.av_frame_free(&frame);
_frame = null;
}
if (_packet != null)
{
var packet = _packet;
ffmpeg.av_packet_free(&packet);
_packet = null;
}
if (_swr != null)
{
var swr = _swr;
ffmpeg.swr_free(&swr);
_swr = null;
}
if (_codec != null)
{
var codec = _codec;
ffmpeg.avcodec_free_context(&codec);
_codec = null;
}
if (_format != null)
{
var format = _format;
ffmpeg.avformat_close_input(&format);
_format = null;
}
}
private static string ErrorString(int error)
{
var buffer = stackalloc byte[1024];
ffmpeg.av_strerror(error, buffer, 1024);
return $"{Marshal.PtrToStringAnsi((IntPtr)buffer)} ({error})";
}
}
+42
View File
@@ -0,0 +1,42 @@
using System;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Sequential reader of decoded PCM at the clip's own sample rate and channel count.
/// Samples are interleaved 32-bit float. Positions and counts are in frames (one frame = one sample per channel).
/// Used by one thread at a time (the store's decoder thread). <see cref="IDisposable.Dispose"/> may be called more than once.
/// </summary>
internal interface IPCMReader : IDisposable
{
/// <summary>Channel count of the decoded audio (1 or more).</summary>
int Channels { get; }
/// <summary>Sample rate of the decoded audio in Hz.</summary>
int SampleRate { get; }
/// <summary>Total frame count of the clip at <see cref="SampleRate"/>.</summary>
long TotalFrames { get; }
/// <summary>
/// True when the data can be read only once, from frame 0 to the end (data is released as it is read).
/// The source then keeps every chunk (<see cref="ChunkPolicy.Resident"/>).
/// </summary>
bool IsForwardOnly { get; }
/// <summary>Frame index that the next <see cref="Read"/> starts at.</summary>
long Position { get; }
/// <summary>
/// Moves the read position to <paramref name="frame"/> (0 ~ <see cref="TotalFrames"/>).
/// A forward-only reader accepts only frames whose data has not been released.
/// </summary>
void Seek(long frame);
/// <summary>
/// Reads up to <paramref name="frameCount"/> frames into <paramref name="buffer"/> starting at index 0 and advances the position.
/// <paramref name="buffer"/> must hold at least frameCount * Channels floats.
/// </summary>
/// <returns>Frames read. 0 at the end of the clip.</returns>
int Read(float[] buffer, int frameCount);
}
+107
View File
@@ -0,0 +1,107 @@
using System;
using System.IO;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Detects the VBR/info header frame ("Xing", "Info" or "VBRI") at the start of an MP3 file.
///
/// The game decodes MP3 files with MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.MP3Stream), which has no header-frame handling:
/// it decodes that frame as one frame of silence and keeps the encoder delay. FFmpeg's MP3 demuxer consumes the header frame
/// without output. Measured with the clip harness (LAME 192k CBR, LAME VBR, 44.1/48 kHz, with and without ID3v2):
/// MP3Sharp output is FFmpeg's untrimmed output preceded by one frame of silence when the header frame exists,
/// and identical in timing when it does not.
///
/// Frame layout used here (ISO/IEC 11172-3 and 13818-3 frame header; Xing/Info tag after the Layer III side information,
/// VBRI tag 32 bytes after the frame header):
/// - Header: 11 sync bits, version (2 bits: 3 = MPEG-1, 2 = MPEG-2, 0 = MPEG-2.5), layer (2 bits: 1 = III, 2 = II, 3 = I),
/// protection bit (0 = 16-bit CRC follows), bitrate index, sample rate index, padding, private, channel mode (3 = mono).
/// - Layer III side information: MPEG-1 32 bytes (17 mono), MPEG-2/2.5 17 bytes (9 mono).
/// - Samples per frame: Layer I 384, Layer II 1152, Layer III 1152 (MPEG-1) or 576 (MPEG-2/2.5).
/// - ID3v2 tags before the first frame: "ID3", version (2 bytes), flags (bit 0x10 = 10-byte footer), size (4 bytes, 7 bits each).
/// </summary>
internal static class MP3HeaderFrame
{
/// <summary>Bytes searched for the first frame sync after the ID3v2 tags.</summary>
private const int SyncSearchBytes = 64 * 1024;
/// <summary>
/// Frames of silence MP3Sharp outputs for the header frame of <paramref name="path"/>:
/// the samples per frame of the first frame when it is a Xing/Info/VBRI header frame, otherwise 0.
/// </summary>
public static int SilentFrames(string path)
{
using var file = File.OpenRead(path);
return SilentFrames(file);
}
public static int SilentFrames(Stream stream)
{
var tag = new byte[10];
long offset = 0;
while (true)
{
stream.Position = offset;
if (ReadFully(stream, tag, 10) < 10 || tag[0] != 'I' || tag[1] != 'D' || tag[2] != '3') break;
var size = (tag[6] & 0x7F) << 21 | (tag[7] & 0x7F) << 14 | (tag[8] & 0x7F) << 7 | (tag[9] & 0x7F);
offset += 10 + size + ((tag[5] & 0x10) != 0 ? 10 : 0);
}
stream.Position = offset;
var data = new byte[SyncSearchBytes];
var length = ReadFully(stream, data, data.Length);
for (var i = 0; i + 4 <= length; i++)
{
if (!TryParseHeader(data, i, out var samplesPerFrame, out var tagOffset)) continue;
// Xing/Info after the side information; VBRI at a fixed offset of 32 bytes after the 4-byte header.
if (HasTag(data, length, i + tagOffset, "Xing") || HasTag(data, length, i + tagOffset, "Info")
|| HasTag(data, length, i + 4 + 32, "VBRI"))
return samplesPerFrame;
return 0;
}
return 0;
}
private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset)
{
samplesPerFrame = 0;
tagOffset = 0;
if (data[i] != 0xFF || (data[i + 1] & 0xE0) != 0xE0) return false;
var version = (data[i + 1] >> 3) & 0x3;
var layer = (data[i + 1] >> 1) & 0x3;
var hasCrc = (data[i + 1] & 0x1) == 0;
var bitrateIndex = (data[i + 2] >> 4) & 0xF;
var sampleRateIndex = (data[i + 2] >> 2) & 0x3;
var mono = ((data[i + 3] >> 6) & 0x3) == 3;
if (version == 1 || layer == 0 || bitrateIndex == 0xF || sampleRateIndex == 3) return false;
var mpeg1 = version == 3;
samplesPerFrame = layer switch
{
3 => 384,
2 => 1152,
_ => mpeg1 ? 1152 : 576
};
var sideInfo = layer == 1 ? (mpeg1 ? (mono ? 17 : 32) : (mono ? 9 : 17)) : 0;
tagOffset = 4 + (hasCrc ? 2 : 0) + sideInfo;
return true;
}
private static bool HasTag(byte[] data, int length, int at, string tag)
{
if (at < 0 || at + tag.Length > length) return false;
for (var k = 0; k < tag.Length; k++)
if (data[at + k] != tag[k]) return false;
return true;
}
private static int ReadFully(Stream stream, byte[] buffer, int count)
{
var total = 0;
int read;
while (total < count && (read = stream.Read(buffer, total, count - total)) > 0) total += read;
return total;
}
}
+73
View File
@@ -0,0 +1,73 @@
using System;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// <see cref="IPCMReader"/> that reproduces the game's output for mono MP3 files.
///
/// MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.Buffer16BitStereo) writes every frame in stereo layout and fills only the
/// channel 0 slot for mono streams, while MP3Stream.ChannelCount reports 1. The game creates a 1-channel clip from that data,
/// so a mono MP3 plays as [s0, 0, s1, 0, ...]: twice as long, at half speed and one octave lower
/// (confirmed in game with TestMaps/mp3_mono, 2026-10-13).
///
/// Output: 1 channel at the inner reader's sample rate. Frame 2k is inner frame k, frame 2k + 1 is 0.
/// <see cref="TotalFrames"/> = 2 * inner TotalFrames.
/// </summary>
internal sealed class MP3MonoLayoutReader : IPCMReader
{
private readonly IPCMReader _inner;
private float[] _innerBuffer = Array.Empty<float>();
/// <param name="inner">Mono reader of the MP3 file with the game's MP3 timing (<see cref="FFmpegPCMReader.IsMP3"/>). Owned by this reader.</param>
public MP3MonoLayoutReader(IPCMReader inner)
{
if (inner.Channels != 1) throw new ArgumentException("Inner reader must be mono.", nameof(inner));
_inner = inner;
Position = inner.Position * 2;
}
public int Channels => 1;
public int SampleRate => _inner.SampleRate;
public long TotalFrames => _inner.TotalFrames * 2;
public bool IsForwardOnly => _inner.IsForwardOnly;
public long Position { get; private set; }
/// <summary>Moves to <paramref name="frame"/>; the inner reader moves to the inner frame of the next even position.</summary>
public void Seek(long frame)
{
_inner.Seek((frame + 1) / 2);
Position = frame;
}
public int Read(float[] buffer, int frameCount)
{
var limit = (int)Math.Min(frameCount, TotalFrames - Position);
var written = 0;
while (written < limit)
{
if ((Position & 1) == 1)
{
buffer[written++] = 0f;
Position++;
continue;
}
var want = (limit - written + 1) / 2;
if (_innerBuffer.Length < want) _innerBuffer = new float[want];
var got = _inner.Read(_innerBuffer, want);
if (got == 0) break;
for (var i = 0; i < got && written < limit; i++)
{
buffer[written++] = _innerBuffer[i];
Position++;
if (written >= limit) break;
buffer[written++] = 0f;
Position++;
}
}
return written;
}
public void Dispose() => _inner.Dispose();
}
+83
View File
@@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Forward-only <see cref="IPCMReader"/> over PCM held in memory as a list of blocks.
/// Block b holds frames [b * FramesPerBlock, (b + 1) * FramesPerBlock) interleaved; the last block may be shorter.
/// Each block is released (passed to the release callback and dropped) as soon as the read position passes its end,
/// and all remaining blocks are released on <see cref="Dispose"/>.
/// </summary>
internal sealed class MemoryPCMReader : IPCMReader
{
private readonly float[]?[] _blocks;
private readonly Action<float[]>? _onBlockReleased;
private int _releasedCount; // Blocks [0, _releasedCount) are released
public int Channels { get; }
public int SampleRate { get; }
public long TotalFrames { get; }
public bool IsForwardOnly => true;
public long Position { get; private set; }
/// <summary>Frames per block (all blocks except the last).</summary>
public int FramesPerBlock { get; }
public MemoryPCMReader(IReadOnlyList<float[]> blocks, int framesPerBlock, int channels, int sampleRate, long totalFrames,
Action<float[]>? onBlockReleased)
{
if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels));
if (framesPerBlock <= 0) throw new ArgumentOutOfRangeException(nameof(framesPerBlock));
if (totalFrames > (long)blocks.Count * framesPerBlock) throw new ArgumentException("Blocks hold fewer frames than totalFrames.", nameof(blocks));
_blocks = new float[blocks.Count][];
for (var i = 0; i < blocks.Count; i++) _blocks[i] = blocks[i];
_onBlockReleased = onBlockReleased;
FramesPerBlock = framesPerBlock;
Channels = channels;
SampleRate = sampleRate;
TotalFrames = totalFrames;
}
public void Seek(long frame)
{
if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame));
if (frame < (long)_releasedCount * FramesPerBlock)
throw new InvalidOperationException($"Frame {frame} was already released by the forward-only reader.");
Position = frame;
}
public int Read(float[] buffer, int frameCount)
{
var total = (int)Math.Min(frameCount, TotalFrames - Position);
if (total <= 0) return 0;
var written = 0;
while (written < total)
{
var blockIndex = (int)(Position / FramesPerBlock);
var block = _blocks[blockIndex] ?? throw new InvalidOperationException($"Block {blockIndex} was already released.");
var offset = (int)(Position - (long)blockIndex * FramesPerBlock);
var count = Math.Min(total - written, FramesPerBlock - offset);
Array.Copy(block, offset * Channels, buffer, written * Channels, count * Channels);
written += count;
Position += count;
ReleaseBlocksBefore((int)(Position / FramesPerBlock));
}
return written;
}
private void ReleaseBlocksBefore(int blockIndex)
{
while (_releasedCount < blockIndex && _releasedCount < _blocks.Length)
{
var block = _blocks[_releasedCount];
_blocks[_releasedCount] = null;
_releasedCount++;
if (block != null) _onBlockReleased?.Invoke(block);
}
}
public void Dispose() => ReleaseBlocksBefore(_blocks.Length);
}
+106
View File
@@ -0,0 +1,106 @@
using System;
using FFmpeg.AutoGen;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Sample rate converter for interleaved 32-bit float PCM (libswresample).
/// Input and output have the same channel count; only the sample rate changes.
/// One instance keeps filter state between calls, so consecutive <see cref="Convert"/> calls form one continuous stream.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// </summary>
internal sealed unsafe class PCMResampler : IDisposable
{
private SwrContext* _swr;
private bool _flushed;
public int Channels { get; }
public int InputRate { get; }
public int OutputRate { get; }
/// <summary>Value of the SwrContext "filter_size" option (filter length per phase in taps at a 1:1 ratio).</summary>
public int FilterSize { get; }
public PCMResampler(int channels, int inputRate, int outputRate)
{
if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels));
if (inputRate <= 0) throw new ArgumentOutOfRangeException(nameof(inputRate));
if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate));
Channels = channels;
InputRate = inputRate;
OutputRate = outputRate;
AVChannelLayout layout;
ffmpeg.av_channel_layout_default(&layout, channels);
SwrContext* swr = null;
var ret = ffmpeg.swr_alloc_set_opts2(
&swr,
&layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, outputRate,
&layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, inputRate,
0, null);
ffmpeg.av_channel_layout_uninit(&layout);
if (ret < 0 || swr == null)
throw new InvalidOperationException($"swr_alloc_set_opts2 failed ({ret}).");
_swr = swr;
long filterSize;
FilterSize = ffmpeg.av_opt_get_int(_swr, "filter_size", 0, &filterSize) >= 0 ? (int)filterSize : 32;
Reset();
}
/// <summary>Discards all buffered input and filter history. The next call starts a new stream.</summary>
public void Reset()
{
// swr_init on an initialized context re-initializes it.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_init, swr_close)
var ret = ffmpeg.swr_init(_swr);
if (ret < 0)
throw new InvalidOperationException($"swr_init failed ({ret}).");
_flushed = false;
}
/// <summary>
/// Feeds <paramref name="inputFrames"/> frames and writes up to <paramref name="outputCapacity"/> frames.
/// Input that does not fit in the output is buffered inside the context and returned by later calls.
/// <paramref name="inputFrames"/> may be 0 to take only buffered output.
/// </summary>
/// <returns>Frames written to <paramref name="output"/>.</returns>
public int Convert(float* input, int inputFrames, float* output, int outputCapacity)
{
if (_flushed) throw new InvalidOperationException("Resampler was flushed. Call Reset first.");
// swr_convert treats a null input array as a flush request, so a non-null pointer is passed even for 0 frames.
float dummy = 0;
var inPlane = (byte*)(input != null ? input : &dummy);
var outPlane = (byte*)output;
var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, &inPlane, inputFrames);
if (ret < 0)
throw new InvalidOperationException($"swr_convert failed ({ret}).");
return ret;
}
/// <summary>
/// Ends the input stream and writes up to <paramref name="outputCapacity"/> remaining frames.
/// Call repeatedly until it returns 0. <see cref="Convert"/> is not allowed afterwards until <see cref="Reset"/>.
/// </summary>
public int Flush(float* output, int outputCapacity)
{
_flushed = true;
var outPlane = (byte*)output;
var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, null, 0);
if (ret < 0)
throw new InvalidOperationException($"swr_convert (flush) failed ({ret}).");
return ret;
}
public void Dispose()
{
if (_swr == null) return;
var swr = _swr;
ffmpeg.swr_free(&swr);
_swr = null;
}
}
+55
View File
@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Copies a Unity <see cref="AudioClip"/> into memory for clips without a file on disk
/// (built-in hit sounds from Resources, official/DLC level audio).
/// <see cref="AudioClip.GetData(float[], int)"/> runs on the main thread only, so the whole clip is copied once on the main thread
/// and the decoder thread reads the copy through a <see cref="MemoryPCMReader"/>.
/// Streaming clips (<see cref="AudioClipLoadType.Streaming"/>) are rejected because GetData does not read them.
/// Reference: https://docs.unity3d.com/ScriptReference/AudioClip.GetData.html
/// </summary>
internal static class UnityClipLoader
{
/// <summary>
/// Copies <paramref name="clip"/> in blocks of <see cref="AudioClipSource.ChunkFloats"/> floats or less
/// (FramesPerBlock = ChunkFloats / channels frames per GetData call). Main thread only.
/// Block memory is counted in <paramref name="store"/> and released block by block as the decoder thread reads it.
/// </summary>
public static MemoryPCMReader Load(AudioClip clip, AudioClipStore store)
{
if (!clip) throw new ArgumentNullException(nameof(clip));
if (clip.loadType == AudioClipLoadType.Streaming)
throw new NotSupportedException($"AudioClip '{clip.name}' is a streaming clip; GetData cannot read it.");
if (clip.loadState == AudioDataLoadState.Unloaded)
clip.LoadAudioData();
var channels = clip.channels;
var totalFrames = (long)clip.samples;
var framesPerBlock = AudioClipSource.ChunkFloats / channels;
var blocks = new List<float[]>();
try
{
for (long start = 0; start < totalFrames; start += framesPerBlock)
{
var frames = (int)Math.Min(framesPerBlock, totalFrames - start);
var block = store.AllocateExactArray(frames * channels);
blocks.Add(block);
// GetData fills the whole array; the offset is in frames (samples per channel).
if (!clip.GetData(block, (int)start))
throw new InvalidOperationException($"AudioClip.GetData failed for '{clip.name}' at frame {start} (load state {clip.loadState}).");
}
}
catch
{
foreach (var block in blocks) store.FreeExactArray(block);
throw;
}
return new MemoryPCMReader(blocks, framesPerBlock, channels, clip.frequency, totalFrames, store.FreeExactArray);
}
}
+156
View File
@@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using ADOFAIRenderer.Util.Audio.Clip;
namespace ADOFAIRenderer.Util.Audio.Mixer;
/// <summary>
/// One playing sound of a <see cref="VoiceMixer"/> (the song, or one effect sound), created by <see cref="VoiceMixer.Play"/>.
///
/// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start,
/// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order.
/// Render thread: <see cref="VoiceMixer.Mix"/> applies each change at sample round(time * OutputRate).
///
/// Playback model (output samples n, clip positions in frames of the source at the output sample rate):
/// - The voice is silent before its start sample and from its end sample on.
/// - Clip position at sample n = anchor position + (n - anchor sample) * pitch. A position or pitch change sets a new anchor
/// at the sample where it takes effect, so a pitch change keeps the position continuous.
/// - Looping: the position wraps at the clip length (<see cref="AudioClipSource.Duration"/> * output rate, fractional).
/// Not looping: the voice ends when the position reaches <see cref="AudioClipSource.TotalFrames"/>.
/// </summary>
internal sealed class Voice
{
internal enum ChangeKind
{
Pitch,
Volume,
Loop,
Position,
EndTime,
Stop
}
internal readonly struct Change
{
public readonly ChangeKind Kind;
public readonly long Sample;
public readonly double Value;
public Change(ChangeKind kind, long sample, double value)
{
Kind = kind;
Sample = sample;
Value = value;
}
}
private readonly VoiceMixer _mixer;
// ── Render thread state ──
/// <summary>Changes not yet applied, in time order. Render thread only.</summary>
internal readonly Queue<Change> Pending = new();
/// <summary>First output sample of playback.</summary>
internal long StartSample;
/// <summary>Output sample at which playback stops (exclusive); long.MaxValue when not set.</summary>
internal long EndSample = long.MaxValue;
internal long AnchorSample;
internal double AnchorPosition;
internal double Pitch;
internal double Volume;
internal bool Loop;
internal bool Finished;
/// <summary>Loop length in frames at the output sample rate (source duration * output rate).</summary>
internal double Period;
internal AudioCursor? Cursor;
internal ChunkView View;
internal Voice(VoiceMixer mixer, AudioClipSource? source, string name)
{
_mixer = mixer;
Source = source;
Name = name;
}
/// <summary>Clip played by this voice; null plays silence (the clip could not be loaded).</summary>
public AudioClipSource? Source { get; }
/// <summary>Name for logs (the game's clip key).</summary>
public string Name { get; }
/// <summary>Sets the playback rate (1 = original speed). Negative values are treated as 0.</summary>
public void SetPitch(double pitch, double time) => _mixer.Enqueue(this, ChangeKind.Pitch, time, Math.Max(0.0, pitch));
/// <summary>Sets the linear gain applied to the clip samples.</summary>
public void SetVolume(double volume, double time) => _mixer.Enqueue(this, ChangeKind.Volume, time, volume);
/// <summary>Turns looping on or off. Turning it off lets the current repetition play to the end of the clip.</summary>
public void SetLoop(bool loop, double time) => _mixer.Enqueue(this, ChangeKind.Loop, time, loop ? 1.0 : 0.0);
/// <summary>
/// Moves playback to <paramref name="clipSeconds"/> seconds into the clip.
/// Before the voice starts, this sets the position playback starts from.
/// </summary>
public void SetPosition(double clipSeconds, double time) => _mixer.Enqueue(this, ChangeKind.Position, time, Math.Max(0.0, clipSeconds));
/// <summary>Stops playback at <paramref name="endTime"/> (seconds relative to the render start).</summary>
public void SetEndTime(double endTime, double time) => _mixer.Enqueue(this, ChangeKind.EndTime, time, endTime);
/// <summary>Stops playback at <paramref name="time"/>.</summary>
public void Stop(double time) => _mixer.Enqueue(this, ChangeKind.Stop, time, 0.0);
/// <summary>Unwrapped clip position at output sample <paramref name="sample"/> (sample at or after the anchor).</summary>
internal double PositionAt(long sample) => AnchorPosition + (sample - AnchorSample) * Pitch;
/// <summary>Clip position at <paramref name="sample"/>, wrapped into the loop when looping.</summary>
internal double WrappedPositionAt(long sample)
{
var position = PositionAt(sample);
if (Loop && Period > 0 && position >= Period) position %= Period;
return position;
}
/// <summary>
/// Applies <paramref name="change"/> at output sample <paramref name="sample"/> (its own sample, or a later one when the change
/// arrived after its sample was mixed). Before the start sample, changes replace the starting state. Render thread only.
/// </summary>
internal void Apply(in Change change, long sample, int outputRate)
{
var at = Math.Max(sample, StartSample);
switch (change.Kind)
{
case ChangeKind.Pitch:
Reanchor(at);
Pitch = change.Value;
break;
case ChangeKind.Volume:
Volume = change.Value;
break;
case ChangeKind.Loop:
Reanchor(at);
Loop = change.Value != 0.0;
break;
case ChangeKind.Position:
AnchorSample = at;
AnchorPosition = change.Value * outputRate;
break;
case ChangeKind.EndTime:
EndSample = VoiceMixer.ToSample(change.Value, outputRate);
break;
case ChangeKind.Stop:
EndSample = Math.Min(EndSample, sample);
break;
}
}
private void Reanchor(long sample)
{
AnchorPosition = WrappedPositionAt(sample);
AnchorSample = sample;
}
}
+341
View File
@@ -0,0 +1,341 @@
using System;
using System.Collections.Generic;
using System.Threading;
using ADOFAIRenderer.Util.Audio.Clip;
namespace ADOFAIRenderer.Util.Audio.Mixer;
/// <summary>
/// Mixes every <see cref="Voice"/> into interleaved double samples at the output sample rate and channel count.
///
/// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed
/// to <see cref="Play"/> and the Voice Set methods). A time t maps to sample round(t * OutputRate).
///
/// Threads:
/// - Main thread: <see cref="Play"/> and the Voice Set methods queue commands under a lock. Commands must be queued in
/// non-decreasing time order per voice.
/// - Render thread: <see cref="Mix"/> takes the queued commands and mixes a block of samples. A command takes effect at its sample;
/// a command whose sample is before the block being mixed takes effect at the start of that block.
/// Chunks are read through <see cref="AudioClipStore.Acquire"/>, which waits for the decoder thread.
///
/// Sample reading: clip positions are in frames of the source at the output rate (sources are resampled when loaded).
/// An integer position copies the stored sample; a fractional position interpolates linearly between the frame and the next one
/// (the next frame of a looping voice after the last frame is frame 0; after the end of a non-looping clip it is 0).
/// Channel mapping follows <see cref="ChunkView.GetSample"/>.
/// </summary>
internal sealed class VoiceMixer : IDisposable
{
private enum CommandKind
{
Play,
Change
}
private readonly struct Command
{
public readonly CommandKind Kind;
public readonly Voice Voice;
public readonly Voice.Change Change;
public Command(CommandKind kind, Voice voice, Voice.Change change)
{
Kind = kind;
Voice = voice;
Change = change;
}
}
private readonly AudioClipStore _store;
private readonly CancellationToken _cancellationToken;
private readonly object _commandLock = new();
private List<Command> _commands = new();
private List<Command> _takenCommands = new();
// Render thread only.
private readonly List<Voice> _voices = new();
private bool _disposed;
/// <param name="store">Store the voices' sources belong to; its OutputRate is the mixer's sample rate.</param>
/// <param name="outputChannels">Interleaved channel count of the mixed output.</param>
/// <param name="cancellationToken">Cancels waits for chunks in <see cref="Mix"/>.</param>
public VoiceMixer(AudioClipStore store, int outputChannels, CancellationToken cancellationToken = default)
{
if (outputChannels < 1) throw new ArgumentOutOfRangeException(nameof(outputChannels));
_store = store;
OutputRate = store.OutputRate;
OutputChannels = outputChannels;
_cancellationToken = cancellationToken;
}
public int OutputRate { get; }
public int OutputChannels { get; }
/// <summary>Voices currently mixed (started or waiting to start). Render thread only.</summary>
public int ActiveVoiceCount => _voices.Count;
/// <summary>Output sample nearest to <paramref name="seconds"/>.</summary>
public static long ToSample(double seconds, int rate) => (long)Math.Round(seconds * rate, MidpointRounding.AwayFromZero);
// ── Main thread ──
/// <summary>
/// Schedules <paramref name="source"/> to start at <paramref name="startTime"/>.
/// A start time before <paramref name="time"/> (the current time) starts the voice at <paramref name="time"/>.
/// </summary>
/// <param name="source">Clip to play; null plays silence.</param>
/// <param name="name">Name for logs.</param>
/// <param name="startTime">Start of playback, seconds relative to the render start.</param>
/// <param name="volume">Linear gain.</param>
/// <param name="pitch">Playback rate (1 = original speed); negative values are treated as 0.</param>
/// <param name="loop">Repeats the clip until the voice is stopped.</param>
/// <param name="time">Current time, seconds relative to the render start.</param>
public Voice Play(AudioClipSource? source, string name, double startTime, double volume, double pitch, bool loop, double time)
{
var voice = new Voice(this, source, name)
{
StartSample = Math.Max(ToSample(startTime, OutputRate), ToSample(time, OutputRate)),
Volume = volume,
Pitch = Math.Max(0.0, pitch),
Loop = loop,
Period = source != null ? source.Duration * OutputRate : 0.0
};
voice.AnchorSample = voice.StartSample;
lock (_commandLock)
{
_commands.Add(new Command(CommandKind.Play, voice, default));
}
return voice;
}
internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value)
{
var change = new Voice.Change(kind, ToSample(time, OutputRate), value);
lock (_commandLock)
{
_commands.Add(new Command(CommandKind.Change, voice, change));
}
}
// ── Render thread ──
/// <summary>
/// Adds the mix of output samples [<paramref name="firstSample"/>, firstSample + frames) to <paramref name="destination"/>
/// (interleaved, frames = destination.Length / OutputChannels). Blocks are expected in increasing sample order.
/// </summary>
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (see <see cref="AudioClipStore.Acquire"/>).</exception>
/// <exception cref="OperationCanceledException">The cancellation token was canceled while waiting for a chunk.</exception>
public void Mix(long firstSample, Span<double> destination)
{
if (_disposed) throw new ObjectDisposedException(nameof(VoiceMixer));
var frames = destination.Length / OutputChannels;
var endSample = firstSample + frames;
TakeCommands();
for (var i = _voices.Count - 1; i >= 0; i--)
{
var voice = _voices[i];
var sample = firstSample;
while (!voice.Finished)
{
var segmentEnd = endSample;
if (voice.Pending.Count > 0)
{
var next = voice.Pending.Peek();
if (next.Sample <= sample)
{
voice.Apply(voice.Pending.Dequeue(), sample, OutputRate);
continue;
}
segmentEnd = Math.Min(segmentEnd, next.Sample);
}
if (segmentEnd > sample)
MixSegment(voice, sample, segmentEnd, destination.Slice((int)(sample - firstSample) * OutputChannels));
sample = segmentEnd;
if (sample >= endSample) break;
}
if (!voice.Finished && endSample >= voice.EndSample)
voice.Finished = true;
if (voice.Finished)
{
ReleaseVoice(voice);
_voices.RemoveAt(i);
}
}
}
private void TakeCommands()
{
lock (_commandLock)
{
(_commands, _takenCommands) = (_takenCommands, _commands);
}
foreach (var command in _takenCommands)
{
var voice = command.Voice;
if (command.Kind == CommandKind.Play)
{
if (voice.Source is { IsFailed: false } source)
voice.Cursor = source.AddCursor(0, voice.Loop);
_voices.Add(voice);
}
else if (!voice.Finished)
{
voice.Pending.Enqueue(command.Change);
}
}
_takenCommands.Clear();
}
/// <summary>Mixes <paramref name="voice"/> over samples [from, to) with its current state into <paramref name="destination"/> (starting at sample from).</summary>
private void MixSegment(Voice voice, long from, long to, Span<double> destination)
{
var start = Math.Max(from, voice.StartSample);
var end = Math.Min(to, voice.EndSample);
if (start >= end) return;
var source = voice.Source;
if (source == null || source.IsFailed)
{
voice.Finished = true;
return;
}
var totalFrames = source.TotalFrames;
if (voice.Volume == 0.0)
{
// Nothing to add; a non-looping voice still ends at the end of the clip.
if (!voice.Loop && voice.PositionAt(end - 1) >= totalFrames) voice.Finished = true;
return;
}
var channels = OutputChannels;
var volume = voice.Volume;
var offset = (int)(start - from) * channels;
UpdateCursor(voice, (long)voice.WrappedPositionAt(start));
for (var n = start; n < end; n++, offset += channels)
{
var position = voice.WrappedPositionAt(n);
if (position >= totalFrames)
{
if (!voice.Loop)
{
voice.Finished = true;
return;
}
position = totalFrames - 1;
}
var index = (long)position;
var fraction = position - index;
if (!EnsureView(voice, index)) return;
var view = voice.View;
if (fraction == 0.0)
{
for (var c = 0; c < channels; c++)
destination[offset + c] += view.GetSample(index, c, channels) * volume;
continue;
}
var nextIndex = index + 1;
var nextInView = view.Contains(nextIndex);
for (var c = 0; c < channels; c++)
{
double low = view.GetSample(index, c, channels);
double high = nextInView ? view.GetSample(nextIndex, c, channels) : ReadOutside(voice, nextIndex, c);
destination[offset + c] += (low + fraction * (high - low)) * volume;
}
}
}
/// <summary>Sample of frame <paramref name="frame"/> outside the voice's current view (next chunk, loop start, or past the end).</summary>
private double ReadOutside(Voice voice, long frame, int channel)
{
var source = voice.Source!;
if (frame >= source.TotalFrames)
{
if (!voice.Loop) return 0.0;
frame = 0;
}
var view = _store.Acquire(source, frame, _cancellationToken);
if (!view.IsValid) return 0.0;
try
{
return view.GetSample(frame, channel, OutputChannels);
}
finally
{
source.Release(view);
}
}
/// <summary>Makes voice.View the chunk containing <paramref name="frame"/>. False when the source cannot provide it (voice finished).</summary>
private bool EnsureView(Voice voice, long frame)
{
if (voice.View.IsValid && voice.View.Contains(frame)) return true;
var source = voice.Source!;
if (voice.View.IsValid)
{
source.Release(voice.View);
voice.View = default;
}
UpdateCursor(voice, frame);
voice.View = _store.Acquire(source, frame, _cancellationToken);
if (voice.View.IsValid) return true;
voice.Finished = true;
return false;
}
private static void UpdateCursor(Voice voice, long frame)
{
var cursor = voice.Cursor;
if (cursor == null) return;
if (cursor.Loop != voice.Loop) cursor.Loop = voice.Loop;
if (cursor.Position != frame) cursor.Position = frame;
}
private static void ReleaseVoice(Voice voice)
{
var source = voice.Source;
if (source == null) return;
if (voice.View.IsValid)
{
source.Release(voice.View);
voice.View = default;
}
if (voice.Cursor != null)
{
source.RemoveCursor(voice.Cursor);
voice.Cursor = null;
}
voice.Pending.Clear();
}
/// <summary>Releases every voice's chunk and cursor. Render thread (or after the render thread stopped).</summary>
public void Dispose()
{
if (_disposed) return;
_disposed = true;
foreach (var voice in _voices)
ReleaseVoice(voice);
_voices.Clear();
lock (_commandLock)
{
_commands.Clear();
}
}
}
+3 -3
View File
@@ -1125,7 +1125,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// </summary>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
/// <param name="yStride">Row stride in bytes for Y plane.</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes).</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes).</param>
/// <param name="uvStride">Row stride in bytes for UV plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
@@ -1178,7 +1178,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = ((w + 1) / 2) * 2;
int uvRowBytes = (w / 2) * 2;
// Copy Y plane (row by row to safely skip linesize GPU padding)
for (int y = 0; y < h; y++)
@@ -1239,7 +1239,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width;
int h = Height;
int uvH = (h + 1) / 2;
int uvRowBytes = ((w + 1) / 2) * 2;
int uvRowBytes = (w / 2) * 2;
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
{
+35 -147
View File
@@ -201,12 +201,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary>
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
/// <summary>
/// Color range of the encoder input and output (codec context color_range of the selected encoder).
/// Full range (JPEG) for every encoder except libvvenc, limited range (MPEG) for libvvenc.
/// </summary>
private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG;
/// <summary>Pixel format of the finally selected and running encoder.</summary>
public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
@@ -261,9 +255,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Whether audio encoding is enabled.</summary>
private bool _audioEnabled;
/// <summary>Bytes of one input sample of every channel (interleaved input format × channel count).</summary>
private int _audioInputBytesPerSample;
/// <summary>Encoding settings passed at construction time (immutable).</summary>
private readonly EncoderSettings _settings;
@@ -276,6 +267,26 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Utility Methods
// ================================================================
/// <summary>
/// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg.
/// AVCodecID identifies "which codec it is (e.g., H.264, AV1)",
/// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately.
/// </summary>
// ReSharper disable once InconsistentNaming
// ReSharper disable once UnusedMember.Local
private AVCodecID GetAVCodecID()
{
return _settings.Codec switch
{
Codec.H264 => AVCodecID.AV_CODEC_ID_H264,
Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC,
Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1,
Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9,
Codec.VVC => AVCodecID.AV_CODEC_ID_VVC,
_ => throw new ArgumentOutOfRangeException()
};
}
/// <summary>
/// Returns a list of candidate encoders to try for the selected codec in priority order.
///
@@ -405,19 +416,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Basic video parameters configuration ──
_pCodecContext->width = _settings.Width;
_pCodecContext->height = _settings.Height;
// ── Output color tags ──
// Matrix: BT.709, the RGB→YUV matrix of both conversion paths
// (GPU: RGB2YUV_BT709_Full in ColorConversion.compute, CPU: ConfigureSwsColorspace).
// Players use this tag to convert the YUV frames back to RGB.
// Range: full (0-255) for every encoder except libvvenc, limited (16-235) for libvvenc.
// The FFmpeg libvvenc wrapper passes the matrix/primaries/transfer tags to VVenC but not the range,
// so a VVC stream is read as limited range; its input is converted to limited range to match.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_color_format)
_pCodecContext->color_range = encoder == VideoEncoder.LibVvenc
? AVColorRange.AVCOL_RANGE_MPEG
: AVColorRange.AVCOL_RANGE_JPEG;
_pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709;
_pCodecContext->color_range = AVColorRange.AVCOL_RANGE_JPEG;
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
// so setting 0 (auto) is required for parallel encoding utilizing all cores.
@@ -691,11 +690,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null)
{
// The codec context (and the frames context it holds) is released by Dispose in the catch block of Start.
ffmpeg.av_buffer_unref(&hwDeviceCtx);
throw new Exception("Failed to allocate QSV hardware frame structure.");
}
break;
}
@@ -774,7 +769,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage";
Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})");
_dstPixFmt = selectedPixFmt;
_colorRange = _pCodecContext->color_range;
openedSuccessfully = true;
break;
}
@@ -803,9 +797,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Copy parameters set in codec context (resolution, codec ID, profile, etc.)
// to stream's codecpar. Information to be written to container header.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext);
if (paramResult < 0)
throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})");
if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0)
throw new Exception("Failed to copy codec parameters to stream.");
// Stream time base: unit of the timestamps stored in the container.
// Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks.
@@ -830,17 +823,15 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
{
var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE);
if (avioResult < 0)
throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0)
throw new Exception($"Cannot open output file {outputPath}.");
}
// Write file header (ftyp, moov, container metadata).
// Packets can be written with av_interleaved_write_frame after this point.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
if (headerResult < 0)
throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})");
if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0)
throw new Exception("Failed to write file header.");
// ── Step 6: Initialize frame buffers and colorspace converter ──
// av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet).
@@ -874,8 +865,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSwsContext == null)
throw new Exception("Failed to initialize software scaler.");
ConfigureSwsColorspace();
_isInitialized = true;
_loggedConversionMode = false;
}
@@ -1171,42 +1160,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock)
{
if (_disposed || !_isInitialized) return;
ValidateInputLength(buffer.Length, isColorConverted: false);
WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr());
}
}
/// <summary>
/// Number of bytes read from the input buffer of one frame.
/// RGBA input (isColorConverted = false): Width × Height × 4
/// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV)
/// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
/// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
/// </summary>
private long GetRequiredInputLength(bool isColorConverted)
{
long w = _settings.Width, h = _settings.Height;
if (!isColorConverted) return w * h * 4;
return _dstPixFmt switch
{
AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2),
AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2),
_ => 0
};
}
/// <summary>
/// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it
/// (GetRequiredInputLength), so that the copy does not read outside the buffer.
/// </summary>
private void ValidateInputLength(long length, bool isColorConverted)
{
var required = GetRequiredInputLength(isColorConverted);
if (length < required)
throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input.");
}
/// <summary>
/// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources.
@@ -1225,7 +1182,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
}
ValidateInputLength(length, isColorConverted);
if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
else WriteFrameFromBuffer((byte*)pointer.ToPointer());
}
@@ -1240,10 +1196,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock)
{
if (_disposed || !_isInitialized) return;
if (length > buffer.Length)
throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes).");
ValidateInputLength(length, isColorConverted);
fixed (byte* pBuffer = buffer)
{
if (!_loggedConversionMode)
@@ -1261,51 +1213,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
}
/// <summary>
/// Sets the RGB→YUV conversion of the CPU conversion path (_pSwsContext) to the BT.709 matrix
/// and the color range of the encoder (_colorRange), so that it produces the same colors as the GPU path
/// (RGB2YUV_BT709_Full in ColorConversion.compute) and as the tags of the output stream.
/// A new libswscale context uses the BT.601 matrix (SWS_CS_DEFAULT = SWS_CS_ITU601) and the limited range.
///
/// For RGB input, "table" holds the coefficients of the output YUV space and dstRange its range (1 = full, 0 = limited).
/// The source (RGB) values, brightness, contrast and saturation keep the values of the context.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails, sws_getCoefficients)
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libswscale/swscale.h (SWS_CS_ITU709, SWS_CS_DEFAULT)
/// </summary>
private void ConfigureSwsColorspace()
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
var ret = ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot read the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
return;
}
var coefficients = ffmpeg.sws_getCoefficients(ffmpeg.SWS_CS_ITU709);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = currentInvTable[i];
table[i] = coefficients[i];
}
var fullRange = _colorRange == AVColorRange.AVCOL_RANGE_JPEG;
ret = ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, srcRange, table, fullRange ? 1 : 0,
brightness, contrast, saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot set the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
}
}
/// <summary>
/// Sets the format, size, color range and matrix of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
/// Second argument 0 means default alignment.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
@@ -1315,8 +1223,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height;
_pDstFrame->color_range = _colorRange;
_pDstFrame->colorspace = AVColorSpace.AVCOL_SPC_BT709;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG;
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
if (ret < 0)
@@ -1355,7 +1262,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
switch (_dstPixFmt)
{
case AVPixelFormat.AV_PIX_FMT_NV12:
case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV:
{
// Copy Y Plane
var pSrcY = buffer;
@@ -1578,7 +1485,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame);
if (response < 0)
{
throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})");
throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})");
}
// Pop all encoded packets and write to file.
@@ -1592,7 +1499,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// EOF: Encoder flush complete (all delayed frames outputted) → exit loop
if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
if (response < 0)
throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})");
throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})");
// Convert PTS/DTS from codec timebase → stream timebase
ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base);
@@ -1603,7 +1510,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// and written in time order. This function is used even when only video is present.
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
if (response < 0)
throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
throw new Exception($"Error occurred while writing packet to file. (Code: {response})");
// Unref packet data reference for reuse in next receive_packet.
ffmpeg.av_packet_unref(_pPacket);
@@ -1868,10 +1775,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
? audioSettings.InputSampleFormat
: AVSampleFormat.AV_SAMPLE_FMT_FLT;
// Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
_audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
// Allocate SwrContext
fixed (SwrContext** ppSwr = &_pSwrContext)
{
@@ -1935,8 +1838,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock)
{
if (_disposed || !_isInitialized || !_audioEnabled) return;
if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float))
throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats).");
fixed (float* pSamples = samples)
{
@@ -1971,19 +1872,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary>
/// <summary>
/// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
/// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
/// Throws an exception when the buffer cannot be allocated.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
/// </summary>
private void MakeAudioFrameWritable()
{
var ret = ffmpeg.av_frame_make_writable(_pAudioFrame);
if (ret < 0)
throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
}
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
{
// ── 1. Resampling: Input format → Encoder format ──
@@ -2022,7 +1910,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
{
MakeAudioFrameWritable();
ffmpeg.av_frame_make_writable(_pAudioFrame);
_pAudioFrame->nb_samples = frameSize;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize);
@@ -2101,7 +1989,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo);
if (remaining > 0)
{
MakeAudioFrameWritable();
ffmpeg.av_frame_make_writable(_pAudioFrame);
_pAudioFrame->nb_samples = remaining;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);
+5 -42
View File
@@ -73,16 +73,6 @@ namespace ADOFAIRenderer.Util
}
}
/// <summary>
/// Bytes of the readback buffer of one frame: Width x Height x 3 / 2 when the GPU conversion to NV12 / YUV420P
/// is used, Width x Height x 4 (RGBA) otherwise. The same size as ReadbackRequestState.Setup uses.
/// </summary>
public long GetFrameBufferBytes(int width, int height)
{
var pixels = (long)width * height;
return _useGPUConversion ? pixels * 3 / 2 : pixels * 4;
}
public void RequestFrameCapture(Texture texture, long currentFrame)
{
if (!RenderManager.IsCapturingActive) return;
@@ -164,11 +154,14 @@ namespace ADOFAIRenderer.Util
}
Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame.");
var blackFrame = CreateBlackFrame(state, out var byteSize);
var byteSize = state.Width * state.Height * 3 / 2;
var blackFrame = ArrayPool<byte>.Shared.Rent(byteSize);
Array.Clear(blackFrame, 0, byteSize);
var capFrame = new RenderManager.CapturedFrame
{
ManagedBuffer = blackFrame, Length = byteSize,
IsColorConverted = state.IsGPUConversion
IsColorConverted = true
};
PushToQueue(frame, capFrame);
RecycleState(state);
@@ -197,36 +190,6 @@ namespace ADOFAIRenderer.Util
}
}
/// <summary>
/// Creates a black frame in the same format as the frames of this reader, from ArrayPool&lt;byte&gt;.Shared
/// (returned to the pool by the render thread after encoding).
///
/// GPU conversion (NV12 / YUV420P, full range): Width × Height × 3 / 2 bytes.
/// Y plane (Width × Height bytes) = 0, U/V data (the rest) = 128.
/// 128 is the zero chroma value of the 8-bit full range: the U/V offset 0.5 of RGB2YUV_BT709_Full
/// in ColorConversion.compute, which the encoder input of this path uses.
/// Both NV12 (interleaved UV) and YUV420P (U plane, then V plane) store all chroma after the Y plane,
/// so the same fill applies to both.
/// No GPU conversion (RGBA input, converted by the encoder): Width × Height × 4 bytes, all 0.
/// </summary>
private static byte[] CreateBlackFrame(ReadbackRequestState state, out int byteSize)
{
var lumaSize = state.Width * state.Height;
byteSize = state.IsGPUConversion ? lumaSize * 3 / 2 : lumaSize * 4;
var frame = ArrayPool<byte>.Shared.Rent(byteSize);
if (state.IsGPUConversion)
{
Array.Clear(frame, 0, lumaSize);
frame.AsSpan(lumaSize, byteSize - lumaSize).Fill(128);
}
else
{
Array.Clear(frame, 0, byteSize);
}
return frame;
}
internal void RecycleState(ReadbackRequestState state)
{
lock (_poolLock)
+3 -3
View File
@@ -97,7 +97,7 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureUV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.RG16, false);
_textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureUV);
@@ -118,8 +118,8 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureU = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
_textureV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
_textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
_textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false);
ConfigureTexture(_textureY);
ConfigureTexture(_textureU);
ConfigureTexture(_textureV);