9 Commits
32 changed files with 1033 additions and 3619 deletions

No files matched your search

+3 -1
View File
@@ -520,7 +520,9 @@ public class RenderSession
} }
RenderManager.Instance.levelPath = levelPath; RenderManager.Instance.levelPath = levelPath;
RenderManager.Instance.Configuration = renderConfig; // The render uses its own copy of the configuration: values adjusted during the render
// (e.g. the audio sample rate in SetupPlay) stay in this copy and do not change the caller's object.
RenderManager.Instance.Configuration = renderConfig.Clone();
RenderManager.Instance.stage = RenderStage.Load; RenderManager.Instance.stage = RenderStage.Load;
+111 -62
View File
@@ -5,11 +5,9 @@ using System.IO;
using System.Threading; using System.Threading;
using System.Buffers; using System.Buffers;
using System.Diagnostics; using System.Diagnostics;
using System.Linq;
using ADOFAIRenderer.API; using ADOFAIRenderer.API;
using ADOFAIRenderer.Core.Data; using ADOFAIRenderer.Core.Data;
using ADOFAIRenderer.Data; using ADOFAIRenderer.Data;
using ADOFAIRenderer.Patch.Engine;
using ADOFAIRenderer.UI; using ADOFAIRenderer.UI;
using ADOFAIRenderer.UI.Window; using ADOFAIRenderer.UI.Window;
using ADOFAIRenderer.Util; using ADOFAIRenderer.Util;
@@ -82,7 +80,27 @@ internal class RenderManager : MonoBehaviour
public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene; public static bool IsActive => Instance && Instance.stage >= RenderStage.OnScene;
public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render; public static bool IsCapturingActive => Instance && Instance.stage >= RenderStage.Render;
/// <summary>
/// Maximum number of frames in ImageQueue. The main thread waits (throttles) while the queue holds more frames.
/// </summary>
public const int MaxQueuedFrames = 64; public const int MaxQueuedFrames = 64;
/// <summary>
/// Upper bound of the readback buffer bytes of the frames in ImageQueue (4 GiB).
/// The frame limit becomes min(MaxQueuedFrames, MaxQueuedBytes / bytes per frame) (see ComputeQueueFrameLimit),
/// so it only lowers the frame count at resolutions where MaxQueuedFrames frames exceed this size.
/// </summary>
public const long MaxQueuedBytes = 4L * 1024 * 1024 * 1024;
/// <summary>
/// Wait time of the throttle loop (OnPostFrame) for the render thread to take a frame from ImageQueue.
/// When it elapses without a dequeue, AsyncGPUReadback.WaitAllRequests() is called once so the readbacks
/// still in flight complete (the render thread may be waiting for one of them to fill a gap in ImageQueue).
/// </summary>
private const int ThrottleWaitMilliseconds = 10;
/// <summary>Frame limit of ImageQueue applied by the last throttle check (MaxQueuedFrames or lower).</summary>
public int QueueFrameLimit { get; private set; } = MaxQueuedFrames;
// Statistics // Statistics
public long framesSent; public long framesSent;
@@ -147,9 +165,6 @@ internal class RenderManager : MonoBehaviour
public int frameCountBase; public int frameCountBase;
public OfflineAudioMixer? AudioMixer { get; private set; } public OfflineAudioMixer? AudioMixer { get; private set; }
/// <summary>Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands.</summary>
public double AudioTime => frameCount / Configuration.RenderFPS;
private AudioSyncCalculator? SyncCalculator { get; set; } private AudioSyncCalculator? SyncCalculator { get; set; }
private FrameReader? _frameReader; private FrameReader? _frameReader;
@@ -217,6 +232,8 @@ internal class RenderManager : MonoBehaviour
// Thread and Queue State // Thread and Queue State
internal SortedList<long, CapturedFrame> ImageQueue = new(); internal SortedList<long, CapturedFrame> ImageQueue = new();
internal AutoResetEvent FrameAvailableEvent = new(false); internal AutoResetEvent FrameAvailableEvent = new(false);
// Set by the render thread each time it takes a frame from ImageQueue; the throttle loop waits on it.
internal AutoResetEvent FrameDequeuedEvent = new(false);
public Thread? RenderThread; public Thread? RenderThread;
public long passCount; public long passCount;
public volatile bool endRequested; public volatile bool endRequested;
@@ -259,10 +276,8 @@ internal class RenderManager : MonoBehaviour
if (stage != RenderStage.None) if (stage != RenderStage.None)
MarkFailed("the game exited or RenderManager was destroyed during rendering."); MarkFailed("the game exited or RenderManager was destroyed during rendering.");
ResetRenderState(success: false); ResetRenderState(success: false);
AudioSpectrumGenerator.Clear(); AudioSpectrumGenerator.Clear();
// Cancels the render thread's audio waits, then releases the clip store (decoder thread, readers, chunks).
AudioMixer?.Dispose(); AudioMixer = null;
ResetFrameHookState(); ResetFrameHookState();
@@ -402,37 +417,74 @@ internal class RenderManager : MonoBehaviour
frameCount++; frameCount++;
} }
// Floors that own a hold renderer, collected from scrLevelMaker.listFloors (see RefreshHoldFloors).
private readonly List<scrFloor> _holdFloors = new();
private List<scrFloor>? _holdFloorsSource;
private int _holdFloorsSourceCount = -1;
public void OnLastUpdate() public void OnLastUpdate()
{ {
if (Instance.stage < RenderStage.Load) return; if (Instance.stage < RenderStage.Load) return;
if (!Configuration.AdvancedConfig.NonTile) return;
foreach (var floor in scrLevelMaker.instance.listFloors if (Configuration.AdvancedConfig.NonTile)
.Where(floor => floor.holdRenderer && floor.holdRenderer.m_meshRenderer.enabled))
{ {
floor.holdRenderer.m_meshRenderer.enabled = false; // scrHoldRenderer.UpdateColor sets m_meshRenderer.enabled = true every time it runs
// (scrHoldRenderer.cs, called from scrPlanet/scrPlayer), so the hold meshes are disabled again every frame.
RefreshHoldFloors();
foreach (var floor in _holdFloors)
{
if (!floor || !floor.holdRenderer) continue;
var meshRenderer = floor.holdRenderer.m_meshRenderer;
if (meshRenderer && meshRenderer.enabled) meshRenderer.enabled = false;
}
} }
TriggerOnLastUpdate(); TriggerOnLastUpdate();
} }
/// <summary>
/// Rebuilds the hold floor list when scrLevelMaker.listFloors is a different list or has a different count
/// (hold renderers are created together with the floors in scrLevelMaker).
/// </summary>
private void RefreshHoldFloors()
{
var floors = scrLevelMaker.instance.listFloors;
if (ReferenceEquals(floors, _holdFloorsSource) && floors.Count == _holdFloorsSourceCount) return;
_holdFloors.Clear();
foreach (var floor in floors)
{
if (floor && floor.holdRenderer) _holdFloors.Add(floor);
}
_holdFloorsSource = floors;
_holdFloorsSourceCount = floors.Count;
}
/// <summary>
/// Frame limit of ImageQueue for the given capture texture: MaxQueuedBytes divided by the readback bytes of
/// one frame (YUV 4:2:0 when the GPU conversion is active, RGBA otherwise), within 4 to MaxQueuedFrames.
/// </summary>
private int ComputeQueueFrameLimit(RenderTexture texture)
{
var frameBytes = _frameReader?.GetFrameBufferBytes(texture.width, texture.height)
?? (long)texture.width * texture.height * 4;
if (frameBytes <= 0) return MaxQueuedFrames;
return (int)Math.Min(MaxQueuedFrames, Math.Max(4, MaxQueuedBytes / frameBytes));
}
// ReSharper disable once Unity.IncorrectMethodSignature // ReSharper disable once Unity.IncorrectMethodSignature
public void OnPostFrame(RenderTexture? texture) public void OnPostFrame(RenderTexture? texture)
{ {
if (Instance.stage < RenderStage.Render) return; if (Instance.stage < RenderStage.Render || !texture) return;
// Audio commands of this frame are queued: update the song voice and let the render thread mix up to the next frame.
if (AudioMixer != null)
{
AudioMixer.UpdateBGM(AudioTime, dspTimeBase);
AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS);
EffectSource.DeactivateMappedSources(AudioMixer);
}
if (!texture) return;
// [Backpressure Throttling] // [Backpressure Throttling]
// The main thread waits while ImageQueue holds more than QueueFrameLimit frames. Only the render thread
// removes frames, and it signals FrameDequeuedEvent for each one. The wait has a timeout so that the loop
// re-checks the render thread state and pumps pending readback completions (see ThrottleWaitMilliseconds).
isThrottling = false; isThrottling = false;
var frameLimit = QueueFrameLimit = ComputeQueueFrameLimit(texture!);
while (Instance.stage >= RenderStage.Render) while (Instance.stage >= RenderStage.Render)
{ {
if (RenderThread is not { IsAlive: true }) if (RenderThread is not { IsAlive: true })
@@ -447,7 +499,7 @@ internal class RenderManager : MonoBehaviour
count = ImageQueue.Count; count = ImageQueue.Count;
} }
if (count <= MaxQueuedFrames) break; if (count <= frameLimit) break;
if (!isThrottling) if (!isThrottling)
{ {
@@ -455,7 +507,10 @@ internal class RenderManager : MonoBehaviour
throttleCount++; throttleCount++;
} }
AsyncGPUReadback.WaitAllRequests(); if (!FrameDequeuedEvent.WaitOne(ThrottleWaitMilliseconds))
{
AsyncGPUReadback.WaitAllRequests();
}
} }
isThrottling = false; isThrottling = false;
@@ -596,19 +651,19 @@ internal class RenderManager : MonoBehaviour
scrController.instance.levelWasSkipped = true; scrController.instance.levelWasSkipped = true;
if (!scrConductor.instance || !scrConductor.instance.song) yield break; if (!scrConductor.instance || !scrConductor.instance.song) yield break;
if (Configuration.AdvancedConfig.ProcessAudio)
{
AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
}
if (Configuration.AudioConfig.RenderAudio) if (Configuration.AudioConfig.RenderAudio)
{ {
AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate); AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate);
AudioMixer.InitializeBGM(scrConductor.instance.song.clip); AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps); SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps);
} }
if (Configuration.AdvancedConfig.ProcessAudio)
{
AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer);
}
scrConductor.instance.song.clip = null; scrConductor.instance.song.clip = null;
scrConductor.instance.song.time = -1000; scrConductor.instance.song.time = -1000;
@@ -635,6 +690,7 @@ internal class RenderManager : MonoBehaviour
endRequested = false; endRequested = false;
endSuccessful = false; endSuccessful = false;
FrameAvailableEvent.Reset(); FrameAvailableEvent.Reset();
FrameDequeuedEvent.Reset();
var renderFPS = Configuration.RenderFPS; var renderFPS = Configuration.RenderFPS;
var videoFPS = Configuration.VideoFPS; var videoFPS = Configuration.VideoFPS;
@@ -730,46 +786,38 @@ internal class RenderManager : MonoBehaviour
endSuccessful = !IsRenderAborted; endSuccessful = !IsRenderAborted;
} }
catch (OperationCanceledException) when (IsRenderAborted)
{
// Audio waits end with OperationCanceledException when the render is cancelled or has failed (OfflineAudioMixer.Cancel).
Melon<RenderMod>.Logger.Msg("Render thread stopped: the render was aborted.");
DisposeEncoderAfterStop(videoFFmpeg);
}
catch (Exception ex) catch (Exception ex)
{ {
Melon<RenderMod>.Logger.Error($"FFmpeg rendering thread error: {ex}"); Melon<RenderMod>.Logger.Error($"FFmpeg rendering thread error: {ex}");
MarkFailed("encoder thread error. See the log for details."); MarkFailed("encoder thread error. See the log for details.");
DisposeEncoderAfterStop(videoFFmpeg); try
{
videoFFmpeg.Dispose();
}
catch (Exception disposeEx)
{
Melon<RenderMod>.Logger.Error($"Encoder dispose after thread error also failed: {disposeEx.Message}");
}
UnregisterEncoder(videoFFmpeg);
} }
} }
/// <summary>Releases the encoder after the render thread stopped early (abort or error) and removes it from the encoder list.</summary>
private void DisposeEncoderAfterStop(FFmpegNativeEncoder videoFFmpeg)
{
try
{
videoFFmpeg.Dispose();
}
catch (Exception disposeEx)
{
Melon<RenderMod>.Logger.Error($"Encoder dispose after the render thread stopped also failed: {disposeEx.Message}");
}
UnregisterEncoder(videoFFmpeg);
}
/// <summary> /// <summary>
/// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio /// Takes frames from ImageQueue in passCount order and sends each one to the encoder, followed by the audio
/// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready. /// samples of that frame. Waits on FrameAvailableEvent (50ms) while no frame is ready.
/// Returns when the render is aborted or all requested frames have been sent after endRequested. /// Returns when the render is aborted (checked before every frame, frames left in ImageQueue are released
/// by the cleanup after the thread ends) or all requested frames have been sent after endRequested.
/// </summary> /// </summary>
private void EncodeQueuedFrames(FFmpegNativeEncoder videoFFmpeg) private void EncodeQueuedFrames(FFmpegNativeEncoder videoFFmpeg)
{ {
while (true) while (true)
{ {
if (IsRenderAborted) return;
if (TryDequeueNextFrame(out var frame)) if (TryDequeueNextFrame(out var frame))
{ {
FrameDequeuedEvent.Set();
SendVideoFrame(videoFFmpeg, frame); SendVideoFrame(videoFFmpeg, frame);
WriteAudioForFrame(videoFFmpeg); WriteAudioForFrame(videoFFmpeg);
continue; continue;
@@ -844,13 +892,17 @@ internal class RenderManager : MonoBehaviour
{ {
if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return; if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return;
var frameIdx = (int)(passCount - 1);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
var frameTime = frameIdx / effectiveVideoFps;
var channels = Configuration.AudioConfig.AudioChannels; var channels = Configuration.AudioConfig.AudioChannels;
var samplesNeeded = syncCalculator.GetNextFrameSamples(); var samplesNeeded = syncCalculator.GetNextFrameSamples();
var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels); var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
try try
{ {
AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded); AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded); videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
} }
finally finally
@@ -893,9 +945,6 @@ internal class RenderManager : MonoBehaviour
{ {
MarkCancelled(); MarkCancelled();
} }
// The abort state is set first: the render thread reads it when its audio wait is cancelled.
if (success) AudioMixer?.Complete();
else AudioMixer?.Cancel();
FrameAvailableEvent.Set(); FrameAvailableEvent.Set();
skipPressToStart = false; skipPressToStart = false;
_renderSetup = false; _renderSetup = false;
@@ -911,7 +960,7 @@ internal class RenderManager : MonoBehaviour
var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED"; var reason = IsRenderFailed ? $"FAILED ({_failureReason})" : "CANCELLED";
Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources."); Melon<RenderMod>.Logger.Error($"Rendering {reason}. Safely cleaning up state and resources.");
// 1. Wait for render background thread to finish processing remaining queued frames and terminate // 1. Wait (up to 2s) for the render background thread to see the abort flag and terminate
if (RenderThread is { IsAlive: true }) if (RenderThread is { IsAlive: true })
{ {
RenderThread.Join(2000); RenderThread.Join(2000);
@@ -1038,10 +1087,10 @@ internal class RenderManager : MonoBehaviour
_frameReader?.Dispose(); _frameReader?.Dispose();
ResetFrameHookState(); ResetFrameHookState();
AudioSpectrumGenerator.Clear(); AudioMixer?.Reset(); AudioMixer = null;
AudioMixer?.Dispose(); AudioMixer = null;
SyncCalculator = null; SyncCalculator = null;
AudioSpectrumGenerator.Clear();
RenderContext.Reset(); RenderContext.Reset();
} }
+41 -251
View File
@@ -1,33 +1,11 @@
using System; using System;
using System.Collections.Generic;
using ADOFAIRenderer.Core; using ADOFAIRenderer.Core;
using ADOFAIRenderer.Util.Audio;
using ADOFAIRenderer.Util.Audio.Clip;
using HarmonyLib; using HarmonyLib;
using UnityEngine; using UnityEngine;
using UnityEngine.Audio; using UnityEngine.Audio;
namespace ADOFAIRenderer.Patch.Engine; namespace ADOFAIRenderer.Patch.Engine;
/// <summary>
/// Records the file path of each external clip the game loads (always active, also outside rendering,
/// because levels load their song and PlaySound files before a render starts).
/// FindOrLoadAudioClipExternal returns the clip already in audioLib when its key exists, so the path is recorded
/// only when the key is missing and the file is about to be loaded.
/// </summary>
[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.FindOrLoadAudioClipExternal))]
public static class AudioManagerFindOrLoadAudioClipExternalPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static void Prefix(AudioManager __instance, string path)
{
if (string.IsNullOrEmpty(path)) return;
if (__instance.audioLib.ContainsKey(AudioClipPaths.KeyOf(path))) return;
AudioClipPaths.Record(path);
}
}
[HarmonyPatch(typeof(AudioManager), "_Play")] [HarmonyPatch(typeof(AudioManager), "_Play")]
public static class AudioManagerPlayPatch public static class AudioManagerPlayPatch
{ {
@@ -52,27 +30,14 @@ public static class AudioManagerPlayPatch
double relativeStartTime = time - RenderManager.Instance.dspTimeBase; double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
// Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource
var mixer = RenderManager.Instance.AudioMixer; RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
if (mixer == null) return;
mixer.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f, RenderManager.Instance.AudioTime);
EffectSource.DeactivateAtFrameEnd(__result);
} }
} }
} }
/// <summary>
/// scrConductor.PlayWithEndTime: registers a looping effect voice until the end time. The AudioSource it creates with
/// AudioManager.MakeSource is mapped to that voice (<see cref="AudioManagerMakeSourcePatch"/>).
/// </summary>
[HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")]
public static class PlayWithEndTimePatch public static class PlayWithEndTimePatch
{ {
/// <summary>State registered by the running PlayWithEndTime call, until its MakeSource returns. Main thread only.</summary>
internal static SourceState? PendingState;
[HarmonyFinalizer]
public static void Finalizer() => PendingState = null;
[HarmonyPrefix] [HarmonyPrefix]
public static bool Prefix(string snd, double time, double endTime, float volume, int priority) public static bool Prefix(string snd, double time, double endTime, float volume, int priority)
{ {
@@ -89,234 +54,59 @@ public static class PlayWithEndTimePatch
double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
// Register looping sound event to the offline mixer (default pitch = 1.0f) // Register looping sound event to the offline mixer (default pitch = 1.0f)
PendingState = mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
} }
} }
return true; // Let the original method execute return true; // Let the original method execute
} }
} }
/// <summary>Maps the AudioSource created inside scrConductor.PlayWithEndTime to its state (<see cref="PlayWithEndTimePatch.PendingState"/>).</summary>
[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.MakeSource))]
public static class AudioManagerMakeSourcePatch
{
[HarmonyPostfix]
// ReSharper disable once InconsistentNaming
public static void Postfix(AudioSource __result)
{
var state = PlayWithEndTimePatch.PendingState;
if (state == null || !__result) return;
PlayWithEndTimePatch.PendingState = null;
var mixer = RenderManager.Instance.AudioMixer;
if (mixer == null) return;
mixer.MapSource(__result, state);
EffectSource.DeactivateAtFrameEnd(__result);
}
}
/// <summary>
/// AudioSource members of an effect source mapped to a <see cref="SourceState"/> during rendering: the patches below skip the
/// Unity implementation and read or change the state instead (isPlaying, time, timeSamples, pitch, SetScheduledEndTime, Stop).
/// The patches apply only while <see cref="TryGet"/> returns true: rendering is capturing (RenderManager.IsCapturingActive,
/// stage Render), the render has an audio mixer (created when the render starts, cleared when it ends), and the source is mapped
/// in that mixer (mappings are created only when RenderAudio is on). Otherwise they run the Unity implementation, so sources
/// without a mapping (the song, menu sounds) and all sources outside rendering behave as in the game.
///
/// The game object of a mapped source is deactivated at the end of the frame it was mapped in: the mixer produces its sound and the
/// state answers every member above, so Unity does not process the source. The game activates it again when AudioManager.MakeSource
/// reuses it. Active AudioSources add main-thread time per frame in proportion to their count, and dense levels keep thousands of
/// scheduled effect sources alive.
/// </summary>
internal static class EffectSource
{
/// <summary>Sources mapped during the current frame, deactivated by <see cref="DeactivateMappedSources"/>. Main thread only.</summary>
private static readonly List<AudioSource> PendingDeactivation = new();
/// <summary>Deactivates the game object of <paramref name="source"/> at the end of the current frame (<see cref="DeactivateMappedSources"/>).</summary>
public static void DeactivateAtFrameEnd(AudioSource source) => PendingDeactivation.Add(source);
/// <summary>
/// Deactivates the game objects of the sources mapped this frame that are still mapped and not destroyed.
/// Called by RenderManager.OnPostFrame after the audio commands of the frame are committed, which runs only in stage Render with
/// an audio mixer; sources are added only by the render-time registrations (AudioManager._Play and scrConductor.PlayWithEndTime
/// patches with RenderAudio on).
/// </summary>
public static void DeactivateMappedSources(OfflineAudioMixer mixer)
{
foreach (var source in PendingDeactivation)
{
if (source && mixer.TryGetSourceState(source, out _)) source.gameObject.SetActive(false);
}
PendingDeactivation.Clear();
}
/// <summary>
/// State mapped to <paramref name="source"/> in the mixer of the current render, with the current render time in <paramref name="now"/>.
/// False outside rendering (stage below Render), without a mixer, or for a source that is not mapped.
/// </summary>
public static bool TryGet(AudioSource source, out SourceState state, out double now)
{
state = null!;
now = 0;
if (!RenderManager.IsCapturingActive) return false;
var mixer = RenderManager.Instance.AudioMixer;
if (mixer == null || !mixer.TryGetSourceState(source, out state)) return false;
now = RenderManager.Instance.AudioTime;
return true;
}
}
/// <summary>
/// AudioSource.isPlaying getter (<see cref="EffectSource"/>): true from the scheduling call until Stop(), the scheduled end time or
/// the end of the clip, including before the scheduled start. AudioManager.Update returns a source to its reusable pool once this is false.
/// </summary>
[HarmonyPatch(typeof(AudioSource), "isPlaying", MethodType.Getter)]
public static class AudioSourceIsPlayingPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, ref bool __result)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
__result = state.IsPlaying(now);
return false;
}
}
/// <summary>AudioSource.time getter (<see cref="EffectSource"/>): clip position in seconds while playing, 0 while not playing.</summary>
[HarmonyPatch(typeof(AudioSource), "time", MethodType.Getter)]
public static class AudioSourceTimeGetterPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, ref float __result)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
__result = state.GetTime(now);
return false;
}
}
/// <summary>
/// AudioSource.time setter (<see cref="EffectSource"/>): moves the playback position (clip seconds) from the current render time.
/// Set before the scheduled start (ffxPlaySound sets it right after AudioManager.Play), it becomes the start position.
/// </summary>
[HarmonyPatch(typeof(AudioSource), "time", MethodType.Setter)]
public static class AudioSourceTimePatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, float value)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
state.SetTime(value, now);
return false;
}
}
/// <summary>AudioSource.timeSamples getter (<see cref="EffectSource"/>): clip position in samples of the clip.</summary>
[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Getter)]
public static class AudioSourceTimeSamplesGetterPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, ref int __result)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
__result = state.GetTimeSamples(now);
return false;
}
}
/// <summary>AudioSource.timeSamples setter (<see cref="EffectSource"/>): moves the playback position (samples of the clip).</summary>
[HarmonyPatch(typeof(AudioSource), "timeSamples", MethodType.Setter)]
public static class AudioSourceTimeSamplesPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, int value)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
state.SetTimeSamples(value, now);
return false;
}
}
/// <summary>AudioSource.pitch getter (<see cref="EffectSource"/>): the last pitch set.</summary>
[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Getter)]
public static class AudioSourcePitchGetterPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, ref float __result)
{
if (!EffectSource.TryGet(__instance, out var state, out _)) return true;
__result = state.GetPitch();
return false;
}
}
/// <summary>AudioSource.pitch setter (<see cref="EffectSource"/>): changes the playback rate from the current render time.</summary>
[HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)] [HarmonyPatch(typeof(AudioSource), "pitch", MethodType.Setter)]
public static class AudioSourcePitchPatch public static class AudioSourcePitchPatch
{ {
[HarmonyPrefix] [HarmonyPostfix]
// ReSharper disable once InconsistentNaming // ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, float value) public static void Postfix(AudioSource __instance, float value)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
state.SetPitch(value, now);
return false;
}
}
/// <summary>AudioSource.SetScheduledEndTime (<see cref="EffectSource"/>): DSP time (absolute) at which playback stops.</summary>
[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))]
public static class AudioSourceSetScheduledEndTimePatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance, double time)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
state.SetEndTime(time - RenderManager.Instance.dspTimeBase, now);
return false;
}
}
/// <summary>
/// AudioSource.Stop() (<see cref="EffectSource"/>): stops playback at the current render time. The mapping stays, so the source
/// keeps reporting the stopped state. AudioManager.Update calls Stop() on sources it returns to the pool, which are no longer
/// playing, so their voices are left unchanged.
/// </summary>
[HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])]
public static class AudioSourceStopPatch
{
[HarmonyPrefix]
// ReSharper disable once InconsistentNaming
public static bool Prefix(AudioSource __instance)
{
if (!EffectSource.TryGet(__instance, out var state, out var now)) return true;
state.Stop(now);
return false;
}
}
/// <summary>
/// AudioManager.StopAllSounds destroys every source in liveSources (all AudioManager._Play and scrConductor.PlayWithEndTime sources),
/// so every effect voice stops at the current render time, including ones scheduled to start later.
/// Callers include scrConductor.PlayHitTimes (from its second call), ScrubMusicToTile/Time, KillAllSoundsCoroutine,
/// scrController.FailAction and scnGame.Play.
/// </summary>
[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.StopAllSounds))]
public static class AudioManagerStopAllSoundsPatch
{
[HarmonyPrefix]
public static void Prefix()
{ {
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
{ {
RenderManager.Instance.AudioMixer?.StopAllEffects(RenderManager.Instance.AudioTime); var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
{
soundEvent.Pitch = value;
}
}
}
}
[HarmonyPatch(typeof(AudioSource), "SetScheduledEndTime", typeof(double))]
public static class AudioSourceSetScheduledEndTimePatch
{
[HarmonyPostfix]
// ReSharper disable once InconsistentNaming
public static void Postfix(AudioSource __instance, double time)
{
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
{
var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
{
soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase;
}
}
}
}
[HarmonyPatch(typeof(AudioSource), "Stop", new Type[0])]
public static class AudioSourceStopPatch
{
[HarmonyPostfix]
// ReSharper disable once InconsistentNaming
public static void Postfix(AudioSource __instance)
{
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
{
RenderManager.Instance.AudioMixer?.RemoveSourceMapping(__instance);
} }
} }
} }
+1 -1
View File
@@ -9,7 +9,7 @@ using MelonLoader.Utils;
using UnityEngine; using UnityEngine;
using Object = UnityEngine.Object; using Object = UnityEngine.Object;
[assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.2.8-INDEV", "ADOFAI Editor Forum")] [assembly: MelonInfo(typeof(RenderMod), "ADOFAI Renderer", "0.4.0-BETA", "ADOFAI Editor Forum")]
[assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")] [assembly: MelonGame("7th Beat Games", "A Dance of Fire and Ice")]
namespace ADOFAIRenderer; namespace ADOFAIRenderer;
+1 -1
View File
@@ -164,7 +164,7 @@ internal class DebugMonitorUI : MonoBehaviour
} }
} }
sb.AppendLine($"• Queue: <b>{queueCount}</b> / {RenderManager.MaxQueuedFrames} frames"); sb.AppendLine($"• Queue: <b>{queueCount}</b> / {(RenderManager.Instance ? RenderManager.Instance.QueueFrameLimit : RenderManager.MaxQueuedFrames)} frames");
sb.AppendLine($"• Rendered Frames: <b>{(RenderManager.Instance ? RenderManager.Instance.frameCount : 0)}</b>"); sb.AppendLine($"• Rendered Frames: <b>{(RenderManager.Instance ? RenderManager.Instance.frameCount : 0)}</b>");
sb.AppendLine($"• Frames Sent: <b>{(RenderManager.Instance ? Interlocked.Read(ref RenderManager.Instance.framesSent) : 0)}</b>"); sb.AppendLine($"• Frames Sent: <b>{(RenderManager.Instance ? Interlocked.Read(ref RenderManager.Instance.framesSent) : 0)}</b>");
+60 -16
View File
@@ -9,6 +9,7 @@ using ADOFAIRenderer.Data;
using ADOFAIRenderer.Localization; using ADOFAIRenderer.Localization;
using ADOFAIRenderer.Util.OS; using ADOFAIRenderer.Util.OS;
using MelonLoader; using MelonLoader;
using Newtonsoft.Json;
using O5Kit.Control; using O5Kit.Control;
using O5Kit.Core; using O5Kit.Core;
using O5Kit.Factory; using O5Kit.Factory;
@@ -32,6 +33,14 @@ internal static class RenderConfigWindow
private static Coroutine? _deleteResetCoroutine; private static Coroutine? _deleteResetCoroutine;
private static string? _lastLoadedPresetName; private static string? _lastLoadedPresetName;
/// <summary>
/// Configuration edited by this window. Kept for the whole game session; a render started from the window
/// receives a copy of it (RenderSession.Start), so values adjusted during the render are not written back here.
/// </summary>
private static RenderConfig? _uiConfig;
private static RenderConfig UiConfig => _uiConfig ??= new RenderConfig();
// Controls cache for validation & sync // Controls cache for validation & sync
private static O5Slider? _renderFpsSlider; private static O5Slider? _renderFpsSlider;
private static O5Slider? _videoFPSSlider; private static O5Slider? _videoFPSSlider;
@@ -111,8 +120,7 @@ internal static class RenderConfigWindow
ReloadPresetDropdown(GlobalSettings.Current.LastPresetName); ReloadPresetDropdown(GlobalSettings.Current.LastPresetName);
} }
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract var config = UiConfig;
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
ApplyConfigToUI(config); ApplyConfigToUI(config);
SetWindowActive(true); SetWindowActive(true);
@@ -125,8 +133,7 @@ internal static class RenderConfigWindow
var (_, scrollContent, _) = var (_, scrollContent, _) =
O5Factory.ScrollView(ctx, content, spacing: 10f, padding: 8f); O5Factory.ScrollView(ctx, content, spacing: 10f, padding: 8f);
// ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract var config = UiConfig;
var config = RenderManager.Instance.Configuration ??= new RenderConfig();
EnsureDefaultConfig(config); EnsureDefaultConfig(config);
double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144, double[] renderFpsSnap = { 23.976, 24, 29.97, 30, 59.94, 60, 144,
@@ -744,11 +751,14 @@ internal static class RenderConfigWindow
{ {
EnsureDefaultConfig(config); EnsureDefaultConfig(config);
_renderFpsSlider?.Set((float) config.RenderFPS, false); // Sliders show the config values as they are (noFilter: true).
_videoFPSSlider?.Set((float) config.VideoFPS, false); // O5Slider.Set applies the slider filter (snapping to the nearest snap value) unless noFilter is set;
_renderWidthSlider?.Set(config.RenderWidth, false); // the filter is meant for dragging, and typed values are not snapped either (O5Slider.SetFromInput).
_renderHeightSlider?.Set(config.RenderHeight, false); _renderFpsSlider?.Set(config.RenderFPS, false, noFilter: true);
_delayAfterEndSlider?.Set((float) config.DelayAfterEnd, false); _videoFPSSlider?.Set(config.VideoFPS, false, noFilter: true);
_renderWidthSlider?.Set(config.RenderWidth, false, noFilter: true);
_renderHeightSlider?.Set(config.RenderHeight, false, noFilter: true);
_delayAfterEndSlider?.Set(config.DelayAfterEnd, false, noFilter: true);
_showJudgementToggle?.Set(config.ShowJudgementText, false); _showJudgementToggle?.Set(config.ShowJudgementText, false);
_codecDropdown?.Set(config.VideoConfig.Codec, false); _codecDropdown?.Set(config.VideoConfig.Codec, false);
@@ -756,13 +766,13 @@ internal static class RenderConfigWindow
_bitrateInput?.Set(config.VideoConfig.Bitrate.ToString(), false); _bitrateInput?.Set(config.VideoConfig.Bitrate.ToString(), false);
_maxBitrateInput?.Set(config.VideoConfig.MaxBitrate.ToString(), false); _maxBitrateInput?.Set(config.VideoConfig.MaxBitrate.ToString(), false);
_qpInput?.Set(config.VideoConfig.QP.ToString(), false); _qpInput?.Set(config.VideoConfig.QP.ToString(), false);
_keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false); _keyframeIntervalSlider?.Set(config.VideoConfig.KeyframeInterval, false, noFilter: true);
_forceSoftwareToggle?.Set(config.VideoConfig.ForceSoftwareEncoder, false); _forceSoftwareToggle?.Set(config.VideoConfig.ForceSoftwareEncoder, false);
_renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false); _renderAudioToggle?.Set(config.AudioConfig.RenderAudio, false);
_sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false); _sampleRateDropdown?.Set(config.AudioConfig.AudioSampleRate, false);
_channelsDropdown?.Set(config.AudioConfig.AudioChannels, false); _channelsDropdown?.Set(config.AudioConfig.AudioChannels, false);
_audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false); _audioBitrateSlider?.Set(config.AudioConfig.AudioBitrate / 1000.0, false, noFilter: true);
_outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false); _outputDirInput?.Set(GlobalSettings.EffectiveOutputDirectory, false);
@@ -789,17 +799,51 @@ internal static class RenderConfigWindow
UpdateVideoCodecUI(config.VideoConfig.Codec, config); UpdateVideoCodecUI(config.VideoConfig.Codec, config);
UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true); UpdateRateControlUI(config.VideoConfig.RateControlMode, noAnimate: true);
if (!string.IsNullOrEmpty(GlobalSettings.Current.LastPresetName)) UpdatePresetLabel(config);
}
/// <summary>
/// Sets the preset dropdown from the config values.
/// Every value equals the last loaded preset (_lastLoadedPresetName): the preset name is shown.
/// Any value differs (edited, or the preset file changed): the Custom label is shown (MarkConfigModified),
/// the same state as after editing a value in the window.
/// </summary>
private static void UpdatePresetLabel(RenderConfig config)
{
if (_presetDropdown == null) return;
var presetName = _lastLoadedPresetName;
var preset = string.IsNullOrEmpty(presetName)
? null
: PresetManager.GetPresets().FirstOrDefault(p => string.Equals(p.Name, presetName, StringComparison.OrdinalIgnoreCase));
if (preset == null || !ConfigValuesEqual(config, preset.Config))
{ {
_presetDropdown?.Set(GlobalSettings.Current.LastPresetName, false); MarkConfigModified();
_lastLoadedPresetName = GlobalSettings.Current.LastPresetName; return;
UpdateDeleteButtonState();
} }
if (_presetDropdown.Values.Contains(CustomPresetLabel))
{
_presetDropdown.SetValues(_presetDropdown.Values.Where(v => v != CustomPresetLabel).ToList());
}
_presetDropdown.Set(preset.Name, false);
UpdateDeleteButtonState();
}
/// <summary>
/// Compares two configs by their serialized JSON (the same form as the preset files), so every saved value is compared.
/// </summary>
private static bool ConfigValuesEqual(RenderConfig a, RenderConfig b)
{
var settings = new JsonSerializerSettings();
settings.Converters.Add(new ColorJsonConverter());
return JsonConvert.SerializeObject(a, settings) == JsonConvert.SerializeObject(b, settings);
} }
private static void TryGoRender() private static void TryGoRender()
{ {
var config = RenderManager.Instance.Configuration; var config = UiConfig;
SyncUItoConfig(config); SyncUItoConfig(config);
// Sync Output Directory // Sync Output Directory
+100
View File
@@ -0,0 +1,100 @@
using System;
namespace ADOFAIRenderer.Util.Audio;
internal static class AudioResampler
{
public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
{
// ── Safety Validation ──
if (inputSamples.Length == 0 ||
inputChannels <= 0 || targetChannels <= 0 ||
inputFrequency <= 0 || targetFrequency <= 0)
{
return Array.Empty<float>();
}
// If format is identical, return original samples
if (inputFrequency == targetFrequency && inputChannels == targetChannels)
return inputSamples;
int inputLength = inputSamples.Length / inputChannels;
int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
float[] outputSamples = new float[targetLength * targetChannels];
for (int i = 0; i < targetLength; i++)
{
double srcIndex = (double)i * inputFrequency / targetFrequency;
long idxLow = (long)Math.Floor(srcIndex);
long idxHigh = idxLow + 1;
float t = (float)(srcIndex - idxLow);
if (idxHigh >= inputLength) idxHigh = inputLength - 1;
if (idxLow >= inputLength) idxLow = inputLength - 1;
if (targetChannels == 1) // 1. Mono Output
{
float valLow, valHigh;
if (inputChannels == 1) // Mono -> Mono
{
valLow = inputSamples[idxLow];
valHigh = inputSamples[idxHigh];
}
else // Multi -> Mono (Downmix: Average all channels)
{
float sumLow = 0f;
float sumHigh = 0f;
for (int c = 0; c < inputChannels; c++)
{
sumLow += inputSamples[idxLow * inputChannels + c];
sumHigh += inputSamples[idxHigh * inputChannels + c];
}
valLow = sumLow / inputChannels;
valHigh = sumHigh / inputChannels;
}
outputSamples[i] = valLow + t * (valHigh - valLow);
}
else if (targetChannels == 2) // 2. Stereo Output
{
if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
{
float valLow = inputSamples[idxLow];
float valHigh = inputSamples[idxHigh];
float val = valLow + t * (valHigh - valLow);
outputSamples[i * 2] = val;
outputSamples[i * 2 + 1] = val;
}
else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
{
float valLowL = inputSamples[idxLow * inputChannels];
float valHighL = inputSamples[idxHigh * inputChannels];
outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
float valLowR = inputSamples[idxLow * inputChannels + 1];
float valHighR = inputSamples[idxHigh * inputChannels + 1];
outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
}
}
else // 3. Multi-channel Output (e.g., 5.1 Surround)
{
for (int c = 0; c < targetChannels; c++)
{
float valLow, valHigh;
if (c < inputChannels)
{
valLow = inputSamples[idxLow * inputChannels + c];
valHigh = inputSamples[idxHigh * inputChannels + c];
}
else
{
valLow = inputSamples[idxLow * inputChannels];
valHigh = inputSamples[idxHigh * inputChannels];
}
outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
}
}
}
return outputSamples;
}
}
+98 -123
View File
@@ -1,188 +1,164 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using ADOFAIRenderer.Core; using ADOFAIRenderer.Core;
using ADOFAIRenderer.Util.Audio.Clip;
using MelonLoader;
using UnityEngine; using UnityEngine;
namespace ADOFAIRenderer.Util.Audio; namespace ADOFAIRenderer.Util.Audio;
/// <summary>
/// Replacement for AudioSource.GetSpectrumData during rendering: the spectrum of the song as the song AudioSource would output it.
///
/// Signal: the song clip from an <see cref="AudioClipStore"/> at the output sample rate (AudioSettings.outputSampleRate),
/// played at conductor.song.pitch (linear interpolation between frames) and scaled by conductor.song.volume.
/// Unity analyzes the output signal of the source: bin spacing follows the output rate, pitch moves the frequencies,
/// and volume scales the magnitudes (measured with a Unity 6000.3.10f1 test project).
/// Channel: stereo clips use the requested channel (0 = left, 1 = right); a mono clip is scaled by <see cref="MonoGain"/>.
/// Window: twice the requested spectrum length in output samples, centered on the current render time.
///
/// The song source is shared with the <see cref="OfflineAudioMixer"/> store when its output rate matches; otherwise a separate
/// store is created. A cursor at the analysis position keeps the needed chunks loaded. Main thread only.
/// </summary>
internal static class AudioSpectrumGenerator internal static class AudioSpectrumGenerator
{ {
/// <summary> // ReSharper disable MemberCanBePrivate.Global
/// Gain of a mono clip relative to one channel of a stereo clip with the same samples. public static float[]? AudioClipData { get; private set; }
/// Measured with a Unity 6000.3.10f1 test project: the spectrum of a mono clip is about 0.70 times that of each stereo channel (1/√2). public static int Channels { get; private set; }
/// </summary> public static int Frequency { get; private set; }
private const double MonoGain = 0.70710678118654752; public static int TotalSamples { get; private set; }
// ReSharper restore MemberCanBePrivate.Global
private static AudioClipStore? _ownStore;
private static AudioClipStore? _store;
private static AudioClipSource? _source;
private static AudioCursor? _cursor;
private static ChunkView _view;
private static bool _loadErrorLogged;
private static long _cachedFrame = -1; private static long _cachedFrame = -1;
private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new(); private static readonly Dictionary<int, float[]> CachedSpectrums = new();
/// <summary>Registers the song clip. Call after the <see cref="OfflineAudioMixer"/> is created (null when audio is not rendered).</summary> public static void Initialize(AudioClip? clip)
public static void Initialize(AudioClip? clip, OfflineAudioMixer? mixer)
{ {
Clear(); if (!clip)
if (!clip) return; {
Clear();
return;
}
try try
{ {
var rate = AudioSettings.outputSampleRate; Channels = clip.channels;
if (mixer != null && mixer.Store.OutputRate == rate) Frequency = clip.frequency;
int samplesCount = clip.samples;
float[] rawSamples = new float[samplesCount * Channels];
clip.GetData(rawSamples, 0);
if (Channels == 1)
{ {
_store = mixer.Store; AudioClipData = rawSamples;
TotalSamples = samplesCount;
} }
else else
{ {
_ownStore = new AudioClipStore(rate); AudioClipData = new float[samplesCount];
_ownStore.Start(); TotalSamples = samplesCount;
_store = _ownStore; for (int i = 0; i < samplesCount; i++)
{
float sum = 0f;
for (int c = 0; c < Channels; c++)
{
sum += rawSamples[i * Channels + c];
}
AudioClipData[i] = sum / Channels;
}
} }
MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}");
_source = AudioClipResolver.GetOrAdd(_store, clip!.name, clip);
if (_source == null)
{
Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Song clip '{clip.name}' could not be loaded, the spectrum will be empty.");
return;
}
_cursor = _source.AddCursor(0, false);
Melon<RenderMod>.Logger.Msg($"[AudioSpectrumGenerator] Song '{clip.name}': {_source.Channels} ch at {rate} Hz ({(_ownStore != null ? "own store" : "shared with the mixer")})");
} }
catch (Exception e) catch (Exception e)
{ {
Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Failed to initialize: {e}"); MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}");
Clear(); Clear();
} }
} }
/// <summary>Releases the chunk, the cursor and the own store. Call before the mixer store is disposed.</summary>
public static void Clear() public static void Clear()
{ {
if (_source != null) AudioClipData = null;
TotalSamples = 0;
Channels = 0;
Frequency = 0;
lock (CachedSpectrums)
{ {
_source.Release(_view); _cachedFrame = -1;
if (_cursor != null) _source.RemoveCursor(_cursor); CachedSpectrums.Clear();
} }
_view = default;
_cursor = null;
_source = null;
_store = null;
_ownStore?.Dispose();
_ownStore = null;
_loadErrorLogged = false;
_cachedFrame = -1;
CachedSpectrums.Clear();
} }
public static void GetSpectrumData(float[] samples, int channel, FFTWindow window) public static void GetSpectrumData(float[] samples, int channel, FFTWindow window)
{ {
var n = samples.Length; if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song)
{
Array.Clear(samples, 0, samples.Length);
return;
}
var conductor = scrConductor.instance; var conductor = scrConductor.instance;
if (_source == null || _store == null || !conductor || !conductor!.hasSongStarted || !conductor.song var pitch = conductor.song.pitch;
|| n == 0 || (n & (n - 1)) != 0) var separateCountdownOffset = conductor.separateCountdownTime ?
conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
// Calculate the actual scheduled start time of the song
var songScheduledStartTime = conductor.skipOffset ?
conductor.dspTimeSong + conductor.addoffset / pitch :
conductor.dspTimeSong + separateCountdownOffset / pitch;
if (conductor.dspTime < songScheduledStartTime)
{ {
Array.Clear(samples, 0, n); Array.Clear(samples, 0, samples.Length);
return; return;
} }
var render = RenderManager.Instance; var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset;
var currentFrame = render.frameCount;
if (currentFrame != _cachedFrame)
{
_cachedFrame = currentFrame;
CachedSpectrums.Clear();
}
var sourceChannel = _source.Channels == 1 ? 0 : Math.Min(Math.Max(channel, 0), _source.Channels - 1); var n = samples.Length;
if (CachedSpectrums.TryGetValue((n, sourceChannel), out var cached)) if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2
{ {
Array.Copy(cached, samples, n); Array.Clear(samples, 0, samples.Length);
return; return;
} }
// Song position (seconds of song time) at the current render time, as in scrConductor.StartMusicCo: var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1;
// the song starts at dspTimeSong + separate countdown (crotchetAtStart * adjustedCountdownTicks in song time).
double pitch = conductor.song.pitch; lock (CachedSpectrums)
double volume = conductor.song.volume; {
var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0; if (currentFrame != _cachedFrame)
var now = render.dspTimeBase + render.AudioTime; {
var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown; _cachedFrame = currentFrame;
CachedSpectrums.Clear();
}
if (CachedSpectrums.TryGetValue(n, out var cached))
{
Array.Copy(cached, samples, n);
return;
}
}
var fftSize = 2 * n; var fftSize = 2 * n;
var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0); var frequency = Frequency;
var first = songSeconds * _store.OutputRate - n * pitch; var totalSamples = TotalSamples;
var currentSampleIndex = (long)(songTime * frequency);
var windowSamples = new float[fftSize]; var windowSamples = new float[fftSize];
try for (var i = 0; i < fftSize; i++)
{ {
_cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1); var idx = currentSampleIndex - fftSize / 2 + i;
for (var i = 0; i < fftSize; i++) if (idx >= 0 && idx < totalSamples)
{ {
var position = first + i * pitch; windowSamples[i] = AudioClipData[idx];
var index = (long)Math.Floor(position);
var fraction = position - index;
var s0 = ReadSample(index, sourceChannel);
var s1 = fraction > 0 ? ReadSample(index + 1, sourceChannel) : 0.0;
windowSamples[i] = (float)((s0 + (s1 - s0) * fraction) * gain);
} }
} else
catch (InvalidOperationException e)
{
if (!_loadErrorLogged)
{ {
_loadErrorLogged = true; windowSamples[i] = 0f;
Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Song samples are unavailable, the spectrum will be empty: {e.Message}");
} }
Array.Clear(samples, 0, n);
return;
} }
var spectrum = new float[n]; var spectrum = new float[n];
CalculateFFT(windowSamples, spectrum); CalculateFFT(windowSamples, spectrum);
CachedSpectrums[(n, sourceChannel)] = spectrum;
Array.Copy(spectrum, samples, n);
}
/// <summary>Sample of source channel <paramref name="channel"/> at <paramref name="frame"/>; 0 outside the clip or when the source failed.</summary> lock (CachedSpectrums)
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (see <see cref="AudioClipStore.Acquire"/>).</exception>
private static double ReadSample(long frame, int channel)
{
var source = _source!;
if (frame < 0 || frame >= source.TotalFrames) return 0.0;
if (!_view.IsValid || !_view.Contains(frame))
{ {
source.Release(_view); CachedSpectrums[n] = spectrum;
_view = default; Array.Copy(spectrum, samples, n);
_view = _store!.Acquire(source, frame);
if (!_view.IsValid) return 0.0;
} }
return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel];
} }
private static void CalculateFFT(float[] samples, float[] spectrum) private static void CalculateFFT(float[] samples, float[] spectrum)
{ {
var n = samples.Length; var n = samples.Length;
// Blackman-Harris window, periodic form: w(i) = a0 - a1 cos(2πi/n) + a2 cos(4πi/n) - a3 cos(6πi/n), i = 0 ~ n - 1 // Apply Blackman-Harris window
const float a0 = 0.35875f; const float a0 = 0.35875f;
const float a1 = 0.48829f; const float a1 = 0.48829f;
const float a2 = 0.14128f; const float a2 = 0.14128f;
@@ -193,7 +169,7 @@ internal static class AudioSpectrumGenerator
for (var i = 0; i < n; i++) for (var i = 0; i < n; i++)
{ {
var angle = 2.0 * Math.PI * i / n; var angle = 2.0 * Math.PI * i / (n - 1);
var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle)); var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle));
re[i] = samples[i] * w; re[i] = samples[i] * w;
im[i] = 0f; im[i] = 0f;
@@ -254,13 +230,12 @@ internal static class AudioSpectrumGenerator
} }
} }
// Magnitude scale 2.5 / n (n = FFT size, twice the spectrum length), clamped to 1.0: the scale of Unity GetSpectrumData. // Normalize and fill spectrum
// A sine of amplitude A centered on a bin gives A * a0 * 1.25 with the Blackman-Harris window. var factor = 2.0f / n;
var factor = 2.5f / n;
var outLen = Math.Min(spectrum.Length, n / 2); var outLen = Math.Min(spectrum.Length, n / 2);
for (var i = 0; i < outLen; i++) for (var i = 0; i < outLen; i++)
{ {
spectrum[i] = Math.Min((float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor, 1f); spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor;
} }
} }
} }
-46
View File
@@ -1,46 +0,0 @@
using System.Collections.Generic;
using System.IO;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// File path of every external audio clip the game loads, keyed by the game's audioLib key.
///
/// The game loads external clips with AudioManager.FindOrLoadAudioClipExternal(path, ...) and stores them in audioLib under
/// Path.GetFileName(path) + "*external" (the same string becomes AudioClip.name). Recorded paths:
/// - Song: Path.Combine(Path.GetDirectoryName(levelPath), levelData.songFilename) (scnGame song loading).
/// - PlaySound event sound: Path.Combine(Path.GetDirectoryName(levelPath), hitsound) (scnGame.LoadSoundCo);
/// ffxPlaySound plays it as hitsound + "*external".
/// <see cref="Record"/> is called only when the game actually loads the file (the key is not in audioLib yet),
/// so the recorded path is the file of the clip audioLib holds under that key.
/// Thread-safe.
/// </summary>
internal static class AudioClipPaths
{
/// <summary>Suffix of the game's audioLib keys for external clips.</summary>
public const string ExternalSuffix = "*external";
private static readonly Dictionary<string, string> Paths = new();
/// <summary>audioLib key of an external clip loaded from <paramref name="path"/>.</summary>
public static string KeyOf(string path) => Path.GetFileName(path) + ExternalSuffix;
/// <summary>Records <paramref name="path"/> under <see cref="KeyOf"/>(path), replacing an earlier path of the same key.</summary>
public static void Record(string path)
{
if (string.IsNullOrEmpty(path)) return;
lock (Paths)
{
Paths[KeyOf(path)] = path;
}
}
/// <summary>Recorded path for an audioLib key (or AudioClip.name) of an external clip.</summary>
public static bool TryGetPath(string key, out string path)
{
lock (Paths)
{
return Paths.TryGetValue(key, out path!);
}
}
}
-60
View File
@@ -1,60 +0,0 @@
using System;
using System.IO;
using MelonLoader;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Opens the <see cref="AudioClipSource"/> for a clip the game plays, keyed by the game's audioLib key (also AudioClip.name):
/// - Key recorded in <see cref="AudioClipPaths"/> and the file exists: decoded from the file with <see cref="FFmpegPCMReader"/>
/// (mono MP3 through <see cref="MP3MonoLayoutReader"/>)
/// (song files and PlaySound files of custom levels).
/// - Otherwise, or when FFmpeg cannot open the file: the Unity clip is copied with <see cref="UnityClipLoader"/>
/// (built-in hit sounds from Resources, official/DLC level songs from Addressables).
/// Main thread only (the Unity copy calls AudioClip.GetData).
/// </summary>
internal static class AudioClipResolver
{
/// <param name="store">Store of the current render.</param>
/// <param name="key">audioLib key of the clip (for example "song.ogg*external" or "sndKick").</param>
/// <param name="clip">Loaded Unity clip for the key, used when the clip has no file; may be null when the key has a file.</param>
/// <returns>The source, or null when neither the file nor the clip can be read.</returns>
public static AudioClipSource? GetOrAdd(AudioClipStore store, string key, AudioClip? clip)
{
if (store.TryGet(key, out var existing)) return existing;
if (AudioClipPaths.TryGetPath(key, out var path) && File.Exists(path))
{
IPCMReader? fileReader = null;
try
{
fileReader = OpenFile(path);
if (fileReader is MP3MonoLayoutReader)
Melon<RenderMod>.Logger.Warning($"[AudioClipResolver] Mono MP3 '{path}' is rendered like the game: half speed, one octave lower.");
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Warning($"[AudioClipResolver] FFmpeg could not open '{path}' for '{key}', using the Unity clip: {e.Message}");
}
if (fileReader != null)
{
var source = store.GetOrAdd(key, () => fileReader, () => OpenFile(path));
if (source == null) fileReader.Dispose();
return source;
}
}
if (!clip) return store.GetOrAdd(key, () => null);
return store.GetOrAdd(key, () => UnityClipLoader.Load(clip!, store));
}
/// <summary>Opens <paramref name="path"/> with FFmpeg; mono MP3 files are wrapped in <see cref="MP3MonoLayoutReader"/> (game layout).</summary>
private static IPCMReader OpenFile(string path)
{
var reader = new FFmpegPCMReader(path);
if (!reader.IsMP3 || reader.Channels != 1) return reader;
return new MP3MonoLayoutReader(reader);
}
}
-613
View File
@@ -1,613 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading;
using MelonLoader;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Read position registered on an <see cref="AudioClipSource"/> by one consumer (a playing voice or the spectrum generator).
/// For a windowed source, the chunk containing <see cref="Position"/> and the following chunks
/// (<see cref="AudioClipSource.SlotCount"/> chunks in total) are kept loaded.
/// </summary>
internal sealed class AudioCursor
{
private readonly AudioClipSource _source;
private long _position;
private volatile bool _loop;
internal AudioCursor(AudioClipSource source, long position, bool loop)
{
_source = source;
_position = position;
_loop = loop;
}
/// <summary>
/// Frame in the source at the output sample rate (0 ~ TotalFrames - 1).
/// Written by the consumer that reads the source; read by the decoder thread.
/// Moving into another chunk wakes the decoder thread.
/// </summary>
public long Position
{
get => Interlocked.Read(ref _position);
set
{
var previous = Interlocked.Exchange(ref _position, value);
if (previous / _source.FramesPerChunk != value / _source.FramesPerChunk)
_source.OnCursorMoved();
}
}
/// <summary>When true, the chunk after the last chunk is chunk 0.</summary>
public bool Loop
{
get => _loop;
set
{
_loop = value;
_source.OnCursorMoved();
}
}
}
/// <summary>
/// One loaded chunk of an <see cref="AudioClipSource"/>, held by a consumer between
/// <see cref="AudioClipSource.TryAcquire"/> and <see cref="AudioClipSource.Release"/>.
/// The chunk is not freed while held.
/// </summary>
internal readonly struct ChunkView
{
/// <summary>Interleaved samples. Frame f of the source is at (f - StartFrame) * Channels.</summary>
public readonly float[] Data;
/// <summary>Source frame of the first frame in <see cref="Data"/>.</summary>
public readonly long StartFrame;
/// <summary>Valid frames in this chunk. Frames past the end of the source are not included.</summary>
public readonly int FrameCount;
/// <summary>Channel count of the stored samples (same as the source clip).</summary>
public readonly int Channels;
public readonly int ChunkIndex;
internal ChunkView(float[] data, long startFrame, int frameCount, int channels, int chunkIndex)
{
Data = data;
StartFrame = startFrame;
FrameCount = frameCount;
Channels = channels;
ChunkIndex = chunkIndex;
}
public bool IsValid => Data != null;
public long EndFrame => StartFrame + FrameCount;
public bool Contains(long frame) => frame >= StartFrame && frame < StartFrame + FrameCount;
/// <summary>
/// Sample of <paramref name="frame"/> mapped to output channel <paramref name="outChannel"/> of <paramref name="outChannels"/>.
/// Mapping: mono output averages all source channels; a mono source is copied to every output channel;
/// otherwise output channel c takes source channel c, and output channels beyond the source take source channel 0.
/// <paramref name="frame"/> must be inside this chunk.
/// </summary>
public float GetSample(long frame, int outChannel, int outChannels)
{
var baseIndex = (int)(frame - StartFrame) * Channels;
if (Channels == 1)
return Data[baseIndex];
if (outChannels == 1)
{
var sum = 0f;
for (var c = 0; c < Channels; c++)
sum += Data[baseIndex + c];
return sum / Channels;
}
return Data[baseIndex + (outChannel < Channels ? outChannel : 0)];
}
}
/// <summary>
/// How an <see cref="AudioClipSource"/> keeps its chunks.
/// </summary>
internal enum ChunkPolicy
{
/// <summary>
/// Only the chunks required by registered cursors (current chunk and the next SlotCount - 1 chunks per cursor),
/// chunks requested by a waiting consumer, and chunks currently held are kept. Other chunks are freed and reloaded on demand.
/// Used for readers that can seek (files decoded with FFmpeg) when the clip is longer than one chunk.
/// </summary>
Windowed,
/// <summary>
/// All chunks are loaded once, in order from chunk 0, starting when the source is created, and kept until the store is disposed.
/// Used for forward-only readers (clip data copied from Unity, released as it is consumed) and for clips of one chunk or less.
/// </summary>
Resident
}
/// <summary>
/// Decoded audio of one clip, resampled to the output sample rate and stored in fixed-size chunks.
/// Channel count stays as in the clip.
///
/// Chunks: chunk k covers output frames [k * FramesPerChunk, (k + 1) * FramesPerChunk), where
/// <see cref="FramesPerChunk"/> = <see cref="ChunkFloats"/> / Channels (rounded down; a frame never spans two chunks).
/// Windowed chunks use pooled arrays of ChunkFloats floats; resident chunks use arrays of their exact length.
/// See <see cref="ChunkPolicy"/> for which chunks are kept.
///
/// Threads: cursors and TryAcquire/Release may be used from any thread.
/// Loading and freeing run only on the store's decoder thread (<see cref="AudioClipStore"/>), through
/// <see cref="FreeUnused"/>, <see cref="TryGetNextLoad"/> and <see cref="LoadChunk"/>.
/// </summary>
internal sealed class AudioClipSource
{
/// <summary>Floats per chunk array: 2^22 floats = 16 MiB.</summary>
public const int ChunkFloats = 1 << 22;
/// <summary>Chunks kept per cursor of a windowed source: the chunk containing the cursor and the next SlotCount - 1 chunks.</summary>
public const int SlotCount = 3;
/// <summary>Load priority of a chunk a consumer is waiting for (lowest value is loaded first).</summary>
internal const int PriorityDemanded = 0;
/// <summary>Load priority of slot s of a cursor window is PriorityWindow + s.</summary>
internal const int PriorityWindow = 1;
/// <summary>Load priority of resident chunks nobody is waiting for.</summary>
internal const int PriorityResident = PriorityWindow + SlotCount;
/// <summary>
/// Most decode streams (reader + resampler) a windowed source opens. A load continues the stream positioned at the chunk start,
/// otherwise opens another stream up to this count, otherwise restarts the least recently used stream.
/// Each stream holds one open reader (file handle and decoder state, measured 0.3 ~ 2.7 MiB for ogg/flac/mp3 with the clip harness).
/// </summary>
public const int MaxDecodeStreams = 4;
/// <summary>Decode stream limit of this source (<see cref="MaxDecodeStreams"/> unless changed for tests).</summary>
internal int DecodeStreamLimit { get; set; } = MaxDecodeStreams;
private readonly AudioClipStore _store;
private readonly IPCMReader _firstReader;
private readonly Func<IPCMReader?>? _openReader;
// Guarded by _sync: chunk table, hold counts, cursors, demand set, loaded set.
private readonly object _sync = new();
private readonly float[]?[] _chunks;
private readonly int[] _holdCounts;
private readonly List<AudioCursor> _cursors = new();
private readonly HashSet<int> _demanded = new();
private readonly HashSet<int> _loaded = new();
private int _residentLoadedCount; // Resident: chunks [0, _residentLoadedCount) are loaded
// Guarded by _loadLock: decode streams.
private readonly object _loadLock = new();
private readonly List<DecodeStream> _streams = new();
private long _streamUseCounter;
private bool _openFailed;
private volatile bool _isFailed;
public string Key { get; }
/// <summary>Channel count of the stored samples (same as the clip).</summary>
public int Channels { get; }
/// <summary>Sample rate of the stored samples in Hz (the output sample rate).</summary>
public int Frequency { get; }
/// <summary>Frame count at <see cref="Frequency"/>: ceil(clip frames * Frequency / clip rate).</summary>
public long TotalFrames { get; }
/// <summary>Clip length in seconds: clip frames / clip rate.</summary>
public double Duration { get; }
public int FramesPerChunk { get; }
public int ChunkCount { get; }
public ChunkPolicy Policy { get; }
public bool IsResident => Policy == ChunkPolicy.Resident;
/// <summary>True after loading failed. A failed source returns no chunks.</summary>
public bool IsFailed => _isFailed;
/// <summary>True when every chunk of a resident source is loaded.</summary>
public bool IsFullyLoaded
{
get
{
lock (_sync) return _loaded.Count == ChunkCount;
}
}
/// <summary>Number of chunk loads performed (for diagnostics).</summary>
public int LoadCount { get; private set; }
/// <summary>Decode streams opened so far (for diagnostics).</summary>
public int StreamCount
{
get
{
lock (_loadLock) return _streams.Count;
}
}
/// <param name="store">Store that owns this source.</param>
/// <param name="key">Clip name or path the source is registered under.</param>
/// <param name="reader">First reader; also defines the clip format.</param>
/// <param name="openReader">Opens another reader of the same clip for an additional decode stream, or null when not possible.</param>
/// <param name="outputRate">Output sample rate in Hz.</param>
internal AudioClipSource(AudioClipStore store, string key, IPCMReader reader, Func<IPCMReader?>? openReader, int outputRate)
{
if (reader.Channels <= 0) throw new ArgumentException("Reader channel count must be positive.", nameof(reader));
if (reader.SampleRate <= 0) throw new ArgumentException("Reader sample rate must be positive.", nameof(reader));
if (reader.TotalFrames < 0) throw new ArgumentException("Reader frame count must not be negative.", nameof(reader));
_store = store;
_firstReader = reader;
_openReader = openReader;
Key = key;
Channels = reader.Channels;
Frequency = outputRate;
Duration = (double)reader.TotalFrames / reader.SampleRate;
TotalFrames = CeilDiv(reader.TotalFrames * outputRate, reader.SampleRate);
FramesPerChunk = ChunkFloats / Channels;
ChunkCount = (int)CeilDiv(TotalFrames, FramesPerChunk);
Policy = reader.IsForwardOnly || ChunkCount <= 1 ? ChunkPolicy.Resident : ChunkPolicy.Windowed;
_chunks = new float[ChunkCount][];
_holdCounts = new int[ChunkCount];
_streams.Add(new DecodeStream(reader, outputRate));
}
// ── Cursors ──
/// <summary>Registers a read position. The decoder thread loads the chunks it requires.</summary>
public AudioCursor AddCursor(long position, bool loop)
{
var cursor = new AudioCursor(this, position, loop);
lock (_sync)
{
_cursors.Add(cursor);
}
_store.RequestService();
return cursor;
}
public void RemoveCursor(AudioCursor cursor)
{
lock (_sync)
{
_cursors.Remove(cursor);
}
_store.RequestService();
}
internal void OnCursorMoved()
{
if (!IsResident) _store.RequestService();
}
// ── Consumer access ──
/// <summary>Chunk index containing <paramref name="frame"/> (0 ~ TotalFrames - 1).</summary>
public int ChunkIndexOf(long frame) => (int)(frame / FramesPerChunk);
/// <summary>
/// Holds the chunk containing <paramref name="frame"/> when it is loaded.
/// When it is not loaded, records a load request (kept until the chunk is acquired), wakes the decoder thread and returns false.
/// Every successful call must be paired with <see cref="Release"/>.
/// </summary>
public bool TryAcquire(long frame, out ChunkView view)
{
view = default;
if (_isFailed || frame < 0 || frame >= TotalFrames) return false;
var index = ChunkIndexOf(frame);
lock (_sync)
{
var data = _chunks[index];
if (data != null)
{
_demanded.Remove(index);
_holdCounts[index]++;
view = CreateView(index, data);
return true;
}
_demanded.Add(index);
}
_store.RequestService();
return false;
}
public void Release(in ChunkView view)
{
if (!view.IsValid) return;
lock (_sync)
{
if (_holdCounts[view.ChunkIndex] > 0)
_holdCounts[view.ChunkIndex]--;
}
}
private ChunkView CreateView(int index, float[] data)
{
var start = (long)index * FramesPerChunk;
var count = (int)Math.Min(FramesPerChunk, TotalFrames - start);
return new ChunkView(data, start, count, Channels, index);
}
// ── Decoder thread ──
/// <summary>
/// Windowed: frees loaded chunks that are not required (see <see cref="ChunkPolicy.Windowed"/>) and not held.
/// Resident: does nothing. Decoder thread only.
/// </summary>
internal void FreeUnused()
{
if (IsResident) return;
List<float[]>? freed = null;
lock (_sync)
{
if (_loaded.Count == 0) return;
List<int>? remove = null;
foreach (var index in _loaded)
{
if (_holdCounts[index] > 0 || IsRequired(index)) continue;
(remove ??= new List<int>()).Add(index);
}
if (remove == null) return;
foreach (var index in remove)
{
(freed ??= new List<float[]>()).Add(_chunks[index]!);
_chunks[index] = null;
_loaded.Remove(index);
}
}
foreach (var data in freed!)
_store.ReturnChunkArray(data);
}
/// <summary>
/// Next chunk to load and its priority (lower loads first), or false when nothing is missing.
/// Windowed: demanded chunks, then each cursor's window by slot.
/// Resident: the next chunk in order; priority is <see cref="PriorityDemanded"/> while a consumer is waiting on this source.
/// Decoder thread only.
/// </summary>
internal bool TryGetNextLoad(out int index, out int priority)
{
index = -1;
priority = int.MaxValue;
if (_isFailed) return false;
lock (_sync)
{
if (IsResident)
{
if (_residentLoadedCount >= ChunkCount) return false;
index = _residentLoadedCount;
priority = _demanded.Count > 0 ? PriorityDemanded : PriorityResident;
return true;
}
foreach (var demanded in _demanded)
{
if (_chunks[demanded] != null) continue;
index = demanded;
priority = PriorityDemanded;
return true;
}
foreach (var cursor in _cursors)
{
if (!TryGetWindowStart(cursor, out var first, out var loop)) continue;
for (var slot = 0; slot < SlotCount; slot++)
{
if (!TryWindowIndex(first, slot, loop, out var candidate)) break;
if (_chunks[candidate] != null) continue;
if (PriorityWindow + slot < priority)
{
index = candidate;
priority = PriorityWindow + slot;
}
break;
}
}
return index >= 0;
}
}
/// <summary>True when a demand or a cursor window covers <paramref name="index"/>. Caller holds _sync.</summary>
private bool IsRequired(int index)
{
if (_demanded.Contains(index)) return true;
foreach (var cursor in _cursors)
{
if (!TryGetWindowStart(cursor, out var first, out var loop)) continue;
for (var slot = 0; slot < SlotCount; slot++)
{
if (!TryWindowIndex(first, slot, loop, out var candidate)) break;
if (candidate == index) return true;
}
}
return false;
}
private bool TryGetWindowStart(AudioCursor cursor, out int first, out bool loop)
{
var position = cursor.Position;
loop = cursor.Loop;
first = 0;
if (position < 0) position = 0;
if (position >= TotalFrames)
{
if (!loop || TotalFrames == 0) return false;
position %= TotalFrames;
}
first = ChunkIndexOf(position);
return true;
}
private bool TryWindowIndex(int first, int slot, bool loop, out int index)
{
index = first + slot;
if (index < ChunkCount) return true;
if (!loop) return false;
index %= ChunkCount;
return true;
}
/// <summary>
/// Loads chunk <paramref name="index"/> if it is not loaded. Resident sources load only the next chunk in order.
/// Decoder thread only.
/// </summary>
/// <returns>True when this call loaded the chunk.</returns>
internal bool LoadChunk(int index)
{
if (_isFailed || index < 0 || index >= ChunkCount) return false;
lock (_loadLock)
{
lock (_sync)
{
if (_chunks[index] != null) return false;
if (IsResident && index != _residentLoadedCount)
throw new InvalidOperationException($"Resident clip '{Key}' loads chunks in order: expected {_residentLoadedCount}, got {index}.");
}
float[]? data = null;
try
{
var start = (long)index * FramesPerChunk;
var frames = (int)Math.Min(FramesPerChunk, TotalFrames - start);
data = IsResident ? _store.AllocateExactArray(frames * Channels) : _store.RentChunkArray();
var stream = SelectStream(start);
stream.LastUse = ++_streamUseCounter;
stream.Fill(start, frames, data);
}
catch (OutOfMemoryException e)
{
ReturnArray(data);
Fail();
_store.ReportFatal(this, e);
return false;
}
catch (Exception e)
{
ReturnArray(data);
Fail();
_store.ReportSourceFailure(this, e);
return false;
}
lock (_sync)
{
_chunks[index] = data;
_loaded.Add(index);
if (IsResident) _residentLoadedCount++;
}
LoadCount++;
// A resident source never reads again after its last chunk; the reader releases its data now.
if (IsResident && _residentLoadedCount == ChunkCount)
DisposeStreams();
}
_store.NotifyChunkLoaded();
return true;
}
/// <summary>
/// Decode stream for a load starting at output frame <paramref name="start"/>:
/// the stream positioned at <paramref name="start"/>; otherwise a new stream while fewer than <see cref="DecodeStreamLimit"/>
/// are open (windowed sources only); otherwise the least recently used stream. Caller holds _loadLock.
/// </summary>
private DecodeStream SelectStream(long start)
{
DecodeStream? oldest = null;
foreach (var stream in _streams)
{
if (stream.NextOutFrame == start) return stream;
if (oldest == null || stream.LastUse < oldest.LastUse) oldest = stream;
}
if (!IsResident && !_openFailed && _openReader != null && _streams.Count < DecodeStreamLimit)
{
IPCMReader? reader = null;
try
{
reader = _openReader();
if (reader is { IsForwardOnly: true })
{
reader.Dispose();
reader = null;
}
if (reader != null)
{
var stream = new DecodeStream(reader, Frequency);
_streams.Add(stream);
return stream;
}
}
catch (Exception e)
{
reader?.Dispose();
Melon<RenderMod>.Logger.Warning($"[AudioClipSource] Could not open another decode stream for '{Key}', reusing open streams: {e.Message}");
}
_openFailed = true;
}
return oldest!;
}
private void Fail()
{
_isFailed = true;
if (IsResident) DisposeStreams();
}
private void DisposeStreams()
{
foreach (var stream in _streams)
stream.Dispose();
_streams.Clear();
}
private void ReturnArray(float[]? data)
{
if (data == null) return;
if (IsResident) _store.FreeExactArray(data);
else _store.ReturnChunkArray(data);
}
/// <summary>Releases all chunk arrays and readers. Called by the store after the decoder thread stopped.</summary>
internal void DisposeInternal()
{
lock (_loadLock)
{
lock (_sync)
{
foreach (var index in _loaded)
{
ReturnArray(_chunks[index]);
_chunks[index] = null;
}
_loaded.Clear();
_cursors.Clear();
_demanded.Clear();
}
DisposeStreams();
_firstReader.Dispose();
}
}
private static long CeilDiv(long a, long b) => (a + b - 1) / b;
}
-348
View File
@@ -1,348 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading;
using MelonLoader;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Clip sources of one render, keyed by clip name or path, with a shared pool of chunk arrays and one decoder thread.
///
/// Threads:
/// - Decoder thread (<see cref="Start"/>): the only thread that decodes, resamples, loads and frees chunks.
/// Each step loads one chunk, the one with the lowest priority value over all sources
/// (a chunk a consumer waits for, then cursor windows by slot, then resident chunks in order).
/// - Consumers (render thread, main thread for the spectrum): <see cref="Acquire"/> waits for the decoder thread and never loads.
/// - Main thread: <see cref="GetOrAdd"/> creates readers (for Unity clips this copies the clip data).
///
/// Memory has no limit; current and peak usage are tracked and logged by <see cref="LogMemoryUsage"/>.
/// An <see cref="OutOfMemoryException"/> while loading is kept in <see cref="FatalError"/> and rethrown by <see cref="Acquire"/>.
/// </summary>
internal sealed class AudioClipStore : IDisposable
{
/// <summary>Longest wait of the decoder thread between checks when nothing signals it, in milliseconds.</summary>
private const int DecoderIdleWaitMs = 50;
/// <summary>Longest wait of a consumer between checks when no chunk is loaded, in milliseconds.</summary>
private const int ConsumerWaitMs = 50;
/// <summary>Longest wait for the decoder thread to stop in <see cref="Dispose"/>, in milliseconds.</summary>
private const int DecoderStopTimeoutMs = 3000;
private readonly Dictionary<string, AudioClipSource?> _sources = new();
private readonly List<AudioClipSource> _sourceList = new();
private readonly Stack<float[]> _chunkPool = new();
private readonly object _poolLock = new();
private readonly object _signal = new();
private readonly AutoResetEvent _serviceRequested = new(false);
private Thread? _decoderThread;
private volatile bool _stopRequested;
private long _allocatedBytes;
private long _peakBytes;
private volatile Exception? _fatalError;
private volatile bool _disposed;
/// <summary>Sample rate every source is resampled to, in Hz.</summary>
public int OutputRate { get; }
/// <summary>Bytes of sample arrays currently allocated (loaded chunks, pooled chunk arrays and copied Unity clip data).</summary>
public long AllocatedBytes => Interlocked.Read(ref _allocatedBytes);
/// <summary>Highest value of <see cref="AllocatedBytes"/> since creation.</summary>
public long PeakBytes => Interlocked.Read(ref _peakBytes);
/// <summary>Out-of-memory error raised while loading, or null.</summary>
public Exception? FatalError => _fatalError;
public AudioClipStore(int outputRate)
{
if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate));
OutputRate = outputRate;
}
// ── Decoder thread ──
/// <summary>Starts the decoder thread. Called once.</summary>
public void Start()
{
if (_decoderThread != null) throw new InvalidOperationException("Decoder thread already started.");
_decoderThread = new Thread(DecoderLoop)
{
Name = "Audio Clip Decoder",
IsBackground = true
};
_decoderThread.Start();
}
private void DecoderLoop()
{
try
{
while (!_stopRequested)
{
if (!LoadNext())
_serviceRequested.WaitOne(DecoderIdleWaitMs);
}
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Decoder thread stopped: {e}");
_fatalError ??= e;
NotifyChunkLoaded();
}
}
/// <summary>
/// Frees unused windowed chunks, then loads the single chunk with the lowest priority value over all sources.
/// Runs on the decoder thread (tests may call it directly while the thread is not started).
/// </summary>
/// <returns>True when a chunk was loaded or a load was attempted.</returns>
internal bool LoadNext()
{
AudioClipSource[] sources;
lock (_sources)
{
if (_disposed) return false;
sources = _sourceList.ToArray();
}
AudioClipSource? best = null;
var bestIndex = -1;
var bestPriority = int.MaxValue;
foreach (var source in sources)
{
source.FreeUnused();
if (!source.TryGetNextLoad(out var index, out var priority) || priority >= bestPriority) continue;
best = source;
bestIndex = index;
bestPriority = priority;
}
if (best == null) return false;
best.LoadChunk(bestIndex);
return true;
}
internal void RequestService() => _serviceRequested.Set();
// ── Sources ──
/// <summary>
/// Returns the source registered under <paramref name="key"/>, creating it with <paramref name="createReader"/> on first use.
/// A key whose reader could not be created is remembered as null and not retried.
/// For readers that can seek, additional decode streams (see <see cref="AudioClipSource.MaxDecodeStreams"/>) are opened on the
/// decoder thread with <paramref name="openAdditionalReader"/>, or with <paramref name="createReader"/> when it is null.
/// </summary>
public AudioClipSource? GetOrAdd(string key, Func<IPCMReader?> createReader, Func<IPCMReader?>? openAdditionalReader = null)
{
lock (_sources)
{
if (_disposed) return null;
if (_sources.TryGetValue(key, out var existing)) return existing;
}
AudioClipSource? source = null;
try
{
var reader = createReader();
if (reader != null)
{
try
{
source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : openAdditionalReader ?? createReader, OutputRate);
}
catch
{
reader.Dispose();
throw;
}
}
}
catch (OutOfMemoryException e)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Out of memory while opening clip '{key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {e}");
_fatalError ??= e;
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Failed to open clip '{key}': {e}");
}
lock (_sources)
{
if (_disposed || _sources.TryGetValue(key, out var raced))
{
source?.DisposeInternal();
return _disposed ? null : _sources[key];
}
_sources[key] = source;
if (source != null) _sourceList.Add(source);
}
if (source != null)
{
Melon<RenderMod>.Logger.Msg($"[AudioClipStore] Opened clip '{key}': {source.Channels}ch, {source.Duration:F3}s, {source.TotalFrames} frames at {OutputRate}Hz, {source.ChunkCount} chunk(s), {source.Policy}");
RequestService();
}
return source;
}
public bool TryGet(string key, out AudioClipSource? source)
{
lock (_sources)
{
return _sources.TryGetValue(key, out source);
}
}
// ── Consumers ──
/// <summary>
/// Holds the chunk containing <paramref name="frame"/>, waiting for the decoder thread until it is loaded.
/// Returns an invalid view when the source failed, the frame is outside the source, or the store was disposed.
/// </summary>
/// <exception cref="InvalidOperationException">Loading ran out of memory or the decoder thread stopped.</exception>
/// <exception cref="OperationCanceledException"><paramref name="cancellationToken"/> was canceled while waiting.</exception>
public ChunkView Acquire(AudioClipSource source, long frame, CancellationToken cancellationToken = default)
{
if (frame < 0 || frame >= source.TotalFrames) return default;
while (true)
{
ThrowIfFatal();
if (source.IsFailed || _disposed) return default;
if (source.TryAcquire(frame, out var view)) return view;
lock (_signal)
{
if (source.TryAcquire(frame, out view)) return view;
if (source.IsFailed || _fatalError != null || _disposed) continue;
Monitor.Wait(_signal, ConsumerWaitMs);
}
cancellationToken.ThrowIfCancellationRequested();
}
}
private void ThrowIfFatal()
{
var error = _fatalError;
if (error != null)
throw new InvalidOperationException("Audio clip loading failed: " + error.Message, error);
}
internal void NotifyChunkLoaded()
{
lock (_signal)
{
Monitor.PulseAll(_signal);
}
}
internal void ReportSourceFailure(AudioClipSource source, Exception error)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Failed to load clip '{source.Key}', it will be silent: {error}");
NotifyChunkLoaded();
}
internal void ReportFatal(AudioClipSource source, OutOfMemoryException error)
{
Melon<RenderMod>.Logger.Error($"[AudioClipStore] Out of memory while loading clip '{source.Key}' ({AllocatedBytes / 1048576.0:F1} MiB allocated): {error}");
_fatalError ??= error;
NotifyChunkLoaded();
}
// ── Arrays ──
internal float[] RentChunkArray()
{
lock (_poolLock)
{
if (_chunkPool.Count > 0) return _chunkPool.Pop();
}
var data = new float[AudioClipSource.ChunkFloats];
AddAllocated(data.LongLength * sizeof(float));
return data;
}
internal void ReturnChunkArray(float[] data)
{
if (_disposed)
{
AddAllocated(-data.LongLength * sizeof(float));
return;
}
lock (_poolLock)
{
_chunkPool.Push(data);
}
}
internal float[] AllocateExactArray(int floats)
{
var data = new float[floats];
AddAllocated(data.LongLength * sizeof(float));
return data;
}
internal void FreeExactArray(float[] data) => AddAllocated(-data.LongLength * sizeof(float));
private void AddAllocated(long bytes)
{
var current = Interlocked.Add(ref _allocatedBytes, bytes);
long peak;
while (current > (peak = Interlocked.Read(ref _peakBytes)))
{
if (Interlocked.CompareExchange(ref _peakBytes, current, peak) == peak) break;
}
}
public void LogMemoryUsage()
{
int sourceCount;
lock (_sources) sourceCount = _sourceList.Count;
Melon<RenderMod>.Logger.Msg($"[AudioClipStore] Clips: {sourceCount}, sample memory now {AllocatedBytes / 1048576.0:F1} MiB, peak {PeakBytes / 1048576.0:F1} MiB");
}
/// <summary>Stops the decoder thread, then releases every source and pooled array.</summary>
public void Dispose()
{
AudioClipSource[] sources;
lock (_sources)
{
if (_disposed) return;
_disposed = true;
sources = _sourceList.ToArray();
}
_stopRequested = true;
_serviceRequested.Set();
NotifyChunkLoaded();
// A decoder thread blocked inside a read (for example a file on an unresponsive drive) is left running:
// it is a background thread, and its sources keep their readers so the blocked call does not touch freed FFmpeg contexts.
var decoderStopped = _decoderThread == null || _decoderThread.Join(DecoderStopTimeoutMs);
if (!decoderStopped)
Melon<RenderMod>.Logger.Warning($"[AudioClipStore] Decoder thread did not stop within {DecoderStopTimeoutMs} ms; its readers are left open.");
LogMemoryUsage();
lock (_sources)
{
_sourceList.Clear();
_sources.Clear();
}
if (decoderStopped)
{
foreach (var source in sources)
source.DisposeInternal();
}
lock (_poolLock)
{
while (_chunkPool.Count > 0)
AddAllocated(-_chunkPool.Pop().LongLength * sizeof(float));
}
}
}
-181
View File
@@ -1,181 +0,0 @@
using System;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// One reader plus its resampler, producing output-rate frames of a clip as a continuous stream.
/// Consecutive <see cref="Fill"/> calls whose ranges follow each other continue the stream (reader position and resampler state);
/// any other range starts a new stream near the requested position.
///
/// Restart: the input position is moved back by a preroll and down to a multiple of in_rate / gcd(in_rate, out_rate),
/// where input frame k * (in_rate / gcd) maps exactly to output frame k * (out_rate / gcd);
/// the resampler is reset and output before the requested frame is discarded.
/// Measured with the clip harness: output is bit-identical to one continuous conversion of the whole clip.
/// Used by one thread at a time (the store's decoder thread).
/// </summary>
internal sealed class DecodeStream : IDisposable
{
/// <summary>Input frames read from the reader per resampler call.</summary>
private const int ReadBlockFrames = 4096;
private readonly PCMResampler? _resampler;
private readonly int _channels;
private readonly int _outputRate;
private readonly float[] _readBuffer;
private float[]? _discardBuffer;
private bool _flushing;
private bool _ended;
private readonly long _inputStep;
private readonly long _outputStep;
private readonly long _prerollInputFrames;
public IPCMReader Reader { get; }
/// <summary>Output frame the stream continues at, or -1 when the stream is not positioned.</summary>
public long NextOutFrame { get; private set; } = -1;
/// <summary>Value of the owner's use counter at the last <see cref="Fill"/> (for least-recently-used selection).</summary>
public long LastUse { get; set; }
/// <summary>Number of times the stream started anew (seek + resampler reset).</summary>
public int RestartCount { get; private set; }
public DecodeStream(IPCMReader reader, int outputRate)
{
Reader = reader;
_channels = reader.Channels;
_outputRate = outputRate;
_readBuffer = new float[ReadBlockFrames * _channels];
if (reader.SampleRate != outputRate)
{
_resampler = new PCMResampler(_channels, reader.SampleRate, outputRate);
var gcd = GCD(reader.SampleRate, outputRate);
_inputStep = reader.SampleRate / gcd;
_outputStep = outputRate / gcd;
// History fed before the first kept output frame when a stream starts mid-clip.
// The filter spans about filter_size input frames per output frame at ratios up to 1:1
// and filter_size * (input rate / output rate) input frames when downsampling.
var ratio = Math.Max(1L, (reader.SampleRate + outputRate - 1L) / outputRate);
_prerollInputFrames = 4L * _resampler.FilterSize * ratio + _inputStep;
}
else
{
_inputStep = 1;
_outputStep = 1;
}
}
/// <summary>
/// Writes output frames [start, start + frames) into <paramref name="data"/> and zeros the rest of the array.
/// Frames past the end of the decoded input are zero.
/// </summary>
public unsafe void Fill(long start, int frames, float[] data)
{
var channels = _channels;
var filled = 0;
if (_resampler == null)
{
if (Reader.Position != start)
{
Reader.Seek(start);
RestartCount++;
}
while (filled < frames)
{
var request = Math.Min(ReadBlockFrames, frames - filled);
var read = Reader.Read(_readBuffer, request);
if (read <= 0) break;
Array.Copy(_readBuffer, 0, data, filled * channels, read * channels);
filled += read;
}
Array.Clear(data, filled * channels, data.Length - filled * channels);
NextOutFrame = start + frames;
return;
}
long discard = 0;
if (NextOutFrame != start)
{
// Input frame of the requested output frame, moved back by the preroll and down to a multiple of _inputStep.
var inputFrame = start * Reader.SampleRate / _outputRate - _prerollInputFrames;
var step = inputFrame > 0 ? inputFrame / _inputStep : 0;
_resampler.Reset();
Reader.Seek(step * _inputStep);
_flushing = false;
_ended = false;
discard = start - step * _outputStep;
RestartCount++;
}
fixed (float* pIn = _readBuffer)
fixed (float* pOut = data)
{
while (discard > 0 && !_ended)
{
_discardBuffer ??= new float[ReadBlockFrames * channels];
fixed (float* pDiscard = _discardBuffer)
{
var capacity = (int)Math.Min(ReadBlockFrames, discard);
discard -= Pull(pIn, pDiscard, capacity);
}
}
while (filled < frames && !_ended)
filled += Pull(pIn, pOut + (long)filled * channels, frames - filled);
}
Array.Clear(data, filled * channels, data.Length - filled * channels);
NextOutFrame = _ended ? -1 : start + frames;
}
/// <summary>
/// Writes 1 ~ <paramref name="capacity"/> resampled frames to <paramref name="output"/>, reading input as needed.
/// Returns 0 and sets _ended once the input and the resampler's flushed tail are exhausted.
/// </summary>
private unsafe int Pull(float* input, float* output, int capacity)
{
var resampler = _resampler!;
if (!_flushing)
{
while (true)
{
var produced = resampler.Convert(input, 0, output, capacity);
if (produced > 0) return produced;
var read = Reader.Read(_readBuffer, ReadBlockFrames);
if (read <= 0)
{
_flushing = true;
break;
}
produced = resampler.Convert(input, read, output, capacity);
if (produced > 0) return produced;
}
}
// Input is exhausted: take the resampler's remaining tail until it returns 0.
var flushed = resampler.Flush(output, capacity);
if (flushed > 0) return flushed;
_ended = true;
return 0;
}
public void Dispose()
{
_resampler?.Dispose();
Reader.Dispose();
}
private static int GCD(int a, int b)
{
while (b != 0) (a, b) = (b, a % b);
return a;
}
}
-390
View File
@@ -1,390 +0,0 @@
using System;
using System.Runtime.InteropServices;
using FFmpeg.AutoGen;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// <see cref="IPCMReader"/> that decodes the best audio stream of a file with libavformat/libavcodec.
/// Output keeps the stream's sample rate and channel count; libswresample converts only the sample format to interleaved float.
///
/// Frame positions: frame 0 is the first sample the decoder outputs for the stream's start time.
/// The position of the first frame decoded after opening or seeking is taken from its timestamp
/// (best_effort_timestamp - stream start_time, in samples); following frames are counted sequentially.
/// Seek moves the demuxer to <see cref="SeekMarginSeconds"/> before the target and discards decoded samples before the target.
///
/// <see cref="TotalFrames"/> comes from the stream duration (or the container duration) and may differ from the decoded length;
/// reads stop at whichever ends first.
///
/// MP3: decoded with the game's timing (MP3Sharp, see <see cref="MP3HeaderFrame"/>): encoder delay and padding are kept
/// (AV_CODEC_FLAG2_SKIP_MANUAL), frame 0 is the first sample of the first audio frame, and one frame of silence is placed
/// before it when the file starts with a Xing/Info/VBRI header frame. <see cref="TotalFrames"/> is the sum of all packet
/// durations plus that silence, counted once when the file is opened.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__decoding.html (send/receive API)
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__decoding.html (av_read_frame, av_seek_frame)
/// </summary>
internal sealed unsafe class FFmpegPCMReader : IPCMReader
{
/// <summary>
/// Time the demuxer is moved before a seek target, in seconds. Decoders that depend on previous packets
/// (for example the MP3 bit reservoir) produce settled output after this margin; the samples in it are discarded.
/// </summary>
private const double SeekMarginSeconds = 0.25;
private AVFormatContext* _format;
private AVCodecContext* _codec;
private SwrContext* _swr;
private AVPacket* _packet;
private AVFrame* _frame;
private readonly int _streamIndex;
private readonly AVRational _timeBase;
private readonly long _ptsOrigin; // Stream timestamp of output frame _leadFrames
private readonly long _leadFrames; // Silent frames placed before the first decoded frame (MP3 header frame, see class summary)
// Decoded samples not yet returned: frames [_pendingStart, _pendingStart + _pendingCount) at _pending[_pendingOffset * Channels].
private float[] _pending = Array.Empty<float>();
private int _pendingOffset;
private int _pendingCount;
private long _pendingStart;
private long _nextDecodedFrame; // Frame index of the next decoded frame when counting sequentially
private bool _syncToTimestamp; // True until the first frame after open/seek sets _nextDecodedFrame from its timestamp
private bool _flushSent;
private bool _endOfStream;
public string Path { get; }
public int Channels { get; }
public int SampleRate { get; }
public long TotalFrames { get; }
public bool IsForwardOnly => false;
public long Position { get; private set; }
/// <summary>True when the stream is MP3 and is decoded with the game's MP3 timing (see class summary).</summary>
public bool IsMP3 { get; }
/// <summary>Number of demuxer seeks performed (for diagnostics).</summary>
public int SeekCount { get; private set; }
public FFmpegPCMReader(string path)
{
Path = path;
try
{
AVFormatContext* format = null;
var ret = ffmpeg.avformat_open_input(&format, path, null, null);
if (ret < 0) throw new InvalidOperationException($"Failed to open audio file '{path}': {ErrorString(ret)}");
_format = format;
ret = ffmpeg.avformat_find_stream_info(_format, null);
if (ret < 0) throw new InvalidOperationException($"Failed to read stream info of '{path}': {ErrorString(ret)}");
AVCodec* decoder = null;
_streamIndex = ffmpeg.av_find_best_stream(_format, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, -1, &decoder, 0);
if (_streamIndex < 0 || decoder == null) throw new InvalidOperationException($"No decodable audio stream in '{path}': {ErrorString(_streamIndex)}");
for (var i = 0; i < (int)_format->nb_streams; i++)
{
if (i != _streamIndex) _format->streams[i]->discard = AVDiscard.AVDISCARD_ALL;
}
var stream = _format->streams[_streamIndex];
_timeBase = stream->time_base;
_ptsOrigin = stream->start_time != ffmpeg.AV_NOPTS_VALUE ? stream->start_time : 0;
_codec = ffmpeg.avcodec_alloc_context3(decoder);
if (_codec == null) throw new InvalidOperationException("avcodec_alloc_context3 failed.");
ret = ffmpeg.avcodec_parameters_to_context(_codec, stream->codecpar);
if (ret < 0) throw new InvalidOperationException($"avcodec_parameters_to_context failed: {ErrorString(ret)}");
_codec->pkt_timebase = _timeBase;
IsMP3 = stream->codecpar->codec_id == AVCodecID.AV_CODEC_ID_MP3;
if (IsMP3)
{
// Keep the encoder delay and padding the decoder would otherwise trim (exported as side data instead).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html (AV_CODEC_FLAG2_SKIP_MANUAL)
_codec->flags2 |= ffmpeg.AV_CODEC_FLAG2_SKIP_MANUAL;
}
ret = ffmpeg.avcodec_open2(_codec, decoder, null);
if (ret < 0) throw new InvalidOperationException($"Failed to open audio decoder for '{path}': {ErrorString(ret)}");
Channels = _codec->ch_layout.nb_channels;
SampleRate = _codec->sample_rate;
if (Channels <= 0 || SampleRate <= 0) throw new InvalidOperationException($"Invalid audio format in '{path}': {Channels}ch {SampleRate}Hz.");
if (IsMP3)
{
_leadFrames = MP3HeaderFrame.SilentFrames(path);
ScanPackets(out _ptsOrigin, out var decodedFrames);
TotalFrames = _leadFrames + decodedFrames;
}
else
{
TotalFrames = EstimateTotalFrames(stream);
}
SwrContext* swr = null;
ret = ffmpeg.swr_alloc_set_opts2(&swr,
&_codec->ch_layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, SampleRate,
&_codec->ch_layout, _codec->sample_fmt, SampleRate, 0, null);
if (ret < 0 || swr == null) throw new InvalidOperationException($"swr_alloc_set_opts2 failed: {ErrorString(ret)}");
_swr = swr;
ret = ffmpeg.swr_init(_swr);
if (ret < 0) throw new InvalidOperationException($"swr_init failed: {ErrorString(ret)}");
_packet = ffmpeg.av_packet_alloc();
_frame = ffmpeg.av_frame_alloc();
if (_packet == null || _frame == null) throw new InvalidOperationException("Failed to allocate packet/frame.");
_syncToTimestamp = true;
}
catch
{
Dispose();
throw;
}
}
/// <summary>
/// Reads every packet of the audio stream once (no decoding) and returns the first packet's timestamp and the summed
/// packet durations in frames, then moves the demuxer back to the first packet. Used for MP3, where every packet decodes
/// to a full frame when trimming is off.
/// </summary>
private void ScanPackets(out long firstPts, out long frames)
{
firstPts = ffmpeg.AV_NOPTS_VALUE;
long durationTicks = 0;
var packet = ffmpeg.av_packet_alloc();
try
{
int ret;
while ((ret = ffmpeg.av_read_frame(_format, packet)) >= 0)
{
if (packet->stream_index == _streamIndex)
{
if (firstPts == ffmpeg.AV_NOPTS_VALUE && packet->pts != ffmpeg.AV_NOPTS_VALUE) firstPts = packet->pts;
durationTicks += packet->duration;
}
ffmpeg.av_packet_unref(packet);
}
if (ret != ffmpeg.AVERROR_EOF)
throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}");
}
finally
{
ffmpeg.av_packet_free(&packet);
}
if (firstPts == ffmpeg.AV_NOPTS_VALUE) firstPts = 0;
frames = ffmpeg.av_rescale_q(durationTicks, _timeBase, new AVRational { num = 1, den = SampleRate });
var seek = ffmpeg.av_seek_frame(_format, _streamIndex, firstPts, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (seek < 0)
throw new InvalidOperationException($"Failed to rewind '{Path}': {ErrorString(seek)}");
}
private long EstimateTotalFrames(AVStream* stream)
{
if (stream->duration != ffmpeg.AV_NOPTS_VALUE && stream->duration > 0)
return ffmpeg.av_rescale_q(stream->duration, _timeBase, new AVRational { num = 1, den = SampleRate });
if (_format->duration != ffmpeg.AV_NOPTS_VALUE && _format->duration > 0)
return ffmpeg.av_rescale(_format->duration, SampleRate, ffmpeg.AV_TIME_BASE);
throw new InvalidOperationException($"Audio file '{Path}' has no duration.");
}
public void Seek(long frame)
{
if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame));
if (frame == Position) return;
var marginFrames = (long)(SeekMarginSeconds * SampleRate);
// Target inside the pending block or less than the seek margin after it: decode forward and discard.
if (!_syncToTimestamp && frame >= _pendingStart + _pendingOffset
&& frame <= _pendingStart + _pendingOffset + _pendingCount + marginFrames)
{
Position = frame;
return;
}
var targetTs = _ptsOrigin + ffmpeg.av_rescale_q(Math.Max(0, frame - _leadFrames - marginFrames), new AVRational { num = 1, den = SampleRate }, _timeBase);
var ret = ffmpeg.av_seek_frame(_format, _streamIndex, targetTs, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
ret = ffmpeg.av_seek_frame(_format, _streamIndex, _ptsOrigin, ffmpeg.AVSEEK_FLAG_BACKWARD);
if (ret < 0)
throw new InvalidOperationException($"Failed to seek '{Path}' to frame {frame}: {ErrorString(ret)}");
ffmpeg.avcodec_flush_buffers(_codec);
SeekCount++;
_pendingCount = 0;
_pendingOffset = 0;
_syncToTimestamp = true;
_flushSent = false;
_endOfStream = false;
Position = frame;
}
public int Read(float[] buffer, int frameCount)
{
var limit = (int)Math.Min(frameCount, TotalFrames - Position);
var written = 0;
while (written < limit)
{
if (_pendingCount == 0 || Position >= _pendingStart + _pendingOffset + _pendingCount)
{
if (!DecodeNextFrame()) break;
continue;
}
var pendingPosition = _pendingStart + _pendingOffset;
if (Position > pendingPosition)
{
// Discard decoded samples before the read position (after a seek).
var skip = (int)Math.Min(Position - pendingPosition, _pendingCount);
_pendingOffset += skip;
_pendingCount -= skip;
continue;
}
if (Position < pendingPosition)
{
// A gap in the decoded timestamps: fill with silence up to the next decoded sample.
var gap = (int)Math.Min(pendingPosition - Position, limit - written);
Array.Clear(buffer, written * Channels, gap * Channels);
written += gap;
Position += gap;
continue;
}
var count = Math.Min(_pendingCount, limit - written);
Array.Copy(_pending, _pendingOffset * Channels, buffer, written * Channels, count * Channels);
_pendingOffset += count;
_pendingCount -= count;
written += count;
Position += count;
}
return written;
}
/// <summary>Decodes the next audio frame into the pending buffer. Returns false at the end of the stream.</summary>
private bool DecodeNextFrame()
{
while (!_endOfStream)
{
var ret = ffmpeg.avcodec_receive_frame(_codec, _frame);
if (ret == 0)
{
StoreFrame();
ffmpeg.av_frame_unref(_frame);
return true;
}
if (ret == ffmpeg.AVERROR_EOF)
{
_endOfStream = true;
break;
}
if (ret != ffmpeg.AVERROR(ffmpeg.EAGAIN))
throw new InvalidOperationException($"Audio decoding failed in '{Path}': {ErrorString(ret)}");
if (_flushSent)
{
_endOfStream = true;
break;
}
ret = ffmpeg.av_read_frame(_format, _packet);
if (ret < 0)
{
if (ret != ffmpeg.AVERROR_EOF)
throw new InvalidOperationException($"Failed to read '{Path}': {ErrorString(ret)}");
ffmpeg.avcodec_send_packet(_codec, null);
_flushSent = true;
continue;
}
try
{
if (_packet->stream_index != _streamIndex) continue;
ret = ffmpeg.avcodec_send_packet(_codec, _packet);
// A corrupt packet is skipped; decoding continues with the next one.
if (ret < 0 && ret != ffmpeg.AVERROR_INVALIDDATA && ret != ffmpeg.AVERROR(ffmpeg.EAGAIN))
throw new InvalidOperationException($"Failed to send audio packet of '{Path}': {ErrorString(ret)}");
}
finally
{
ffmpeg.av_packet_unref(_packet);
}
}
return false;
}
private void StoreFrame()
{
var samples = _frame->nb_samples;
if (samples <= 0) return;
if (_syncToTimestamp)
{
var pts = _frame->best_effort_timestamp;
_nextDecodedFrame = pts != ffmpeg.AV_NOPTS_VALUE
? _leadFrames + ffmpeg.av_rescale_q(pts - _ptsOrigin, _timeBase, new AVRational { num = 1, den = SampleRate })
: Position;
_syncToTimestamp = false;
}
if (_pending.Length < samples * Channels)
_pending = new float[samples * Channels];
fixed (float* pPending = _pending)
{
var outPlane = (byte*)pPending;
var converted = ffmpeg.swr_convert(_swr, &outPlane, samples, _frame->extended_data, samples);
if (converted < 0)
throw new InvalidOperationException($"Sample format conversion failed in '{Path}': {ErrorString(converted)}");
samples = converted;
}
_pendingStart = _nextDecodedFrame;
_pendingOffset = 0;
_pendingCount = samples;
_nextDecodedFrame += samples;
}
public void Dispose()
{
if (_frame != null)
{
var frame = _frame;
ffmpeg.av_frame_free(&frame);
_frame = null;
}
if (_packet != null)
{
var packet = _packet;
ffmpeg.av_packet_free(&packet);
_packet = null;
}
if (_swr != null)
{
var swr = _swr;
ffmpeg.swr_free(&swr);
_swr = null;
}
if (_codec != null)
{
var codec = _codec;
ffmpeg.avcodec_free_context(&codec);
_codec = null;
}
if (_format != null)
{
var format = _format;
ffmpeg.avformat_close_input(&format);
_format = null;
}
}
private static string ErrorString(int error)
{
var buffer = stackalloc byte[1024];
ffmpeg.av_strerror(error, buffer, 1024);
return $"{Marshal.PtrToStringAnsi((IntPtr)buffer)} ({error})";
}
}
-42
View File
@@ -1,42 +0,0 @@
using System;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Sequential reader of decoded PCM at the clip's own sample rate and channel count.
/// Samples are interleaved 32-bit float. Positions and counts are in frames (one frame = one sample per channel).
/// Used by one thread at a time (the store's decoder thread). <see cref="IDisposable.Dispose"/> may be called more than once.
/// </summary>
internal interface IPCMReader : IDisposable
{
/// <summary>Channel count of the decoded audio (1 or more).</summary>
int Channels { get; }
/// <summary>Sample rate of the decoded audio in Hz.</summary>
int SampleRate { get; }
/// <summary>Total frame count of the clip at <see cref="SampleRate"/>.</summary>
long TotalFrames { get; }
/// <summary>
/// True when the data can be read only once, from frame 0 to the end (data is released as it is read).
/// The source then keeps every chunk (<see cref="ChunkPolicy.Resident"/>).
/// </summary>
bool IsForwardOnly { get; }
/// <summary>Frame index that the next <see cref="Read"/> starts at.</summary>
long Position { get; }
/// <summary>
/// Moves the read position to <paramref name="frame"/> (0 ~ <see cref="TotalFrames"/>).
/// A forward-only reader accepts only frames whose data has not been released.
/// </summary>
void Seek(long frame);
/// <summary>
/// Reads up to <paramref name="frameCount"/> frames into <paramref name="buffer"/> starting at index 0 and advances the position.
/// <paramref name="buffer"/> must hold at least frameCount * Channels floats.
/// </summary>
/// <returns>Frames read. 0 at the end of the clip.</returns>
int Read(float[] buffer, int frameCount);
}
-109
View File
@@ -1,109 +0,0 @@
using System;
using System.IO;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Detects the VBR/info header frame ("Xing", "Info" or "VBRI") at the start of an MP3 file.
///
/// The game decodes MP3 files with MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.MP3Stream), which has no header-frame handling:
/// it decodes that frame as one frame of silence and keeps the encoder delay. FFmpeg's MP3 demuxer consumes the header frame
/// without output. Measured with the clip harness (LAME 192k CBR, LAME VBR, 44.1/48 kHz, with and without ID3v2):
/// MP3Sharp output is FFmpeg's untrimmed output preceded by one frame of silence when the header frame exists,
/// and identical in timing when it does not.
///
/// Frame layout used here (ISO/IEC 11172-3 and 13818-3 frame header; Xing/Info tag after the Layer III side information,
/// VBRI tag 32 bytes after the frame header):
/// - Header: 11 sync bits, version (2 bits: 3 = MPEG-1, 2 = MPEG-2, 0 = MPEG-2.5), layer (2 bits: 1 = III, 2 = II, 3 = I),
/// protection bit (0 = 16-bit CRC follows), bitrate index, sample rate index, padding, private, channel mode (3 = mono).
/// - Layer III side information: MPEG-1 32 bytes (17 mono), MPEG-2/2.5 17 bytes (9 mono).
/// - Samples per frame: Layer I 384, Layer II 1152, Layer III 1152 (MPEG-1) or 576 (MPEG-2/2.5).
/// - ID3v2 tags before the first frame: "ID3", version (2 bytes), flags (bit 0x10 = 10-byte footer), size (4 bytes, 7 bits each).
/// </summary>
internal static class MP3HeaderFrame
{
/// <summary>Bytes searched for the first frame sync after the ID3v2 tags.</summary>
private const int SyncSearchBytes = 64 * 1024;
/// <summary>
/// Frames of silence MP3Sharp outputs for the header frame of <paramref name="path"/>:
/// the samples per frame of the first frame when it is a Xing/Info/VBRI header frame, otherwise 0.
/// </summary>
public static int SilentFrames(string path)
{
// The game keeps the song open while it streams it (MP3Sharp.MP3Stream: new FileStream(path, FileMode.Open),
// read/write access with read sharing), so the file is opened with read/write/delete sharing.
using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
return SilentFrames(file);
}
public static int SilentFrames(Stream stream)
{
var tag = new byte[10];
long offset = 0;
while (true)
{
stream.Position = offset;
if (ReadFully(stream, tag, 10) < 10 || tag[0] != 'I' || tag[1] != 'D' || tag[2] != '3') break;
var size = (tag[6] & 0x7F) << 21 | (tag[7] & 0x7F) << 14 | (tag[8] & 0x7F) << 7 | (tag[9] & 0x7F);
offset += 10 + size + ((tag[5] & 0x10) != 0 ? 10 : 0);
}
stream.Position = offset;
var data = new byte[SyncSearchBytes];
var length = ReadFully(stream, data, data.Length);
for (var i = 0; i + 4 <= length; i++)
{
if (!TryParseHeader(data, i, out var samplesPerFrame, out var tagOffset)) continue;
// Xing/Info after the side information; VBRI at a fixed offset of 32 bytes after the 4-byte header.
if (HasTag(data, length, i + tagOffset, "Xing") || HasTag(data, length, i + tagOffset, "Info")
|| HasTag(data, length, i + 4 + 32, "VBRI"))
return samplesPerFrame;
return 0;
}
return 0;
}
private static bool TryParseHeader(byte[] data, int i, out int samplesPerFrame, out int tagOffset)
{
samplesPerFrame = 0;
tagOffset = 0;
if (data[i] != 0xFF || (data[i + 1] & 0xE0) != 0xE0) return false;
var version = (data[i + 1] >> 3) & 0x3;
var layer = (data[i + 1] >> 1) & 0x3;
var hasCrc = (data[i + 1] & 0x1) == 0;
var bitrateIndex = (data[i + 2] >> 4) & 0xF;
var sampleRateIndex = (data[i + 2] >> 2) & 0x3;
var mono = ((data[i + 3] >> 6) & 0x3) == 3;
if (version == 1 || layer == 0 || bitrateIndex == 0xF || sampleRateIndex == 3) return false;
var mpeg1 = version == 3;
samplesPerFrame = layer switch
{
3 => 384,
2 => 1152,
_ => mpeg1 ? 1152 : 576
};
var sideInfo = layer == 1 ? (mpeg1 ? (mono ? 17 : 32) : (mono ? 9 : 17)) : 0;
tagOffset = 4 + (hasCrc ? 2 : 0) + sideInfo;
return true;
}
private static bool HasTag(byte[] data, int length, int at, string tag)
{
if (at < 0 || at + tag.Length > length) return false;
for (var k = 0; k < tag.Length; k++)
if (data[at + k] != tag[k]) return false;
return true;
}
private static int ReadFully(Stream stream, byte[] buffer, int count)
{
var total = 0;
int read;
while (total < count && (read = stream.Read(buffer, total, count - total)) > 0) total += read;
return total;
}
}
-73
View File
@@ -1,73 +0,0 @@
using System;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// <see cref="IPCMReader"/> that reproduces the game's output for mono MP3 files.
///
/// MP3Sharp (Assembly-CSharp-firstpass, MP3Sharp.Buffer16BitStereo) writes every frame in stereo layout and fills only the
/// channel 0 slot for mono streams, while MP3Stream.ChannelCount reports 1. The game creates a 1-channel clip from that data,
/// so a mono MP3 plays as [s0, 0, s1, 0, ...]: twice as long, at half speed and one octave lower
/// (confirmed in game with TestMaps/mp3_mono, 2026-10-13).
///
/// Output: 1 channel at the inner reader's sample rate. Frame 2k is inner frame k, frame 2k + 1 is 0.
/// <see cref="TotalFrames"/> = 2 * inner TotalFrames.
/// </summary>
internal sealed class MP3MonoLayoutReader : IPCMReader
{
private readonly IPCMReader _inner;
private float[] _innerBuffer = Array.Empty<float>();
/// <param name="inner">Mono reader of the MP3 file with the game's MP3 timing (<see cref="FFmpegPCMReader.IsMP3"/>). Owned by this reader.</param>
public MP3MonoLayoutReader(IPCMReader inner)
{
if (inner.Channels != 1) throw new ArgumentException("Inner reader must be mono.", nameof(inner));
_inner = inner;
Position = inner.Position * 2;
}
public int Channels => 1;
public int SampleRate => _inner.SampleRate;
public long TotalFrames => _inner.TotalFrames * 2;
public bool IsForwardOnly => _inner.IsForwardOnly;
public long Position { get; private set; }
/// <summary>Moves to <paramref name="frame"/>; the inner reader moves to the inner frame of the next even position.</summary>
public void Seek(long frame)
{
_inner.Seek((frame + 1) / 2);
Position = frame;
}
public int Read(float[] buffer, int frameCount)
{
var limit = (int)Math.Min(frameCount, TotalFrames - Position);
var written = 0;
while (written < limit)
{
if ((Position & 1) == 1)
{
buffer[written++] = 0f;
Position++;
continue;
}
var want = (limit - written + 1) / 2;
if (_innerBuffer.Length < want) _innerBuffer = new float[want];
var got = _inner.Read(_innerBuffer, want);
if (got == 0) break;
for (var i = 0; i < got && written < limit; i++)
{
buffer[written++] = _innerBuffer[i];
Position++;
if (written >= limit) break;
buffer[written++] = 0f;
Position++;
}
}
return written;
}
public void Dispose() => _inner.Dispose();
}
-83
View File
@@ -1,83 +0,0 @@
using System;
using System.Collections.Generic;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Forward-only <see cref="IPCMReader"/> over PCM held in memory as a list of blocks.
/// Block b holds frames [b * FramesPerBlock, (b + 1) * FramesPerBlock) interleaved; the last block may be shorter.
/// Each block is released (passed to the release callback and dropped) as soon as the read position passes its end,
/// and all remaining blocks are released on <see cref="Dispose"/>.
/// </summary>
internal sealed class MemoryPCMReader : IPCMReader
{
private readonly float[]?[] _blocks;
private readonly Action<float[]>? _onBlockReleased;
private int _releasedCount; // Blocks [0, _releasedCount) are released
public int Channels { get; }
public int SampleRate { get; }
public long TotalFrames { get; }
public bool IsForwardOnly => true;
public long Position { get; private set; }
/// <summary>Frames per block (all blocks except the last).</summary>
public int FramesPerBlock { get; }
public MemoryPCMReader(IReadOnlyList<float[]> blocks, int framesPerBlock, int channels, int sampleRate, long totalFrames,
Action<float[]>? onBlockReleased)
{
if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels));
if (framesPerBlock <= 0) throw new ArgumentOutOfRangeException(nameof(framesPerBlock));
if (totalFrames > (long)blocks.Count * framesPerBlock) throw new ArgumentException("Blocks hold fewer frames than totalFrames.", nameof(blocks));
_blocks = new float[blocks.Count][];
for (var i = 0; i < blocks.Count; i++) _blocks[i] = blocks[i];
_onBlockReleased = onBlockReleased;
FramesPerBlock = framesPerBlock;
Channels = channels;
SampleRate = sampleRate;
TotalFrames = totalFrames;
}
public void Seek(long frame)
{
if (frame < 0 || frame > TotalFrames) throw new ArgumentOutOfRangeException(nameof(frame));
if (frame < (long)_releasedCount * FramesPerBlock)
throw new InvalidOperationException($"Frame {frame} was already released by the forward-only reader.");
Position = frame;
}
public int Read(float[] buffer, int frameCount)
{
var total = (int)Math.Min(frameCount, TotalFrames - Position);
if (total <= 0) return 0;
var written = 0;
while (written < total)
{
var blockIndex = (int)(Position / FramesPerBlock);
var block = _blocks[blockIndex] ?? throw new InvalidOperationException($"Block {blockIndex} was already released.");
var offset = (int)(Position - (long)blockIndex * FramesPerBlock);
var count = Math.Min(total - written, FramesPerBlock - offset);
Array.Copy(block, offset * Channels, buffer, written * Channels, count * Channels);
written += count;
Position += count;
ReleaseBlocksBefore((int)(Position / FramesPerBlock));
}
return written;
}
private void ReleaseBlocksBefore(int blockIndex)
{
while (_releasedCount < blockIndex && _releasedCount < _blocks.Length)
{
var block = _blocks[_releasedCount];
_blocks[_releasedCount] = null;
_releasedCount++;
if (block != null) _onBlockReleased?.Invoke(block);
}
}
public void Dispose() => ReleaseBlocksBefore(_blocks.Length);
}
-106
View File
@@ -1,106 +0,0 @@
using System;
using FFmpeg.AutoGen;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Sample rate converter for interleaved 32-bit float PCM (libswresample).
/// Input and output have the same channel count; only the sample rate changes.
/// One instance keeps filter state between calls, so consecutive <see cref="Convert"/> calls form one continuous stream.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// </summary>
internal sealed unsafe class PCMResampler : IDisposable
{
private SwrContext* _swr;
private bool _flushed;
public int Channels { get; }
public int InputRate { get; }
public int OutputRate { get; }
/// <summary>Value of the SwrContext "filter_size" option (filter length per phase in taps at a 1:1 ratio).</summary>
public int FilterSize { get; }
public PCMResampler(int channels, int inputRate, int outputRate)
{
if (channels <= 0) throw new ArgumentOutOfRangeException(nameof(channels));
if (inputRate <= 0) throw new ArgumentOutOfRangeException(nameof(inputRate));
if (outputRate <= 0) throw new ArgumentOutOfRangeException(nameof(outputRate));
Channels = channels;
InputRate = inputRate;
OutputRate = outputRate;
AVChannelLayout layout;
ffmpeg.av_channel_layout_default(&layout, channels);
SwrContext* swr = null;
var ret = ffmpeg.swr_alloc_set_opts2(
&swr,
&layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, outputRate,
&layout, AVSampleFormat.AV_SAMPLE_FMT_FLT, inputRate,
0, null);
ffmpeg.av_channel_layout_uninit(&layout);
if (ret < 0 || swr == null)
throw new InvalidOperationException($"swr_alloc_set_opts2 failed ({ret}).");
_swr = swr;
long filterSize;
FilterSize = ffmpeg.av_opt_get_int(_swr, "filter_size", 0, &filterSize) >= 0 ? (int)filterSize : 32;
Reset();
}
/// <summary>Discards all buffered input and filter history. The next call starts a new stream.</summary>
public void Reset()
{
// swr_init on an initialized context re-initializes it.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lswr.html (swr_init, swr_close)
var ret = ffmpeg.swr_init(_swr);
if (ret < 0)
throw new InvalidOperationException($"swr_init failed ({ret}).");
_flushed = false;
}
/// <summary>
/// Feeds <paramref name="inputFrames"/> frames and writes up to <paramref name="outputCapacity"/> frames.
/// Input that does not fit in the output is buffered inside the context and returned by later calls.
/// <paramref name="inputFrames"/> may be 0 to take only buffered output.
/// </summary>
/// <returns>Frames written to <paramref name="output"/>.</returns>
public int Convert(float* input, int inputFrames, float* output, int outputCapacity)
{
if (_flushed) throw new InvalidOperationException("Resampler was flushed. Call Reset first.");
// swr_convert treats a null input array as a flush request, so a non-null pointer is passed even for 0 frames.
float dummy = 0;
var inPlane = (byte*)(input != null ? input : &dummy);
var outPlane = (byte*)output;
var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, &inPlane, inputFrames);
if (ret < 0)
throw new InvalidOperationException($"swr_convert failed ({ret}).");
return ret;
}
/// <summary>
/// Ends the input stream and writes up to <paramref name="outputCapacity"/> remaining frames.
/// Call repeatedly until it returns 0. <see cref="Convert"/> is not allowed afterwards until <see cref="Reset"/>.
/// </summary>
public int Flush(float* output, int outputCapacity)
{
_flushed = true;
var outPlane = (byte*)output;
var ret = ffmpeg.swr_convert(_swr, &outPlane, outputCapacity, null, 0);
if (ret < 0)
throw new InvalidOperationException($"swr_convert (flush) failed ({ret}).");
return ret;
}
public void Dispose()
{
if (_swr == null) return;
var swr = _swr;
ffmpeg.swr_free(&swr);
_swr = null;
}
}
-55
View File
@@ -1,55 +0,0 @@
using System;
using System.Collections.Generic;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio.Clip;
/// <summary>
/// Copies a Unity <see cref="AudioClip"/> into memory for clips without a file on disk
/// (built-in hit sounds from Resources, official/DLC level audio).
/// <see cref="AudioClip.GetData(float[], int)"/> runs on the main thread only, so the whole clip is copied once on the main thread
/// and the decoder thread reads the copy through a <see cref="MemoryPCMReader"/>.
/// Streaming clips (<see cref="AudioClipLoadType.Streaming"/>) are rejected because GetData does not read them.
/// Reference: https://docs.unity3d.com/ScriptReference/AudioClip.GetData.html
/// </summary>
internal static class UnityClipLoader
{
/// <summary>
/// Copies <paramref name="clip"/> in blocks of <see cref="AudioClipSource.ChunkFloats"/> floats or less
/// (FramesPerBlock = ChunkFloats / channels frames per GetData call). Main thread only.
/// Block memory is counted in <paramref name="store"/> and released block by block as the decoder thread reads it.
/// </summary>
public static MemoryPCMReader Load(AudioClip clip, AudioClipStore store)
{
if (!clip) throw new ArgumentNullException(nameof(clip));
if (clip.loadType == AudioClipLoadType.Streaming)
throw new NotSupportedException($"AudioClip '{clip.name}' is a streaming clip; GetData cannot read it.");
if (clip.loadState == AudioDataLoadState.Unloaded)
clip.LoadAudioData();
var channels = clip.channels;
var totalFrames = (long)clip.samples;
var framesPerBlock = AudioClipSource.ChunkFloats / channels;
var blocks = new List<float[]>();
try
{
for (long start = 0; start < totalFrames; start += framesPerBlock)
{
var frames = (int)Math.Min(framesPerBlock, totalFrames - start);
var block = store.AllocateExactArray(frames * channels);
blocks.Add(block);
// GetData fills the whole array; the offset is in frames (samples per channel).
if (!clip.GetData(block, (int)start))
throw new InvalidOperationException($"AudioClip.GetData failed for '{clip.name}' at frame {start} (load state {clip.loadState}).");
}
}
catch
{
foreach (var block in blocks) store.FreeExactArray(block);
throw;
}
return new MemoryPCMReader(blocks, framesPerBlock, channels, clip.frequency, totalFrames, store.FreeExactArray);
}
}
+4 -4
View File
@@ -4,14 +4,14 @@ namespace ADOFAIRenderer.Util.Audio.Filter;
internal class Gain : IAudioFilter internal class Gain : IAudioFilter
{ {
public double Level = 0; public float Level = 0f;
public void Initialize(int sampleRate, int channels) { } public void Initialize(int sampleRate, int channels) { }
public double ProcessSample(double sample, int channel) => public float ProcessSample(float sample, int channel) =>
sample * Math.Pow(10, Level / 20); sample * (float)Math.Pow(10, Level / 20);
public void ProcessBlock(Span<double> buffer) public void ProcessBlock(Span<float> buffer)
{ {
for (var i = 0; i < buffer.Length; i++) for (var i = 0; i < buffer.Length; i++)
buffer[i] = ProcessSample(buffer[i], 0); buffer[i] = ProcessSample(buffer[i], 0);
+3 -3
View File
@@ -8,19 +8,19 @@ internal class HardClip : IAudioFilter
// ReSharper disable once MemberCanBePrivate.Global // ReSharper disable once MemberCanBePrivate.Global
// ReSharper disable once FieldCanBeMadeReadOnly.Global // ReSharper disable once FieldCanBeMadeReadOnly.Global
// ReSharper disable once ConvertToConstant.Global // ReSharper disable once ConvertToConstant.Global
public double Threshold = 1.0; public float Threshold = 1.0f;
public void Initialize(int sampleRate, int channels) { } public void Initialize(int sampleRate, int channels) { }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public double ProcessSample(double sample, int channel) public float ProcessSample(float sample, int channel)
{ {
if (sample > Threshold) return Threshold; if (sample > Threshold) return Threshold;
if (sample < -Threshold) return -Threshold; if (sample < -Threshold) return -Threshold;
return sample; return sample;
} }
public void ProcessBlock(Span<double> buffer) public void ProcessBlock(Span<float> buffer)
{ {
for (var i = 0; i < buffer.Length; i++) for (var i = 0; i < buffer.Length; i++)
buffer[i] = ProcessSample(buffer[i], 0); buffer[i] = ProcessSample(buffer[i], 0);
+2 -2
View File
@@ -6,8 +6,8 @@ internal interface IAudioFilter
{ {
void Initialize(int sampleRate, int channels); void Initialize(int sampleRate, int channels);
double ProcessSample(double sample, int channel); float ProcessSample(float sample, int channel);
void ProcessBlock(Span<double> buffer); void ProcessBlock(Span<float> buffer);
void Reset(); void Reset();
+53 -53
View File
@@ -6,84 +6,84 @@ namespace ADOFAIRenderer.Util.Audio.Filter;
// ReSharper disable once InconsistentNaming // ReSharper disable once InconsistentNaming
internal class RMSCompressor : IAudioFilter internal class RMSCompressor : IAudioFilter
{ {
private double _thresholdDb = -20; private float _thresholdDb = -20f;
private double _ratio = 4; private float _ratio = 4f;
private double _attackTime = 0.010; // in seconds private float _attackTime = 0.010f; // in seconds
private double _releaseTime = 0.100; // in seconds private float _releaseTime = 0.100f; // in seconds
private double _kneeDb = 5; // in dB private float _kneeDb = 5f; // in dB
private double _rmsWindowTime = 0.030; // in seconds private float _rmsWindowTime = 0.030f; // in seconds
private double _lookAheadTime; // in seconds private float _lookAheadTime; // in seconds
private int _lookAheadSamples; private int _lookAheadSamples;
private int _sampleRate = 44100; private int _sampleRate = 44100;
private int _channels = 2; private int _channels = 2;
private double _attackCoef; private float _attackCoef;
private double _releaseCoef; private float _releaseCoef;
private double _rmsCoef; private float _rmsCoef;
// Per-channel state // Per-channel state
private double[] _rmsState = Array.Empty<double>(); // running mean-square (MS) state private float[] _rmsState = Array.Empty<float>(); // running mean-square (MS) state
private double[] _gainState = Array.Empty<double>(); // smoothed gain reduction in dB private float[] _gainState = Array.Empty<float>(); // smoothed gain reduction in dB
private double[][] _delayBuffers = Array.Empty<double[]>(); // circular delay buffers private float[][] _delayBuffers = Array.Empty<float[]>(); // circular delay buffers
private int[] _delayWriteIndices = Array.Empty<int>(); private int[] _delayWriteIndices = Array.Empty<int>();
public double LookAheadTime public float LookAheadTime
{ {
get => _lookAheadTime; get => _lookAheadTime;
set set
{ {
_lookAheadTime = Math.Max(0, Math.Min(1.0, value)); _lookAheadTime = Math.Max(0f, Math.Min(1.0f, value));
UpdateLookAheadSettings(); UpdateLookAheadSettings();
} }
} }
public double Latency => _lookAheadTime; public double Latency => _lookAheadTime;
public double ThresholdDb public float ThresholdDb
{ {
get => _thresholdDb; get => _thresholdDb;
set => _thresholdDb = value; set => _thresholdDb = value;
} }
public double Ratio public float Ratio
{ {
get => _ratio; get => _ratio;
set => _ratio = Math.Max(1, value); set => _ratio = Math.Max(1f, value);
} }
public double AttackTime public float AttackTime
{ {
get => _attackTime; get => _attackTime;
set set
{ {
_attackTime = Math.Max(0.0001, value); _attackTime = Math.Max(0.0001f, value);
UpdateAttackCoef(); UpdateAttackCoef();
} }
} }
public double ReleaseTime public float ReleaseTime
{ {
get => _releaseTime; get => _releaseTime;
set set
{ {
_releaseTime = Math.Max(0.0001, value); _releaseTime = Math.Max(0.0001f, value);
UpdateReleaseCoef(); UpdateReleaseCoef();
} }
} }
public double KneeDb public float KneeDb
{ {
get => _kneeDb; get => _kneeDb;
set => _kneeDb = Math.Max(0, value); set => _kneeDb = Math.Max(0f, value);
} }
public double RmsWindowTime public float RmsWindowTime
{ {
get => _rmsWindowTime; get => _rmsWindowTime;
set set
{ {
_rmsWindowTime = Math.Max(0.0001, value); _rmsWindowTime = Math.Max(0.0001f, value);
UpdateRmsCoef(); UpdateRmsCoef();
} }
} }
@@ -98,8 +98,8 @@ internal class RMSCompressor : IAudioFilter
UpdateRmsCoef(); UpdateRmsCoef();
UpdateLookAheadSettings(); UpdateLookAheadSettings();
_rmsState = new double[channels]; _rmsState = new float[channels];
_gainState = new double[channels]; _gainState = new float[channels];
Reset(); Reset();
} }
@@ -114,33 +114,33 @@ internal class RMSCompressor : IAudioFilter
{ {
if (_channels <= 0) return; if (_channels <= 0) return;
_delayBuffers = new double[_channels][]; _delayBuffers = new float[_channels][];
_delayWriteIndices = new int[_channels]; _delayWriteIndices = new int[_channels];
for (int i = 0; i < _channels; i++) for (int i = 0; i < _channels; i++)
{ {
_delayBuffers[i] = _lookAheadSamples > 0 ? new double[_lookAheadSamples] : Array.Empty<double>(); _delayBuffers[i] = _lookAheadSamples > 0 ? new float[_lookAheadSamples] : Array.Empty<float>();
_delayWriteIndices[i] = 0; _delayWriteIndices[i] = 0;
} }
} }
private void UpdateAttackCoef() private void UpdateAttackCoef()
{ {
_attackCoef = Math.Exp(-1.0 / (_sampleRate * _attackTime)); _attackCoef = (float)Math.Exp(-1.0 / (_sampleRate * _attackTime));
} }
private void UpdateReleaseCoef() private void UpdateReleaseCoef()
{ {
_releaseCoef = Math.Exp(-1.0 / (_sampleRate * _releaseTime)); _releaseCoef = (float)Math.Exp(-1.0 / (_sampleRate * _releaseTime));
} }
private void UpdateRmsCoef() private void UpdateRmsCoef()
{ {
_rmsCoef = Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); _rmsCoef = (float)Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime));
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public double ProcessSample(double sample, int channel) public float ProcessSample(float sample, int channel)
{ {
if (channel < 0 || channel >= _channels) if (channel < 0 || channel >= _channels)
{ {
@@ -149,54 +149,54 @@ internal class RMSCompressor : IAudioFilter
// Calculate running Mean-Square (MS) level // Calculate running Mean-Square (MS) level
// v_c[n] = alpha * v_c[n-1] + (1 - alpha) * x^2[n] // v_c[n] = alpha * v_c[n-1] + (1 - alpha) * x^2[n]
double sq = sample * sample; float sq = sample * sample;
_rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0 - _rmsCoef) * sq; _rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0f - _rmsCoef) * sq;
double rmsSq = Math.Max(0, _rmsState[channel]); float rmsSq = Math.Max(0f, _rmsState[channel]);
// Convert to dB. 10 * log10(rmsSq) is equivalent to 20 * log10(sqrt(rmsSq)) // Convert to dB. 10 * log10(rmsSq) is equivalent to 20 * log10(sqrt(rmsSq))
double rmsDb = 10.0 * Math.Log10(rmsSq + 1e-12); float rmsDb = 10.0f * (float)Math.Log10(rmsSq + 1e-12f);
// Calculate static envelope curve (gain reduction in dB) // Calculate static envelope curve (gain reduction in dB)
double gStatic = 0; float gStatic = 0f;
if (_kneeDb > 0) if (_kneeDb > 0f)
{ {
if (rmsDb > _thresholdDb + _kneeDb * 0.5) if (rmsDb > _thresholdDb + _kneeDb * 0.5f)
{ {
gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb);
} }
else if (rmsDb >= _thresholdDb - _kneeDb * 0.5) else if (rmsDb >= _thresholdDb - _kneeDb * 0.5f)
{ {
double diff = rmsDb - _thresholdDb + _kneeDb * 0.5; float diff = rmsDb - _thresholdDb + _kneeDb * 0.5f;
gStatic = (1 / _ratio - 1) * diff * diff / (2 * _kneeDb); gStatic = (1f / _ratio - 1f) * diff * diff / (2f * _kneeDb);
} }
} }
else else
{ {
if (rmsDb > _thresholdDb) if (rmsDb > _thresholdDb)
{ {
gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb);
} }
} }
// Apply ballistics (attack and release) on the dB gain reduction // Apply ballistics (attack and release) on the dB gain reduction
double lastGain = _gainState[channel]; float lastGain = _gainState[channel];
double targetGain = gStatic; float targetGain = gStatic;
// Note: Since targetGain and lastGain are <= 0 (gain reduction in dB), // Note: Since targetGain and lastGain are <= 0 (gain reduction in dB),
// targetGain < lastGain means target gain reduction is larger (more negative), // targetGain < lastGain means target gain reduction is larger (more negative),
// so we are in the attack phase. // so we are in the attack phase.
double coef = targetGain < lastGain ? _attackCoef : _releaseCoef; float coef = targetGain < lastGain ? _attackCoef : _releaseCoef;
double currentGain = coef * lastGain + (1 - coef) * targetGain; float currentGain = coef * lastGain + (1f - coef) * targetGain;
_gainState[channel] = currentGain; _gainState[channel] = currentGain;
// Convert back to linear gain multiplier // Convert back to linear gain multiplier
double linearGain = Math.Pow(10.0, currentGain / 20.0); float linearGain = (float)Math.Pow(10.0, currentGain / 20.0);
if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0) if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0)
{ {
var buffer = _delayBuffers[channel]; var buffer = _delayBuffers[channel];
int writeIdx = _delayWriteIndices[channel]; int writeIdx = _delayWriteIndices[channel];
double delayedSample = buffer[writeIdx]; float delayedSample = buffer[writeIdx];
buffer[writeIdx] = sample; buffer[writeIdx] = sample;
_delayWriteIndices[channel] = (writeIdx + 1) % _lookAheadSamples; _delayWriteIndices[channel] = (writeIdx + 1) % _lookAheadSamples;
return delayedSample * linearGain; return delayedSample * linearGain;
@@ -205,7 +205,7 @@ internal class RMSCompressor : IAudioFilter
return sample * linearGain; return sample * linearGain;
} }
public void ProcessBlock(Span<double> buffer) public void ProcessBlock(Span<float> buffer)
{ {
for (var i = 0; i < buffer.Length; i++) for (var i = 0; i < buffer.Length; i++)
{ {
-163
View File
@@ -1,163 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading;
using ADOFAIRenderer.Util.Audio.Clip;
namespace ADOFAIRenderer.Util.Audio.Mixer;
/// <summary>
/// One playing sound of a <see cref="VoiceMixer"/> (the song, or one effect sound), created by <see cref="VoiceMixer.Play"/>.
///
/// Main thread: the Set methods queue a change that takes effect at a given time (seconds relative to the render start,
/// the time at which the game made the change). Changes of one voice must be queued in non-decreasing time order.
/// Render thread: <see cref="VoiceMixer.Mix"/> applies each change at sample round(time * OutputRate).
///
/// Playback model (output samples n, clip positions in frames of the source at the output sample rate):
/// - The voice is silent before its start sample and from its end sample on.
/// - Clip position at sample n = anchor position + (n - anchor sample) * pitch. A position or pitch change sets a new anchor
/// at the sample where it takes effect, so a pitch change keeps the position continuous.
/// - Looping: the position wraps at the clip length (<see cref="AudioClipSource.Duration"/> * output rate, fractional).
/// Not looping: the voice ends when the position reaches <see cref="AudioClipSource.TotalFrames"/>.
/// </summary>
internal sealed class Voice
{
internal enum ChangeKind
{
Pitch,
Volume,
Loop,
Position,
EndTime,
Stop
}
internal readonly struct Change
{
public readonly ChangeKind Kind;
public readonly long Sample;
public readonly double Value;
public Change(ChangeKind kind, long sample, double value)
{
Kind = kind;
Sample = sample;
Value = value;
}
}
private readonly VoiceMixer _mixer;
// ── Render thread state ──
/// <summary>Changes not yet applied, in time order. Render thread only.</summary>
internal readonly Queue<Change> Pending = new();
/// <summary>First output sample of playback.</summary>
internal long StartSample;
/// <summary>Output sample at which playback stops (exclusive); long.MaxValue when not set.</summary>
internal long EndSample = long.MaxValue;
internal long AnchorSample;
internal double AnchorPosition;
internal double Pitch;
internal double Volume;
internal bool Loop;
internal bool Finished;
/// <summary>
/// Playback has ended and the voice was released by the render thread. Readable from any thread;
/// the value can lag behind the render thread.
/// </summary>
public bool IsFinished => Volatile.Read(ref Finished);
/// <summary>Loop length in frames at the output sample rate (source duration * output rate).</summary>
internal double Period;
internal AudioCursor? Cursor;
internal ChunkView View;
internal Voice(VoiceMixer mixer, AudioClipSource? source, string name)
{
_mixer = mixer;
Source = source;
Name = name;
}
/// <summary>Clip played by this voice; null plays silence (the clip could not be loaded).</summary>
public AudioClipSource? Source { get; }
/// <summary>Name for logs (the game's clip key).</summary>
public string Name { get; }
/// <summary>Sets the playback rate (1 = original speed). Negative values are treated as 0.</summary>
public void SetPitch(double pitch, double time) => _mixer.Enqueue(this, ChangeKind.Pitch, time, Math.Max(0.0, pitch));
/// <summary>Sets the linear gain applied to the clip samples.</summary>
public void SetVolume(double volume, double time) => _mixer.Enqueue(this, ChangeKind.Volume, time, volume);
/// <summary>Turns looping on or off. Turning it off lets the current repetition play to the end of the clip.</summary>
public void SetLoop(bool loop, double time) => _mixer.Enqueue(this, ChangeKind.Loop, time, loop ? 1.0 : 0.0);
/// <summary>
/// Moves playback to <paramref name="clipSeconds"/> seconds into the clip.
/// Before the voice starts, this sets the position playback starts from.
/// </summary>
public void SetPosition(double clipSeconds, double time) => _mixer.Enqueue(this, ChangeKind.Position, time, Math.Max(0.0, clipSeconds));
/// <summary>Stops playback at <paramref name="endTime"/> (seconds relative to the render start).</summary>
public void SetEndTime(double endTime, double time) => _mixer.Enqueue(this, ChangeKind.EndTime, time, endTime);
/// <summary>Stops playback at <paramref name="time"/>.</summary>
public void Stop(double time) => _mixer.Enqueue(this, ChangeKind.Stop, time, 0.0);
/// <summary>Unwrapped clip position at output sample <paramref name="sample"/> (sample at or after the anchor).</summary>
internal double PositionAt(long sample) => AnchorPosition + (sample - AnchorSample) * Pitch;
/// <summary>Clip position at <paramref name="sample"/>, wrapped into the loop when looping.</summary>
internal double WrappedPositionAt(long sample)
{
var position = PositionAt(sample);
if (Loop && Period > 0 && position >= Period) position %= Period;
return position;
}
/// <summary>
/// Applies <paramref name="change"/> at output sample <paramref name="sample"/> (its own sample, or a later one when the change
/// arrived after its sample was mixed). Before the start sample, changes replace the starting state. Render thread only.
/// </summary>
internal void Apply(in Change change, long sample, int outputRate)
{
var at = Math.Max(sample, StartSample);
switch (change.Kind)
{
case ChangeKind.Pitch:
Reanchor(at);
Pitch = change.Value;
break;
case ChangeKind.Volume:
Volume = change.Value;
break;
case ChangeKind.Loop:
Reanchor(at);
Loop = change.Value != 0.0;
break;
case ChangeKind.Position:
AnchorSample = at;
AnchorPosition = change.Value * outputRate;
break;
case ChangeKind.EndTime:
EndSample = VoiceMixer.ToSample(change.Value, outputRate);
break;
case ChangeKind.Stop:
EndSample = Math.Min(EndSample, sample);
break;
}
}
private void Reanchor(long sample)
{
AnchorPosition = WrappedPositionAt(sample);
AnchorSample = sample;
}
}
-344
View File
@@ -1,344 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading;
using ADOFAIRenderer.Util.Audio.Clip;
namespace ADOFAIRenderer.Util.Audio.Mixer;
/// <summary>
/// Mixes every <see cref="Voice"/> into interleaved double samples at the output sample rate and channel count.
///
/// Time: output sample n is at n / OutputRate seconds relative to the render start (the same time base as the times passed
/// to <see cref="Play"/> and the Voice Set methods). A time t maps to sample round(t * OutputRate).
///
/// Threads:
/// - Main thread: <see cref="Play"/> and the Voice Set methods queue commands under a lock. Commands must be queued in
/// non-decreasing time order per voice.
/// - Render thread: <see cref="Mix"/> takes the queued commands and mixes a block of samples. A command takes effect at its sample;
/// a command whose sample is before the block being mixed takes effect at the start of that block.
/// Chunks are read through <see cref="AudioClipStore.Acquire"/>, which waits for the decoder thread.
///
/// Sample reading: clip positions are in frames of the source at the output rate (sources are resampled when loaded).
/// An integer position copies the stored sample; a fractional position interpolates linearly between the frame and the next one
/// (the next frame of a looping voice after the last frame is frame 0; after the end of a non-looping clip it is 0).
/// Channel mapping follows <see cref="ChunkView.GetSample"/>.
/// </summary>
internal sealed class VoiceMixer : IDisposable
{
private enum CommandKind
{
Play,
Change
}
private readonly struct Command
{
public readonly CommandKind Kind;
public readonly Voice Voice;
public readonly Voice.Change Change;
public Command(CommandKind kind, Voice voice, Voice.Change change)
{
Kind = kind;
Voice = voice;
Change = change;
}
}
private readonly AudioClipStore _store;
private readonly CancellationToken _cancellationToken;
private readonly object _commandLock = new();
private List<Command> _commands = new();
private List<Command> _takenCommands = new();
// Render thread only.
private readonly List<Voice> _voices = new();
private bool _disposed;
/// <param name="store">Store the voices' sources belong to; its OutputRate is the mixer's sample rate.</param>
/// <param name="outputChannels">Interleaved channel count of the mixed output.</param>
/// <param name="cancellationToken">Cancels waits for chunks in <see cref="Mix"/>.</param>
public VoiceMixer(AudioClipStore store, int outputChannels, CancellationToken cancellationToken = default)
{
if (outputChannels < 1) throw new ArgumentOutOfRangeException(nameof(outputChannels));
_store = store;
OutputRate = store.OutputRate;
OutputChannels = outputChannels;
_cancellationToken = cancellationToken;
}
public int OutputRate { get; }
public int OutputChannels { get; }
/// <summary>Voices currently mixed (started or waiting to start). Render thread only.</summary>
public int ActiveVoiceCount => _voices.Count;
/// <summary>
/// Output sample containing <paramref name="seconds"/> (seconds × rate rounded down), the sample at which
/// AudioSource.PlayScheduled starts a sound scheduled between two output samples.
/// </summary>
public static long ToSample(double seconds, int rate) => (long)Math.Floor(seconds * rate);
// ── Main thread ──
/// <summary>
/// Schedules <paramref name="source"/> to start at <paramref name="startTime"/>.
/// A start time before <paramref name="time"/> (the current time) starts the voice at <paramref name="time"/>.
/// </summary>
/// <param name="source">Clip to play; null plays silence.</param>
/// <param name="name">Name for logs.</param>
/// <param name="startTime">Start of playback, seconds relative to the render start.</param>
/// <param name="volume">Linear gain.</param>
/// <param name="pitch">Playback rate (1 = original speed); negative values are treated as 0.</param>
/// <param name="loop">Repeats the clip until the voice is stopped.</param>
/// <param name="time">Current time, seconds relative to the render start.</param>
public Voice Play(AudioClipSource? source, string name, double startTime, double volume, double pitch, bool loop, double time)
{
var voice = new Voice(this, source, name)
{
StartSample = Math.Max(ToSample(startTime, OutputRate), ToSample(time, OutputRate)),
Volume = volume,
Pitch = Math.Max(0.0, pitch),
Loop = loop,
Period = source != null ? source.Duration * OutputRate : 0.0
};
voice.AnchorSample = voice.StartSample;
lock (_commandLock)
{
_commands.Add(new Command(CommandKind.Play, voice, default));
}
return voice;
}
internal void Enqueue(Voice voice, Voice.ChangeKind kind, double time, double value)
{
var change = new Voice.Change(kind, ToSample(time, OutputRate), value);
lock (_commandLock)
{
_commands.Add(new Command(CommandKind.Change, voice, change));
}
}
// ── Render thread ──
/// <summary>
/// Adds the mix of output samples [<paramref name="firstSample"/>, firstSample + frames) to <paramref name="destination"/>
/// (interleaved, frames = destination.Length / OutputChannels). Blocks are expected in increasing sample order.
/// </summary>
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (see <see cref="AudioClipStore.Acquire"/>).</exception>
/// <exception cref="OperationCanceledException">The cancellation token was canceled while waiting for a chunk.</exception>
public void Mix(long firstSample, Span<double> destination)
{
if (_disposed) throw new ObjectDisposedException(nameof(VoiceMixer));
var frames = destination.Length / OutputChannels;
var endSample = firstSample + frames;
TakeCommands();
for (var i = _voices.Count - 1; i >= 0; i--)
{
var voice = _voices[i];
var sample = firstSample;
while (!voice.Finished)
{
var segmentEnd = endSample;
if (voice.Pending.Count > 0)
{
var next = voice.Pending.Peek();
if (next.Sample <= sample)
{
voice.Apply(voice.Pending.Dequeue(), sample, OutputRate);
continue;
}
segmentEnd = Math.Min(segmentEnd, next.Sample);
}
if (segmentEnd > sample)
MixSegment(voice, sample, segmentEnd, destination.Slice((int)(sample - firstSample) * OutputChannels));
sample = segmentEnd;
if (sample >= endSample) break;
}
if (!voice.Finished && endSample >= voice.EndSample)
voice.Finished = true;
if (voice.Finished)
{
ReleaseVoice(voice);
_voices.RemoveAt(i);
}
}
}
private void TakeCommands()
{
lock (_commandLock)
{
(_commands, _takenCommands) = (_takenCommands, _commands);
}
foreach (var command in _takenCommands)
{
var voice = command.Voice;
if (command.Kind == CommandKind.Play)
{
if (voice.Source is { IsFailed: false } source)
voice.Cursor = source.AddCursor(0, voice.Loop);
_voices.Add(voice);
}
else if (!voice.Finished)
{
voice.Pending.Enqueue(command.Change);
}
}
_takenCommands.Clear();
}
/// <summary>Mixes <paramref name="voice"/> over samples [from, to) with its current state into <paramref name="destination"/> (starting at sample from).</summary>
private void MixSegment(Voice voice, long from, long to, Span<double> destination)
{
var start = Math.Max(from, voice.StartSample);
var end = Math.Min(to, voice.EndSample);
if (start >= end) return;
var source = voice.Source;
if (source == null || source.IsFailed)
{
voice.Finished = true;
return;
}
var totalFrames = source.TotalFrames;
if (voice.Volume == 0.0)
{
// Nothing to add; a non-looping voice still ends at the end of the clip.
if (!voice.Loop && voice.PositionAt(end - 1) >= totalFrames) voice.Finished = true;
return;
}
var channels = OutputChannels;
var volume = voice.Volume;
var offset = (int)(start - from) * channels;
UpdateCursor(voice, (long)voice.WrappedPositionAt(start));
for (var n = start; n < end; n++, offset += channels)
{
var position = voice.WrappedPositionAt(n);
if (position >= totalFrames)
{
if (!voice.Loop)
{
voice.Finished = true;
return;
}
position = totalFrames - 1;
}
var index = (long)position;
var fraction = position - index;
if (!EnsureView(voice, index)) return;
var view = voice.View;
if (fraction == 0.0)
{
for (var c = 0; c < channels; c++)
destination[offset + c] += view.GetSample(index, c, channels) * volume;
continue;
}
var nextIndex = index + 1;
var nextInView = view.Contains(nextIndex);
for (var c = 0; c < channels; c++)
{
double low = view.GetSample(index, c, channels);
double high = nextInView ? view.GetSample(nextIndex, c, channels) : ReadOutside(voice, nextIndex, c);
destination[offset + c] += (low + fraction * (high - low)) * volume;
}
}
}
/// <summary>Sample of frame <paramref name="frame"/> outside the voice's current view (next chunk, loop start, or past the end).</summary>
private double ReadOutside(Voice voice, long frame, int channel)
{
var source = voice.Source!;
if (frame >= source.TotalFrames)
{
if (!voice.Loop) return 0.0;
frame = 0;
}
var view = _store.Acquire(source, frame, _cancellationToken);
if (!view.IsValid) return 0.0;
try
{
return view.GetSample(frame, channel, OutputChannels);
}
finally
{
source.Release(view);
}
}
/// <summary>Makes voice.View the chunk containing <paramref name="frame"/>. False when the source cannot provide it (voice finished).</summary>
private bool EnsureView(Voice voice, long frame)
{
if (voice.View.IsValid && voice.View.Contains(frame)) return true;
var source = voice.Source!;
if (voice.View.IsValid)
{
source.Release(voice.View);
voice.View = default;
}
UpdateCursor(voice, frame);
voice.View = _store.Acquire(source, frame, _cancellationToken);
if (voice.View.IsValid) return true;
voice.Finished = true;
return false;
}
private static void UpdateCursor(Voice voice, long frame)
{
var cursor = voice.Cursor;
if (cursor == null) return;
if (cursor.Loop != voice.Loop) cursor.Loop = voice.Loop;
if (cursor.Position != frame) cursor.Position = frame;
}
private static void ReleaseVoice(Voice voice)
{
var source = voice.Source;
if (source == null) return;
if (voice.View.IsValid)
{
source.Release(voice.View);
voice.View = default;
}
if (voice.Cursor != null)
{
source.RemoveCursor(voice.Cursor);
voice.Cursor = null;
}
voice.Pending.Clear();
}
/// <summary>Releases every voice's chunk and cursor. Render thread (or after the render thread stopped).</summary>
public void Dispose()
{
if (_disposed) return;
_disposed = true;
foreach (var voice in _voices)
ReleaseVoice(voice);
_voices.Clear();
lock (_commandLock)
{
_commands.Clear();
}
}
}
+361 -303
View File
@@ -1,358 +1,416 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Threading;
using ADOFAIRenderer.Util.Audio.Clip;
using ADOFAIRenderer.Util.Audio.Filter; using ADOFAIRenderer.Util.Audio.Filter;
using ADOFAIRenderer.Util.Audio.Mixer;
using MelonLoader; using MelonLoader;
using UnityEngine; using UnityEngine;
namespace ADOFAIRenderer.Util.Audio; namespace ADOFAIRenderer.Util.Audio;
/// <summary> internal class AudioClipCache
/// Audio of one render: the song and effect sounds as <see cref="Voice"/>s of a <see cref="VoiceMixer"/>, clips from an
/// <see cref="AudioClipStore"/>, followed by the filter chain (RMS compressor → look-ahead limiter → hard clip → -1 dB gain).
///
/// Time: seconds relative to the render start (RenderManager.dspTimeBase); output sample n is at n / sample rate.
/// The limiter delays its output by its look-ahead (<see cref="IAudioFilter.Latency"/>), so output sample n is mixed from
/// voices at sample n + latency, and the filters are primed with samples [0, latency) before the first block.
///
/// Threads:
/// - Main thread: <see cref="InitializeBGM"/>, <see cref="CacheClip"/>, <see cref="RegisterSoundEvent"/>, <see cref="UpdateBGM"/>,
/// source/voice mapping, and <see cref="Commit"/> after each frame (all commands with an earlier time have been queued).
/// - Render thread: <see cref="GenerateAudioForFrame"/> waits until the main thread has committed the end of the samples it mixes,
/// so every command takes effect at its own sample. <see cref="Complete"/> ends the waiting when the main thread stops.
/// </summary>
internal sealed class OfflineAudioMixer : IDisposable
{ {
/// <summary>Gain applied to every clip.</summary> private readonly Dictionary<string, CachedClipData> _cache = new();
private const double ClipVolume = 0.5;
/// <summary>Additional gain of effect sounds (registered with <see cref="RegisterSoundEvent"/>).</summary> public class CachedClipData
private const double EffectVolume = 2.0 / 3.0; {
public float[] Samples = null!; // Pre-resampled to target channels and sample rate
public double Duration;
}
/// <summary> public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
/// Smallest number of tracked effect voices at which finished ones are removed from the list used by <see cref="StopAllEffects"/>. {
/// After each removal the next threshold is twice the remaining count (at least this value). if (string.IsNullOrEmpty(clipName)) return null;
/// </summary>
private const int EffectVoicePruneCount = 1024;
/// <summary>Longest wait for a commit before checking for cancellation again, in milliseconds.</summary> lock (_cache)
private const int CommitWaitMs = 50; {
if (_cache.TryGetValue(clipName, out var cached))
return cached;
private readonly int _channels; if (!allowLoad) return null;
private readonly int _sampleRate;
private readonly AudioClipStore _store;
private readonly VoiceMixer _mixer;
private readonly CancellationTokenSource _cancellation = new();
private readonly List<IAudioFilter> _filters = new();
private readonly int _latencySamples;
// Main thread. if (!AudioManager.Instance) return null;
private ConditionalWeakTable<AudioSource, SourceState> _sourceStates = new();
private readonly List<Voice> _effectVoices = new();
private int _effectVoicePruneAt = EffectVoicePruneCount;
private AudioClipSource? _bgmSource;
private string _bgmKey = "";
private Voice? _bgmVoice;
private double _bgmStart = double.NaN;
private double _bgmPitch = double.NaN;
private double _bgmVolume = double.NaN;
// Shared: committed output sample (voices are final before it), guarded by _commitLock. var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
private readonly object _commitLock = new(); if (!clip) return null;
private long _committedSample = long.MinValue;
// Render thread (and Dispose, under _renderLock). try
private readonly object _renderLock = new(); {
private long _nextOutputSample; var totalSamples = clip.samples;
private bool _primed; var channels = clip.channels;
private double[] _mixBuffer = Array.Empty<double>(); var rawSamples = new float[totalSamples * channels];
private bool _disposed; clip.GetData(rawSamples, 0);
// Auto-resample during cache generation
var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
var data = new CachedClipData
{
Samples = resampled,
Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
};
_cache[clipName] = data;
return data;
}
catch (Exception e)
{
MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}");
return null;
}
}
}
public void Clear()
{
lock (_cache)
{
_cache.Clear();
}
}
}
internal class OfflineAudioMixer
{
public class SoundEvent
{
public string ClipName = null!;
public double StartTime; // In seconds relative to dspTimeBase
public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot)
public float Volume;
public float Pitch;
}
private readonly List<SoundEvent> _activeEvents = new();
private ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
private readonly object _lock = new();
// Target Audio Settings
private readonly int _targetChannels;
private readonly int _targetSampleRate;
private const float ClipVolume = .5f;
// Audio Filter Settings
private readonly LinkedList<IAudioFilter> _filters = new();
// BGM Cached Samples (Pre-resampled)
private float[]? _bgmSamples;
private double _bgmDuration;
// Clip Cache
private readonly AudioClipCache _clipCache = new();
private bool _isPrimed;
public OfflineAudioMixer(int targetChannels, int targetSampleRate) public OfflineAudioMixer(int targetChannels, int targetSampleRate)
{ {
_channels = targetChannels; _targetChannels = targetChannels;
_sampleRate = targetSampleRate; _targetSampleRate = targetSampleRate;
_store = new AudioClipStore(targetSampleRate);
_mixer = new VoiceMixer(_store, targetChannels, _cancellation.Token);
var rmsCompressor = new RMSCompressor var rmsCompressor = new RMSCompressor
{ {
ThresholdDb = -10, Ratio = 2, ThresholdDb = -10, Ratio = 2,
AttackTime = 0.1, ReleaseTime = 0.1, AttackTime = 0.1f, ReleaseTime = 0.1f,
KneeDb = 10, RmsWindowTime = 0.02 KneeDb = 10, RmsWindowTime = 0.02f
}; };
rmsCompressor.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(rmsCompressor);
var limiter = new RMSCompressor var limiter = new RMSCompressor
{ {
ThresholdDb = 0, Ratio = double.PositiveInfinity, ThresholdDb = 0, Ratio = float.PositiveInfinity,
AttackTime = 0, ReleaseTime = 0.05, AttackTime = 0f, ReleaseTime = 0.05f,
KneeDb = 0, RmsWindowTime = 0.02, KneeDb = 0, RmsWindowTime = 0.02f,
LookAheadTime = 0.01 LookAheadTime = 0.01f
}; };
_filters.Add(rmsCompressor); limiter.Initialize(_targetSampleRate, _targetChannels);
_filters.Add(limiter); _filters.AddLast(limiter);
_filters.Add(new HardClip());
_filters.Add(new Gain { Level = -1 }); var hardClip = new HardClip();
hardClip.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(hardClip);
double latency = 0; var truePeakSafetyGain = new Gain { Level = -1f };
foreach (var filter in _filters) truePeakSafetyGain.Initialize(_targetSampleRate, _targetChannels);
{ _filters.AddLast(truePeakSafetyGain);
filter.Initialize(_sampleRate, _channels);
latency += filter.Latency;
}
_latencySamples = (int)Math.Round(latency * _sampleRate);
_store.Start();
} }
/// <summary>Clip store of this render (output rate = mixer sample rate), also read by <see cref="AudioSpectrumGenerator"/>.</summary>
internal AudioClipStore Store => _store;
// ── Main thread ──
/// <summary>Registers the song clip. The song voice is created and kept in sync by <see cref="UpdateBGM"/>.</summary>
public void InitializeBGM(AudioClip? clip) public void InitializeBGM(AudioClip? clip)
{ {
if (!clip) lock (_lock)
{ {
_bgmSource = null; if (!clip)
return; {
} _bgmSamples = null;
_bgmDuration = 0;
return;
}
_bgmKey = clip!.name; try
_bgmSource = AudioClipResolver.GetOrAdd(_store, _bgmKey, clip); {
if (_bgmSource == null) int originalSamples = clip.samples;
{ int originalChannels = clip.channels;
Melon<RenderMod>.Logger.Error($"[OfflineAudioMixer] Song clip '{_bgmKey}' could not be loaded, the song will be silent."); float[] rawSamples = new float[originalSamples * originalChannels];
return; clip.GetData(rawSamples, 0);
// Pre-resample the BGM to the target format
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate);
_bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate);
Melon<RenderMod>.Logger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}");
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}");
_bgmSamples = null;
_bgmDuration = 0;
}
} }
Melon<RenderMod>.Logger.Msg($"[OfflineAudioMixer] Song '{_bgmKey}': {_bgmSource.Channels} ch, {_bgmSource.Duration:F2} s, {_bgmSource.Policy}");
} }
/// <summary> public void CacheClip(string clipName)
/// Updates the song voice from the conductor (start time, pitch) and the level volume.
/// A changed start time restarts the song voice at position (now - start) * pitch; pitch and volume changes take effect at <paramref name="now"/>.
/// Start time: dspTimeSong + separate countdown (crotchetAtStart / pitch * adjustedCountdownTicks when separateCountdownTime),
/// the time scrConductor.StartMusicCo schedules the song at.
/// </summary>
/// <param name="now">Current time, seconds relative to the render start.</param>
/// <param name="dspTimeBase">AudioSettings.dspTime at the render start.</param>
public void UpdateBGM(double now, double dspTimeBase)
{ {
if (_bgmSource == null) return; _clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true);
}
public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
{
lock (_lock)
{
var ev = new SoundEvent
{
ClipName = clipName,
StartTime = startTime,
EndTime = endTime,
Volume = volume,
Pitch = pitch
};
_activeEvents.Add(ev);
if (!source) return;
_sourceMappings.Remove(source);
_sourceMappings.Add(source, ev);
}
}
public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
{
RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
}
public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
{
if (source == null)
{
ev = null!;
return false;
}
lock (_lock)
{
return _sourceMappings.TryGetValue(source, out ev);
}
}
public bool RemoveSourceMapping(AudioSource source)
{
lock (_lock)
{
return _sourceMappings.Remove(source);
}
}
public void Reset()
{
lock (_lock)
{
_activeEvents.Clear();
_sourceMappings = new ConditionalWeakTable<AudioSource, SoundEvent>();
_bgmSamples = null;
_bgmDuration = 0;
_clipCache.Clear();
_isPrimed = false;
foreach (var filter in _filters)
{
filter.Reset();
}
}
}
private void PrimeFilters(double totalLatency, double dspTimeBase)
{
int latencySamples = (int)Math.Round(totalLatency * _targetSampleRate);
if (latencySamples <= 0) return;
// Rent temporary buffer to prime filters with initial audio
float[] tempBuffer = System.Buffers.ArrayPool<float>.Shared.Rent(latencySamples * _targetChannels);
try
{
Array.Clear(tempBuffer, 0, latencySamples * _targetChannels);
// Mix starting from -totalLatency to 0
MixBGM(tempBuffer, latencySamples, -totalLatency, dspTimeBase, totalLatency);
MixEffectSounds(tempBuffer, latencySamples, -totalLatency, totalLatency);
// Feed the buffer into filters to prime circular delay buffers
foreach (var filter in _filters)
{
filter.ProcessBlock(tempBuffer.AsSpan(0, latencySamples * _targetChannels));
}
}
finally
{
System.Buffers.ArrayPool<float>.Shared.Return(tempBuffer);
}
}
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
// mixedBuffer size must be at least samplesNeeded * _targetChannels
Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
lock (_lock)
{
double totalLatency = 0;
foreach (var filter in _filters)
{
totalLatency += filter.Latency;
}
// Prime filters on first frame execution if latency exists
if (!_isPrimed && totalLatency > 0)
{
PrimeFilters(totalLatency, dspTimeBase);
_isPrimed = true;
}
// 1. Mix BGM
MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase, totalLatency);
// 2. Mix Hitsounds/Effect Sounds
MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime, totalLatency);
foreach (var filter in _filters)
filter.ProcessBlock(mixedBuffer.AsSpan(0, samplesNeeded * _targetChannels));
for (var i = 0; i < samplesNeeded * _targetChannels; i++)
{
if (Math.Abs(mixedBuffer[i]) > 1) Melon<RenderMod>.Logger.Msg("Warning Why Audio Sample Is Larger Than 1");
}
}
}
private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase, double totalLatency)
{
if (_bgmSamples == null || _bgmSamples.Length == 0) return;
var conductor = scrConductor.instance; var conductor = scrConductor.instance;
if (!conductor) return; if (!conductor) return;
double pitch = conductor.song.pitch; double pitch = conductor.song.pitch;
var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0; var dspTimeSong = conductor.dspTimeSong;
var start = conductor.dspTimeSong + countdown - dspTimeBase; var separateCountdownOffset = conductor.separateCountdownTime
var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0; ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks
var volume = levelVolume * ClipVolume; : 0.0;
if (_bgmVoice == null || start != _bgmStart) // BGM Playback start time relative to dspTimeBase
var time1 = dspTimeSong + separateCountdownOffset;
var relativeTime1 = time1 - dspTimeBase;
var adjustedRelativeTime1 = relativeTime1 - totalLatency;
var totalSamples = _bgmSamples.Length / _targetChannels;
for (int s = 0; s < samplesNeeded; s++)
{ {
// Song position at time t is (t - start) * pitch, also when the start is already past. double sampleTime = frameStartTime + (double)s / _targetSampleRate;
_bgmVoice?.Stop(now); double clipTime = (sampleTime - adjustedRelativeTime1) * pitch;
_bgmVoice = _mixer.Play(_bgmSource, _bgmKey, Math.Max(start, now), volume, pitch, false, now);
if (start < now) _bgmVoice.SetPosition((now - start) * pitch, now);
_bgmStart = start;
_bgmPitch = pitch;
_bgmVolume = volume;
return;
}
if (pitch != _bgmPitch) if (clipTime >= 0 && clipTime < _bgmDuration)
{
_bgmVoice.SetPitch(pitch, now);
_bgmPitch = pitch;
}
if (volume != _bgmVolume)
{
_bgmVoice.SetVolume(volume, now);
_bgmVolume = volume;
}
}
/// <summary>Prepares the source of the game clip <paramref name="clipName"/> (audioLib key) so its first chunk starts loading.</summary>
public void CacheClip(string clipName)
{
GetOrAddSource(clipName);
}
private AudioClipSource? GetOrAddSource(string clipName)
{
if (string.IsNullOrEmpty(clipName)) return null;
if (_store.TryGet(clipName, out var existing)) return existing;
var clip = AudioManager.Instance ? AudioManager.Instance.FindOrLoadAudioClip(clipName) : null;
return AudioClipResolver.GetOrAdd(_store, clipName, clip);
}
/// <summary>
/// Starts an effect sound at <paramref name="startTime"/>. An <paramref name="endTime"/> of 0 or more makes it loop until then.
/// <paramref name="source"/> (when given) is mapped to the returned state, which reports and applies later changes through the AudioSource.
/// </summary>
public SourceState RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
{
var clipSource = GetOrAddSource(clipName);
var loop = endTime >= 0;
var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now);
if (loop) voice.SetEndTime(endTime, now);
if (_effectVoices.Count >= _effectVoicePruneAt)
{
_effectVoices.RemoveAll(v => v.IsFinished);
_effectVoicePruneAt = Math.Max(EffectVoicePruneCount, _effectVoices.Count * 2);
}
_effectVoices.Add(voice);
var state = new SourceState(voice, Math.Max(startTime, now), pitch, loop, loop ? endTime : double.PositiveInfinity,
clipSource?.Duration ?? 0.0, _sampleRate);
if (source) MapSource(source!, state);
return state;
}
/// <summary>
/// Maps <paramref name="source"/> to <paramref name="state"/> (replaces an earlier mapping) and takes the clip length and
/// sample rate from the clip of the source.
/// </summary>
public void MapSource(AudioSource source, SourceState state)
{
state.BindClip(source.clip);
_sourceStates.Remove(source);
_sourceStates.Add(source, state);
}
/// <summary>State of the effect sound last started for <paramref name="source"/>. The mapping stays after the sound ends or stops.</summary>
public bool TryGetSourceState(AudioSource? source, out SourceState state)
{
state = null!;
return source != null && _sourceStates.TryGetValue(source, out state);
}
/// <summary>
/// Stops every effect sound at <paramref name="now"/>, including ones scheduled to start later
/// (AudioManager.StopAllSounds destroys every AudioSource in liveSources, which holds the sources of
/// AudioManager._Play and scrConductor.PlayWithEndTime). The song voice keeps playing.
/// </summary>
public void StopAllEffects(double now)
{
foreach (var voice in _effectVoices)
voice.Stop(now);
_effectVoices.Clear();
_sourceStates = new ConditionalWeakTable<AudioSource, SourceState>();
}
/// <summary>
/// Marks every command with a time before <paramref name="time"/> as queued
/// (called after each frame with the time of the next frame).
/// </summary>
public void Commit(double time)
{
var sample = VoiceMixer.ToSample(time, _sampleRate);
lock (_commitLock)
{
if (sample <= _committedSample) return;
_committedSample = sample;
Monitor.PulseAll(_commitLock);
}
}
/// <summary>No more commands will be queued (render ending): mixing no longer waits for commits.</summary>
public void Complete()
{
lock (_commitLock)
{
_committedSample = long.MaxValue;
Monitor.PulseAll(_commitLock);
}
}
/// <summary>Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException.</summary>
public void Cancel()
{
_cancellation.Cancel();
Complete();
}
// ── Render thread ──
/// <summary>
/// Writes the next <paramref name="samplesNeeded"/> output frames (interleaved float, continuing from the previous call).
/// </summary>
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (out of memory or the decoder thread stopped).</exception>
/// <exception cref="OperationCanceledException">The render was canceled (<see cref="Cancel"/>).</exception>
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded)
{
lock (_renderLock)
{
if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer));
if (!_primed)
{ {
_primed = true; double sampleIdxDouble = clipTime * _targetSampleRate;
if (_latencySamples > 0) long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (var c = 0; c < _targetChannels; c++)
{ {
var prime = MixBlock(0, _latencySamples); float valLow = _bgmSamples[idxLow * _targetChannels + c];
foreach (var filter in _filters) float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
filter.ProcessBlock(prime); mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * (scnGame.instance.levelData.volume * 0.01f)
* ClipVolume;
}
}
}
}
private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double totalLatency)
{
for (var i = _activeEvents.Count - 1; i >= 0; i--)
{
var ev = _activeEvents[i];
var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
if (clipData == null)
{
_activeEvents.RemoveAt(i);
continue;
}
var isSoundFinished = false;
var totalSamples = clipData.Samples.Length / _targetChannels;
double adjustedStartTime = ev.StartTime - totalLatency;
double adjustedEndTime = ev.EndTime >= 0 ? ev.EndTime - totalLatency : -1.0;
for (var s = 0; s < samplesNeeded; s++)
{
var sampleTime = frameStartTime + (double)s / _targetSampleRate;
// If it is a looping sound and has reached or exceeded its end time, stop playing.
if (adjustedEndTime >= 0 && sampleTime >= adjustedEndTime)
{
isSoundFinished = true;
break;
}
var elapsed = sampleTime - adjustedStartTime;
if (elapsed < 0) continue; // Sound hasn't started yet
var soundTime = elapsed * ev.Pitch;
if (soundTime >= clipData.Duration)
{
if (ev.EndTime < 0) // One-shot
{
isSoundFinished = true;
break;
}
// Looping
soundTime %= clipData.Duration;
}
// Linear interpolation for pitch shifting in the mixer loop
double sampleIdxDouble = soundTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (var c = 0; c < _targetChannels; c++)
{
float valLow = clipData.Samples[idxLow * _targetChannels + c];
float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume
* ClipVolume * 2 / 3;
} }
} }
var block = MixBlock(_nextOutputSample + _latencySamples, samplesNeeded); // Clean up finished sound events (both one-shot and looping)
foreach (var filter in _filters) if (isSoundFinished)
filter.ProcessBlock(block);
_nextOutputSample += samplesNeeded;
for (var i = 0; i < block.Length; i++)
mixedBuffer[i] = (float)block[i];
}
}
/// <summary>Mixes voices at samples [first, first + frames) into the reused double buffer, after the main thread committed them.</summary>
private Span<double> MixBlock(long first, int frames)
{
WaitForCommit(first + frames);
var length = frames * _channels;
if (_mixBuffer.Length < length) _mixBuffer = new double[length];
var block = _mixBuffer.AsSpan(0, length);
block.Clear();
_mixer.Mix(first, block);
return block;
}
private void WaitForCommit(long endSample)
{
lock (_commitLock)
{
while (_committedSample < endSample)
{ {
_cancellation.Token.ThrowIfCancellationRequested(); _activeEvents.RemoveAt(i);
Monitor.Wait(_commitLock, CommitWaitMs);
} }
} }
_cancellation.Token.ThrowIfCancellationRequested();
} }
}
/// <summary>
/// Cancels waits, then releases the voices, the clip store (decoder thread) and the filter state.
/// Waits for a running <see cref="GenerateAudioForFrame"/> to return.
/// </summary>
public void Dispose()
{
Cancel();
lock (_renderLock)
{
if (_disposed) return;
_disposed = true;
_mixer.Dispose();
_store.Dispose();
foreach (var filter in _filters)
filter.Reset();
}
_sourceStates = new ConditionalWeakTable<AudioSource, SourceState>();
_effectVoices.Clear();
}
}
-140
View File
@@ -1,140 +0,0 @@
using System;
using ADOFAIRenderer.Util.Audio.Mixer;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio;
/// <summary>
/// Playback state of a game AudioSource mapped to an effect <see cref="Voice"/>, as the AudioSource reports it to the game
/// (isPlaying, time, timeSamples, pitch). Main thread only. Times are seconds relative to the render start (the render clock that
/// also drives AudioSettings.dspTime during rendering); clip positions are seconds into the clip.
///
/// Reported values:
/// - Playing: from the scheduled start until Stop(), the scheduled end time, or the natural end of the clip (not looping).
/// Before the start, isPlaying is true and time is the position playback will start from.
/// - time while playing: start position + elapsed time * pitch since the last change, wrapped at the clip length when looping.
/// time is 0 while not playing.
/// - timeSamples while playing: floor(time * clip frequency). While not playing: the last value set through time or timeSamples,
/// reset to 0 by Stop().
/// - pitch: the last finite value set (NaN and infinity are ignored). Playback uses max(0, pitch).
/// Changes made while not playing are stored and do not reach the voice.
/// </summary>
internal sealed class SourceState
{
private readonly double _start;
private double _anchorTime;
private double _anchorPosition;
private double _pitch;
private readonly bool _loop;
private double _endTime = double.PositiveInfinity;
private bool _stopped;
private int _offsetSamples;
private double _length;
private int _frequency;
/// <param name="voice">Voice that plays the sound.</param>
/// <param name="start">Start of playback (seconds relative to the render start, not before the time it was scheduled).</param>
/// <param name="pitch">Pitch at the start.</param>
/// <param name="loop">Repeats the clip.</param>
/// <param name="endTime">Scheduled end time (seconds relative to the render start), or infinity.</param>
/// <param name="length">Clip length in seconds (replaced by the game clip in <see cref="BindClip"/>).</param>
/// <param name="frequency">Clip sample rate for timeSamples (replaced by the game clip in <see cref="BindClip"/>).</param>
public SourceState(Voice voice, double start, double pitch, bool loop, double endTime, double length, int frequency)
{
Voice = voice;
_start = start;
_anchorTime = start;
_pitch = pitch;
_loop = loop;
_endTime = endTime;
_length = length;
_frequency = Math.Max(1, frequency);
}
public Voice Voice { get; }
/// <summary>Takes the clip length and sample rate from the clip of the game AudioSource.</summary>
public void BindClip(AudioClip? clip)
{
if (!clip) return;
_length = clip!.length;
_frequency = Math.Max(1, clip.frequency);
}
/// <summary>Scheduled end time (see AudioSource.SetScheduledEndTime), seconds relative to the render start.</summary>
public void SetEndTime(double endTime, double now)
{
if (!IsPlaying(now)) return;
_endTime = endTime;
Voice.SetEndTime(endTime, now);
}
public bool IsPlaying(double now) => !_stopped && now < Math.Min(_endTime, NaturalEnd());
public float GetTime(double now) => IsPlaying(now) ? (float)PositionAt(now) : 0f;
public int GetTimeSamples(double now) => IsPlaying(now) ? (int)Math.Floor(PositionAt(now) * _frequency) : _offsetSamples;
public float GetPitch() => (float)_pitch;
public void SetTime(float seconds, double now)
{
_offsetSamples = (int)Math.Floor((double)seconds * _frequency);
Seek(seconds, now);
}
public void SetTimeSamples(int samples, double now)
{
_offsetSamples = samples;
Seek((double)samples / _frequency, now);
}
public void SetPitch(float pitch, double now)
{
if (float.IsNaN(pitch) || float.IsInfinity(pitch)) return;
var playing = IsPlaying(now);
if (playing) Reanchor(now);
_pitch = pitch;
if (playing) Voice.SetPitch(pitch, now);
}
public void Stop(double now)
{
if (IsPlaying(now)) Voice.Stop(now);
_stopped = true;
_offsetSamples = 0;
}
private double PlaybackPitch => Math.Max(0.0, _pitch);
private void Seek(double seconds, double now)
{
if (!IsPlaying(now)) return;
var position = Math.Max(0.0, seconds);
_anchorTime = Math.Max(now, _start);
_anchorPosition = position;
Voice.SetPosition(position, now);
}
private void Reanchor(double now)
{
var time = Math.Max(now, _start);
_anchorPosition = PositionAt(time);
_anchorTime = time;
}
/// <summary>Clip position at <paramref name="now"/> (the start position before the start), wrapped when looping.</summary>
private double PositionAt(double now)
{
var position = _anchorPosition + (Math.Max(now, _anchorTime) - _anchorTime) * PlaybackPitch;
if (_loop && _length > 0 && position >= _length) position %= _length;
return position;
}
/// <summary>Time at which the clip position reaches the clip length (infinity when looping or at pitch 0).</summary>
private double NaturalEnd()
{
if (_loop || PlaybackPitch <= 0) return double.PositiveInfinity;
return _anchorTime + Math.Max(0.0, _length - _anchorPosition) / PlaybackPitch;
}
}
+3 -3
View File
@@ -1125,7 +1125,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
/// </summary> /// </summary>
/// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param> /// <param name="yBuffer">Pointer to destination buffer for Y plane (Width * Height bytes).</param>
/// <param name="yStride">Row stride in bytes for Y plane.</param> /// <param name="yStride">Row stride in bytes for Y plane.</param>
/// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (Width * (Height / 2) bytes).</param> /// <param name="uvBuffer">Pointer to destination buffer for interleaved UV plane (((Width + 1) / 2) * 2 * ((Height + 1) / 2) bytes).</param>
/// <param name="uvStride">Row stride in bytes for UV plane.</param> /// <param name="uvStride">Row stride in bytes for UV plane.</param>
/// <param name="timestamp">Timestamp of the output frame in seconds.</param> /// <param name="timestamp">Timestamp of the output frame in seconds.</param>
/// <returns>True if a frame was successfully retrieved; false on EOF.</returns> /// <returns>True if a frame was successfully retrieved; false on EOF.</returns>
@@ -1178,7 +1178,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width; int w = Width;
int h = Height; int h = Height;
int uvH = (h + 1) / 2; int uvH = (h + 1) / 2;
int uvRowBytes = (w / 2) * 2; int uvRowBytes = ((w + 1) / 2) * 2;
// Copy Y plane (row by row to safely skip linesize GPU padding) // Copy Y plane (row by row to safely skip linesize GPU padding)
for (int y = 0; y < h; y++) for (int y = 0; y < h; y++)
@@ -1239,7 +1239,7 @@ internal sealed unsafe class FFmpegNativeDecoder : IDisposable
int w = Width; int w = Width;
int h = Height; int h = Height;
int uvH = (h + 1) / 2; int uvH = (h + 1) / 2;
int uvRowBytes = (w / 2) * 2; int uvRowBytes = ((w + 1) / 2) * 2;
if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH) if (yBuffer.Length < (long)w * h || uvBuffer.Length < (long)uvRowBytes * uvH)
{ {
+147 -35
View File
@@ -201,6 +201,12 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// </summary> /// </summary>
private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P; private AVPixelFormat _dstPixFmt = AVPixelFormat.AV_PIX_FMT_YUV420P;
/// <summary>
/// Color range of the encoder input and output (codec context color_range of the selected encoder).
/// Full range (JPEG) for every encoder except libvvenc, limited range (MPEG) for libvvenc.
/// </summary>
private AVColorRange _colorRange = AVColorRange.AVCOL_RANGE_JPEG;
/// <summary>Pixel format of the finally selected and running encoder.</summary> /// <summary>Pixel format of the finally selected and running encoder.</summary>
public AVPixelFormat SelectedPixelFormat => _dstPixFmt; public AVPixelFormat SelectedPixelFormat => _dstPixFmt;
@@ -255,6 +261,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// <summary>Whether audio encoding is enabled.</summary> /// <summary>Whether audio encoding is enabled.</summary>
private bool _audioEnabled; private bool _audioEnabled;
/// <summary>Bytes of one input sample of every channel (interleaved input format × channel count).</summary>
private int _audioInputBytesPerSample;
/// <summary>Encoding settings passed at construction time (immutable).</summary> /// <summary>Encoding settings passed at construction time (immutable).</summary>
private readonly EncoderSettings _settings; private readonly EncoderSettings _settings;
@@ -267,26 +276,6 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Utility Methods // Utility Methods
// ================================================================ // ================================================================
/// <summary>
/// Converts the application's Codec enum value to the AVCodecID used internally by FFmpeg.
/// AVCodecID identifies "which codec it is (e.g., H.264, AV1)",
/// while the actual encoder implementation (e.g., libx264, h264_nvenc) is selected separately.
/// </summary>
// ReSharper disable once InconsistentNaming
// ReSharper disable once UnusedMember.Local
private AVCodecID GetAVCodecID()
{
return _settings.Codec switch
{
Codec.H264 => AVCodecID.AV_CODEC_ID_H264,
Codec.H265 => AVCodecID.AV_CODEC_ID_HEVC,
Codec.AV1 => AVCodecID.AV_CODEC_ID_AV1,
Codec.VP9 => AVCodecID.AV_CODEC_ID_VP9,
Codec.VVC => AVCodecID.AV_CODEC_ID_VVC,
_ => throw new ArgumentOutOfRangeException()
};
}
/// <summary> /// <summary>
/// Returns a list of candidate encoders to try for the selected codec in priority order. /// Returns a list of candidate encoders to try for the selected codec in priority order.
/// ///
@@ -416,7 +405,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
// ── Basic video parameters configuration ── // ── Basic video parameters configuration ──
_pCodecContext->width = _settings.Width; _pCodecContext->width = _settings.Width;
_pCodecContext->height = _settings.Height; _pCodecContext->height = _settings.Height;
_pCodecContext->color_range = AVColorRange.AVCOL_RANGE_JPEG;
// ── Output color tags ──
// Matrix: BT.709, the RGB→YUV matrix of both conversion paths
// (GPU: RGB2YUV_BT709_Full in ColorConversion.compute, CPU: ConfigureSwsColorspace).
// Players use this tag to convert the YUV frames back to RGB.
// Range: full (0-255) for every encoder except libvvenc, limited (16-235) for libvvenc.
// The FFmpeg libvvenc wrapper passes the matrix/primaries/transfer tags to VVenC but not the range,
// so a VVC stream is read as limited range; its input is converted to limited range to match.
// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libavcodec/libvvenc.c (vvenc_set_color_format)
_pCodecContext->color_range = encoder == VideoEncoder.LibVvenc
? AVColorRange.AVCOL_RANGE_MPEG
: AVColorRange.AVCOL_RANGE_JPEG;
_pCodecContext->colorspace = AVColorSpace.AVCOL_SPC_BT709;
// thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count. // thread_count = 0: FFmpeg automatically allocates threads according to the system CPU core count.
// For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count, // For libvvenc, the FFmpeg wrapper passes thread_count as the VVenC internal thread count,
// so setting 0 (auto) is required for parallel encoding utilizing all cores. // so setting 0 (auto) is required for parallel encoding utilizing all cores.
@@ -690,7 +691,11 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pHwFrame = ffmpeg.av_frame_alloc(); _pHwFrame = ffmpeg.av_frame_alloc();
if (_pHwFrame == null) if (_pHwFrame == null)
{
// The codec context (and the frames context it holds) is released by Dispose in the catch block of Start.
ffmpeg.av_buffer_unref(&hwDeviceCtx);
throw new Exception("Failed to allocate QSV hardware frame structure."); throw new Exception("Failed to allocate QSV hardware frame structure.");
}
break; break;
} }
@@ -769,6 +774,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage"; var accelStatus = isHardware ? "Hardware Acceleration Enabled" : "Software CPU Usage";
Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})"); Melon<RenderMod>.Logger.Msg($"Video encoder successfully initialized: {encoder.Name} ({accelStatus})");
_dstPixFmt = selectedPixFmt; _dstPixFmt = selectedPixFmt;
_colorRange = _pCodecContext->color_range;
openedSuccessfully = true; openedSuccessfully = true;
break; break;
} }
@@ -797,8 +803,9 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Copy parameters set in codec context (resolution, codec ID, profile, etc.) // Copy parameters set in codec context (resolution, codec ID, profile, etc.)
// to stream's codecpar. Information to be written to container header. // to stream's codecpar. Information to be written to container header.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavc__core.html
if (ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext) < 0) var paramResult = ffmpeg.avcodec_parameters_from_context(_pVideoStream->codecpar, _pCodecContext);
throw new Exception("Failed to copy codec parameters to stream."); if (paramResult < 0)
throw new Exception($"Failed to copy codec parameters to stream — {GetFFmpegErrorString(paramResult)} (Code: {paramResult})");
// Stream time base: unit of the timestamps stored in the container. // Stream time base: unit of the timestamps stored in the container.
// Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks. // Set to the codec time base (1 / framerate) so that every frame interval is a whole number of ticks.
@@ -823,15 +830,17 @@ namespace ADOFAIRenderer.Util.FFmpeg
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__io.html
if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0) if ((_pFormatContext->oformat->flags & ffmpeg.AVFMT_NOFILE) == 0)
{ {
if (ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE) < 0) var avioResult = ffmpeg.avio_open(&_pFormatContext->pb, outputPath, ffmpeg.AVIO_FLAG_WRITE);
throw new Exception($"Cannot open output file {outputPath}."); if (avioResult < 0)
throw new Exception($"Cannot open output file {outputPath} — {GetFFmpegErrorString(avioResult)} (Code: {avioResult})");
} }
// Write file header (ftyp, moov, container metadata). // Write file header (ftyp, moov, container metadata).
// Packets can be written with av_interleaved_write_frame after this point. // Packets can be written with av_interleaved_write_frame after this point.
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html // Reference: https://ffmpeg.org/doxygen/trunk/group__lavf__encoding.html
if (ffmpeg.avformat_write_header(_pFormatContext, null) < 0) var headerResult = ffmpeg.avformat_write_header(_pFormatContext, null);
throw new Exception("Failed to write file header."); if (headerResult < 0)
throw new Exception($"Failed to write file header — {GetFFmpegErrorString(headerResult)} (Code: {headerResult})");
// ── Step 6: Initialize frame buffers and colorspace converter ── // ── Step 6: Initialize frame buffers and colorspace converter ──
// av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet). // av_frame_alloc: Allocates AVFrame structure on heap (no pixel data yet).
@@ -865,6 +874,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
if (_pSwsContext == null) if (_pSwsContext == null)
throw new Exception("Failed to initialize software scaler."); throw new Exception("Failed to initialize software scaler.");
ConfigureSwsColorspace();
_isInitialized = true; _isInitialized = true;
_loggedConversionMode = false; _loggedConversionMode = false;
} }
@@ -1160,10 +1171,42 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock) lock (_lock)
{ {
if (_disposed || !_isInitialized) return; if (_disposed || !_isInitialized) return;
ValidateInputLength(buffer.Length, isColorConverted: false);
WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr()); WriteFrameFromBuffer((byte*)buffer.GetUnsafeReadOnlyPtr());
} }
} }
/// <summary>
/// Number of bytes read from the input buffer of one frame.
/// RGBA input (isColorConverted = false): Width × Height × 4
/// Pre-converted NV12: Width × Height (Y) + Width × (Height / 2) (interleaved UV)
/// Pre-converted YUV420P: Width × Height (Y) + 2 × (Width / 2) × (Height / 2) (U, V)
/// Returns 0 for a pre-converted input of another format (WritePreConvertedFrame rejects it).
/// </summary>
private long GetRequiredInputLength(bool isColorConverted)
{
long w = _settings.Width, h = _settings.Height;
if (!isColorConverted) return w * h * 4;
return _dstPixFmt switch
{
AVPixelFormat.AV_PIX_FMT_NV12 => w * h + w * (h / 2),
AVPixelFormat.AV_PIX_FMT_YUV420P => w * h + 2 * (w / 2) * (h / 2),
_ => 0
};
}
/// <summary>
/// Throws an ArgumentException when the input buffer of a frame is smaller than the bytes read from it
/// (GetRequiredInputLength), so that the copy does not read outside the buffer.
/// </summary>
private void ValidateInputLength(long length, bool isColorConverted)
{
var required = GetRequiredInputLength(isColorConverted);
if (length < required)
throw new ArgumentException($"Frame buffer is too small: {length} bytes, {required} bytes required for {_settings.Width}x{_settings.Height} {(isColorConverted ? _dstPixFmt.ToString() : "RGBA")} input.");
}
/// <summary> /// <summary>
/// Encodes a frame from IntPtr (native pointer). /// Encodes a frame from IntPtr (native pointer).
/// Can be used from external native plugins or custom capture sources. /// Can be used from external native plugins or custom capture sources.
@@ -1182,6 +1225,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
: "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)"); : "[Encoder] CPU color conversion fallback enabled (RGBA Input, using sws_scale)");
} }
ValidateInputLength(length, isColorConverted);
if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer()); if (isColorConverted) WritePreConvertedFrame((byte*)pointer.ToPointer());
else WriteFrameFromBuffer((byte*)pointer.ToPointer()); else WriteFrameFromBuffer((byte*)pointer.ToPointer());
} }
@@ -1196,6 +1240,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock) lock (_lock)
{ {
if (_disposed || !_isInitialized) return; if (_disposed || !_isInitialized) return;
if (length > buffer.Length)
throw new ArgumentException($"Frame length ({length} bytes) exceeds the buffer size ({buffer.Length} bytes).");
ValidateInputLength(length, isColorConverted);
fixed (byte* pBuffer = buffer) fixed (byte* pBuffer = buffer)
{ {
if (!_loggedConversionMode) if (!_loggedConversionMode)
@@ -1213,7 +1261,51 @@ namespace ADOFAIRenderer.Util.FFmpeg
} }
/// <summary> /// <summary>
/// Sets the format, size and color range of the destination frame (_pDstFrame) and allocates a new pixel buffer for it. /// Sets the RGB→YUV conversion of the CPU conversion path (_pSwsContext) to the BT.709 matrix
/// and the color range of the encoder (_colorRange), so that it produces the same colors as the GPU path
/// (RGB2YUV_BT709_Full in ColorConversion.compute) and as the tags of the output stream.
/// A new libswscale context uses the BT.601 matrix (SWS_CS_DEFAULT = SWS_CS_ITU601) and the limited range.
///
/// For RGB input, "table" holds the coefficients of the output YUV space and dstRange its range (1 = full, 0 = limited).
/// The source (RGB) values, brightness, contrast and saturation keep the values of the context.
///
/// Reference: https://ffmpeg.org/doxygen/trunk/group__libsws.html (sws_setColorspaceDetails, sws_getCoefficients)
/// Reference: https://github.com/FFmpeg/FFmpeg/blob/release/8.0/libswscale/swscale.h (SWS_CS_ITU709, SWS_CS_DEFAULT)
/// </summary>
private void ConfigureSwsColorspace()
{
int* currentInvTable;
int* currentTable;
int srcRange, dstRange, brightness, contrast, saturation;
var ret = ffmpeg.sws_getColorspaceDetails(_pSwsContext, &currentInvTable, &srcRange, &currentTable, &dstRange,
&brightness, &contrast, &saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot read the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
return;
}
var coefficients = ffmpeg.sws_getCoefficients(ffmpeg.SWS_CS_ITU709);
var invTable = new int_array4();
var table = new int_array4();
for (uint i = 0; i < 4; i++)
{
invTable[i] = currentInvTable[i];
table[i] = coefficients[i];
}
var fullRange = _colorRange == AVColorRange.AVCOL_RANGE_JPEG;
ret = ffmpeg.sws_setColorspaceDetails(_pSwsContext, invTable, srcRange, table, fullRange ? 1 : 0,
brightness, contrast, saturation);
if (ret < 0)
{
Melon<RenderMod>.Logger.Warning($"[Encoder] Cannot set the colorspace details of the software scaler: {GetFFmpegErrorString(ret)} (Code: {ret}). The CPU conversion path keeps the libswscale defaults.");
}
}
/// <summary>
/// Sets the format, size, color range and matrix of the destination frame (_pDstFrame) and allocates a new pixel buffer for it.
/// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers. /// av_frame_get_buffer allocates memory matching format, width and height and sets the data[]/linesize[] pointers.
/// Second argument 0 means default alignment. /// Second argument 0 means default alignment.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer) /// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_get_buffer)
@@ -1223,7 +1315,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
_pDstFrame->format = (int)_dstPixFmt; _pDstFrame->format = (int)_dstPixFmt;
_pDstFrame->width = _settings.Width; _pDstFrame->width = _settings.Width;
_pDstFrame->height = _settings.Height; _pDstFrame->height = _settings.Height;
_pDstFrame->color_range = AVColorRange.AVCOL_RANGE_JPEG; _pDstFrame->color_range = _colorRange;
_pDstFrame->colorspace = AVColorSpace.AVCOL_SPC_BT709;
var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0); var ret = ffmpeg.av_frame_get_buffer(_pDstFrame, 0);
if (ret < 0) if (ret < 0)
@@ -1262,7 +1355,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
switch (_dstPixFmt) switch (_dstPixFmt)
{ {
case AVPixelFormat.AV_PIX_FMT_NV12 or AVPixelFormat.AV_PIX_FMT_QSV: case AVPixelFormat.AV_PIX_FMT_NV12:
{ {
// Copy Y Plane // Copy Y Plane
var pSrcY = buffer; var pSrcY = buffer;
@@ -1485,7 +1578,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame); var response = ffmpeg.avcodec_send_frame(_pCodecContext, pFrame);
if (response < 0) if (response < 0)
{ {
throw new Exception($"Error occurred while sending frame to encoder. (Code: {response})"); throw new Exception($"Error occurred while sending frame to encoder: {GetFFmpegErrorString(response)} (Code: {response})");
} }
// Pop all encoded packets and write to file. // Pop all encoded packets and write to file.
@@ -1499,7 +1592,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// EOF: Encoder flush complete (all delayed frames outputted) → exit loop // EOF: Encoder flush complete (all delayed frames outputted) → exit loop
if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break; if (response == ffmpeg.AVERROR(ffmpeg.EAGAIN) || response == ffmpeg.AVERROR_EOF) break;
if (response < 0) if (response < 0)
throw new Exception($"Error occurred while receiving packet from encoder. (Code: {response})"); throw new Exception($"Error occurred while receiving packet from encoder: {GetFFmpegErrorString(response)} (Code: {response})");
// Convert PTS/DTS from codec timebase → stream timebase // Convert PTS/DTS from codec timebase → stream timebase
ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base); ffmpeg.av_packet_rescale_ts(_pPacket, _pCodecContext->time_base, _pVideoStream->time_base);
@@ -1510,7 +1603,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
// and written in time order. This function is used even when only video is present. // and written in time order. This function is used even when only video is present.
response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket); response = ffmpeg.av_interleaved_write_frame(_pFormatContext, _pPacket);
if (response < 0) if (response < 0)
throw new Exception($"Error occurred while writing packet to file. (Code: {response})"); throw new Exception($"Error occurred while writing packet to file: {GetFFmpegErrorString(response)} (Code: {response})");
// Unref packet data reference for reuse in next receive_packet. // Unref packet data reference for reuse in next receive_packet.
ffmpeg.av_packet_unref(_pPacket); ffmpeg.av_packet_unref(_pPacket);
@@ -1775,6 +1868,10 @@ namespace ADOFAIRenderer.Util.FFmpeg
? audioSettings.InputSampleFormat ? audioSettings.InputSampleFormat
: AVSampleFormat.AV_SAMPLE_FMT_FLT; : AVSampleFormat.AV_SAMPLE_FMT_FLT;
// Bytes of one sample of every channel in the interleaved input (e.g. FLT stereo: 4 × 2 = 8 bytes).
// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__sampfmts.html (av_get_bytes_per_sample)
_audioInputBytesPerSample = ffmpeg.av_get_bytes_per_sample(inputSampleFmt) * audioSettings.Channels;
// Allocate SwrContext // Allocate SwrContext
fixed (SwrContext** ppSwr = &_pSwrContext) fixed (SwrContext** ppSwr = &_pSwrContext)
{ {
@@ -1838,6 +1935,8 @@ namespace ADOFAIRenderer.Util.FFmpeg
lock (_lock) lock (_lock)
{ {
if (_disposed || !_isInitialized || !_audioEnabled) return; if (_disposed || !_isInitialized || !_audioEnabled) return;
if (numSamples < 0 || (long)numSamples * _audioInputBytesPerSample > (long)samples.Length * sizeof(float))
throw new ArgumentException($"Audio sample count ({numSamples} per channel) exceeds the buffer size ({samples.Length} floats).");
fixed (float* pSamples = samples) fixed (float* pSamples = samples)
{ {
@@ -1872,6 +1971,19 @@ namespace ADOFAIRenderer.Util.FFmpeg
/// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html /// swr_convert: https://ffmpeg.org/doxygen/trunk/group__lswr.html
/// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html /// av_audio_fifo: https://ffmpeg.org/doxygen/trunk/group__lavu__audiofifo.html
/// </summary> /// </summary>
/// <summary>
/// Makes the buffer of the reusable audio frame (_pAudioFrame) writable before samples are read into it.
/// When the audio encoder still references the buffer, av_frame_make_writable allocates a new buffer.
/// Throws an exception when the buffer cannot be allocated.
/// Reference: https://ffmpeg.org/doxygen/trunk/group__lavu__frame.html (av_frame_make_writable)
/// </summary>
private void MakeAudioFrameWritable()
{
var ret = ffmpeg.av_frame_make_writable(_pAudioFrame);
if (ret < 0)
throw new Exception($"Failed to make the audio frame writable: {GetFFmpegErrorString(ret)} (Code: {ret})");
}
private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples) private void WriteAudioSamplesInternal(byte* inputBuffer, int numSamples)
{ {
// ── 1. Resampling: Input format → Encoder format ── // ── 1. Resampling: Input format → Encoder format ──
@@ -1910,7 +2022,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024; var frameSize = _pAudioCodecContext->frame_size > 0 ? _pAudioCodecContext->frame_size : 1024;
while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize) while (ffmpeg.av_audio_fifo_size(_pAudioFifo) >= frameSize)
{ {
ffmpeg.av_frame_make_writable(_pAudioFrame); MakeAudioFrameWritable();
_pAudioFrame->nb_samples = frameSize; _pAudioFrame->nb_samples = frameSize;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize); var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, frameSize);
@@ -1989,7 +2101,7 @@ namespace ADOFAIRenderer.Util.FFmpeg
var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo); var remaining = ffmpeg.av_audio_fifo_size(_pAudioFifo);
if (remaining > 0) if (remaining > 0)
{ {
ffmpeg.av_frame_make_writable(_pAudioFrame); MakeAudioFrameWritable();
_pAudioFrame->nb_samples = remaining; _pAudioFrame->nb_samples = remaining;
var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining); var read = ffmpeg.av_audio_fifo_read(_pAudioFifo, (void**)&_pAudioFrame->data, remaining);
+42 -5
View File
@@ -73,6 +73,16 @@ namespace ADOFAIRenderer.Util
} }
} }
/// <summary>
/// Bytes of the readback buffer of one frame: Width x Height x 3 / 2 when the GPU conversion to NV12 / YUV420P
/// is used, Width x Height x 4 (RGBA) otherwise. The same size as ReadbackRequestState.Setup uses.
/// </summary>
public long GetFrameBufferBytes(int width, int height)
{
var pixels = (long)width * height;
return _useGPUConversion ? pixels * 3 / 2 : pixels * 4;
}
public void RequestFrameCapture(Texture texture, long currentFrame) public void RequestFrameCapture(Texture texture, long currentFrame)
{ {
if (!RenderManager.IsCapturingActive) return; if (!RenderManager.IsCapturingActive) return;
@@ -154,14 +164,11 @@ namespace ADOFAIRenderer.Util
} }
Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame."); Melon<RenderMod>.Logger.Warning($"[OOM Prevent] GPU Readback failed for frame {frame}. Queue deadlock prevented by sending a black frame.");
var byteSize = state.Width * state.Height * 3 / 2; var blackFrame = CreateBlackFrame(state, out var byteSize);
var blackFrame = ArrayPool<byte>.Shared.Rent(byteSize);
Array.Clear(blackFrame, 0, byteSize);
var capFrame = new RenderManager.CapturedFrame var capFrame = new RenderManager.CapturedFrame
{ {
ManagedBuffer = blackFrame, Length = byteSize, ManagedBuffer = blackFrame, Length = byteSize,
IsColorConverted = true IsColorConverted = state.IsGPUConversion
}; };
PushToQueue(frame, capFrame); PushToQueue(frame, capFrame);
RecycleState(state); RecycleState(state);
@@ -190,6 +197,36 @@ namespace ADOFAIRenderer.Util
} }
} }
/// <summary>
/// Creates a black frame in the same format as the frames of this reader, from ArrayPool&lt;byte&gt;.Shared
/// (returned to the pool by the render thread after encoding).
///
/// GPU conversion (NV12 / YUV420P, full range): Width × Height × 3 / 2 bytes.
/// Y plane (Width × Height bytes) = 0, U/V data (the rest) = 128.
/// 128 is the zero chroma value of the 8-bit full range: the U/V offset 0.5 of RGB2YUV_BT709_Full
/// in ColorConversion.compute, which the encoder input of this path uses.
/// Both NV12 (interleaved UV) and YUV420P (U plane, then V plane) store all chroma after the Y plane,
/// so the same fill applies to both.
/// No GPU conversion (RGBA input, converted by the encoder): Width × Height × 4 bytes, all 0.
/// </summary>
private static byte[] CreateBlackFrame(ReadbackRequestState state, out int byteSize)
{
var lumaSize = state.Width * state.Height;
byteSize = state.IsGPUConversion ? lumaSize * 3 / 2 : lumaSize * 4;
var frame = ArrayPool<byte>.Shared.Rent(byteSize);
if (state.IsGPUConversion)
{
Array.Clear(frame, 0, lumaSize);
frame.AsSpan(lumaSize, byteSize - lumaSize).Fill(128);
}
else
{
Array.Clear(frame, 0, byteSize);
}
return frame;
}
internal void RecycleState(ReadbackRequestState state) internal void RecycleState(ReadbackRequestState state)
{ {
lock (_poolLock) lock (_poolLock)
+3 -3
View File
@@ -97,7 +97,7 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA"); _kernelIndex = _computeShader.FindKernel("CSMain_NV12ToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false); _textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureUV = new Texture2D(w / 2, h / 2, TextureFormat.RG16, false); _textureUV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.RG16, false);
ConfigureTexture(_textureY); ConfigureTexture(_textureY);
ConfigureTexture(_textureUV); ConfigureTexture(_textureUV);
@@ -118,8 +118,8 @@ internal class VideoBGPlayer : MonoBehaviour
_kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA"); _kernelIndex = _computeShader.FindKernel("CSMain_YUV420PToRGBA");
_textureY = new Texture2D(w, h, TextureFormat.R8, false); _textureY = new Texture2D(w, h, TextureFormat.R8, false);
_textureU = new Texture2D(w / 2, h / 2, TextureFormat.R8, false); _textureU = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
_textureV = new Texture2D(w / 2, h / 2, TextureFormat.R8, false); _textureV = new Texture2D((w + 1) / 2, (h + 1) / 2, TextureFormat.R8, false);
ConfigureTexture(_textureY); ConfigureTexture(_textureY);
ConfigureTexture(_textureU); ConfigureTexture(_textureU);
ConfigureTexture(_textureV); ConfigureTexture(_textureV);