diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs
index 3b94580..c53b3c1 100644
--- a/Core/RenderManager.cs
+++ b/Core/RenderManager.cs
@@ -592,19 +592,19 @@ internal class RenderManager : MonoBehaviour
scrController.instance.levelWasSkipped = true;
if (!scrConductor.instance || !scrConductor.instance.song) yield break;
- if (Configuration.AdvancedConfig.ProcessAudio)
- {
- AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
- }
-
if (Configuration.AudioConfig.RenderAudio)
{
AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate);
AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
-
+
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps);
}
+
+ if (Configuration.AdvancedConfig.ProcessAudio)
+ {
+ AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer);
+ }
scrConductor.instance.song.clip = null;
scrConductor.instance.song.time = -1000;
@@ -1021,10 +1021,10 @@ internal class RenderManager : MonoBehaviour
_frameReader?.Dispose();
ResetFrameHookState();
+ AudioSpectrumGenerator.Clear();
AudioMixer?.Dispose(); AudioMixer = null;
SyncCalculator = null;
-
- AudioSpectrumGenerator.Clear();
+
RenderContext.Reset();
}
diff --git a/Util/Audio/AudioSpectrumGenerator.cs b/Util/Audio/AudioSpectrumGenerator.cs
index 306334a..d8567db 100644
--- a/Util/Audio/AudioSpectrumGenerator.cs
+++ b/Util/Audio/AudioSpectrumGenerator.cs
@@ -1,164 +1,187 @@
using System;
using System.Collections.Generic;
using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Util.Audio.Clip;
+using MelonLoader;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio;
+///
+/// Replacement for AudioSource.GetSpectrumData during rendering: the spectrum of the song as the song AudioSource would output it.
+///
+/// Signal: the song clip from an at the output sample rate (AudioSettings.outputSampleRate),
+/// played at conductor.song.pitch (linear interpolation between frames) and scaled by conductor.song.volume.
+/// Unity analyzes the output signal of the source: bin spacing follows the output rate, pitch moves the frequencies,
+/// and volume scales the magnitudes (measured with a Unity 6000.3.10f1 test project).
+/// Channel: stereo clips use the requested channel (0 = left, 1 = right); a mono clip is scaled by .
+/// Window: twice the requested spectrum length in output samples, centered on the current render time.
+///
+/// The song source is shared with the store when its output rate matches; otherwise a separate
+/// store is created. A cursor at the analysis position keeps the needed chunks loaded. Main thread only.
+///
internal static class AudioSpectrumGenerator
{
- // ReSharper disable MemberCanBePrivate.Global
- public static float[]? AudioClipData { get; private set; }
- public static int Channels { get; private set; }
- public static int Frequency { get; private set; }
- public static int TotalSamples { get; private set; }
- // ReSharper restore MemberCanBePrivate.Global
+ ///
+ /// Gain of a mono clip relative to one channel of a stereo clip with the same samples.
+ /// Measured with a Unity 6000.3.10f1 test project: the spectrum of a mono clip is about 0.70 times that of each stereo channel (1/√2).
+ ///
+ private const double MonoGain = 0.70710678118654752;
+
+ private static AudioClipStore? _ownStore;
+ private static AudioClipStore? _store;
+ private static AudioClipSource? _source;
+ private static AudioCursor? _cursor;
+ private static ChunkView _view;
+ private static bool _loadErrorLogged;
private static long _cachedFrame = -1;
- private static readonly Dictionary CachedSpectrums = new();
+ private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new();
- public static void Initialize(AudioClip? clip)
+ /// Registers the song clip. Call after the is created (null when audio is not rendered).
+ public static void Initialize(AudioClip? clip, OfflineAudioMixer? mixer)
{
- if (!clip)
- {
- Clear();
- return;
- }
+ Clear();
+ if (!clip) return;
try
{
- Channels = clip.channels;
- Frequency = clip.frequency;
- int samplesCount = clip.samples;
-
- float[] rawSamples = new float[samplesCount * Channels];
- clip.GetData(rawSamples, 0);
-
- if (Channels == 1)
+ var rate = AudioSettings.outputSampleRate;
+ if (mixer != null && mixer.Store.OutputRate == rate)
{
- AudioClipData = rawSamples;
- TotalSamples = samplesCount;
+ _store = mixer.Store;
}
else
{
- AudioClipData = new float[samplesCount];
- TotalSamples = samplesCount;
- for (int i = 0; i < samplesCount; i++)
- {
- float sum = 0f;
- for (int c = 0; c < Channels; c++)
- {
- sum += rawSamples[i * Channels + c];
- }
- AudioClipData[i] = sum / Channels;
- }
+ _ownStore = new AudioClipStore(rate);
+ _ownStore.Start();
+ _store = _ownStore;
}
- MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}");
+
+ _source = AudioClipResolver.GetOrAdd(_store, clip!.name, clip);
+ if (_source == null)
+ {
+ Melon.Logger.Error($"[AudioSpectrumGenerator] Song clip '{clip.name}' could not be loaded, the spectrum will be empty.");
+ return;
+ }
+ _cursor = _source.AddCursor(0, false);
+ Melon.Logger.Msg($"[AudioSpectrumGenerator] Song '{clip.name}': {_source.Channels} ch at {rate} Hz ({(_ownStore != null ? "own store" : "shared with the mixer")})");
}
catch (Exception e)
{
- MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}");
+ Melon.Logger.Error($"[AudioSpectrumGenerator] Failed to initialize: {e}");
Clear();
}
}
+ /// Releases the chunk, the cursor and the own store. Call before the mixer store is disposed.
public static void Clear()
{
- AudioClipData = null;
- TotalSamples = 0;
- Channels = 0;
- Frequency = 0;
- lock (CachedSpectrums)
+ if (_source != null)
{
- _cachedFrame = -1;
- CachedSpectrums.Clear();
+ _source.Release(_view);
+ if (_cursor != null) _source.RemoveCursor(_cursor);
}
+ _view = default;
+ _cursor = null;
+ _source = null;
+ _store = null;
+ _ownStore?.Dispose();
+ _ownStore = null;
+ _loadErrorLogged = false;
+ _cachedFrame = -1;
+ CachedSpectrums.Clear();
}
public static void GetSpectrumData(float[] samples, int channel, FFTWindow window)
{
- if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song)
- {
- Array.Clear(samples, 0, samples.Length);
- return;
- }
-
- var conductor = scrConductor.instance;
- var pitch = conductor.song.pitch;
- var separateCountdownOffset = conductor.separateCountdownTime ?
- conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
-
- // Calculate the actual scheduled start time of the song
- var songScheduledStartTime = conductor.skipOffset ?
- conductor.dspTimeSong + conductor.addoffset / pitch :
- conductor.dspTimeSong + separateCountdownOffset / pitch;
-
- if (conductor.dspTime < songScheduledStartTime)
- {
- Array.Clear(samples, 0, samples.Length);
- return;
- }
-
- var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset;
-
var n = samples.Length;
- if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2
+ var conductor = scrConductor.instance;
+ if (_source == null || _store == null || !conductor || !conductor!.hasSongStarted || !conductor.song
+ || n == 0 || (n & (n - 1)) != 0)
{
- Array.Clear(samples, 0, samples.Length);
+ Array.Clear(samples, 0, n);
return;
}
- var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1;
-
- lock (CachedSpectrums)
+ var render = RenderManager.Instance;
+ var currentFrame = render.frameCount;
+ if (currentFrame != _cachedFrame)
{
- if (currentFrame != _cachedFrame)
- {
- _cachedFrame = currentFrame;
- CachedSpectrums.Clear();
- }
-
- if (CachedSpectrums.TryGetValue(n, out var cached))
- {
- Array.Copy(cached, samples, n);
- return;
- }
+ _cachedFrame = currentFrame;
+ CachedSpectrums.Clear();
}
+ var sourceChannel = _source.Channels == 1 ? 0 : Math.Min(Math.Max(channel, 0), _source.Channels - 1);
+ if (CachedSpectrums.TryGetValue((n, sourceChannel), out var cached))
+ {
+ Array.Copy(cached, samples, n);
+ return;
+ }
+
+ // Song position (seconds of song time) at the current render time, as in scrConductor.StartMusicCo:
+ // the song starts at dspTimeSong + separate countdown (crotchetAtStart * adjustedCountdownTicks in song time).
+ double pitch = conductor.song.pitch;
+ double volume = conductor.song.volume;
+ var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
+ var now = render.dspTimeBase + render.AudioTime;
+ var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown;
+
var fftSize = 2 * n;
- var frequency = Frequency;
- var totalSamples = TotalSamples;
- var currentSampleIndex = (long)(songTime * frequency);
+ var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0);
+ var first = songSeconds * _store.OutputRate - n * pitch;
var windowSamples = new float[fftSize];
- for (var i = 0; i < fftSize; i++)
+ try
{
- var idx = currentSampleIndex - fftSize / 2 + i;
- if (idx >= 0 && idx < totalSamples)
+ _cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1);
+ for (var i = 0; i < fftSize; i++)
{
- windowSamples[i] = AudioClipData[idx];
+ var position = first + i * pitch;
+ var index = (long)Math.Floor(position);
+ var fraction = position - index;
+ var s0 = ReadSample(index, sourceChannel);
+ var s1 = fraction > 0 ? ReadSample(index + 1, sourceChannel) : 0.0;
+ windowSamples[i] = (float)((s0 + (s1 - s0) * fraction) * gain);
}
- else
+ }
+ catch (InvalidOperationException e)
+ {
+ if (!_loadErrorLogged)
{
- windowSamples[i] = 0f;
+ _loadErrorLogged = true;
+ Melon.Logger.Error($"[AudioSpectrumGenerator] Song samples are unavailable, the spectrum will be empty: {e.Message}");
}
+ Array.Clear(samples, 0, n);
+ return;
}
var spectrum = new float[n];
CalculateFFT(windowSamples, spectrum);
+ CachedSpectrums[(n, sourceChannel)] = spectrum;
+ Array.Copy(spectrum, samples, n);
+ }
- lock (CachedSpectrums)
+ /// Sample of source channel at ; 0 outside the clip or when the source failed.
+ /// Chunk loading failed fatally (see ).
+ private static double ReadSample(long frame, int channel)
+ {
+ var source = _source!;
+ if (frame < 0 || frame >= source.TotalFrames) return 0.0;
+ if (!_view.IsValid || !_view.Contains(frame))
{
- CachedSpectrums[n] = spectrum;
- Array.Copy(spectrum, samples, n);
+ source.Release(_view);
+ _view = _store!.Acquire(source, frame);
+ if (!_view.IsValid) return 0.0;
}
+ return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel];
}
private static void CalculateFFT(float[] samples, float[] spectrum)
{
var n = samples.Length;
- // Apply Blackman-Harris window
+ // Blackman-Harris window, periodic form: w(i) = a0 - a1 cos(2πi/n) + a2 cos(4πi/n) - a3 cos(6πi/n), i = 0 ~ n - 1
const float a0 = 0.35875f;
const float a1 = 0.48829f;
const float a2 = 0.14128f;
@@ -169,7 +192,7 @@ internal static class AudioSpectrumGenerator
for (var i = 0; i < n; i++)
{
- var angle = 2.0 * Math.PI * i / (n - 1);
+ var angle = 2.0 * Math.PI * i / n;
var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle));
re[i] = samples[i] * w;
im[i] = 0f;
@@ -230,12 +253,13 @@ internal static class AudioSpectrumGenerator
}
}
- // Normalize and fill spectrum
- var factor = 2.0f / n;
+ // Magnitude scale 2.5 / n (n = FFT size, twice the spectrum length), clamped to 1.0: the scale of Unity GetSpectrumData.
+ // A sine of amplitude A centered on a bin gives A * a0 * 1.25 with the Blackman-Harris window.
+ var factor = 2.5f / n;
var outLen = Math.Min(spectrum.Length, n / 2);
for (var i = 0; i < outLen; i++)
{
- spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor;
+ spectrum[i] = Math.Min((float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor, 1f);
}
}
-}
\ No newline at end of file
+}
diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs
index 17ee323..bee91d9 100644
--- a/Util/Audio/OfflineAudioMixer.cs
+++ b/Util/Audio/OfflineAudioMixer.cs
@@ -107,6 +107,9 @@ internal sealed class OfflineAudioMixer : IDisposable
_store.Start();
}
+ /// Clip store of this render (output rate = mixer sample rate), also read by .
+ internal AudioClipStore Store => _store;
+
// ── Main thread ──
/// Registers the song clip. The song voice is created and kept in sync by .