Read Spectrum Samples From Clip Store

Core/RenderManager.cs

@@ -592,19 +592,19 @@ internal class RenderManager : MonoBehaviour
         scrController.instance.levelWasSkipped = true;
             
         if (!scrConductor.instance || !scrConductor.instance.song) yield break;
-        if (Configuration.AdvancedConfig.ProcessAudio)
-        {
-            AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
-        }
-            
         if (Configuration.AudioConfig.RenderAudio)
         {
             AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate);
             AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
-                
+
             var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
             SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps);
         }
+
+        if (Configuration.AdvancedConfig.ProcessAudio)
+        {
+            AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer);
+        }
             
         scrConductor.instance.song.clip = null;
         scrConductor.instance.song.time = -1000;
@@ -1021,10 +1021,10 @@ internal class RenderManager : MonoBehaviour
 
         _frameReader?.Dispose();
         ResetFrameHookState();
+        AudioSpectrumGenerator.Clear();
         AudioMixer?.Dispose(); AudioMixer = null;
         SyncCalculator = null;
-        
+
-        AudioSpectrumGenerator.Clear();
         RenderContext.Reset();
             
     }

Util/Audio/AudioSpectrumGenerator.cs

@@ -1,164 +1,187 @@
 using System;
 using System.Collections.Generic;
 using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Util.Audio.Clip;
+using MelonLoader;
 using UnityEngine;
 
 namespace ADOFAIRenderer.Util.Audio;
 
+/// <summary>
+/// Replacement for AudioSource.GetSpectrumData during rendering: the spectrum of the song as the song AudioSource would output it.
+///
+/// Signal: the song clip from an <see cref="AudioClipStore"/> at the output sample rate (AudioSettings.outputSampleRate),
+/// played at conductor.song.pitch (linear interpolation between frames) and scaled by conductor.song.volume.
+/// Unity analyzes the output signal of the source: bin spacing follows the output rate, pitch moves the frequencies,
+/// and volume scales the magnitudes (measured with a Unity 6000.3.10f1 test project).
+/// Channel: stereo clips use the requested channel (0 = left, 1 = right); a mono clip is scaled by <see cref="MonoGain"/>.
+/// Window: twice the requested spectrum length in output samples, centered on the current render time.
+///
+/// The song source is shared with the <see cref="OfflineAudioMixer"/> store when its output rate matches; otherwise a separate
+/// store is created. A cursor at the analysis position keeps the needed chunks loaded. Main thread only.
+/// </summary>
 internal static class AudioSpectrumGenerator
 {
-    // ReSharper disable MemberCanBePrivate.Global
+    /// <summary>
-    public static float[]? AudioClipData { get; private set; }
+    /// Gain of a mono clip relative to one channel of a stereo clip with the same samples.
-    public static int Channels { get; private set; }
+    /// Measured with a Unity 6000.3.10f1 test project: the spectrum of a mono clip is about 0.70 times that of each stereo channel (1/√2).
-    public static int Frequency { get; private set; }
+    /// </summary>
-    public static int TotalSamples { get; private set; }
+    private const double MonoGain = 0.70710678118654752;
-    // ReSharper restore MemberCanBePrivate.Global
+
+    private static AudioClipStore? _ownStore;
+    private static AudioClipStore? _store;
+    private static AudioClipSource? _source;
+    private static AudioCursor? _cursor;
+    private static ChunkView _view;
+    private static bool _loadErrorLogged;
 
     private static long _cachedFrame = -1;
-    private static readonly Dictionary<int, float[]> CachedSpectrums = new();
+    private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new();
 
-    public static void Initialize(AudioClip? clip)
+    /// <summary>Registers the song clip. Call after the <see cref="OfflineAudioMixer"/> is created (null when audio is not rendered).</summary>
+    public static void Initialize(AudioClip? clip, OfflineAudioMixer? mixer)
     {
-        if (!clip)
+        Clear();
-        {
+        if (!clip) return;
-            Clear();
-            return;
-        }
 
         try
         {
-            Channels = clip.channels;
+            var rate = AudioSettings.outputSampleRate;
-            Frequency = clip.frequency;
+            if (mixer != null && mixer.Store.OutputRate == rate)
-            int samplesCount = clip.samples;
-
-            float[] rawSamples = new float[samplesCount * Channels];
-            clip.GetData(rawSamples, 0);
-
-            if (Channels == 1)
             {
-                AudioClipData = rawSamples;
+                _store = mixer.Store;
-                TotalSamples = samplesCount;
             }
             else
             {
-                AudioClipData = new float[samplesCount];
+                _ownStore = new AudioClipStore(rate);
-                TotalSamples = samplesCount;
+                _ownStore.Start();
-                for (int i = 0; i < samplesCount; i++)
+                _store = _ownStore;
-                {
-                    float sum = 0f;
-                    for (int c = 0; c < Channels; c++)
-                    {
-                        sum += rawSamples[i * Channels + c];
-                    }
-                    AudioClipData[i] = sum / Channels;
-                }
             }
-            MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}");
+
+            _source = AudioClipResolver.GetOrAdd(_store, clip!.name, clip);
+            if (_source == null)
+            {
+                Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Song clip '{clip.name}' could not be loaded, the spectrum will be empty.");
+                return;
+            }
+            _cursor = _source.AddCursor(0, false);
+            Melon<RenderMod>.Logger.Msg($"[AudioSpectrumGenerator] Song '{clip.name}': {_source.Channels} ch at {rate} Hz ({(_ownStore != null ? "own store" : "shared with the mixer")})");
         }
         catch (Exception e)
         {
-            MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}");
+            Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Failed to initialize: {e}");
             Clear();
         }
     }
 
+    /// <summary>Releases the chunk, the cursor and the own store. Call before the mixer store is disposed.</summary>
     public static void Clear()
     {
-        AudioClipData = null;
+        if (_source != null)
-        TotalSamples = 0;
-        Channels = 0;
-        Frequency = 0;
-        lock (CachedSpectrums)
         {
-            _cachedFrame = -1;
+            _source.Release(_view);
-            CachedSpectrums.Clear();
+            if (_cursor != null) _source.RemoveCursor(_cursor);
         }
+        _view = default;
+        _cursor = null;
+        _source = null;
+        _store = null;
+        _ownStore?.Dispose();
+        _ownStore = null;
+        _loadErrorLogged = false;
+        _cachedFrame = -1;
+        CachedSpectrums.Clear();
     }
 
     public static void GetSpectrumData(float[] samples, int channel, FFTWindow window)
     {
-        if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song)
-        {
-            Array.Clear(samples, 0, samples.Length);
-            return;
-        }
-
-        var conductor = scrConductor.instance;
-        var pitch = conductor.song.pitch;
-        var separateCountdownOffset = conductor.separateCountdownTime ? 
-            conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
-
-        // Calculate the actual scheduled start time of the song
-        var songScheduledStartTime = conductor.skipOffset ?
-            conductor.dspTimeSong + conductor.addoffset / pitch :
-            conductor.dspTimeSong + separateCountdownOffset / pitch;
-
-        if (conductor.dspTime < songScheduledStartTime)
-        {
-            Array.Clear(samples, 0, samples.Length);
-            return;
-        }
-
-        var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset;
-
         var n = samples.Length;
-        if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2
+        var conductor = scrConductor.instance;
+        if (_source == null || _store == null || !conductor || !conductor!.hasSongStarted || !conductor.song
+            || n == 0 || (n & (n - 1)) != 0)
         {
-            Array.Clear(samples, 0, samples.Length);
+            Array.Clear(samples, 0, n);
             return;
         }
 
-        var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1;
+        var render = RenderManager.Instance;
-
+        var currentFrame = render.frameCount;
-        lock (CachedSpectrums)
+        if (currentFrame != _cachedFrame)
         {
-            if (currentFrame != _cachedFrame)
+            _cachedFrame = currentFrame;
-            {
+            CachedSpectrums.Clear();
-                _cachedFrame = currentFrame;
-                CachedSpectrums.Clear();
-            }
-
-            if (CachedSpectrums.TryGetValue(n, out var cached))
-            {
-                Array.Copy(cached, samples, n);
-                return;
-            }
         }
 
+        var sourceChannel = _source.Channels == 1 ? 0 : Math.Min(Math.Max(channel, 0), _source.Channels - 1);
+        if (CachedSpectrums.TryGetValue((n, sourceChannel), out var cached))
+        {
+            Array.Copy(cached, samples, n);
+            return;
+        }
+
+        // Song position (seconds of song time) at the current render time, as in scrConductor.StartMusicCo:
+        // the song starts at dspTimeSong + separate countdown (crotchetAtStart * adjustedCountdownTicks in song time).
+        double pitch = conductor.song.pitch;
+        double volume = conductor.song.volume;
+        var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
+        var now = render.dspTimeBase + render.AudioTime;
+        var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown;
+
         var fftSize = 2 * n;
-        var frequency = Frequency;
+        var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0);
-        var totalSamples = TotalSamples;
+        var first = songSeconds * _store.OutputRate - n * pitch;
-        var currentSampleIndex = (long)(songTime * frequency);
 
         var windowSamples = new float[fftSize];
-        for (var i = 0; i < fftSize; i++)
+        try
         {
-            var idx = currentSampleIndex - fftSize / 2 + i;
+            _cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1);
-            if (idx >= 0 && idx < totalSamples)
+            for (var i = 0; i < fftSize; i++)
             {
-                windowSamples[i] = AudioClipData[idx];
+                var position = first + i * pitch;
+                var index = (long)Math.Floor(position);
+                var fraction = position - index;
+                var s0 = ReadSample(index, sourceChannel);
+                var s1 = fraction > 0 ? ReadSample(index + 1, sourceChannel) : 0.0;
+                windowSamples[i] = (float)((s0 + (s1 - s0) * fraction) * gain);
             }
-            else
+        }
+        catch (InvalidOperationException e)
+        {
+            if (!_loadErrorLogged)
             {
-                windowSamples[i] = 0f;
+                _loadErrorLogged = true;
+                Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Song samples are unavailable, the spectrum will be empty: {e.Message}");
             }
+            Array.Clear(samples, 0, n);
+            return;
         }
 
         var spectrum = new float[n];
         CalculateFFT(windowSamples, spectrum);
+        CachedSpectrums[(n, sourceChannel)] = spectrum;
+        Array.Copy(spectrum, samples, n);
+    }
 
-        lock (CachedSpectrums)
+    /// <summary>Sample of source channel <paramref name="channel"/> at <paramref name="frame"/>; 0 outside the clip or when the source failed.</summary>
+    /// <exception cref="InvalidOperationException">Chunk loading failed fatally (see <see cref="AudioClipStore.Acquire"/>).</exception>
+    private static double ReadSample(long frame, int channel)
+    {
+        var source = _source!;
+        if (frame < 0 || frame >= source.TotalFrames) return 0.0;
+        if (!_view.IsValid || !_view.Contains(frame))
         {
-            CachedSpectrums[n] = spectrum;
+            source.Release(_view);
-            Array.Copy(spectrum, samples, n);
+            _view = _store!.Acquire(source, frame);
+            if (!_view.IsValid) return 0.0;
         }
+        return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel];
     }
 
     private static void CalculateFFT(float[] samples, float[] spectrum)
     {
         var n = samples.Length;
 
-        // Apply Blackman-Harris window
+        // Blackman-Harris window, periodic form: w(i) = a0 - a1 cos(2πi/n) + a2 cos(4πi/n) - a3 cos(6πi/n), i = 0 ~ n - 1
         const float a0 = 0.35875f;
         const float a1 = 0.48829f;
         const float a2 = 0.14128f;
@@ -169,7 +192,7 @@ internal static class AudioSpectrumGenerator
 
         for (var i = 0; i < n; i++)
         {
-            var angle = 2.0 * Math.PI * i / (n - 1);
+            var angle = 2.0 * Math.PI * i / n;
             var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle));
             re[i] = samples[i] * w;
             im[i] = 0f;
@@ -230,12 +253,13 @@ internal static class AudioSpectrumGenerator
             }
         }
 
-        // Normalize and fill spectrum
+        // Magnitude scale 2.5 / n (n = FFT size, twice the spectrum length), clamped to 1.0: the scale of Unity GetSpectrumData.
-        var factor = 2.0f / n;
+        // A sine of amplitude A centered on a bin gives A * a0 * 1.25 with the Blackman-Harris window.
+        var factor = 2.5f / n;
         var outLen = Math.Min(spectrum.Length, n / 2);
         for (var i = 0; i < outLen; i++)
         {
-            spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor;
+            spectrum[i] = Math.Min((float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor, 1f);
         }
     }
-}
+}

Util/Audio/OfflineAudioMixer.cs

@@ -107,6 +107,9 @@ internal sealed class OfflineAudioMixer : IDisposable
         _store.Start();
     }
 
+    /// <summary>Clip store of this render (output rate = mixer sample rate), also read by <see cref="AudioSpectrumGenerator"/>.</summary>
+    internal AudioClipStore Store => _store;
+
     // ── Main thread ──
 
     /// <summary>Registers the song clip. The song voice is created and kept in sync by <see cref="UpdateBGM"/>.</summary>