diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index 3b94580..c53b3c1 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -592,19 +592,19 @@ internal class RenderManager : MonoBehaviour scrController.instance.levelWasSkipped = true; if (!scrConductor.instance || !scrConductor.instance.song) yield break; - if (Configuration.AdvancedConfig.ProcessAudio) - { - AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip); - } - if (Configuration.AudioConfig.RenderAudio) { AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate); AudioMixer.InitializeBGM(scrConductor.instance.song.clip); - + var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps); } + + if (Configuration.AdvancedConfig.ProcessAudio) + { + AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer); + } scrConductor.instance.song.clip = null; scrConductor.instance.song.time = -1000; @@ -1021,10 +1021,10 @@ internal class RenderManager : MonoBehaviour _frameReader?.Dispose(); ResetFrameHookState(); + AudioSpectrumGenerator.Clear(); AudioMixer?.Dispose(); AudioMixer = null; SyncCalculator = null; - - AudioSpectrumGenerator.Clear(); + RenderContext.Reset(); } diff --git a/Util/Audio/AudioSpectrumGenerator.cs b/Util/Audio/AudioSpectrumGenerator.cs index 306334a..d8567db 100644 --- a/Util/Audio/AudioSpectrumGenerator.cs +++ b/Util/Audio/AudioSpectrumGenerator.cs @@ -1,164 +1,187 @@ using System; using System.Collections.Generic; using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util.Audio.Clip; +using MelonLoader; using UnityEngine; namespace ADOFAIRenderer.Util.Audio; +/// +/// Replacement for AudioSource.GetSpectrumData during rendering: the spectrum of the song as the song AudioSource would output it. +/// +/// Signal: the song clip from an at the output sample rate (AudioSettings.outputSampleRate), +/// played at conductor.song.pitch (linear interpolation between frames) and scaled by conductor.song.volume. +/// Unity analyzes the output signal of the source: bin spacing follows the output rate, pitch moves the frequencies, +/// and volume scales the magnitudes (measured with a Unity 6000.3.10f1 test project). +/// Channel: stereo clips use the requested channel (0 = left, 1 = right); a mono clip is scaled by . +/// Window: twice the requested spectrum length in output samples, centered on the current render time. +/// +/// The song source is shared with the store when its output rate matches; otherwise a separate +/// store is created. A cursor at the analysis position keeps the needed chunks loaded. Main thread only. +/// internal static class AudioSpectrumGenerator { - // ReSharper disable MemberCanBePrivate.Global - public static float[]? AudioClipData { get; private set; } - public static int Channels { get; private set; } - public static int Frequency { get; private set; } - public static int TotalSamples { get; private set; } - // ReSharper restore MemberCanBePrivate.Global + /// + /// Gain of a mono clip relative to one channel of a stereo clip with the same samples. + /// Measured with a Unity 6000.3.10f1 test project: the spectrum of a mono clip is about 0.70 times that of each stereo channel (1/√2). + /// + private const double MonoGain = 0.70710678118654752; + + private static AudioClipStore? _ownStore; + private static AudioClipStore? _store; + private static AudioClipSource? _source; + private static AudioCursor? _cursor; + private static ChunkView _view; + private static bool _loadErrorLogged; private static long _cachedFrame = -1; - private static readonly Dictionary CachedSpectrums = new(); + private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new(); - public static void Initialize(AudioClip? clip) + /// Registers the song clip. Call after the is created (null when audio is not rendered). + public static void Initialize(AudioClip? clip, OfflineAudioMixer? mixer) { - if (!clip) - { - Clear(); - return; - } + Clear(); + if (!clip) return; try { - Channels = clip.channels; - Frequency = clip.frequency; - int samplesCount = clip.samples; - - float[] rawSamples = new float[samplesCount * Channels]; - clip.GetData(rawSamples, 0); - - if (Channels == 1) + var rate = AudioSettings.outputSampleRate; + if (mixer != null && mixer.Store.OutputRate == rate) { - AudioClipData = rawSamples; - TotalSamples = samplesCount; + _store = mixer.Store; } else { - AudioClipData = new float[samplesCount]; - TotalSamples = samplesCount; - for (int i = 0; i < samplesCount; i++) - { - float sum = 0f; - for (int c = 0; c < Channels; c++) - { - sum += rawSamples[i * Channels + c]; - } - AudioClipData[i] = sum / Channels; - } + _ownStore = new AudioClipStore(rate); + _ownStore.Start(); + _store = _ownStore; } - MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}"); + + _source = AudioClipResolver.GetOrAdd(_store, clip!.name, clip); + if (_source == null) + { + Melon.Logger.Error($"[AudioSpectrumGenerator] Song clip '{clip.name}' could not be loaded, the spectrum will be empty."); + return; + } + _cursor = _source.AddCursor(0, false); + Melon.Logger.Msg($"[AudioSpectrumGenerator] Song '{clip.name}': {_source.Channels} ch at {rate} Hz ({(_ownStore != null ? "own store" : "shared with the mixer")})"); } catch (Exception e) { - MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}"); + Melon.Logger.Error($"[AudioSpectrumGenerator] Failed to initialize: {e}"); Clear(); } } + /// Releases the chunk, the cursor and the own store. Call before the mixer store is disposed. public static void Clear() { - AudioClipData = null; - TotalSamples = 0; - Channels = 0; - Frequency = 0; - lock (CachedSpectrums) + if (_source != null) { - _cachedFrame = -1; - CachedSpectrums.Clear(); + _source.Release(_view); + if (_cursor != null) _source.RemoveCursor(_cursor); } + _view = default; + _cursor = null; + _source = null; + _store = null; + _ownStore?.Dispose(); + _ownStore = null; + _loadErrorLogged = false; + _cachedFrame = -1; + CachedSpectrums.Clear(); } public static void GetSpectrumData(float[] samples, int channel, FFTWindow window) { - if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song) - { - Array.Clear(samples, 0, samples.Length); - return; - } - - var conductor = scrConductor.instance; - var pitch = conductor.song.pitch; - var separateCountdownOffset = conductor.separateCountdownTime ? - conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0; - - // Calculate the actual scheduled start time of the song - var songScheduledStartTime = conductor.skipOffset ? - conductor.dspTimeSong + conductor.addoffset / pitch : - conductor.dspTimeSong + separateCountdownOffset / pitch; - - if (conductor.dspTime < songScheduledStartTime) - { - Array.Clear(samples, 0, samples.Length); - return; - } - - var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset; - var n = samples.Length; - if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2 + var conductor = scrConductor.instance; + if (_source == null || _store == null || !conductor || !conductor!.hasSongStarted || !conductor.song + || n == 0 || (n & (n - 1)) != 0) { - Array.Clear(samples, 0, samples.Length); + Array.Clear(samples, 0, n); return; } - var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1; - - lock (CachedSpectrums) + var render = RenderManager.Instance; + var currentFrame = render.frameCount; + if (currentFrame != _cachedFrame) { - if (currentFrame != _cachedFrame) - { - _cachedFrame = currentFrame; - CachedSpectrums.Clear(); - } - - if (CachedSpectrums.TryGetValue(n, out var cached)) - { - Array.Copy(cached, samples, n); - return; - } + _cachedFrame = currentFrame; + CachedSpectrums.Clear(); } + var sourceChannel = _source.Channels == 1 ? 0 : Math.Min(Math.Max(channel, 0), _source.Channels - 1); + if (CachedSpectrums.TryGetValue((n, sourceChannel), out var cached)) + { + Array.Copy(cached, samples, n); + return; + } + + // Song position (seconds of song time) at the current render time, as in scrConductor.StartMusicCo: + // the song starts at dspTimeSong + separate countdown (crotchetAtStart * adjustedCountdownTicks in song time). + double pitch = conductor.song.pitch; + double volume = conductor.song.volume; + var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0; + var now = render.dspTimeBase + render.AudioTime; + var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown; + var fftSize = 2 * n; - var frequency = Frequency; - var totalSamples = TotalSamples; - var currentSampleIndex = (long)(songTime * frequency); + var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0); + var first = songSeconds * _store.OutputRate - n * pitch; var windowSamples = new float[fftSize]; - for (var i = 0; i < fftSize; i++) + try { - var idx = currentSampleIndex - fftSize / 2 + i; - if (idx >= 0 && idx < totalSamples) + _cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1); + for (var i = 0; i < fftSize; i++) { - windowSamples[i] = AudioClipData[idx]; + var position = first + i * pitch; + var index = (long)Math.Floor(position); + var fraction = position - index; + var s0 = ReadSample(index, sourceChannel); + var s1 = fraction > 0 ? ReadSample(index + 1, sourceChannel) : 0.0; + windowSamples[i] = (float)((s0 + (s1 - s0) * fraction) * gain); } - else + } + catch (InvalidOperationException e) + { + if (!_loadErrorLogged) { - windowSamples[i] = 0f; + _loadErrorLogged = true; + Melon.Logger.Error($"[AudioSpectrumGenerator] Song samples are unavailable, the spectrum will be empty: {e.Message}"); } + Array.Clear(samples, 0, n); + return; } var spectrum = new float[n]; CalculateFFT(windowSamples, spectrum); + CachedSpectrums[(n, sourceChannel)] = spectrum; + Array.Copy(spectrum, samples, n); + } - lock (CachedSpectrums) + /// Sample of source channel at ; 0 outside the clip or when the source failed. + /// Chunk loading failed fatally (see ). + private static double ReadSample(long frame, int channel) + { + var source = _source!; + if (frame < 0 || frame >= source.TotalFrames) return 0.0; + if (!_view.IsValid || !_view.Contains(frame)) { - CachedSpectrums[n] = spectrum; - Array.Copy(spectrum, samples, n); + source.Release(_view); + _view = _store!.Acquire(source, frame); + if (!_view.IsValid) return 0.0; } + return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel]; } private static void CalculateFFT(float[] samples, float[] spectrum) { var n = samples.Length; - // Apply Blackman-Harris window + // Blackman-Harris window, periodic form: w(i) = a0 - a1 cos(2πi/n) + a2 cos(4πi/n) - a3 cos(6πi/n), i = 0 ~ n - 1 const float a0 = 0.35875f; const float a1 = 0.48829f; const float a2 = 0.14128f; @@ -169,7 +192,7 @@ internal static class AudioSpectrumGenerator for (var i = 0; i < n; i++) { - var angle = 2.0 * Math.PI * i / (n - 1); + var angle = 2.0 * Math.PI * i / n; var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle)); re[i] = samples[i] * w; im[i] = 0f; @@ -230,12 +253,13 @@ internal static class AudioSpectrumGenerator } } - // Normalize and fill spectrum - var factor = 2.0f / n; + // Magnitude scale 2.5 / n (n = FFT size, twice the spectrum length), clamped to 1.0: the scale of Unity GetSpectrumData. + // A sine of amplitude A centered on a bin gives A * a0 * 1.25 with the Blackman-Harris window. + var factor = 2.5f / n; var outLen = Math.Min(spectrum.Length, n / 2); for (var i = 0; i < outLen; i++) { - spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor; + spectrum[i] = Math.Min((float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor, 1f); } } -} \ No newline at end of file +} diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 17ee323..bee91d9 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -107,6 +107,9 @@ internal sealed class OfflineAudioMixer : IDisposable _store.Start(); } + /// Clip store of this render (output rate = mixer sample rate), also read by . + internal AudioClipStore Store => _store; + // ── Main thread ── /// Registers the song clip. The song voice is created and kept in sync by .