ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Read Spectrum Samples From Clip Store

Browse Source
This commit is contained in:
copecone committed 2026-10-13 15:20:04 +09:00
1 parent 154c35219f
commit 7ba8944fbe
3 files changed
+136 -109
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Core/RenderManager.cs
+8 -8
View File
@@ -592,19 +592,19 @@ internal class RenderManager : MonoBehaviour
scrController.instance.levelWasSkipped = true;
if (!scrConductor.instance || !scrConductor.instance.song) yield break;
if (Configuration.AdvancedConfig.ProcessAudio)
{
AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
}
if (Configuration.AudioConfig.RenderAudio)
{
AudioMixer = new OfflineAudioMixer(Configuration.AudioConfig.AudioChannels, Configuration.AudioConfig.AudioSampleRate);
AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
SyncCalculator = new AudioSyncCalculator(Configuration.AudioConfig.AudioSampleRate, effectiveVideoFps);
}
if (Configuration.AdvancedConfig.ProcessAudio)
{
AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip, AudioMixer);
}
scrConductor.instance.song.clip = null;
scrConductor.instance.song.time = -1000;
@@ -1021,10 +1021,10 @@ internal class RenderManager : MonoBehaviour
_frameReader?.Dispose();
ResetFrameHookState();
AudioSpectrumGenerator.Clear();
AudioMixer?.Dispose(); AudioMixer = null;
SyncCalculator = null;
AudioSpectrumGenerator.Clear();
RenderContext.Reset();
}
Util/Audio/AudioSpectrumGenerator.cs
+125 -101
View File
@@ -1,164 +1,187 @@
using System;
using System.Collections.Generic;
using ADOFAIRenderer.Core;
using ADOFAIRenderer.Util.Audio.Clip;
using MelonLoader;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio;
/// <summary>
/// Replacement for AudioSource.GetSpectrumData during rendering: the spectrum of the song as the song AudioSource would output it.
///
/// Signal: the song clip from an <see cref="AudioClipStore"/> at the output sample rate (AudioSettings.outputSampleRate),
/// played at conductor.song.pitch (linear interpolation between frames) and scaled by conductor.song.volume.
/// Unity analyzes the output signal of the source: bin spacing follows the output rate, pitch moves the frequencies,
/// and volume scales the magnitudes (measured with a Unity 6000.3.10f1 test project).
/// Channel: stereo clips use the requested channel (0 = left, 1 = right); a mono clip is scaled by <see cref="MonoGain"/>.
/// Window: twice the requested spectrum length in output samples, centered on the current render time.
///
/// The song source is shared with the <see cref="OfflineAudioMixer"/> store when its output rate matches; otherwise a separate
/// store is created. A cursor at the analysis position keeps the needed chunks loaded. Main thread only.
/// </summary>
internal static class AudioSpectrumGenerator
{
// ReSharper disable MemberCanBePrivate.Global
public static float[]? AudioClipData { get; private set; }
public static int Channels { get; private set; }
public static int Frequency { get; private set; }
public static int TotalSamples { get; private set; }
// ReSharper restore MemberCanBePrivate.Global
/// <summary>
/// Gain of a mono clip relative to one channel of a stereo clip with the same samples.
/// Measured with a Unity 6000.3.10f1 test project: the spectrum of a mono clip is about 0.70 times that of each stereo channel (1/√2).
/// </summary>
private const double MonoGain = 0.70710678118654752;
private static AudioClipStore? _ownStore;
private static AudioClipStore? _store;
private static AudioClipSource? _source;
private static AudioCursor? _cursor;
private static ChunkView _view;
private static bool _loadErrorLogged;
private static long _cachedFrame = -1;
private static readonly Dictionary<int, float[]> CachedSpectrums = new();
private static readonly Dictionary<(int Size, int Channel), float[]> CachedSpectrums = new();
public static void Initialize(AudioClip? clip)
/// <summary>Registers the song clip. Call after the <see cref="OfflineAudioMixer"/> is created (null when audio is not rendered).</summary>
public static void Initialize(AudioClip? clip, OfflineAudioMixer? mixer)
{
if (!clip)
{
Clear();
return;
}
Clear();
if (!clip) return;
try
{
Channels = clip.channels;
Frequency = clip.frequency;
int samplesCount = clip.samples;
float[] rawSamples = new float[samplesCount * Channels];
clip.GetData(rawSamples, 0);
if (Channels == 1)
var rate = AudioSettings.outputSampleRate;
if (mixer != null && mixer.Store.OutputRate == rate)
{
AudioClipData = rawSamples;
TotalSamples = samplesCount;
_store = mixer.Store;
}
else
{
AudioClipData = new float[samplesCount];
TotalSamples = samplesCount;
for (int i = 0; i < samplesCount; i++)
{
float sum = 0f;
for (int c = 0; c < Channels; c++)
{
sum += rawSamples[i * Channels + c];
}
AudioClipData[i] = sum / Channels;
}
_ownStore = new AudioClipStore(rate);
_ownStore.Start();
_store = _ownStore;
}
MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}");
_source = AudioClipResolver.GetOrAdd(_store, clip!.name, clip);
if (_source == null)
{
Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Song clip '{clip.name}' could not be loaded, the spectrum will be empty.");
return;
}
_cursor = _source.AddCursor(0, false);
Melon<RenderMod>.Logger.Msg($"[AudioSpectrumGenerator] Song '{clip.name}': {_source.Channels} ch at {rate} Hz ({(_ownStore != null ? "own store" : "shared with the mixer")})");
}
catch (Exception e)
{
MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}");
Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Failed to initialize: {e}");
Clear();
}
}
/// <summary>Releases the chunk, the cursor and the own store. Call before the mixer store is disposed.</summary>
public static void Clear()
{
AudioClipData = null;
TotalSamples = 0;
Channels = 0;
Frequency = 0;
lock (CachedSpectrums)
if (_source != null)
{
_cachedFrame = -1;
CachedSpectrums.Clear();
_source.Release(_view);
if (_cursor != null) _source.RemoveCursor(_cursor);
}
_view = default;
_cursor = null;
_source = null;
_store = null;
_ownStore?.Dispose();
_ownStore = null;
_loadErrorLogged = false;
_cachedFrame = -1;
CachedSpectrums.Clear();
}
public static void GetSpectrumData(float[] samples, int channel, FFTWindow window)
{
if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song)
{
Array.Clear(samples, 0, samples.Length);
return;
}
var conductor = scrConductor.instance;
var pitch = conductor.song.pitch;
var separateCountdownOffset = conductor.separateCountdownTime ?
conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
// Calculate the actual scheduled start time of the song
var songScheduledStartTime = conductor.skipOffset ?
conductor.dspTimeSong + conductor.addoffset / pitch :
conductor.dspTimeSong + separateCountdownOffset / pitch;
if (conductor.dspTime < songScheduledStartTime)
{
Array.Clear(samples, 0, samples.Length);
return;
}
var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset;
var n = samples.Length;
if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2
var conductor = scrConductor.instance;
if (_source == null || _store == null || !conductor || !conductor!.hasSongStarted || !conductor.song
|| n == 0 || (n & (n - 1)) != 0)
{
Array.Clear(samples, 0, samples.Length);
Array.Clear(samples, 0, n);
return;
}
var currentFrame = RenderManager.Instance ? RenderManager.Instance.frameCount : -1;
lock (CachedSpectrums)
var render = RenderManager.Instance;
var currentFrame = render.frameCount;
if (currentFrame != _cachedFrame)
{
if (currentFrame != _cachedFrame)
{
_cachedFrame = currentFrame;
CachedSpectrums.Clear();
}
if (CachedSpectrums.TryGetValue(n, out var cached))
{
Array.Copy(cached, samples, n);
return;
}
_cachedFrame = currentFrame;
CachedSpectrums.Clear();
}
var sourceChannel = _source.Channels == 1 ? 0 : Math.Min(Math.Max(channel, 0), _source.Channels - 1);
if (CachedSpectrums.TryGetValue((n, sourceChannel), out var cached))
{
Array.Copy(cached, samples, n);
return;
}
// Song position (seconds of song time) at the current render time, as in scrConductor.StartMusicCo:
// the song starts at dspTimeSong + separate countdown (crotchetAtStart * adjustedCountdownTicks in song time).
double pitch = conductor.song.pitch;
double volume = conductor.song.volume;
var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
var now = render.dspTimeBase + render.AudioTime;
var songSeconds = (now - conductor.dspTimeSong) * pitch - countdown;
var fftSize = 2 * n;
var frequency = Frequency;
var totalSamples = TotalSamples;
var currentSampleIndex = (long)(songTime * frequency);
var gain = volume * (_source.Channels == 1 ? MonoGain : 1.0);
var first = songSeconds * _store.OutputRate - n * pitch;
var windowSamples = new float[fftSize];
for (var i = 0; i < fftSize; i++)
try
{
var idx = currentSampleIndex - fftSize / 2 + i;
if (idx >= 0 && idx < totalSamples)
_cursor!.Position = Math.Min(Math.Max((long)first, 0), _source.TotalFrames - 1);
for (var i = 0; i < fftSize; i++)
{
windowSamples[i] = AudioClipData[idx];
var position = first + i * pitch;
var index = (long)Math.Floor(position);
var fraction = position - index;
var s0 = ReadSample(index, sourceChannel);
var s1 = fraction > 0 ? ReadSample(index + 1, sourceChannel) : 0.0;
windowSamples[i] = (float)((s0 + (s1 - s0) * fraction) * gain);
}
else
}
catch (InvalidOperationException e)
{
if (!_loadErrorLogged)
{
windowSamples[i] = 0f;
_loadErrorLogged = true;
Melon<RenderMod>.Logger.Error($"[AudioSpectrumGenerator] Song samples are unavailable, the spectrum will be empty: {e.Message}");
}
Array.Clear(samples, 0, n);
return;
}
var spectrum = new float[n];
CalculateFFT(windowSamples, spectrum);
CachedSpectrums[(n, sourceChannel)] = spectrum;
Array.Copy(spectrum, samples, n);
}
lock (CachedSpectrums)
/// <summary>Sample of source channel <paramref name="channel"/> at <paramref name="frame"/>; 0 outside the clip or when the source failed.</summary>
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (see <see cref="AudioClipStore.Acquire"/>).</exception>
private static double ReadSample(long frame, int channel)
{
var source = _source!;
if (frame < 0 || frame >= source.TotalFrames) return 0.0;
if (!_view.IsValid || !_view.Contains(frame))
{
CachedSpectrums[n] = spectrum;
Array.Copy(spectrum, samples, n);
source.Release(_view);
_view = _store!.Acquire(source, frame);
if (!_view.IsValid) return 0.0;
}
return _view.Data[(int)(frame - _view.StartFrame) * _view.Channels + channel];
}
private static void CalculateFFT(float[] samples, float[] spectrum)
{
var n = samples.Length;
// Apply Blackman-Harris window
// Blackman-Harris window, periodic form: w(i) = a0 - a1 cos(2πi/n) + a2 cos(4πi/n) - a3 cos(6πi/n), i = 0 ~ n - 1
const float a0 = 0.35875f;
const float a1 = 0.48829f;
const float a2 = 0.14128f;
@@ -169,7 +192,7 @@ internal static class AudioSpectrumGenerator
for (var i = 0; i < n; i++)
{
var angle = 2.0 * Math.PI * i / (n - 1);
var angle = 2.0 * Math.PI * i / n;
var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle));
re[i] = samples[i] * w;
im[i] = 0f;
@@ -230,12 +253,13 @@ internal static class AudioSpectrumGenerator
}
}
// Normalize and fill spectrum
var factor = 2.0f / n;
// Magnitude scale 2.5 / n (n = FFT size, twice the spectrum length), clamped to 1.0: the scale of Unity GetSpectrumData.
// A sine of amplitude A centered on a bin gives A * a0 * 1.25 with the Blackman-Harris window.
var factor = 2.5f / n;
var outLen = Math.Min(spectrum.Length, n / 2);
for (var i = 0; i < outLen; i++)
{
spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor;
spectrum[i] = Math.Min((float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor, 1f);
}
}
}
}
Util/Audio/OfflineAudioMixer.cs
+3
View File
@@ -107,6 +107,9 @@ internal sealed class OfflineAudioMixer : IDisposable
_store.Start();
}
/// <summary>Clip store of this render (output rate = mixer sample rate), also read by <see cref="AudioSpectrumGenerator"/>.</summary>
internal AudioClipStore Store => _store;
// ── Main thread ──
/// <summary>Registers the song clip. The song voice is created and kept in sync by <see cref="UpdateBGM"/>.</summary>