Add Sync Between AudioSource and SoundEvent

This commit is contained in:
copecone committed 2026-07-09 01:44:59 +09:00
1 parent af8bb439f5
commit 1e7dcd508a
7 files changed
+213 -110

No files matched your search

+101
View File
@@ -0,0 +1,101 @@
using System;
namespace ADOFAIRenderer.Util.Audio
{
public static class AudioResampler
{
public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
{
// ── Safety Validation ──
if (inputSamples.Length == 0 ||
inputChannels <= 0 || targetChannels <= 0 ||
inputFrequency <= 0 || targetFrequency <= 0)
{
return Array.Empty<float>();
}
// If format is identical, return original samples
if (inputFrequency == targetFrequency && inputChannels == targetChannels)
return inputSamples;
int inputLength = inputSamples.Length / inputChannels;
int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
float[] outputSamples = new float[targetLength * targetChannels];
for (int i = 0; i < targetLength; i++)
{
double srcIndex = (double)i * inputFrequency / targetFrequency;
long idxLow = (long)Math.Floor(srcIndex);
long idxHigh = idxLow + 1;
float t = (float)(srcIndex - idxLow);
if (idxHigh >= inputLength) idxHigh = inputLength - 1;
if (idxLow >= inputLength) idxLow = inputLength - 1;
if (targetChannels == 1) // 1. Mono Output
{
float valLow, valHigh;
if (inputChannels == 1) // Mono -> Mono
{
valLow = inputSamples[idxLow];
valHigh = inputSamples[idxHigh];
}
else // Multi -> Mono (Downmix: Average all channels)
{
float sumLow = 0f;
float sumHigh = 0f;
for (int c = 0; c < inputChannels; c++)
{
sumLow += inputSamples[idxLow * inputChannels + c];
sumHigh += inputSamples[idxHigh * inputChannels + c];
}
valLow = sumLow / inputChannels;
valHigh = sumHigh / inputChannels;
}
outputSamples[i] = valLow + t * (valHigh - valLow);
}
else if (targetChannels == 2) // 2. Stereo Output
{
if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
{
float valLow = inputSamples[idxLow];
float valHigh = inputSamples[idxHigh];
float val = valLow + t * (valHigh - valLow);
outputSamples[i * 2] = val;
outputSamples[i * 2 + 1] = val;
}
else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
{
float valLowL = inputSamples[idxLow * inputChannels];
float valHighL = inputSamples[idxHigh * inputChannels];
outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
float valLowR = inputSamples[idxLow * inputChannels + 1];
float valHighR = inputSamples[idxHigh * inputChannels + 1];
outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
}
}
else // 3. Multi-channel Output (e.g., 5.1 Surround)
{
for (int c = 0; c < targetChannels; c++)
{
float valLow, valHigh;
if (c < inputChannels)
{
valLow = inputSamples[idxLow * inputChannels + c];
valHigh = inputSamples[idxHigh * inputChannels + c];
}
else
{
valLow = inputSamples[idxLow * inputChannels];
valHigh = inputSamples[idxHigh * inputChannels];
}
outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
}
}
}
return outputSamples;
}
}
}
+240
View File
@@ -0,0 +1,240 @@
using System;
using System.Collections.Generic;
using ADOFAIRenderer.Core;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio
{
public static class AudioSpectrumGenerator
{
public static float[]? AudioClipData { get; private set; }
public static int Channels { get; private set; }
public static int Frequency { get; private set; }
public static int TotalSamples { get; private set; }
private static long _cachedFrame = -1;
private static readonly Dictionary<int, float[]> CachedSpectrums = new();
public static void Initialize(AudioClip? clip)
{
if (!clip)
{
Clear();
return;
}
try
{
Channels = clip.channels;
Frequency = clip.frequency;
int samplesCount = clip.samples;
float[] rawSamples = new float[samplesCount * Channels];
clip.GetData(rawSamples, 0);
if (Channels == 1)
{
AudioClipData = rawSamples;
TotalSamples = samplesCount;
}
else
{
AudioClipData = new float[samplesCount];
TotalSamples = samplesCount;
for (int i = 0; i < samplesCount; i++)
{
float sum = 0f;
for (int c = 0; c < Channels; c++)
{
sum += rawSamples[i * Channels + c];
}
AudioClipData[i] = sum / Channels;
}
}
MelonLoader.MelonLogger.Msg($"[AudioSpectrumGenerator] Cached audio clip: {clip.name}, Samples: {TotalSamples}, Frequency: {Frequency}Hz, Channels: {Channels}");
}
catch (Exception e)
{
MelonLoader.MelonLogger.Error($"[AudioSpectrumGenerator] Failed to initialize audio cache: {e}");
Clear();
}
}
public static void Clear()
{
AudioClipData = null;
TotalSamples = 0;
Channels = 0;
Frequency = 0;
lock (CachedSpectrums)
{
_cachedFrame = -1;
CachedSpectrums.Clear();
}
}
public static void GetSpectrumData(float[] samples, int channel, FFTWindow window)
{
if (AudioClipData == null || AudioClipData.Length == 0 || !scrConductor.instance || !scrConductor.instance.hasSongStarted || !scrConductor.instance.song)
{
Array.Clear(samples, 0, samples.Length);
return;
}
var conductor = scrConductor.instance;
var pitch = conductor.song.pitch;
var separateCountdownOffset = conductor.separateCountdownTime ?
conductor.crotchetAtStart * conductor.adjustedCountdownTicks : 0.0;
// Calculate the actual scheduled start time of the song
var songScheduledStartTime = conductor.skipOffset ?
conductor.dspTimeSong + conductor.addoffset / pitch :
conductor.dspTimeSong + separateCountdownOffset / pitch;
if (conductor.dspTime < songScheduledStartTime)
{
Array.Clear(samples, 0, samples.Length);
return;
}
var songTime = (conductor.dspTime - conductor.dspTimeSong) * pitch - separateCountdownOffset;
var n = samples.Length;
if (n == 0 || (n & (n - 1)) != 0) // Ensure it's a power of 2
{
Array.Clear(samples, 0, samples.Length);
return;
}
var currentFrame = (RenderManager.Instance != null) ? RenderManager.Instance.frameCount : -1;
lock (CachedSpectrums)
{
if (currentFrame != _cachedFrame)
{
_cachedFrame = currentFrame;
CachedSpectrums.Clear();
}
if (CachedSpectrums.TryGetValue(n, out var cached))
{
Array.Copy(cached, samples, n);
return;
}
}
var fftSize = 2 * n;
var frequency = Frequency;
var totalSamples = TotalSamples;
var currentSampleIndex = (long)(songTime * frequency);
var windowSamples = new float[fftSize];
for (var i = 0; i < fftSize; i++)
{
var idx = currentSampleIndex - fftSize / 2 + i;
if (idx >= 0 && idx < totalSamples)
{
windowSamples[i] = AudioClipData[idx];
}
else
{
windowSamples[i] = 0f;
}
}
var spectrum = new float[n];
CalculateFFT(windowSamples, spectrum);
lock (CachedSpectrums)
{
CachedSpectrums[n] = spectrum;
Array.Copy(spectrum, samples, n);
}
}
private static void CalculateFFT(float[] samples, float[] spectrum)
{
var n = samples.Length;
// Apply Blackman-Harris window
const float a0 = 0.35875f;
const float a1 = 0.48829f;
const float a2 = 0.14128f;
const float a3 = 0.01168f;
var re = new float[n];
var im = new float[n];
for (var i = 0; i < n; i++)
{
var angle = 2.0 * Math.PI * i / (n - 1);
var w = (float)(a0 - a1 * Math.Cos(angle) + a2 * Math.Cos(2.0 * angle) - a3 * Math.Cos(3.0 * angle));
re[i] = samples[i] * w;
im[i] = 0f;
}
// Cooley-Tukey Radix-2 FFT
var m = (int)Math.Round(Math.Log(n, 2));
// Bit-reversal
var j = 0;
for (var i = 0; i < n - 1; i++)
{
if (i < j)
{
var temp = re[i];
re[i] = re[j];
re[j] = temp;
temp = im[i];
im[i] = im[j];
im[j] = temp;
}
var k = n / 2;
while (k <= j)
{
j -= k;
k /= 2;
}
j += k;
}
// FFT stages
for (var stage = 1; stage <= m; stage++)
{
var le = 1 << stage;
var le2 = le / 2;
var ur = 1f;
var ui = 0f;
var wr = (float)Math.Cos(Math.PI / le2);
var wi = -(float)Math.Sin(Math.PI / le2);
for (var sub = 0; sub < le2; sub++)
{
for (var item = sub; item < n; item += le)
{
var ip = item + le2;
var tr = re[ip] * ur - im[ip] * ui;
var ti = re[ip] * ui + im[ip] * ur;
re[ip] = re[item] - tr;
im[ip] = im[item] - ti;
re[item] += tr;
im[item] += ti;
}
var tempUr = ur;
ur = tempUr * wr - ui * wi;
ui = tempUr * wi + ui * wr;
}
}
// Normalize and fill spectrum
var factor = 2.0f / n;
var outLen = Math.Min(spectrum.Length, n / 2);
for (var i = 0; i < outLen; i++)
{
spectrum[i] = (float)Math.Sqrt(re[i] * re[i] + im[i] * im[i]) * factor;
}
}
}
}
+322
View File
@@ -0,0 +1,322 @@
using System;
using System.Collections.Generic;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio
{
public static class AudioClipCache
{
private static readonly Dictionary<string, CachedClipData> Cache = new();
public class CachedClipData
{
public float[] Samples = null!; // Pre-resampled to target channels and sample rate
public double Duration;
}
public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
{
if (string.IsNullOrEmpty(clipName)) return null;
lock (Cache)
{
if (Cache.TryGetValue(clipName, out var cached))
return cached;
if (!allowLoad) return null;
if (AudioManager.Instance == null) return null;
AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
if (clip == null) return null;
try
{
int totalSamples = clip.samples;
int channels = clip.channels;
float[] rawSamples = new float[totalSamples * channels];
clip.GetData(rawSamples, 0);
// Auto-resample during cache generation
float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
var data = new CachedClipData
{
Samples = resampled,
Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
};
Cache[clipName] = data;
return data;
}
catch (Exception e)
{
MelonLoader.MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}");
return null;
}
}
}
public static void Clear()
{
lock (Cache)
{
Cache.Clear();
}
}
}
public static class OfflineAudioMixer
{
public class SoundEvent
{
public string ClipName = null!;
public double StartTime; // In seconds relative to dspTimeBase
public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot)
public float Volume;
public float Pitch;
}
private static readonly List<SoundEvent> ActiveEvents = new();
private static System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
private static readonly object Lock = new();
// Target Audio Settings
private static int _targetChannels = 2;
private static int _targetSampleRate = 44100;
private static readonly float MasterVolume = 0.2f;
// BGM Cached Samples (Pre-resampled)
private static float[]? _bgmSamples;
private static double _bgmDuration;
public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate)
{
lock (Lock)
{
_targetChannels = targetChannels;
_targetSampleRate = targetSampleRate;
if (!clip)
{
_bgmSamples = null;
_bgmDuration = 0;
return;
}
try
{
int originalSamples = clip.samples;
int originalChannels = clip.channels;
float[] rawSamples = new float[originalSamples * originalChannels];
clip.GetData(rawSamples, 0);
// Pre-resample the BGM to the target format
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate);
_bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate);
MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}");
}
catch (Exception e)
{
MelonLoader.MelonLogger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}");
_bgmSamples = null;
_bgmDuration = 0;
}
}
}
public static void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
{
lock (Lock)
{
var ev = new SoundEvent
{
ClipName = clipName,
StartTime = startTime,
EndTime = endTime,
Volume = volume,
Pitch = pitch
};
ActiveEvents.Add(ev);
if (source != null)
{
_sourceMappings.Remove(source);
_sourceMappings.Add(source, ev);
}
}
}
public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
{
RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
}
public static bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
{
if (source == null)
{
ev = null!;
return false;
}
lock (Lock)
{
return _sourceMappings.TryGetValue(source, out ev);
}
}
public static bool RemoveSourceMapping(AudioSource source)
{
lock (Lock)
{
return _sourceMappings.Remove(source);
}
}
public static void Reset()
{
lock (Lock)
{
ActiveEvents.Clear();
_sourceMappings = new System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent>();
_bgmSamples = null;
_bgmDuration = 0;
AudioClipCache.Clear();
}
}
public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
// mixedBuffer size must be at least samplesNeeded * _targetChannels
Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
lock (Lock)
{
// 1. Mix BGM
MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase);
// 2. Mix Hitsounds/Effect Sounds
MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime);
// 3. Saturation Guard (Clamping)
for (int i = 0; i < samplesNeeded * _targetChannels; i++)
{
if (mixedBuffer[i] > 1.0f) mixedBuffer[i] = 1.0f;
else if (mixedBuffer[i] < -1.0f) mixedBuffer[i] = -1.0f;
}
}
}
private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
if (_bgmSamples == null || _bgmSamples.Length == 0) return;
var conductor = scrConductor.instance;
if (!conductor) return;
double pitch = conductor.song.pitch;
double dspTimeSong = conductor.dspTimeSong;
double separateCountdownOffset = conductor.separateCountdownTime
? (conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks)
: 0.0;
// BGM Playback start time relative to dspTimeBase
double time1 = dspTimeSong + separateCountdownOffset;
double relativeTime1 = time1 - dspTimeBase;
int totalSamples = _bgmSamples.Length / _targetChannels;
for (int s = 0; s < samplesNeeded; s++)
{
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
double clipTime = (sampleTime - relativeTime1) * pitch;
if (clipTime >= 0 && clipTime < _bgmDuration)
{
double sampleIdxDouble = clipTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (int c = 0; c < _targetChannels; c++)
{
float valLow = _bgmSamples[idxLow * _targetChannels + c];
float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * MasterVolume;
}
}
}
}
private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
{
for (int i = ActiveEvents.Count - 1; i >= 0; i--)
{
var ev = ActiveEvents[i];
var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
if (clipData == null)
{
ActiveEvents.RemoveAt(i);
continue;
}
bool isSoundFinished = false;
int totalSamples = clipData.Samples.Length / _targetChannels;
for (int s = 0; s < samplesNeeded; s++)
{
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
// If it is a looping sound and has reached or exceeded its end time, stop playing.
if (ev.EndTime >= 0 && sampleTime >= ev.EndTime)
{
isSoundFinished = true;
break;
}
double elapsed = sampleTime - ev.StartTime;
if (elapsed < 0) continue; // Sound hasn't started yet
double soundTime = elapsed * ev.Pitch;
if (soundTime >= clipData.Duration)
{
if (ev.EndTime < 0) // One-shot
{
isSoundFinished = true;
break;
}
// Looping
soundTime %= clipData.Duration;
}
// Linear interpolation for pitch shifting in the mixer loop
double sampleIdxDouble = soundTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (int c = 0; c < _targetChannels; c++)
{
float valLow = clipData.Samples[idxLow * _targetChannels + c];
float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume * MasterVolume;
}
}
// Clean up finished sound events (both one-shot and looping)
if (isSoundFinished)
{
ActiveEvents.RemoveAt(i);
}
}
}
}
}