SHA256
Add First Prototype of Audio Render Feature
This commit is contained in:
4 files changed
+493
-8
No files matched your search
@@ -0,0 +1,379 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
|
||||
namespace ADOFAIRenderer.Util
|
||||
{
|
||||
public static class AudioResampler
|
||||
{
|
||||
public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
|
||||
{
|
||||
// ── Safety Validation ──
|
||||
if (inputSamples.Length == 0 ||
|
||||
inputChannels <= 0 || targetChannels <= 0 ||
|
||||
inputFrequency <= 0 || targetFrequency <= 0)
|
||||
{
|
||||
return Array.Empty<float>();
|
||||
}
|
||||
|
||||
// If format is identical, return original samples
|
||||
if (inputFrequency == targetFrequency && inputChannels == targetChannels)
|
||||
return inputSamples;
|
||||
|
||||
int inputLength = inputSamples.Length / inputChannels;
|
||||
int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
|
||||
float[] outputSamples = new float[targetLength * targetChannels];
|
||||
|
||||
for (int i = 0; i < targetLength; i++)
|
||||
{
|
||||
double srcIndex = (double)i * inputFrequency / targetFrequency;
|
||||
long idxLow = (long)Math.Floor(srcIndex);
|
||||
long idxHigh = idxLow + 1;
|
||||
float t = (float)(srcIndex - idxLow);
|
||||
|
||||
if (idxHigh >= inputLength) idxHigh = inputLength - 1;
|
||||
if (idxLow >= inputLength) idxLow = inputLength - 1;
|
||||
|
||||
if (targetChannels == 1) // 1. Mono Output
|
||||
{
|
||||
float valLow, valHigh;
|
||||
if (inputChannels == 1) // Mono -> Mono
|
||||
{
|
||||
valLow = inputSamples[idxLow];
|
||||
valHigh = inputSamples[idxHigh];
|
||||
}
|
||||
else // Multi -> Mono (Downmix: Average all channels)
|
||||
{
|
||||
float sumLow = 0f;
|
||||
float sumHigh = 0f;
|
||||
for (int c = 0; c < inputChannels; c++)
|
||||
{
|
||||
sumLow += inputSamples[idxLow * inputChannels + c];
|
||||
sumHigh += inputSamples[idxHigh * inputChannels + c];
|
||||
}
|
||||
valLow = sumLow / inputChannels;
|
||||
valHigh = sumHigh / inputChannels;
|
||||
}
|
||||
outputSamples[i] = valLow + t * (valHigh - valLow);
|
||||
}
|
||||
else if (targetChannels == 2) // 2. Stereo Output
|
||||
{
|
||||
if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
|
||||
{
|
||||
float valLow = inputSamples[idxLow];
|
||||
float valHigh = inputSamples[idxHigh];
|
||||
float val = valLow + t * (valHigh - valLow);
|
||||
outputSamples[i * 2] = val;
|
||||
outputSamples[i * 2 + 1] = val;
|
||||
}
|
||||
else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
|
||||
{
|
||||
float valLowL = inputSamples[idxLow * inputChannels];
|
||||
float valHighL = inputSamples[idxHigh * inputChannels];
|
||||
outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
|
||||
|
||||
float valLowR = inputSamples[idxLow * inputChannels + 1];
|
||||
float valHighR = inputSamples[idxHigh * inputChannels + 1];
|
||||
outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
|
||||
}
|
||||
}
|
||||
else // 3. Multi-channel Output (e.g., 5.1 Surround)
|
||||
{
|
||||
for (int c = 0; c < targetChannels; c++)
|
||||
{
|
||||
float valLow, valHigh;
|
||||
if (c < inputChannels)
|
||||
{
|
||||
valLow = inputSamples[idxLow * inputChannels + c];
|
||||
valHigh = inputSamples[idxHigh * inputChannels + c];
|
||||
}
|
||||
else
|
||||
{
|
||||
valLow = inputSamples[idxLow * inputChannels];
|
||||
valHigh = inputSamples[idxHigh * inputChannels];
|
||||
}
|
||||
outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return outputSamples;
|
||||
}
|
||||
}
|
||||
|
||||
public static class AudioClipCache
|
||||
{
|
||||
private static readonly Dictionary<string, CachedClipData> Cache = new();
|
||||
|
||||
public class CachedClipData
|
||||
{
|
||||
public float[] Samples = null!; // Pre-resampled to target channels and sample rate
|
||||
public double Duration;
|
||||
}
|
||||
|
||||
public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
|
||||
{
|
||||
if (string.IsNullOrEmpty(clipName)) return null;
|
||||
|
||||
lock (Cache)
|
||||
{
|
||||
if (Cache.TryGetValue(clipName, out var cached))
|
||||
return cached;
|
||||
|
||||
if (!allowLoad) return null;
|
||||
|
||||
if (AudioManager.Instance == null) return null;
|
||||
|
||||
AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
|
||||
if (clip == null) return null;
|
||||
|
||||
try
|
||||
{
|
||||
int totalSamples = clip.samples;
|
||||
int channels = clip.channels;
|
||||
float[] rawSamples = new float[totalSamples * channels];
|
||||
clip.GetData(rawSamples, 0);
|
||||
|
||||
// Auto-resample during cache generation
|
||||
float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
|
||||
|
||||
var data = new CachedClipData
|
||||
{
|
||||
Samples = resampled,
|
||||
Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
|
||||
};
|
||||
|
||||
Cache[clipName] = data;
|
||||
return data;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
MelonLoader.MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void Clear()
|
||||
{
|
||||
lock (Cache)
|
||||
{
|
||||
Cache.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class OfflineAudioMixer
|
||||
{
|
||||
public struct SoundEvent
|
||||
{
|
||||
public string ClipName;
|
||||
public double StartTime; // In seconds relative to dspTimeBase
|
||||
public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot)
|
||||
public float Volume;
|
||||
public float Pitch;
|
||||
}
|
||||
|
||||
private static readonly List<SoundEvent> ActiveEvents = new();
|
||||
private static readonly object Lock = new();
|
||||
|
||||
// Target Audio Settings
|
||||
private static int _targetChannels = 2;
|
||||
private static int _targetSampleRate = 44100;
|
||||
|
||||
// BGM Cached Samples (Pre-resampled)
|
||||
private static float[]? _bgmSamples;
|
||||
private static double _bgmDuration;
|
||||
|
||||
public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate)
|
||||
{
|
||||
lock (Lock)
|
||||
{
|
||||
_targetChannels = targetChannels;
|
||||
_targetSampleRate = targetSampleRate;
|
||||
|
||||
if (!clip)
|
||||
{
|
||||
_bgmSamples = null;
|
||||
_bgmDuration = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
int originalSamples = clip.samples;
|
||||
int originalChannels = clip.channels;
|
||||
float[] rawSamples = new float[originalSamples * originalChannels];
|
||||
clip.GetData(rawSamples, 0);
|
||||
|
||||
// Pre-resample the BGM to the target format
|
||||
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate);
|
||||
_bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate);
|
||||
|
||||
MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
MelonLoader.MelonLogger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}");
|
||||
_bgmSamples = null;
|
||||
_bgmDuration = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
|
||||
{
|
||||
lock (Lock)
|
||||
{
|
||||
ActiveEvents.Add(new SoundEvent
|
||||
{
|
||||
ClipName = clipName,
|
||||
StartTime = startTime,
|
||||
EndTime = endTime,
|
||||
Volume = volume,
|
||||
Pitch = pitch
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public static void Reset()
|
||||
{
|
||||
lock (Lock)
|
||||
{
|
||||
ActiveEvents.Clear();
|
||||
_bgmSamples = null;
|
||||
_bgmDuration = 0;
|
||||
AudioClipCache.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
|
||||
{
|
||||
// mixedBuffer size must be at least samplesNeeded * _targetChannels
|
||||
Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
|
||||
|
||||
lock (Lock)
|
||||
{
|
||||
// 1. Mix BGM
|
||||
MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase);
|
||||
|
||||
// 2. Mix Hitsounds/Effect Sounds
|
||||
MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime);
|
||||
|
||||
// 3. Saturation Guard (Clamping)
|
||||
for (int i = 0; i < samplesNeeded * _targetChannels; i++)
|
||||
{
|
||||
if (mixedBuffer[i] > 1.0f) mixedBuffer[i] = 1.0f;
|
||||
else if (mixedBuffer[i] < -1.0f) mixedBuffer[i] = -1.0f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
|
||||
{
|
||||
if (_bgmSamples == null || _bgmSamples.Length == 0) return;
|
||||
|
||||
var conductor = scrConductor.instance;
|
||||
if (!conductor) return;
|
||||
|
||||
double pitch = conductor.song.pitch;
|
||||
double dspTimeSong = conductor.dspTimeSong;
|
||||
double separateCountdownOffset = conductor.separateCountdownTime
|
||||
? (conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks)
|
||||
: 0.0;
|
||||
|
||||
// BGM Playback start time relative to dspTimeBase
|
||||
double time1 = dspTimeSong + separateCountdownOffset;
|
||||
double relativeTime1 = time1 - dspTimeBase;
|
||||
|
||||
int totalSamples = _bgmSamples.Length / _targetChannels;
|
||||
|
||||
for (int s = 0; s < samplesNeeded; s++)
|
||||
{
|
||||
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
|
||||
double clipTime = (sampleTime - relativeTime1) * pitch;
|
||||
|
||||
if (clipTime >= 0 && clipTime < _bgmDuration)
|
||||
{
|
||||
double sampleIdxDouble = clipTime * _targetSampleRate;
|
||||
long idxLow = (long)Math.Floor(sampleIdxDouble);
|
||||
long idxHigh = idxLow + 1;
|
||||
float t = (float)(sampleIdxDouble - idxLow);
|
||||
|
||||
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
|
||||
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
|
||||
|
||||
for (int c = 0; c < _targetChannels; c++)
|
||||
{
|
||||
float valLow = _bgmSamples[idxLow * _targetChannels + c];
|
||||
float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
|
||||
mixedBuffer[s * _targetChannels + c] += valLow + t * (valHigh - valLow);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
|
||||
{
|
||||
for (int i = ActiveEvents.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var ev = ActiveEvents[i];
|
||||
var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
|
||||
if (clipData == null) continue;
|
||||
|
||||
bool isSoundFinished = false;
|
||||
int totalSamples = clipData.Samples.Length / _targetChannels;
|
||||
|
||||
for (int s = 0; s < samplesNeeded; s++)
|
||||
{
|
||||
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
|
||||
double elapsed = sampleTime - ev.StartTime;
|
||||
|
||||
if (elapsed < 0) continue; // Sound hasn't started yet
|
||||
|
||||
double soundTime = elapsed * ev.Pitch;
|
||||
|
||||
if (soundTime >= clipData.Duration)
|
||||
{
|
||||
if (ev.EndTime < 0) // One-shot
|
||||
{
|
||||
isSoundFinished = true;
|
||||
break;
|
||||
}
|
||||
else // Looping
|
||||
{
|
||||
if (sampleTime >= ev.EndTime)
|
||||
{
|
||||
isSoundFinished = true;
|
||||
break;
|
||||
}
|
||||
soundTime %= clipData.Duration;
|
||||
}
|
||||
}
|
||||
|
||||
// Linear interpolation for pitch shifting in the mixer loop
|
||||
double sampleIdxDouble = soundTime * _targetSampleRate;
|
||||
long idxLow = (long)Math.Floor(sampleIdxDouble);
|
||||
long idxHigh = idxLow + 1;
|
||||
float t = (float)(sampleIdxDouble - idxLow);
|
||||
|
||||
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
|
||||
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
|
||||
|
||||
for (int c = 0; c < _targetChannels; c++)
|
||||
{
|
||||
float valLow = clipData.Samples[idxLow * _targetChannels + c];
|
||||
float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
|
||||
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up finished one-shot sounds
|
||||
if (isSoundFinished && ev.EndTime < 0)
|
||||
{
|
||||
ActiveEvents.RemoveAt(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user