ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Switch Render Audio To Voice Mixer

Browse Source
This commit is contained in:
copecone committed 2026-10-13 11:30:57 +09:00
1 parent 3d2ffe9dfe
commit cb0fd1bd96
8 files changed
+351 -541
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Util/Audio/OfflineAudioMixer.cs
+265 -365
View File
@@ -1,416 +1,316 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using ADOFAIRenderer.Util.Audio.Clip;
using ADOFAIRenderer.Util.Audio.Filter;
using ADOFAIRenderer.Util.Audio.Mixer;
using MelonLoader;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio;
internal class AudioClipCache
/// <summary>
/// Audio of one render: the song and effect sounds as <see cref="Voice"/>s of a <see cref="VoiceMixer"/>, clips from an
/// <see cref="AudioClipStore"/>, followed by the filter chain (RMS compressor → look-ahead limiter → hard clip → -1 dB gain).
///
/// Time: seconds relative to the render start (RenderManager.dspTimeBase); output sample n is at n / sample rate.
/// The limiter delays its output by its look-ahead (<see cref="IAudioFilter.Latency"/>), so output sample n is mixed from
/// voices at sample n + latency, and the filters are primed with samples [0, latency) before the first block.
///
/// Threads:
/// - Main thread: <see cref="InitializeBGM"/>, <see cref="CacheClip"/>, <see cref="RegisterSoundEvent"/>, <see cref="UpdateBGM"/>,
/// source/voice mapping, and <see cref="Commit"/> after each frame (all commands with an earlier time have been queued).
/// - Render thread: <see cref="GenerateAudioForFrame"/> waits until the main thread has committed the end of the samples it mixes,
/// so every command takes effect at its own sample. <see cref="Complete"/> ends the waiting when the main thread stops.
/// </summary>
internal sealed class OfflineAudioMixer : IDisposable
{
private readonly Dictionary<string, CachedClipData> _cache = new();
/// <summary>Gain applied to every clip.</summary>
private const double ClipVolume = 0.5;
public class CachedClipData
{
public float[] Samples = null!; // Pre-resampled to target channels and sample rate
public double Duration;
}
/// <summary>Additional gain of effect sounds (registered with <see cref="RegisterSoundEvent"/>).</summary>
private const double EffectVolume = 2.0 / 3.0;
public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
{
if (string.IsNullOrEmpty(clipName)) return null;
/// <summary>Longest wait for a commit before checking for cancellation again, in milliseconds.</summary>
private const int CommitWaitMs = 50;
lock (_cache)
{
if (_cache.TryGetValue(clipName, out var cached))
return cached;
private readonly int _channels;
private readonly int _sampleRate;
private readonly AudioClipStore _store;
private readonly VoiceMixer _mixer;
private readonly CancellationTokenSource _cancellation = new();
private readonly List<IAudioFilter> _filters = new();
private readonly int _latencySamples;
if (!allowLoad) return null;
// Main thread.
private ConditionalWeakTable<AudioSource, Voice> _sourceVoices = new();
private AudioClipSource? _bgmSource;
private string _bgmKey = "";
private Voice? _bgmVoice;
private double _bgmStart = double.NaN;
private double _bgmPitch = double.NaN;
private double _bgmVolume = double.NaN;
if (!AudioManager.Instance) return null;
// Shared: committed output sample (voices are final before it), guarded by _commitLock.
private readonly object _commitLock = new();
private long _committedSample = long.MinValue;
var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
if (!clip) return null;
try
{
var totalSamples = clip.samples;
var channels = clip.channels;
var rawSamples = new float[totalSamples * channels];
clip.GetData(rawSamples, 0);
// Auto-resample during cache generation
var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
var data = new CachedClipData
{
Samples = resampled,
Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
};
_cache[clipName] = data;
return data;
}
catch (Exception e)
{
MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}");
return null;
}
}
}
public void Clear()
{
lock (_cache)
{
_cache.Clear();
}
}
}
internal class OfflineAudioMixer
{
public class SoundEvent
{
public string ClipName = null!;
public double StartTime; // In seconds relative to dspTimeBase
public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot)
public float Volume;
public float Pitch;
}
private readonly List<SoundEvent> _activeEvents = new();
private ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
private readonly object _lock = new();
// Target Audio Settings
private readonly int _targetChannels;
private readonly int _targetSampleRate;
private const float ClipVolume = .5f;
// Audio Filter Settings
private readonly LinkedList<IAudioFilter> _filters = new();
// BGM Cached Samples (Pre-resampled)
private float[]? _bgmSamples;
private double _bgmDuration;
// Clip Cache
private readonly AudioClipCache _clipCache = new();
private bool _isPrimed;
// Render thread (and Dispose, under _renderLock).
private readonly object _renderLock = new();
private long _nextOutputSample;
private bool _primed;
private double[] _mixBuffer = Array.Empty<double>();
private bool _disposed;
public OfflineAudioMixer(int targetChannels, int targetSampleRate)
{
_targetChannels = targetChannels;
_targetSampleRate = targetSampleRate;
_channels = targetChannels;
_sampleRate = targetSampleRate;
_store = new AudioClipStore(targetSampleRate);
_mixer = new VoiceMixer(_store, targetChannels, _cancellation.Token);
var rmsCompressor = new RMSCompressor
{
ThresholdDb = -10, Ratio = 2,
AttackTime = 0.1f, ReleaseTime = 0.1f,
KneeDb = 10, RmsWindowTime = 0.02f
AttackTime = 0.1, ReleaseTime = 0.1,
KneeDb = 10, RmsWindowTime = 0.02
};
rmsCompressor.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(rmsCompressor);
var limiter = new RMSCompressor
{
ThresholdDb = 0, Ratio = float.PositiveInfinity,
AttackTime = 0f, ReleaseTime = 0.05f,
KneeDb = 0, RmsWindowTime = 0.02f,
LookAheadTime = 0.01f
ThresholdDb = 0, Ratio = double.PositiveInfinity,
AttackTime = 0, ReleaseTime = 0.05,
KneeDb = 0, RmsWindowTime = 0.02,
LookAheadTime = 0.01
};
limiter.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(limiter);
var hardClip = new HardClip();
hardClip.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(hardClip);
_filters.Add(rmsCompressor);
_filters.Add(limiter);
_filters.Add(new HardClip());
_filters.Add(new Gain { Level = -1 });
var truePeakSafetyGain = new Gain { Level = -1f };
truePeakSafetyGain.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(truePeakSafetyGain);
double latency = 0;
foreach (var filter in _filters)
{
filter.Initialize(_sampleRate, _channels);
latency += filter.Latency;
}
_latencySamples = (int)Math.Round(latency * _sampleRate);
_store.Start();
}
// ── Main thread ──
/// <summary>Registers the song clip. The song voice is created and kept in sync by <see cref="UpdateBGM"/>.</summary>
public void InitializeBGM(AudioClip? clip)
{
lock (_lock)
if (!clip)
{
if (!clip)
{
_bgmSamples = null;
_bgmDuration = 0;
return;
}
try
{
int originalSamples = clip.samples;
int originalChannels = clip.channels;
float[] rawSamples = new float[originalSamples * originalChannels];
clip.GetData(rawSamples, 0);
// Pre-resample the BGM to the target format
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate);
_bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate);
Melon<RenderMod>.Logger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}");
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}");
_bgmSamples = null;
_bgmDuration = 0;
}
_bgmSource = null;
return;
}
_bgmKey = clip!.name;
_bgmSource = AudioClipResolver.GetOrAdd(_store, _bgmKey, clip);
if (_bgmSource == null)
{
Melon<RenderMod>.Logger.Error($"[OfflineAudioMixer] Song clip '{_bgmKey}' could not be loaded, the song will be silent.");
return;
}
Melon<RenderMod>.Logger.Msg($"[OfflineAudioMixer] Song '{_bgmKey}': {_bgmSource.Channels} ch, {_bgmSource.Duration:F2} s, {_bgmSource.Policy}");
}
public void CacheClip(string clipName)
/// <summary>
/// Updates the song voice from the conductor (start time, pitch) and the level volume.
/// A changed start time restarts the song voice at position (now - start) * pitch; pitch and volume changes take effect at <paramref name="now"/>.
/// Start time: dspTimeSong + separate countdown (crotchetAtStart / pitch * adjustedCountdownTicks when separateCountdownTime),
/// the time scrConductor.StartMusicCo schedules the song at.
/// </summary>
/// <param name="now">Current time, seconds relative to the render start.</param>
/// <param name="dspTimeBase">AudioSettings.dspTime at the render start.</param>
public void UpdateBGM(double now, double dspTimeBase)
{
_clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true);
}
public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
{
lock (_lock)
{
var ev = new SoundEvent
{
ClipName = clipName,
StartTime = startTime,
EndTime = endTime,
Volume = volume,
Pitch = pitch
};
_activeEvents.Add(ev);
if (!source) return;
_sourceMappings.Remove(source);
_sourceMappings.Add(source, ev);
}
}
public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
{
RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
}
public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
{
if (source == null)
{
ev = null!;
return false;
}
lock (_lock)
{
return _sourceMappings.TryGetValue(source, out ev);
}
}
public bool RemoveSourceMapping(AudioSource source)
{
lock (_lock)
{
return _sourceMappings.Remove(source);
}
}
public void Reset()
{
lock (_lock)
{
_activeEvents.Clear();
_sourceMappings = new ConditionalWeakTable<AudioSource, SoundEvent>();
_bgmSamples = null;
_bgmDuration = 0;
_clipCache.Clear();
_isPrimed = false;
foreach (var filter in _filters)
{
filter.Reset();
}
}
}
private void PrimeFilters(double totalLatency, double dspTimeBase)
{
int latencySamples = (int)Math.Round(totalLatency * _targetSampleRate);
if (latencySamples <= 0) return;
// Rent temporary buffer to prime filters with initial audio
float[] tempBuffer = System.Buffers.ArrayPool<float>.Shared.Rent(latencySamples * _targetChannels);
try
{
Array.Clear(tempBuffer, 0, latencySamples * _targetChannels);
// Mix starting from -totalLatency to 0
MixBGM(tempBuffer, latencySamples, -totalLatency, dspTimeBase, totalLatency);
MixEffectSounds(tempBuffer, latencySamples, -totalLatency, totalLatency);
// Feed the buffer into filters to prime circular delay buffers
foreach (var filter in _filters)
{
filter.ProcessBlock(tempBuffer.AsSpan(0, latencySamples * _targetChannels));
}
}
finally
{
System.Buffers.ArrayPool<float>.Shared.Return(tempBuffer);
}
}
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
// mixedBuffer size must be at least samplesNeeded * _targetChannels
Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
lock (_lock)
{
double totalLatency = 0;
foreach (var filter in _filters)
{
totalLatency += filter.Latency;
}
// Prime filters on first frame execution if latency exists
if (!_isPrimed && totalLatency > 0)
{
PrimeFilters(totalLatency, dspTimeBase);
_isPrimed = true;
}
// 1. Mix BGM
MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase, totalLatency);
// 2. Mix Hitsounds/Effect Sounds
MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime, totalLatency);
foreach (var filter in _filters)
filter.ProcessBlock(mixedBuffer.AsSpan(0, samplesNeeded * _targetChannels));
for (var i = 0; i < samplesNeeded * _targetChannels; i++)
{
if (Math.Abs(mixedBuffer[i]) > 1) Melon<RenderMod>.Logger.Msg("Warning Why Audio Sample Is Larger Than 1");
}
}
}
private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase, double totalLatency)
{
if (_bgmSamples == null || _bgmSamples.Length == 0) return;
if (_bgmSource == null) return;
var conductor = scrConductor.instance;
if (!conductor) return;
double pitch = conductor.song.pitch;
var dspTimeSong = conductor.dspTimeSong;
var separateCountdownOffset = conductor.separateCountdownTime
? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks
: 0.0;
var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0;
var start = conductor.dspTimeSong + countdown - dspTimeBase;
var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0;
var volume = levelVolume * ClipVolume;
// BGM Playback start time relative to dspTimeBase
var time1 = dspTimeSong + separateCountdownOffset;
var relativeTime1 = time1 - dspTimeBase;
var adjustedRelativeTime1 = relativeTime1 - totalLatency;
var totalSamples = _bgmSamples.Length / _targetChannels;
for (int s = 0; s < samplesNeeded; s++)
if (_bgmVoice == null || start != _bgmStart)
{
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
double clipTime = (sampleTime - adjustedRelativeTime1) * pitch;
// Song position at time t is (t - start) * pitch, also when the start is already past.
_bgmVoice?.Stop(now);
_bgmVoice = _mixer.Play(_bgmSource, _bgmKey, Math.Max(start, now), volume, pitch, false, now);
if (start < now) _bgmVoice.SetPosition((now - start) * pitch, now);
_bgmStart = start;
_bgmPitch = pitch;
_bgmVolume = volume;
return;
}
if (clipTime >= 0 && clipTime < _bgmDuration)
{
double sampleIdxDouble = clipTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (var c = 0; c < _targetChannels; c++)
{
float valLow = _bgmSamples[idxLow * _targetChannels + c];
float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * (scnGame.instance.levelData.volume * 0.01f)
* ClipVolume;
}
}
if (pitch != _bgmPitch)
{
_bgmVoice.SetPitch(pitch, now);
_bgmPitch = pitch;
}
if (volume != _bgmVolume)
{
_bgmVoice.SetVolume(volume, now);
_bgmVolume = volume;
}
}
private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double totalLatency)
/// <summary>Prepares the source of the game clip <paramref name="clipName"/> (audioLib key) so its first chunk starts loading.</summary>
public void CacheClip(string clipName)
{
for (var i = _activeEvents.Count - 1; i >= 0; i--)
GetOrAddSource(clipName);
}
private AudioClipSource? GetOrAddSource(string clipName)
{
if (string.IsNullOrEmpty(clipName)) return null;
if (_store.TryGet(clipName, out var existing)) return existing;
var clip = AudioManager.Instance ? AudioManager.Instance.FindOrLoadAudioClip(clipName) : null;
return AudioClipResolver.GetOrAdd(_store, clipName, clip);
}
/// <summary>
/// Starts an effect sound at <paramref name="startTime"/>. An <paramref name="endTime"/> of 0 or more makes it loop until then.
/// <paramref name="source"/> (when given) is mapped to the voice for later changes through the AudioSource.
/// </summary>
public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
{
var clipSource = GetOrAddSource(clipName);
var loop = endTime >= 0;
var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now);
if (loop) voice.SetEndTime(endTime, now);
if (!source) return;
_sourceVoices.Remove(source!);
_sourceVoices.Add(source!, voice);
}
/// <summary>Voice started for <paramref name="source"/> by <see cref="RegisterSoundEvent"/>.</summary>
public bool TryGetVoice(AudioSource? source, out Voice voice)
{
voice = null!;
return source != null && _sourceVoices.TryGetValue(source, out voice);
}
public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source);
/// <summary>
/// Marks every command with a time before <paramref name="time"/> as queued
/// (called after each frame with the time of the next frame).
/// </summary>
public void Commit(double time)
{
var sample = VoiceMixer.ToSample(time, _sampleRate);
lock (_commitLock)
{
var ev = _activeEvents[i];
var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
if (clipData == null)
{
_activeEvents.RemoveAt(i);
continue;
}
var isSoundFinished = false;
var totalSamples = clipData.Samples.Length / _targetChannels;
double adjustedStartTime = ev.StartTime - totalLatency;
double adjustedEndTime = ev.EndTime >= 0 ? ev.EndTime - totalLatency : -1.0;
for (var s = 0; s < samplesNeeded; s++)
{
var sampleTime = frameStartTime + (double)s / _targetSampleRate;
// If it is a looping sound and has reached or exceeded its end time, stop playing.
if (adjustedEndTime >= 0 && sampleTime >= adjustedEndTime)
{
isSoundFinished = true;
break;
}
var elapsed = sampleTime - adjustedStartTime;
if (elapsed < 0) continue; // Sound hasn't started yet
var soundTime = elapsed * ev.Pitch;
if (soundTime >= clipData.Duration)
{
if (ev.EndTime < 0) // One-shot
{
isSoundFinished = true;
break;
}
// Looping
soundTime %= clipData.Duration;
}
// Linear interpolation for pitch shifting in the mixer loop
double sampleIdxDouble = soundTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (var c = 0; c < _targetChannels; c++)
{
float valLow = clipData.Samples[idxLow * _targetChannels + c];
float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume
* ClipVolume * 2 / 3;
}
}
// Clean up finished sound events (both one-shot and looping)
if (isSoundFinished)
{
_activeEvents.RemoveAt(i);
}
if (sample <= _committedSample) return;
_committedSample = sample;
Monitor.PulseAll(_commitLock);
}
}
}
/// <summary>No more commands will be queued (render ending): mixing no longer waits for commits.</summary>
public void Complete()
{
lock (_commitLock)
{
_committedSample = long.MaxValue;
Monitor.PulseAll(_commitLock);
}
}
/// <summary>Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException.</summary>
public void Cancel()
{
_cancellation.Cancel();
Complete();
}
// ── Render thread ──
/// <summary>
/// Writes the next <paramref name="samplesNeeded"/> output frames (interleaved float, continuing from the previous call).
/// </summary>
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (out of memory or the decoder thread stopped).</exception>
/// <exception cref="OperationCanceledException">The render was canceled (<see cref="Cancel"/>).</exception>
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded)
{
lock (_renderLock)
{
if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer));
if (!_primed)
{
_primed = true;
if (_latencySamples > 0)
{
var prime = MixBlock(0, _latencySamples);
foreach (var filter in _filters)
filter.ProcessBlock(prime);
}
}
var block = MixBlock(_nextOutputSample + _latencySamples, samplesNeeded);
foreach (var filter in _filters)
filter.ProcessBlock(block);
_nextOutputSample += samplesNeeded;
for (var i = 0; i < block.Length; i++)
mixedBuffer[i] = (float)block[i];
}
}
/// <summary>Mixes voices at samples [first, first + frames) into the reused double buffer, after the main thread committed them.</summary>
private Span<double> MixBlock(long first, int frames)
{
WaitForCommit(first + frames);
var length = frames * _channels;
if (_mixBuffer.Length < length) _mixBuffer = new double[length];
var block = _mixBuffer.AsSpan(0, length);
block.Clear();
_mixer.Mix(first, block);
return block;
}
private void WaitForCommit(long endSample)
{
lock (_commitLock)
{
while (_committedSample < endSample)
{
_cancellation.Token.ThrowIfCancellationRequested();
Monitor.Wait(_commitLock, CommitWaitMs);
}
}
_cancellation.Token.ThrowIfCancellationRequested();
}
/// <summary>
/// Cancels waits, then releases the voices, the clip store (decoder thread) and the filter state.
/// Waits for a running <see cref="GenerateAudioForFrame"/> to return.
/// </summary>
public void Dispose()
{
Cancel();
lock (_renderLock)
{
if (_disposed) return;
_disposed = true;
_mixer.Dispose();
_store.Dispose();
foreach (var filter in _filters)
filter.Reset();
}
_sourceVoices = new ConditionalWeakTable<AudioSource, Voice>();
}
}