ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Switch Render Audio To Voice Mixer

Browse Source
This commit is contained in:
copecone committed 2026-10-13 11:30:57 +09:00
1 parent 3d2ffe9dfe
commit cb0fd1bd96
8 files changed
+351 -541
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Core/RenderManager.cs
+18 -8
View File
@@ -146,6 +146,9 @@ internal class RenderManager : MonoBehaviour
public int frameCountBase;
public OfflineAudioMixer? AudioMixer { get; private set; }
/// <summary>Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands.</summary>
public double AudioTime => frameCount / Configuration.RenderFPS;
private AudioSyncCalculator? SyncCalculator { get; set; }
private FrameReader? _frameReader;
@@ -412,7 +415,16 @@ internal class RenderManager : MonoBehaviour
// ReSharper disable once Unity.IncorrectMethodSignature
public void OnPostFrame(RenderTexture? texture)
{
if (Instance.stage < RenderStage.Render || !texture) return;
if (Instance.stage < RenderStage.Render) return;
// Audio commands of this frame are queued: update the song voice and let the render thread mix up to the next frame.
if (AudioMixer != null)
{
AudioMixer.UpdateBGM(AudioTime, dspTimeBase);
AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS);
}
if (!texture) return;
// [Backpressure Throttling]
isThrottling = false;
@@ -682,7 +694,7 @@ internal class RenderManager : MonoBehaviour
throw;
}
var encoder = videoFFmpeg!;
var encoder = videoFFmpeg;
RenderThread = new Thread(() => RunRenderThread(encoder))
{
IsBackground = true,
@@ -816,17 +828,13 @@ internal class RenderManager : MonoBehaviour
{
if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return;
var frameIdx = (int)(passCount - 1);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
var frameTime = frameIdx / effectiveVideoFps;
var channels = Configuration.AudioConfig.AudioChannels;
var samplesNeeded = syncCalculator.GetNextFrameSamples();
var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
try
{
AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded);
videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
}
finally
@@ -865,6 +873,8 @@ internal class RenderManager : MonoBehaviour
private void StopFFmpeg(bool success)
{
endRequested = true;
if (success) AudioMixer?.Complete();
else AudioMixer?.Cancel();
if (!success)
{
MarkCancelled();
@@ -1011,7 +1021,7 @@ internal class RenderManager : MonoBehaviour
_frameReader?.Dispose();
ResetFrameHookState();
AudioMixer?.Reset(); AudioMixer = null;
AudioMixer?.Dispose(); AudioMixer = null;
SyncCalculator = null;
AudioSpectrumGenerator.Clear();
Patch/Engine/AudioPatches.cs
+6 -6
View File
@@ -50,7 +50,7 @@ public static class AudioManagerPlayPatch
double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
// Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource
RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f, RenderManager.Instance.AudioTime);
}
}
}
@@ -74,7 +74,7 @@ public static class PlayWithEndTimePatch
double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
// Register looping sound event to the offline mixer (default pitch = 1.0f)
mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime);
}
}
return true; // Let the original method execute
@@ -91,9 +91,9 @@ public static class AudioSourcePitchPatch
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
{
var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
{
soundEvent.Pitch = value;
voice.SetPitch(value, RenderManager.Instance.AudioTime);
}
}
}
@@ -109,9 +109,9 @@ public static class AudioSourceSetScheduledEndTimePatch
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio)
{
var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
if (mixer != null && mixer.TryGetVoice(__instance, out var voice))
{
soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase;
voice.SetEndTime(time - RenderManager.Instance.dspTimeBase, RenderManager.Instance.AudioTime);
}
}
}
Util/Audio/AudioResampler.cs
-100
View File
@@ -1,100 +0,0 @@
using System;
namespace ADOFAIRenderer.Util.Audio;
internal static class AudioResampler
{
public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency)
{
// ── Safety Validation ──
if (inputSamples.Length == 0 ||
inputChannels <= 0 || targetChannels <= 0 ||
inputFrequency <= 0 || targetFrequency <= 0)
{
return Array.Empty<float>();
}
// If format is identical, return original samples
if (inputFrequency == targetFrequency && inputChannels == targetChannels)
return inputSamples;
int inputLength = inputSamples.Length / inputChannels;
int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency);
float[] outputSamples = new float[targetLength * targetChannels];
for (int i = 0; i < targetLength; i++)
{
double srcIndex = (double)i * inputFrequency / targetFrequency;
long idxLow = (long)Math.Floor(srcIndex);
long idxHigh = idxLow + 1;
float t = (float)(srcIndex - idxLow);
if (idxHigh >= inputLength) idxHigh = inputLength - 1;
if (idxLow >= inputLength) idxLow = inputLength - 1;
if (targetChannels == 1) // 1. Mono Output
{
float valLow, valHigh;
if (inputChannels == 1) // Mono -> Mono
{
valLow = inputSamples[idxLow];
valHigh = inputSamples[idxHigh];
}
else // Multi -> Mono (Downmix: Average all channels)
{
float sumLow = 0f;
float sumHigh = 0f;
for (int c = 0; c < inputChannels; c++)
{
sumLow += inputSamples[idxLow * inputChannels + c];
sumHigh += inputSamples[idxHigh * inputChannels + c];
}
valLow = sumLow / inputChannels;
valHigh = sumHigh / inputChannels;
}
outputSamples[i] = valLow + t * (valHigh - valLow);
}
else if (targetChannels == 2) // 2. Stereo Output
{
if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right)
{
float valLow = inputSamples[idxLow];
float valHigh = inputSamples[idxHigh];
float val = valLow + t * (valHigh - valLow);
outputSamples[i * 2] = val;
outputSamples[i * 2 + 1] = val;
}
else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels)
{
float valLowL = inputSamples[idxLow * inputChannels];
float valHighL = inputSamples[idxHigh * inputChannels];
outputSamples[i * 2] = valLowL + t * (valHighL - valLowL);
float valLowR = inputSamples[idxLow * inputChannels + 1];
float valHighR = inputSamples[idxHigh * inputChannels + 1];
outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR);
}
}
else // 3. Multi-channel Output (e.g., 5.1 Surround)
{
for (int c = 0; c < targetChannels; c++)
{
float valLow, valHigh;
if (c < inputChannels)
{
valLow = inputSamples[idxLow * inputChannels + c];
valHigh = inputSamples[idxHigh * inputChannels + c];
}
else
{
valLow = inputSamples[idxLow * inputChannels];
valHigh = inputSamples[idxHigh * inputChannels];
}
outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow);
}
}
}
return outputSamples;
}
}
Util/Audio/Filter/Gain.cs
+4 -4
View File
@@ -4,14 +4,14 @@ namespace ADOFAIRenderer.Util.Audio.Filter;
internal class Gain : IAudioFilter
{
public float Level = 0f;
public double Level = 0;
public void Initialize(int sampleRate, int channels) { }
public float ProcessSample(float sample, int channel) =>
sample * (float)Math.Pow(10, Level / 20);
public double ProcessSample(double sample, int channel) =>
sample * Math.Pow(10, Level / 20);
public void ProcessBlock(Span<float> buffer)
public void ProcessBlock(Span<double> buffer)
{
for (var i = 0; i < buffer.Length; i++)
buffer[i] = ProcessSample(buffer[i], 0);
Util/Audio/Filter/HardClip.cs
+3 -3
View File
@@ -8,19 +8,19 @@ internal class HardClip : IAudioFilter
// ReSharper disable once MemberCanBePrivate.Global
// ReSharper disable once FieldCanBeMadeReadOnly.Global
// ReSharper disable once ConvertToConstant.Global
public float Threshold = 1.0f;
public double Threshold = 1.0;
public void Initialize(int sampleRate, int channels) { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public float ProcessSample(float sample, int channel)
public double ProcessSample(double sample, int channel)
{
if (sample > Threshold) return Threshold;
if (sample < -Threshold) return -Threshold;
return sample;
}
public void ProcessBlock(Span<float> buffer)
public void ProcessBlock(Span<double> buffer)
{
for (var i = 0; i < buffer.Length; i++)
buffer[i] = ProcessSample(buffer[i], 0);
Util/Audio/Filter/IAudioFilter.cs
+2 -2
View File
@@ -6,8 +6,8 @@ internal interface IAudioFilter
{
void Initialize(int sampleRate, int channels);
float ProcessSample(float sample, int channel);
void ProcessBlock(Span<float> buffer);
double ProcessSample(double sample, int channel);
void ProcessBlock(Span<double> buffer);
void Reset();
Util/Audio/Filter/RMSCompressor.cs
+53 -53
View File
@@ -6,84 +6,84 @@ namespace ADOFAIRenderer.Util.Audio.Filter;
// ReSharper disable once InconsistentNaming
internal class RMSCompressor : IAudioFilter
{
private float _thresholdDb = -20f;
private float _ratio = 4f;
private float _attackTime = 0.010f; // in seconds
private float _releaseTime = 0.100f; // in seconds
private float _kneeDb = 5f; // in dB
private float _rmsWindowTime = 0.030f; // in seconds
private float _lookAheadTime; // in seconds
private double _thresholdDb = -20;
private double _ratio = 4;
private double _attackTime = 0.010; // in seconds
private double _releaseTime = 0.100; // in seconds
private double _kneeDb = 5; // in dB
private double _rmsWindowTime = 0.030; // in seconds
private double _lookAheadTime; // in seconds
private int _lookAheadSamples;
private int _sampleRate = 44100;
private int _channels = 2;
private float _attackCoef;
private float _releaseCoef;
private float _rmsCoef;
private double _attackCoef;
private double _releaseCoef;
private double _rmsCoef;
// Per-channel state
private float[] _rmsState = Array.Empty<float>(); // running mean-square (MS) state
private float[] _gainState = Array.Empty<float>(); // smoothed gain reduction in dB
private float[][] _delayBuffers = Array.Empty<float[]>(); // circular delay buffers
private double[] _rmsState = Array.Empty<double>(); // running mean-square (MS) state
private double[] _gainState = Array.Empty<double>(); // smoothed gain reduction in dB
private double[][] _delayBuffers = Array.Empty<double[]>(); // circular delay buffers
private int[] _delayWriteIndices = Array.Empty<int>();
public float LookAheadTime
public double LookAheadTime
{
get => _lookAheadTime;
set
{
_lookAheadTime = Math.Max(0f, Math.Min(1.0f, value));
_lookAheadTime = Math.Max(0, Math.Min(1.0, value));
UpdateLookAheadSettings();
}
}
public double Latency => _lookAheadTime;
public float ThresholdDb
public double ThresholdDb
{
get => _thresholdDb;
set => _thresholdDb = value;
}
public float Ratio
public double Ratio
{
get => _ratio;
set => _ratio = Math.Max(1f, value);
set => _ratio = Math.Max(1, value);
}
public float AttackTime
public double AttackTime
{
get => _attackTime;
set
{
_attackTime = Math.Max(0.0001f, value);
_attackTime = Math.Max(0.0001, value);
UpdateAttackCoef();
}
}
public float ReleaseTime
public double ReleaseTime
{
get => _releaseTime;
set
{
_releaseTime = Math.Max(0.0001f, value);
_releaseTime = Math.Max(0.0001, value);
UpdateReleaseCoef();
}
}
public float KneeDb
public double KneeDb
{
get => _kneeDb;
set => _kneeDb = Math.Max(0f, value);
set => _kneeDb = Math.Max(0, value);
}
public float RmsWindowTime
public double RmsWindowTime
{
get => _rmsWindowTime;
set
{
_rmsWindowTime = Math.Max(0.0001f, value);
_rmsWindowTime = Math.Max(0.0001, value);
UpdateRmsCoef();
}
}
@@ -98,8 +98,8 @@ internal class RMSCompressor : IAudioFilter
UpdateRmsCoef();
UpdateLookAheadSettings();
_rmsState = new float[channels];
_gainState = new float[channels];
_rmsState = new double[channels];
_gainState = new double[channels];
Reset();
}
@@ -114,33 +114,33 @@ internal class RMSCompressor : IAudioFilter
{
if (_channels <= 0) return;
_delayBuffers = new float[_channels][];
_delayBuffers = new double[_channels][];
_delayWriteIndices = new int[_channels];
for (int i = 0; i < _channels; i++)
{
_delayBuffers[i] = _lookAheadSamples > 0 ? new float[_lookAheadSamples] : Array.Empty<float>();
_delayBuffers[i] = _lookAheadSamples > 0 ? new double[_lookAheadSamples] : Array.Empty<double>();
_delayWriteIndices[i] = 0;
}
}
private void UpdateAttackCoef()
{
_attackCoef = (float)Math.Exp(-1.0 / (_sampleRate * _attackTime));
_attackCoef = Math.Exp(-1.0 / (_sampleRate * _attackTime));
}
private void UpdateReleaseCoef()
{
_releaseCoef = (float)Math.Exp(-1.0 / (_sampleRate * _releaseTime));
_releaseCoef = Math.Exp(-1.0 / (_sampleRate * _releaseTime));
}
private void UpdateRmsCoef()
{
_rmsCoef = (float)Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime));
_rmsCoef = Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public float ProcessSample(float sample, int channel)
public double ProcessSample(double sample, int channel)
{
if (channel < 0 || channel >= _channels)
{
@@ -149,54 +149,54 @@ internal class RMSCompressor : IAudioFilter
// Calculate running Mean-Square (MS) level
// v_c[n] = alpha * v_c[n-1] + (1 - alpha) * x^2[n]
float sq = sample * sample;
_rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0f - _rmsCoef) * sq;
double sq = sample * sample;
_rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0 - _rmsCoef) * sq;
float rmsSq = Math.Max(0f, _rmsState[channel]);
double rmsSq = Math.Max(0, _rmsState[channel]);
// Convert to dB. 10 * log10(rmsSq) is equivalent to 20 * log10(sqrt(rmsSq))
float rmsDb = 10.0f * (float)Math.Log10(rmsSq + 1e-12f);
double rmsDb = 10.0 * Math.Log10(rmsSq + 1e-12);
// Calculate static envelope curve (gain reduction in dB)
float gStatic = 0f;
if (_kneeDb > 0f)
double gStatic = 0;
if (_kneeDb > 0)
{
if (rmsDb > _thresholdDb + _kneeDb * 0.5f)
if (rmsDb > _thresholdDb + _kneeDb * 0.5)
{
gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb);
gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb);
}
else if (rmsDb >= _thresholdDb - _kneeDb * 0.5f)
else if (rmsDb >= _thresholdDb - _kneeDb * 0.5)
{
float diff = rmsDb - _thresholdDb + _kneeDb * 0.5f;
gStatic = (1f / _ratio - 1f) * diff * diff / (2f * _kneeDb);
double diff = rmsDb - _thresholdDb + _kneeDb * 0.5;
gStatic = (1 / _ratio - 1) * diff * diff / (2 * _kneeDb);
}
}
else
{
if (rmsDb > _thresholdDb)
{
gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb);
gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb);
}
}
// Apply ballistics (attack and release) on the dB gain reduction
float lastGain = _gainState[channel];
float targetGain = gStatic;
double lastGain = _gainState[channel];
double targetGain = gStatic;
// Note: Since targetGain and lastGain are <= 0 (gain reduction in dB),
// targetGain < lastGain means target gain reduction is larger (more negative),
// so we are in the attack phase.
float coef = targetGain < lastGain ? _attackCoef : _releaseCoef;
float currentGain = coef * lastGain + (1f - coef) * targetGain;
double coef = targetGain < lastGain ? _attackCoef : _releaseCoef;
double currentGain = coef * lastGain + (1 - coef) * targetGain;
_gainState[channel] = currentGain;
// Convert back to linear gain multiplier
float linearGain = (float)Math.Pow(10.0, currentGain / 20.0);
double linearGain = Math.Pow(10.0, currentGain / 20.0);
if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0)
{
var buffer = _delayBuffers[channel];
int writeIdx = _delayWriteIndices[channel];
float delayedSample = buffer[writeIdx];
double delayedSample = buffer[writeIdx];
buffer[writeIdx] = sample;
_delayWriteIndices[channel] = (writeIdx + 1) % _lookAheadSamples;
return delayedSample * linearGain;
@@ -205,7 +205,7 @@ internal class RMSCompressor : IAudioFilter
return sample * linearGain;
}
public void ProcessBlock(Span<float> buffer)
public void ProcessBlock(Span<double> buffer)
{
for (var i = 0; i < buffer.Length; i++)
{
Util/Audio/OfflineAudioMixer.cs
+265 -365
View File
@@ -1,416 +1,316 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using ADOFAIRenderer.Util.Audio.Clip;
using ADOFAIRenderer.Util.Audio.Filter;
using ADOFAIRenderer.Util.Audio.Mixer;
using MelonLoader;
using UnityEngine;
namespace ADOFAIRenderer.Util.Audio;
internal class AudioClipCache
/// <summary>
/// Audio of one render: the song and effect sounds as <see cref="Voice"/>s of a <see cref="VoiceMixer"/>, clips from an
/// <see cref="AudioClipStore"/>, followed by the filter chain (RMS compressor → look-ahead limiter → hard clip → -1 dB gain).
///
/// Time: seconds relative to the render start (RenderManager.dspTimeBase); output sample n is at n / sample rate.
/// The limiter delays its output by its look-ahead (<see cref="IAudioFilter.Latency"/>), so output sample n is mixed from
/// voices at sample n + latency, and the filters are primed with samples [0, latency) before the first block.
///
/// Threads:
/// - Main thread: <see cref="InitializeBGM"/>, <see cref="CacheClip"/>, <see cref="RegisterSoundEvent"/>, <see cref="UpdateBGM"/>,
/// source/voice mapping, and <see cref="Commit"/> after each frame (all commands with an earlier time have been queued).
/// - Render thread: <see cref="GenerateAudioForFrame"/> waits until the main thread has committed the end of the samples it mixes,
/// so every command takes effect at its own sample. <see cref="Complete"/> ends the waiting when the main thread stops.
/// </summary>
internal sealed class OfflineAudioMixer : IDisposable
{
private readonly Dictionary<string, CachedClipData> _cache = new();
/// <summary>Gain applied to every clip.</summary>
private const double ClipVolume = 0.5;
public class CachedClipData
{
public float[] Samples = null!; // Pre-resampled to target channels and sample rate
public double Duration;
}
/// <summary>Additional gain of effect sounds (registered with <see cref="RegisterSoundEvent"/>).</summary>
private const double EffectVolume = 2.0 / 3.0;
public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
{
if (string.IsNullOrEmpty(clipName)) return null;
/// <summary>Longest wait for a commit before checking for cancellation again, in milliseconds.</summary>
private const int CommitWaitMs = 50;
lock (_cache)
{
if (_cache.TryGetValue(clipName, out var cached))
return cached;
private readonly int _channels;
private readonly int _sampleRate;
private readonly AudioClipStore _store;
private readonly VoiceMixer _mixer;
private readonly CancellationTokenSource _cancellation = new();
private readonly List<IAudioFilter> _filters = new();
private readonly int _latencySamples;
if (!allowLoad) return null;
// Main thread.
private ConditionalWeakTable<AudioSource, Voice> _sourceVoices = new();
private AudioClipSource? _bgmSource;
private string _bgmKey = "";
private Voice? _bgmVoice;
private double _bgmStart = double.NaN;
private double _bgmPitch = double.NaN;
private double _bgmVolume = double.NaN;
if (!AudioManager.Instance) return null;
// Shared: committed output sample (voices are final before it), guarded by _commitLock.
private readonly object _commitLock = new();
private long _committedSample = long.MinValue;
var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
if (!clip) return null;
try
{
var totalSamples = clip.samples;
var channels = clip.channels;
var rawSamples = new float[totalSamples * channels];
clip.GetData(rawSamples, 0);
// Auto-resample during cache generation
var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
var data = new CachedClipData
{
Samples = resampled,
Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
};
_cache[clipName] = data;
return data;
}
catch (Exception e)
{
MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}");
return null;
}
}
}
public void Clear()
{
lock (_cache)
{
_cache.Clear();
}
}
}
internal class OfflineAudioMixer
{
public class SoundEvent
{
public string ClipName = null!;
public double StartTime; // In seconds relative to dspTimeBase
public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot)
public float Volume;
public float Pitch;
}
private readonly List<SoundEvent> _activeEvents = new();
private ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
private readonly object _lock = new();
// Target Audio Settings
private readonly int _targetChannels;
private readonly int _targetSampleRate;
private const float ClipVolume = .5f;
// Audio Filter Settings
private readonly LinkedList<IAudioFilter> _filters = new();
// BGM Cached Samples (Pre-resampled)
private float[]? _bgmSamples;
private double _bgmDuration;
// Clip Cache
private readonly AudioClipCache _clipCache = new();
private bool _isPrimed;
// Render thread (and Dispose, under _renderLock).
private readonly object _renderLock = new();
private long _nextOutputSample;
private bool _primed;
private double[] _mixBuffer = Array.Empty<double>();
private bool _disposed;
public OfflineAudioMixer(int targetChannels, int targetSampleRate)
{
_targetChannels = targetChannels;
_targetSampleRate = targetSampleRate;
_channels = targetChannels;
_sampleRate = targetSampleRate;
_store = new AudioClipStore(targetSampleRate);
_mixer = new VoiceMixer(_store, targetChannels, _cancellation.Token);
var rmsCompressor = new RMSCompressor
{
ThresholdDb = -10, Ratio = 2,
AttackTime = 0.1f, ReleaseTime = 0.1f,
KneeDb = 10, RmsWindowTime = 0.02f
AttackTime = 0.1, ReleaseTime = 0.1,
KneeDb = 10, RmsWindowTime = 0.02
};
rmsCompressor.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(rmsCompressor);
var limiter = new RMSCompressor
{
ThresholdDb = 0, Ratio = float.PositiveInfinity,
AttackTime = 0f, ReleaseTime = 0.05f,
KneeDb = 0, RmsWindowTime = 0.02f,
LookAheadTime = 0.01f
ThresholdDb = 0, Ratio = double.PositiveInfinity,
AttackTime = 0, ReleaseTime = 0.05,
KneeDb = 0, RmsWindowTime = 0.02,
LookAheadTime = 0.01
};
limiter.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(limiter);
var hardClip = new HardClip();
hardClip.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(hardClip);
_filters.Add(rmsCompressor);
_filters.Add(limiter);
_filters.Add(new HardClip());
_filters.Add(new Gain { Level = -1 });
var truePeakSafetyGain = new Gain { Level = -1f };
truePeakSafetyGain.Initialize(_targetSampleRate, _targetChannels);
_filters.AddLast(truePeakSafetyGain);
double latency = 0;
foreach (var filter in _filters)
{
filter.Initialize(_sampleRate, _channels);
latency += filter.Latency;
}
_latencySamples = (int)Math.Round(latency * _sampleRate);
_store.Start();
}
// ── Main thread ──
/// <summary>Registers the song clip. The song voice is created and kept in sync by <see cref="UpdateBGM"/>.</summary>
public void InitializeBGM(AudioClip? clip)
{
lock (_lock)
if (!clip)
{
if (!clip)
{
_bgmSamples = null;
_bgmDuration = 0;
return;
}
try
{
int originalSamples = clip.samples;
int originalChannels = clip.channels;
float[] rawSamples = new float[originalSamples * originalChannels];
clip.GetData(rawSamples, 0);
// Pre-resample the BGM to the target format
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate);
_bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate);
Melon<RenderMod>.Logger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}");
}
catch (Exception e)
{
Melon<RenderMod>.Logger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}");
_bgmSamples = null;
_bgmDuration = 0;
}
_bgmSource = null;
return;
}
_bgmKey = clip!.name;
_bgmSource = AudioClipResolver.GetOrAdd(_store, _bgmKey, clip);
if (_bgmSource == null)
{
Melon<RenderMod>.Logger.Error($"[OfflineAudioMixer] Song clip '{_bgmKey}' could not be loaded, the song will be silent.");
return;
}
Melon<RenderMod>.Logger.Msg($"[OfflineAudioMixer] Song '{_bgmKey}': {_bgmSource.Channels} ch, {_bgmSource.Duration:F2} s, {_bgmSource.Policy}");
}
public void CacheClip(string clipName)
/// <summary>
/// Updates the song voice from the conductor (start time, pitch) and the level volume.
/// A changed start time restarts the song voice at position (now - start) * pitch; pitch and volume changes take effect at <paramref name="now"/>.
/// Start time: dspTimeSong + separate countdown (crotchetAtStart / pitch * adjustedCountdownTicks when separateCountdownTime),
/// the time scrConductor.StartMusicCo schedules the song at.
/// </summary>
/// <param name="now">Current time, seconds relative to the render start.</param>
/// <param name="dspTimeBase">AudioSettings.dspTime at the render start.</param>
public void UpdateBGM(double now, double dspTimeBase)
{
_clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true);
}
public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
{
lock (_lock)
{
var ev = new SoundEvent
{
ClipName = clipName,
StartTime = startTime,
EndTime = endTime,
Volume = volume,
Pitch = pitch
};
_activeEvents.Add(ev);
if (!source) return;
_sourceMappings.Remove(source);
_sourceMappings.Add(source, ev);
}
}
public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
{
RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
}
public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
{
if (source == null)
{
ev = null!;
return false;
}
lock (_lock)
{
return _sourceMappings.TryGetValue(source, out ev);
}
}
public bool RemoveSourceMapping(AudioSource source)
{
lock (_lock)
{
return _sourceMappings.Remove(source);
}
}
public void Reset()
{
lock (_lock)
{
_activeEvents.Clear();
_sourceMappings = new ConditionalWeakTable<AudioSource, SoundEvent>();
_bgmSamples = null;
_bgmDuration = 0;
_clipCache.Clear();
_isPrimed = false;
foreach (var filter in _filters)
{
filter.Reset();
}
}
}
private void PrimeFilters(double totalLatency, double dspTimeBase)
{
int latencySamples = (int)Math.Round(totalLatency * _targetSampleRate);
if (latencySamples <= 0) return;
// Rent temporary buffer to prime filters with initial audio
float[] tempBuffer = System.Buffers.ArrayPool<float>.Shared.Rent(latencySamples * _targetChannels);
try
{
Array.Clear(tempBuffer, 0, latencySamples * _targetChannels);
// Mix starting from -totalLatency to 0
MixBGM(tempBuffer, latencySamples, -totalLatency, dspTimeBase, totalLatency);
MixEffectSounds(tempBuffer, latencySamples, -totalLatency, totalLatency);
// Feed the buffer into filters to prime circular delay buffers
foreach (var filter in _filters)
{
filter.ProcessBlock(tempBuffer.AsSpan(0, latencySamples * _targetChannels));
}
}
finally
{
System.Buffers.ArrayPool<float>.Shared.Return(tempBuffer);
}
}
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
// mixedBuffer size must be at least samplesNeeded * _targetChannels
Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
lock (_lock)
{
double totalLatency = 0;
foreach (var filter in _filters)
{
totalLatency += filter.Latency;
}
// Prime filters on first frame execution if latency exists
if (!_isPrimed && totalLatency > 0)
{
PrimeFilters(totalLatency, dspTimeBase);
_isPrimed = true;
}
// 1. Mix BGM
MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase, totalLatency);
// 2. Mix Hitsounds/Effect Sounds
MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime, totalLatency);
foreach (var filter in _filters)
filter.ProcessBlock(mixedBuffer.AsSpan(0, samplesNeeded * _targetChannels));
for (var i = 0; i < samplesNeeded * _targetChannels; i++)
{
if (Math.Abs(mixedBuffer[i]) > 1) Melon<RenderMod>.Logger.Msg("Warning Why Audio Sample Is Larger Than 1");
}
}
}
private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase, double totalLatency)
{
if (_bgmSamples == null || _bgmSamples.Length == 0) return;
if (_bgmSource == null) return;
var conductor = scrConductor.instance;
if (!conductor) return;
double pitch = conductor.song.pitch;
var dspTimeSong = conductor.dspTimeSong;
var separateCountdownOffset = conductor.separateCountdownTime
? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks
: 0.0;
var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0;
var start = conductor.dspTimeSong + countdown - dspTimeBase;
var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0;
var volume = levelVolume * ClipVolume;
// BGM Playback start time relative to dspTimeBase
var time1 = dspTimeSong + separateCountdownOffset;
var relativeTime1 = time1 - dspTimeBase;
var adjustedRelativeTime1 = relativeTime1 - totalLatency;
var totalSamples = _bgmSamples.Length / _targetChannels;
for (int s = 0; s < samplesNeeded; s++)
if (_bgmVoice == null || start != _bgmStart)
{
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
double clipTime = (sampleTime - adjustedRelativeTime1) * pitch;
// Song position at time t is (t - start) * pitch, also when the start is already past.
_bgmVoice?.Stop(now);
_bgmVoice = _mixer.Play(_bgmSource, _bgmKey, Math.Max(start, now), volume, pitch, false, now);
if (start < now) _bgmVoice.SetPosition((now - start) * pitch, now);
_bgmStart = start;
_bgmPitch = pitch;
_bgmVolume = volume;
return;
}
if (clipTime >= 0 && clipTime < _bgmDuration)
{
double sampleIdxDouble = clipTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (var c = 0; c < _targetChannels; c++)
{
float valLow = _bgmSamples[idxLow * _targetChannels + c];
float valHigh = _bgmSamples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * (scnGame.instance.levelData.volume * 0.01f)
* ClipVolume;
}
}
if (pitch != _bgmPitch)
{
_bgmVoice.SetPitch(pitch, now);
_bgmPitch = pitch;
}
if (volume != _bgmVolume)
{
_bgmVoice.SetVolume(volume, now);
_bgmVolume = volume;
}
}
private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double totalLatency)
/// <summary>Prepares the source of the game clip <paramref name="clipName"/> (audioLib key) so its first chunk starts loading.</summary>
public void CacheClip(string clipName)
{
for (var i = _activeEvents.Count - 1; i >= 0; i--)
GetOrAddSource(clipName);
}
private AudioClipSource? GetOrAddSource(string clipName)
{
if (string.IsNullOrEmpty(clipName)) return null;
if (_store.TryGet(clipName, out var existing)) return existing;
var clip = AudioManager.Instance ? AudioManager.Instance.FindOrLoadAudioClip(clipName) : null;
return AudioClipResolver.GetOrAdd(_store, clipName, clip);
}
/// <summary>
/// Starts an effect sound at <paramref name="startTime"/>. An <paramref name="endTime"/> of 0 or more makes it loop until then.
/// <paramref name="source"/> (when given) is mapped to the voice for later changes through the AudioSource.
/// </summary>
public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now)
{
var clipSource = GetOrAddSource(clipName);
var loop = endTime >= 0;
var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now);
if (loop) voice.SetEndTime(endTime, now);
if (!source) return;
_sourceVoices.Remove(source!);
_sourceVoices.Add(source!, voice);
}
/// <summary>Voice started for <paramref name="source"/> by <see cref="RegisterSoundEvent"/>.</summary>
public bool TryGetVoice(AudioSource? source, out Voice voice)
{
voice = null!;
return source != null && _sourceVoices.TryGetValue(source, out voice);
}
public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source);
/// <summary>
/// Marks every command with a time before <paramref name="time"/> as queued
/// (called after each frame with the time of the next frame).
/// </summary>
public void Commit(double time)
{
var sample = VoiceMixer.ToSample(time, _sampleRate);
lock (_commitLock)
{
var ev = _activeEvents[i];
var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
if (clipData == null)
{
_activeEvents.RemoveAt(i);
continue;
}
var isSoundFinished = false;
var totalSamples = clipData.Samples.Length / _targetChannels;
double adjustedStartTime = ev.StartTime - totalLatency;
double adjustedEndTime = ev.EndTime >= 0 ? ev.EndTime - totalLatency : -1.0;
for (var s = 0; s < samplesNeeded; s++)
{
var sampleTime = frameStartTime + (double)s / _targetSampleRate;
// If it is a looping sound and has reached or exceeded its end time, stop playing.
if (adjustedEndTime >= 0 && sampleTime >= adjustedEndTime)
{
isSoundFinished = true;
break;
}
var elapsed = sampleTime - adjustedStartTime;
if (elapsed < 0) continue; // Sound hasn't started yet
var soundTime = elapsed * ev.Pitch;
if (soundTime >= clipData.Duration)
{
if (ev.EndTime < 0) // One-shot
{
isSoundFinished = true;
break;
}
// Looping
soundTime %= clipData.Duration;
}
// Linear interpolation for pitch shifting in the mixer loop
double sampleIdxDouble = soundTime * _targetSampleRate;
long idxLow = (long)Math.Floor(sampleIdxDouble);
long idxHigh = idxLow + 1;
float t = (float)(sampleIdxDouble - idxLow);
if (idxHigh >= totalSamples) idxHigh = totalSamples - 1;
if (idxLow >= totalSamples) idxLow = totalSamples - 1;
for (var c = 0; c < _targetChannels; c++)
{
float valLow = clipData.Samples[idxLow * _targetChannels + c];
float valHigh = clipData.Samples[idxHigh * _targetChannels + c];
mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume
* ClipVolume * 2 / 3;
}
}
// Clean up finished sound events (both one-shot and looping)
if (isSoundFinished)
{
_activeEvents.RemoveAt(i);
}
if (sample <= _committedSample) return;
_committedSample = sample;
Monitor.PulseAll(_commitLock);
}
}
}
/// <summary>No more commands will be queued (render ending): mixing no longer waits for commits.</summary>
public void Complete()
{
lock (_commitLock)
{
_committedSample = long.MaxValue;
Monitor.PulseAll(_commitLock);
}
}
/// <summary>Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException.</summary>
public void Cancel()
{
_cancellation.Cancel();
Complete();
}
// ── Render thread ──
/// <summary>
/// Writes the next <paramref name="samplesNeeded"/> output frames (interleaved float, continuing from the previous call).
/// </summary>
/// <exception cref="InvalidOperationException">Chunk loading failed fatally (out of memory or the decoder thread stopped).</exception>
/// <exception cref="OperationCanceledException">The render was canceled (<see cref="Cancel"/>).</exception>
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded)
{
lock (_renderLock)
{
if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer));
if (!_primed)
{
_primed = true;
if (_latencySamples > 0)
{
var prime = MixBlock(0, _latencySamples);
foreach (var filter in _filters)
filter.ProcessBlock(prime);
}
}
var block = MixBlock(_nextOutputSample + _latencySamples, samplesNeeded);
foreach (var filter in _filters)
filter.ProcessBlock(block);
_nextOutputSample += samplesNeeded;
for (var i = 0; i < block.Length; i++)
mixedBuffer[i] = (float)block[i];
}
}
/// <summary>Mixes voices at samples [first, first + frames) into the reused double buffer, after the main thread committed them.</summary>
private Span<double> MixBlock(long first, int frames)
{
WaitForCommit(first + frames);
var length = frames * _channels;
if (_mixBuffer.Length < length) _mixBuffer = new double[length];
var block = _mixBuffer.AsSpan(0, length);
block.Clear();
_mixer.Mix(first, block);
return block;
}
private void WaitForCommit(long endSample)
{
lock (_commitLock)
{
while (_committedSample < endSample)
{
_cancellation.Token.ThrowIfCancellationRequested();
Monitor.Wait(_commitLock, CommitWaitMs);
}
}
_cancellation.Token.ThrowIfCancellationRequested();
}
/// <summary>
/// Cancels waits, then releases the voices, the clip store (decoder thread) and the filter state.
/// Waits for a running <see cref="GenerateAudioForFrame"/> to return.
/// </summary>
public void Dispose()
{
Cancel();
lock (_renderLock)
{
if (_disposed) return;
_disposed = true;
_mixer.Dispose();
_store.Dispose();
foreach (var filter in _filters)
filter.Reset();
}
_sourceVoices = new ConditionalWeakTable<AudioSource, Voice>();
}
}