diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index 9afecb8..3b94580 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -146,6 +146,9 @@ internal class RenderManager : MonoBehaviour public int frameCountBase; public OfflineAudioMixer? AudioMixer { get; private set; } + + /// Render time of the current frame in seconds (frameCount / RenderFPS), the time base of audio commands. + public double AudioTime => frameCount / Configuration.RenderFPS; private AudioSyncCalculator? SyncCalculator { get; set; } private FrameReader? _frameReader; @@ -412,7 +415,16 @@ internal class RenderManager : MonoBehaviour // ReSharper disable once Unity.IncorrectMethodSignature public void OnPostFrame(RenderTexture? texture) { - if (Instance.stage < RenderStage.Render || !texture) return; + if (Instance.stage < RenderStage.Render) return; + + // Audio commands of this frame are queued: update the song voice and let the render thread mix up to the next frame. + if (AudioMixer != null) + { + AudioMixer.UpdateBGM(AudioTime, dspTimeBase); + AudioMixer.Commit((frameCount + 1) / Configuration.RenderFPS); + } + + if (!texture) return; // [Backpressure Throttling] isThrottling = false; @@ -682,7 +694,7 @@ internal class RenderManager : MonoBehaviour throw; } - var encoder = videoFFmpeg!; + var encoder = videoFFmpeg; RenderThread = new Thread(() => RunRenderThread(encoder)) { IsBackground = true, @@ -816,17 +828,13 @@ internal class RenderManager : MonoBehaviour { if (!Configuration.AudioConfig.RenderAudio || SyncCalculator is not { } syncCalculator) return; - var frameIdx = (int)(passCount - 1); - var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); - var frameTime = frameIdx / effectiveVideoFps; - var channels = Configuration.AudioConfig.AudioChannels; var samplesNeeded = syncCalculator.GetNextFrameSamples(); var audioBuffer = ArrayPool.Shared.Rent(samplesNeeded * channels); try { - AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase); + AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded); videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded); } finally @@ -865,6 +873,8 @@ internal class RenderManager : MonoBehaviour private void StopFFmpeg(bool success) { endRequested = true; + if (success) AudioMixer?.Complete(); + else AudioMixer?.Cancel(); if (!success) { MarkCancelled(); @@ -1011,7 +1021,7 @@ internal class RenderManager : MonoBehaviour _frameReader?.Dispose(); ResetFrameHookState(); - AudioMixer?.Reset(); AudioMixer = null; + AudioMixer?.Dispose(); AudioMixer = null; SyncCalculator = null; AudioSpectrumGenerator.Clear(); diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs index 5bcf75e..0483521 100644 --- a/Patch/Engine/AudioPatches.cs +++ b/Patch/Engine/AudioPatches.cs @@ -50,7 +50,7 @@ public static class AudioManagerPlayPatch double relativeStartTime = time - RenderManager.Instance.dspTimeBase; // Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource - RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f); + RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f, RenderManager.Instance.AudioTime); } } } @@ -74,7 +74,7 @@ public static class PlayWithEndTimePatch double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; // Register looping sound event to the offline mixer (default pitch = 1.0f) - mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f); + mixer.RegisterSoundEvent(null, snd, relativeStartTime, relativeEndTime, volume, 1.0f, RenderManager.Instance.AudioTime); } } return true; // Let the original method execute @@ -91,9 +91,9 @@ public static class AudioSourcePitchPatch if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) { var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent)) + if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) { - soundEvent.Pitch = value; + voice.SetPitch(value, RenderManager.Instance.AudioTime); } } } @@ -109,9 +109,9 @@ public static class AudioSourceSetScheduledEndTimePatch if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.AudioConfig.RenderAudio) { var mixer = RenderManager.Instance.AudioMixer; - if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent)) + if (mixer != null && mixer.TryGetVoice(__instance, out var voice)) { - soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase; + voice.SetEndTime(time - RenderManager.Instance.dspTimeBase, RenderManager.Instance.AudioTime); } } } diff --git a/Util/Audio/AudioResampler.cs b/Util/Audio/AudioResampler.cs deleted file mode 100644 index 12a8ff0..0000000 --- a/Util/Audio/AudioResampler.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; - -namespace ADOFAIRenderer.Util.Audio; - -internal static class AudioResampler -{ - public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency) - { - // ── Safety Validation ── - if (inputSamples.Length == 0 || - inputChannels <= 0 || targetChannels <= 0 || - inputFrequency <= 0 || targetFrequency <= 0) - { - return Array.Empty(); - } - - // If format is identical, return original samples - if (inputFrequency == targetFrequency && inputChannels == targetChannels) - return inputSamples; - - int inputLength = inputSamples.Length / inputChannels; - int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency); - float[] outputSamples = new float[targetLength * targetChannels]; - - for (int i = 0; i < targetLength; i++) - { - double srcIndex = (double)i * inputFrequency / targetFrequency; - long idxLow = (long)Math.Floor(srcIndex); - long idxHigh = idxLow + 1; - float t = (float)(srcIndex - idxLow); - - if (idxHigh >= inputLength) idxHigh = inputLength - 1; - if (idxLow >= inputLength) idxLow = inputLength - 1; - - if (targetChannels == 1) // 1. Mono Output - { - float valLow, valHigh; - if (inputChannels == 1) // Mono -> Mono - { - valLow = inputSamples[idxLow]; - valHigh = inputSamples[idxHigh]; - } - else // Multi -> Mono (Downmix: Average all channels) - { - float sumLow = 0f; - float sumHigh = 0f; - for (int c = 0; c < inputChannels; c++) - { - sumLow += inputSamples[idxLow * inputChannels + c]; - sumHigh += inputSamples[idxHigh * inputChannels + c]; - } - valLow = sumLow / inputChannels; - valHigh = sumHigh / inputChannels; - } - outputSamples[i] = valLow + t * (valHigh - valLow); - } - else if (targetChannels == 2) // 2. Stereo Output - { - if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right) - { - float valLow = inputSamples[idxLow]; - float valHigh = inputSamples[idxHigh]; - float val = valLow + t * (valHigh - valLow); - outputSamples[i * 2] = val; - outputSamples[i * 2 + 1] = val; - } - else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels) - { - float valLowL = inputSamples[idxLow * inputChannels]; - float valHighL = inputSamples[idxHigh * inputChannels]; - outputSamples[i * 2] = valLowL + t * (valHighL - valLowL); - - float valLowR = inputSamples[idxLow * inputChannels + 1]; - float valHighR = inputSamples[idxHigh * inputChannels + 1]; - outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR); - } - } - else // 3. Multi-channel Output (e.g., 5.1 Surround) - { - for (int c = 0; c < targetChannels; c++) - { - float valLow, valHigh; - if (c < inputChannels) - { - valLow = inputSamples[idxLow * inputChannels + c]; - valHigh = inputSamples[idxHigh * inputChannels + c]; - } - else - { - valLow = inputSamples[idxLow * inputChannels]; - valHigh = inputSamples[idxHigh * inputChannels]; - } - outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow); - } - } - } - - return outputSamples; - } -} \ No newline at end of file diff --git a/Util/Audio/Filter/Gain.cs b/Util/Audio/Filter/Gain.cs index 3c33a22..bafa79c 100644 --- a/Util/Audio/Filter/Gain.cs +++ b/Util/Audio/Filter/Gain.cs @@ -4,14 +4,14 @@ namespace ADOFAIRenderer.Util.Audio.Filter; internal class Gain : IAudioFilter { - public float Level = 0f; + public double Level = 0; public void Initialize(int sampleRate, int channels) { } - public float ProcessSample(float sample, int channel) => - sample * (float)Math.Pow(10, Level / 20); + public double ProcessSample(double sample, int channel) => + sample * Math.Pow(10, Level / 20); - public void ProcessBlock(Span buffer) + public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); diff --git a/Util/Audio/Filter/HardClip.cs b/Util/Audio/Filter/HardClip.cs index adc58c4..2e20f6c 100644 --- a/Util/Audio/Filter/HardClip.cs +++ b/Util/Audio/Filter/HardClip.cs @@ -8,19 +8,19 @@ internal class HardClip : IAudioFilter // ReSharper disable once MemberCanBePrivate.Global // ReSharper disable once FieldCanBeMadeReadOnly.Global // ReSharper disable once ConvertToConstant.Global - public float Threshold = 1.0f; + public double Threshold = 1.0; public void Initialize(int sampleRate, int channels) { } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public float ProcessSample(float sample, int channel) + public double ProcessSample(double sample, int channel) { if (sample > Threshold) return Threshold; if (sample < -Threshold) return -Threshold; return sample; } - public void ProcessBlock(Span buffer) + public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) buffer[i] = ProcessSample(buffer[i], 0); diff --git a/Util/Audio/Filter/IAudioFilter.cs b/Util/Audio/Filter/IAudioFilter.cs index 63c684b..ebf9a00 100644 --- a/Util/Audio/Filter/IAudioFilter.cs +++ b/Util/Audio/Filter/IAudioFilter.cs @@ -6,8 +6,8 @@ internal interface IAudioFilter { void Initialize(int sampleRate, int channels); - float ProcessSample(float sample, int channel); - void ProcessBlock(Span buffer); + double ProcessSample(double sample, int channel); + void ProcessBlock(Span buffer); void Reset(); diff --git a/Util/Audio/Filter/RMSCompressor.cs b/Util/Audio/Filter/RMSCompressor.cs index 571f126..ea07d27 100644 --- a/Util/Audio/Filter/RMSCompressor.cs +++ b/Util/Audio/Filter/RMSCompressor.cs @@ -6,84 +6,84 @@ namespace ADOFAIRenderer.Util.Audio.Filter; // ReSharper disable once InconsistentNaming internal class RMSCompressor : IAudioFilter { - private float _thresholdDb = -20f; - private float _ratio = 4f; - private float _attackTime = 0.010f; // in seconds - private float _releaseTime = 0.100f; // in seconds - private float _kneeDb = 5f; // in dB - private float _rmsWindowTime = 0.030f; // in seconds - private float _lookAheadTime; // in seconds + private double _thresholdDb = -20; + private double _ratio = 4; + private double _attackTime = 0.010; // in seconds + private double _releaseTime = 0.100; // in seconds + private double _kneeDb = 5; // in dB + private double _rmsWindowTime = 0.030; // in seconds + private double _lookAheadTime; // in seconds private int _lookAheadSamples; private int _sampleRate = 44100; private int _channels = 2; - private float _attackCoef; - private float _releaseCoef; - private float _rmsCoef; + private double _attackCoef; + private double _releaseCoef; + private double _rmsCoef; // Per-channel state - private float[] _rmsState = Array.Empty(); // running mean-square (MS) state - private float[] _gainState = Array.Empty(); // smoothed gain reduction in dB - private float[][] _delayBuffers = Array.Empty(); // circular delay buffers + private double[] _rmsState = Array.Empty(); // running mean-square (MS) state + private double[] _gainState = Array.Empty(); // smoothed gain reduction in dB + private double[][] _delayBuffers = Array.Empty(); // circular delay buffers private int[] _delayWriteIndices = Array.Empty(); - public float LookAheadTime + public double LookAheadTime { get => _lookAheadTime; set { - _lookAheadTime = Math.Max(0f, Math.Min(1.0f, value)); + _lookAheadTime = Math.Max(0, Math.Min(1.0, value)); UpdateLookAheadSettings(); } } public double Latency => _lookAheadTime; - public float ThresholdDb + public double ThresholdDb { get => _thresholdDb; set => _thresholdDb = value; } - public float Ratio + public double Ratio { get => _ratio; - set => _ratio = Math.Max(1f, value); + set => _ratio = Math.Max(1, value); } - public float AttackTime + public double AttackTime { get => _attackTime; set { - _attackTime = Math.Max(0.0001f, value); + _attackTime = Math.Max(0.0001, value); UpdateAttackCoef(); } } - public float ReleaseTime + public double ReleaseTime { get => _releaseTime; set { - _releaseTime = Math.Max(0.0001f, value); + _releaseTime = Math.Max(0.0001, value); UpdateReleaseCoef(); } } - public float KneeDb + public double KneeDb { get => _kneeDb; - set => _kneeDb = Math.Max(0f, value); + set => _kneeDb = Math.Max(0, value); } - public float RmsWindowTime + public double RmsWindowTime { get => _rmsWindowTime; set { - _rmsWindowTime = Math.Max(0.0001f, value); + _rmsWindowTime = Math.Max(0.0001, value); UpdateRmsCoef(); } } @@ -98,8 +98,8 @@ internal class RMSCompressor : IAudioFilter UpdateRmsCoef(); UpdateLookAheadSettings(); - _rmsState = new float[channels]; - _gainState = new float[channels]; + _rmsState = new double[channels]; + _gainState = new double[channels]; Reset(); } @@ -114,33 +114,33 @@ internal class RMSCompressor : IAudioFilter { if (_channels <= 0) return; - _delayBuffers = new float[_channels][]; + _delayBuffers = new double[_channels][]; _delayWriteIndices = new int[_channels]; for (int i = 0; i < _channels; i++) { - _delayBuffers[i] = _lookAheadSamples > 0 ? new float[_lookAheadSamples] : Array.Empty(); + _delayBuffers[i] = _lookAheadSamples > 0 ? new double[_lookAheadSamples] : Array.Empty(); _delayWriteIndices[i] = 0; } } private void UpdateAttackCoef() { - _attackCoef = (float)Math.Exp(-1.0 / (_sampleRate * _attackTime)); + _attackCoef = Math.Exp(-1.0 / (_sampleRate * _attackTime)); } private void UpdateReleaseCoef() { - _releaseCoef = (float)Math.Exp(-1.0 / (_sampleRate * _releaseTime)); + _releaseCoef = Math.Exp(-1.0 / (_sampleRate * _releaseTime)); } private void UpdateRmsCoef() { - _rmsCoef = (float)Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); + _rmsCoef = Math.Exp(-1.0 / (_sampleRate * _rmsWindowTime)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public float ProcessSample(float sample, int channel) + public double ProcessSample(double sample, int channel) { if (channel < 0 || channel >= _channels) { @@ -149,54 +149,54 @@ internal class RMSCompressor : IAudioFilter // Calculate running Mean-Square (MS) level // v_c[n] = alpha * v_c[n-1] + (1 - alpha) * x^2[n] - float sq = sample * sample; - _rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0f - _rmsCoef) * sq; + double sq = sample * sample; + _rmsState[channel] = _rmsCoef * _rmsState[channel] + (1.0 - _rmsCoef) * sq; - float rmsSq = Math.Max(0f, _rmsState[channel]); + double rmsSq = Math.Max(0, _rmsState[channel]); // Convert to dB. 10 * log10(rmsSq) is equivalent to 20 * log10(sqrt(rmsSq)) - float rmsDb = 10.0f * (float)Math.Log10(rmsSq + 1e-12f); + double rmsDb = 10.0 * Math.Log10(rmsSq + 1e-12); // Calculate static envelope curve (gain reduction in dB) - float gStatic = 0f; - if (_kneeDb > 0f) + double gStatic = 0; + if (_kneeDb > 0) { - if (rmsDb > _thresholdDb + _kneeDb * 0.5f) + if (rmsDb > _thresholdDb + _kneeDb * 0.5) { - gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb); + gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); } - else if (rmsDb >= _thresholdDb - _kneeDb * 0.5f) + else if (rmsDb >= _thresholdDb - _kneeDb * 0.5) { - float diff = rmsDb - _thresholdDb + _kneeDb * 0.5f; - gStatic = (1f / _ratio - 1f) * diff * diff / (2f * _kneeDb); + double diff = rmsDb - _thresholdDb + _kneeDb * 0.5; + gStatic = (1 / _ratio - 1) * diff * diff / (2 * _kneeDb); } } else { if (rmsDb > _thresholdDb) { - gStatic = (1f / _ratio - 1f) * (rmsDb - _thresholdDb); + gStatic = (1 / _ratio - 1) * (rmsDb - _thresholdDb); } } // Apply ballistics (attack and release) on the dB gain reduction - float lastGain = _gainState[channel]; - float targetGain = gStatic; + double lastGain = _gainState[channel]; + double targetGain = gStatic; // Note: Since targetGain and lastGain are <= 0 (gain reduction in dB), // targetGain < lastGain means target gain reduction is larger (more negative), // so we are in the attack phase. - float coef = targetGain < lastGain ? _attackCoef : _releaseCoef; - float currentGain = coef * lastGain + (1f - coef) * targetGain; + double coef = targetGain < lastGain ? _attackCoef : _releaseCoef; + double currentGain = coef * lastGain + (1 - coef) * targetGain; _gainState[channel] = currentGain; // Convert back to linear gain multiplier - float linearGain = (float)Math.Pow(10.0, currentGain / 20.0); + double linearGain = Math.Pow(10.0, currentGain / 20.0); if (_lookAheadSamples > 0 && _delayBuffers.Length > channel && _delayBuffers[channel].Length > 0) { var buffer = _delayBuffers[channel]; int writeIdx = _delayWriteIndices[channel]; - float delayedSample = buffer[writeIdx]; + double delayedSample = buffer[writeIdx]; buffer[writeIdx] = sample; _delayWriteIndices[channel] = (writeIdx + 1) % _lookAheadSamples; return delayedSample * linearGain; @@ -205,7 +205,7 @@ internal class RMSCompressor : IAudioFilter return sample * linearGain; } - public void ProcessBlock(Span buffer) + public void ProcessBlock(Span buffer) { for (var i = 0; i < buffer.Length; i++) { diff --git a/Util/Audio/OfflineAudioMixer.cs b/Util/Audio/OfflineAudioMixer.cs index 18190be..82fbf9a 100644 --- a/Util/Audio/OfflineAudioMixer.cs +++ b/Util/Audio/OfflineAudioMixer.cs @@ -1,416 +1,316 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Threading; +using ADOFAIRenderer.Util.Audio.Clip; using ADOFAIRenderer.Util.Audio.Filter; +using ADOFAIRenderer.Util.Audio.Mixer; using MelonLoader; using UnityEngine; namespace ADOFAIRenderer.Util.Audio; -internal class AudioClipCache +/// +/// Audio of one render: the song and effect sounds as s of a , clips from an +/// , followed by the filter chain (RMS compressor → look-ahead limiter → hard clip → -1 dB gain). +/// +/// Time: seconds relative to the render start (RenderManager.dspTimeBase); output sample n is at n / sample rate. +/// The limiter delays its output by its look-ahead (), so output sample n is mixed from +/// voices at sample n + latency, and the filters are primed with samples [0, latency) before the first block. +/// +/// Threads: +/// - Main thread: , , , , +/// source/voice mapping, and after each frame (all commands with an earlier time have been queued). +/// - Render thread: waits until the main thread has committed the end of the samples it mixes, +/// so every command takes effect at its own sample. ends the waiting when the main thread stops. +/// +internal sealed class OfflineAudioMixer : IDisposable { - private readonly Dictionary _cache = new(); + /// Gain applied to every clip. + private const double ClipVolume = 0.5; - public class CachedClipData - { - public float[] Samples = null!; // Pre-resampled to target channels and sample rate - public double Duration; - } + /// Additional gain of effect sounds (registered with ). + private const double EffectVolume = 2.0 / 3.0; - public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true) - { - if (string.IsNullOrEmpty(clipName)) return null; + /// Longest wait for a commit before checking for cancellation again, in milliseconds. + private const int CommitWaitMs = 50; - lock (_cache) - { - if (_cache.TryGetValue(clipName, out var cached)) - return cached; + private readonly int _channels; + private readonly int _sampleRate; + private readonly AudioClipStore _store; + private readonly VoiceMixer _mixer; + private readonly CancellationTokenSource _cancellation = new(); + private readonly List _filters = new(); + private readonly int _latencySamples; - if (!allowLoad) return null; + // Main thread. + private ConditionalWeakTable _sourceVoices = new(); + private AudioClipSource? _bgmSource; + private string _bgmKey = ""; + private Voice? _bgmVoice; + private double _bgmStart = double.NaN; + private double _bgmPitch = double.NaN; + private double _bgmVolume = double.NaN; - if (!AudioManager.Instance) return null; + // Shared: committed output sample (voices are final before it), guarded by _commitLock. + private readonly object _commitLock = new(); + private long _committedSample = long.MinValue; - var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName); - if (!clip) return null; - - try - { - var totalSamples = clip.samples; - var channels = clip.channels; - var rawSamples = new float[totalSamples * channels]; - clip.GetData(rawSamples, 0); - - // Auto-resample during cache generation - var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate); - - var data = new CachedClipData - { - Samples = resampled, - Duration = (double)resampled.Length / (targetChannels * targetSampleRate) - }; - - _cache[clipName] = data; - return data; - } - catch (Exception e) - { - MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}"); - return null; - } - } - } - - public void Clear() - { - lock (_cache) - { - _cache.Clear(); - } - } -} - -internal class OfflineAudioMixer -{ - public class SoundEvent - { - public string ClipName = null!; - public double StartTime; // In seconds relative to dspTimeBase - public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot) - public float Volume; - public float Pitch; - } - - private readonly List _activeEvents = new(); - private ConditionalWeakTable _sourceMappings = new(); - private readonly object _lock = new(); - - // Target Audio Settings - private readonly int _targetChannels; - private readonly int _targetSampleRate; - private const float ClipVolume = .5f; - - // Audio Filter Settings - private readonly LinkedList _filters = new(); - - // BGM Cached Samples (Pre-resampled) - private float[]? _bgmSamples; - private double _bgmDuration; - - // Clip Cache - private readonly AudioClipCache _clipCache = new(); - private bool _isPrimed; + // Render thread (and Dispose, under _renderLock). + private readonly object _renderLock = new(); + private long _nextOutputSample; + private bool _primed; + private double[] _mixBuffer = Array.Empty(); + private bool _disposed; public OfflineAudioMixer(int targetChannels, int targetSampleRate) { - _targetChannels = targetChannels; - _targetSampleRate = targetSampleRate; + _channels = targetChannels; + _sampleRate = targetSampleRate; + _store = new AudioClipStore(targetSampleRate); + _mixer = new VoiceMixer(_store, targetChannels, _cancellation.Token); var rmsCompressor = new RMSCompressor { ThresholdDb = -10, Ratio = 2, - AttackTime = 0.1f, ReleaseTime = 0.1f, - KneeDb = 10, RmsWindowTime = 0.02f + AttackTime = 0.1, ReleaseTime = 0.1, + KneeDb = 10, RmsWindowTime = 0.02 }; - rmsCompressor.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(rmsCompressor); - var limiter = new RMSCompressor { - ThresholdDb = 0, Ratio = float.PositiveInfinity, - AttackTime = 0f, ReleaseTime = 0.05f, - KneeDb = 0, RmsWindowTime = 0.02f, - LookAheadTime = 0.01f + ThresholdDb = 0, Ratio = double.PositiveInfinity, + AttackTime = 0, ReleaseTime = 0.05, + KneeDb = 0, RmsWindowTime = 0.02, + LookAheadTime = 0.01 }; - limiter.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(limiter); - - var hardClip = new HardClip(); - hardClip.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(hardClip); + _filters.Add(rmsCompressor); + _filters.Add(limiter); + _filters.Add(new HardClip()); + _filters.Add(new Gain { Level = -1 }); - var truePeakSafetyGain = new Gain { Level = -1f }; - truePeakSafetyGain.Initialize(_targetSampleRate, _targetChannels); - _filters.AddLast(truePeakSafetyGain); + double latency = 0; + foreach (var filter in _filters) + { + filter.Initialize(_sampleRate, _channels); + latency += filter.Latency; + } + _latencySamples = (int)Math.Round(latency * _sampleRate); + + _store.Start(); } + // ── Main thread ── + + /// Registers the song clip. The song voice is created and kept in sync by . public void InitializeBGM(AudioClip? clip) { - lock (_lock) + if (!clip) { - if (!clip) - { - _bgmSamples = null; - _bgmDuration = 0; - return; - } - - try - { - int originalSamples = clip.samples; - int originalChannels = clip.channels; - float[] rawSamples = new float[originalSamples * originalChannels]; - clip.GetData(rawSamples, 0); - - // Pre-resample the BGM to the target format - _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate); - _bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate); - - Melon.Logger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}"); - } - catch (Exception e) - { - Melon.Logger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}"); - _bgmSamples = null; - _bgmDuration = 0; - } + _bgmSource = null; + return; } + + _bgmKey = clip!.name; + _bgmSource = AudioClipResolver.GetOrAdd(_store, _bgmKey, clip); + if (_bgmSource == null) + { + Melon.Logger.Error($"[OfflineAudioMixer] Song clip '{_bgmKey}' could not be loaded, the song will be silent."); + return; + } + Melon.Logger.Msg($"[OfflineAudioMixer] Song '{_bgmKey}': {_bgmSource.Channels} ch, {_bgmSource.Duration:F2} s, {_bgmSource.Policy}"); } - public void CacheClip(string clipName) + /// + /// Updates the song voice from the conductor (start time, pitch) and the level volume. + /// A changed start time restarts the song voice at position (now - start) * pitch; pitch and volume changes take effect at . + /// Start time: dspTimeSong + separate countdown (crotchetAtStart / pitch * adjustedCountdownTicks when separateCountdownTime), + /// the time scrConductor.StartMusicCo schedules the song at. + /// + /// Current time, seconds relative to the render start. + /// AudioSettings.dspTime at the render start. + public void UpdateBGM(double now, double dspTimeBase) { - _clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true); - } - - public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch) - { - lock (_lock) - { - var ev = new SoundEvent - { - ClipName = clipName, - StartTime = startTime, - EndTime = endTime, - Volume = volume, - Pitch = pitch - }; - _activeEvents.Add(ev); - - if (!source) return; - _sourceMappings.Remove(source); - _sourceMappings.Add(source, ev); - } - } - - public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch) - { - RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch); - } - - public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev) - { - if (source == null) - { - ev = null!; - return false; - } - lock (_lock) - { - return _sourceMappings.TryGetValue(source, out ev); - } - } - - public bool RemoveSourceMapping(AudioSource source) - { - lock (_lock) - { - return _sourceMappings.Remove(source); - } - } - - public void Reset() - { - lock (_lock) - { - _activeEvents.Clear(); - _sourceMappings = new ConditionalWeakTable(); - _bgmSamples = null; - _bgmDuration = 0; - _clipCache.Clear(); - _isPrimed = false; - foreach (var filter in _filters) - { - filter.Reset(); - } - } - } - - private void PrimeFilters(double totalLatency, double dspTimeBase) - { - int latencySamples = (int)Math.Round(totalLatency * _targetSampleRate); - if (latencySamples <= 0) return; - - // Rent temporary buffer to prime filters with initial audio - float[] tempBuffer = System.Buffers.ArrayPool.Shared.Rent(latencySamples * _targetChannels); - try - { - Array.Clear(tempBuffer, 0, latencySamples * _targetChannels); - - // Mix starting from -totalLatency to 0 - MixBGM(tempBuffer, latencySamples, -totalLatency, dspTimeBase, totalLatency); - MixEffectSounds(tempBuffer, latencySamples, -totalLatency, totalLatency); - - // Feed the buffer into filters to prime circular delay buffers - foreach (var filter in _filters) - { - filter.ProcessBlock(tempBuffer.AsSpan(0, latencySamples * _targetChannels)); - } - } - finally - { - System.Buffers.ArrayPool.Shared.Return(tempBuffer); - } - } - - public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) - { - // mixedBuffer size must be at least samplesNeeded * _targetChannels - Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels); - - lock (_lock) - { - double totalLatency = 0; - foreach (var filter in _filters) - { - totalLatency += filter.Latency; - } - - // Prime filters on first frame execution if latency exists - if (!_isPrimed && totalLatency > 0) - { - PrimeFilters(totalLatency, dspTimeBase); - _isPrimed = true; - } - - // 1. Mix BGM - MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase, totalLatency); - - // 2. Mix Hitsounds/Effect Sounds - MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime, totalLatency); - - foreach (var filter in _filters) - filter.ProcessBlock(mixedBuffer.AsSpan(0, samplesNeeded * _targetChannels)); - - for (var i = 0; i < samplesNeeded * _targetChannels; i++) - { - if (Math.Abs(mixedBuffer[i]) > 1) Melon.Logger.Msg("Warning Why Audio Sample Is Larger Than 1"); - } - } - } - - private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase, double totalLatency) - { - if (_bgmSamples == null || _bgmSamples.Length == 0) return; - + if (_bgmSource == null) return; var conductor = scrConductor.instance; if (!conductor) return; double pitch = conductor.song.pitch; - var dspTimeSong = conductor.dspTimeSong; - var separateCountdownOffset = conductor.separateCountdownTime - ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks - : 0.0; + var countdown = conductor.separateCountdownTime ? conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks : 0.0; + var start = conductor.dspTimeSong + countdown - dspTimeBase; + var levelVolume = scnGame.instance ? scnGame.instance.levelData.volume * 0.01 : 1.0; + var volume = levelVolume * ClipVolume; - // BGM Playback start time relative to dspTimeBase - var time1 = dspTimeSong + separateCountdownOffset; - var relativeTime1 = time1 - dspTimeBase; - var adjustedRelativeTime1 = relativeTime1 - totalLatency; - - var totalSamples = _bgmSamples.Length / _targetChannels; - - for (int s = 0; s < samplesNeeded; s++) + if (_bgmVoice == null || start != _bgmStart) { - double sampleTime = frameStartTime + (double)s / _targetSampleRate; - double clipTime = (sampleTime - adjustedRelativeTime1) * pitch; + // Song position at time t is (t - start) * pitch, also when the start is already past. + _bgmVoice?.Stop(now); + _bgmVoice = _mixer.Play(_bgmSource, _bgmKey, Math.Max(start, now), volume, pitch, false, now); + if (start < now) _bgmVoice.SetPosition((now - start) * pitch, now); + _bgmStart = start; + _bgmPitch = pitch; + _bgmVolume = volume; + return; + } - if (clipTime >= 0 && clipTime < _bgmDuration) - { - double sampleIdxDouble = clipTime * _targetSampleRate; - long idxLow = (long)Math.Floor(sampleIdxDouble); - long idxHigh = idxLow + 1; - float t = (float)(sampleIdxDouble - idxLow); - - if (idxHigh >= totalSamples) idxHigh = totalSamples - 1; - if (idxLow >= totalSamples) idxLow = totalSamples - 1; - - for (var c = 0; c < _targetChannels; c++) - { - float valLow = _bgmSamples[idxLow * _targetChannels + c]; - float valHigh = _bgmSamples[idxHigh * _targetChannels + c]; - mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * (scnGame.instance.levelData.volume * 0.01f) - * ClipVolume; - } - } + if (pitch != _bgmPitch) + { + _bgmVoice.SetPitch(pitch, now); + _bgmPitch = pitch; + } + if (volume != _bgmVolume) + { + _bgmVoice.SetVolume(volume, now); + _bgmVolume = volume; } } - private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double totalLatency) + /// Prepares the source of the game clip (audioLib key) so its first chunk starts loading. + public void CacheClip(string clipName) { - for (var i = _activeEvents.Count - 1; i >= 0; i--) + GetOrAddSource(clipName); + } + + private AudioClipSource? GetOrAddSource(string clipName) + { + if (string.IsNullOrEmpty(clipName)) return null; + if (_store.TryGet(clipName, out var existing)) return existing; + var clip = AudioManager.Instance ? AudioManager.Instance.FindOrLoadAudioClip(clipName) : null; + return AudioClipResolver.GetOrAdd(_store, clipName, clip); + } + + /// + /// Starts an effect sound at . An of 0 or more makes it loop until then. + /// (when given) is mapped to the voice for later changes through the AudioSource. + /// + public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch, double now) + { + var clipSource = GetOrAddSource(clipName); + var loop = endTime >= 0; + var voice = _mixer.Play(clipSource, clipName, startTime, volume * ClipVolume * EffectVolume, pitch, loop, now); + if (loop) voice.SetEndTime(endTime, now); + + if (!source) return; + _sourceVoices.Remove(source!); + _sourceVoices.Add(source!, voice); + } + + /// Voice started for by . + public bool TryGetVoice(AudioSource? source, out Voice voice) + { + voice = null!; + return source != null && _sourceVoices.TryGetValue(source, out voice); + } + + public bool RemoveSourceMapping(AudioSource source) => _sourceVoices.Remove(source); + + /// + /// Marks every command with a time before as queued + /// (called after each frame with the time of the next frame). + /// + public void Commit(double time) + { + var sample = VoiceMixer.ToSample(time, _sampleRate); + lock (_commitLock) { - var ev = _activeEvents[i]; - var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false); - if (clipData == null) - { - _activeEvents.RemoveAt(i); - continue; - } - - var isSoundFinished = false; - var totalSamples = clipData.Samples.Length / _targetChannels; - double adjustedStartTime = ev.StartTime - totalLatency; - double adjustedEndTime = ev.EndTime >= 0 ? ev.EndTime - totalLatency : -1.0; - - for (var s = 0; s < samplesNeeded; s++) - { - var sampleTime = frameStartTime + (double)s / _targetSampleRate; - - // If it is a looping sound and has reached or exceeded its end time, stop playing. - if (adjustedEndTime >= 0 && sampleTime >= adjustedEndTime) - { - isSoundFinished = true; - break; - } - - var elapsed = sampleTime - adjustedStartTime; - if (elapsed < 0) continue; // Sound hasn't started yet - - var soundTime = elapsed * ev.Pitch; - - if (soundTime >= clipData.Duration) - { - if (ev.EndTime < 0) // One-shot - { - isSoundFinished = true; - break; - } - - // Looping - soundTime %= clipData.Duration; - } - - // Linear interpolation for pitch shifting in the mixer loop - double sampleIdxDouble = soundTime * _targetSampleRate; - long idxLow = (long)Math.Floor(sampleIdxDouble); - long idxHigh = idxLow + 1; - float t = (float)(sampleIdxDouble - idxLow); - - if (idxHigh >= totalSamples) idxHigh = totalSamples - 1; - if (idxLow >= totalSamples) idxLow = totalSamples - 1; - - for (var c = 0; c < _targetChannels; c++) - { - float valLow = clipData.Samples[idxLow * _targetChannels + c]; - float valHigh = clipData.Samples[idxHigh * _targetChannels + c]; - mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume - * ClipVolume * 2 / 3; - } - } - - // Clean up finished sound events (both one-shot and looping) - if (isSoundFinished) - { - _activeEvents.RemoveAt(i); - } + if (sample <= _committedSample) return; + _committedSample = sample; + Monitor.PulseAll(_commitLock); } } -} \ No newline at end of file + + /// No more commands will be queued (render ending): mixing no longer waits for commits. + public void Complete() + { + lock (_commitLock) + { + _committedSample = long.MaxValue; + Monitor.PulseAll(_commitLock); + } + } + + /// Stops waiting for commits and chunks; the render thread's current or next mix throws OperationCanceledException. + public void Cancel() + { + _cancellation.Cancel(); + Complete(); + } + + // ── Render thread ── + + /// + /// Writes the next output frames (interleaved float, continuing from the previous call). + /// + /// Chunk loading failed fatally (out of memory or the decoder thread stopped). + /// The render was canceled (). + public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded) + { + lock (_renderLock) + { + if (_disposed) throw new ObjectDisposedException(nameof(OfflineAudioMixer)); + + if (!_primed) + { + _primed = true; + if (_latencySamples > 0) + { + var prime = MixBlock(0, _latencySamples); + foreach (var filter in _filters) + filter.ProcessBlock(prime); + } + } + + var block = MixBlock(_nextOutputSample + _latencySamples, samplesNeeded); + foreach (var filter in _filters) + filter.ProcessBlock(block); + _nextOutputSample += samplesNeeded; + + for (var i = 0; i < block.Length; i++) + mixedBuffer[i] = (float)block[i]; + } + } + + /// Mixes voices at samples [first, first + frames) into the reused double buffer, after the main thread committed them. + private Span MixBlock(long first, int frames) + { + WaitForCommit(first + frames); + + var length = frames * _channels; + if (_mixBuffer.Length < length) _mixBuffer = new double[length]; + var block = _mixBuffer.AsSpan(0, length); + block.Clear(); + _mixer.Mix(first, block); + return block; + } + + private void WaitForCommit(long endSample) + { + lock (_commitLock) + { + while (_committedSample < endSample) + { + _cancellation.Token.ThrowIfCancellationRequested(); + Monitor.Wait(_commitLock, CommitWaitMs); + } + } + _cancellation.Token.ThrowIfCancellationRequested(); + } + + /// + /// Cancels waits, then releases the voices, the clip store (decoder thread) and the filter state. + /// Waits for a running to return. + /// + public void Dispose() + { + Cancel(); + lock (_renderLock) + { + if (_disposed) return; + _disposed = true; + _mixer.Dispose(); + _store.Dispose(); + foreach (var filter in _filters) + filter.Reset(); + } + _sourceVoices = new ConditionalWeakTable(); + } +}