ADOFAIEditorForum/ADOFAIRenderer
SHA256
2
2 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Refactor Audio Calculation

Browse Source
This commit is contained in:
copecone committed 2026-07-09 12:45:29 +09:00
1 parent 1e7dcd508a
commit cca027ed66
5 files changed
+155 -86
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Core/RenderManager.cs
+19 -12
View File
@@ -49,6 +49,9 @@ namespace ADOFAIRenderer.Core
public double unscaledTimeBase;
public double dspTimeBase;
public int frameCountBase;
public OfflineAudioMixer? AudioMixer { get; private set; }
public AudioSyncCalculator SyncCalculator { get; private set; } = null!;
private FrameReader? _frameReader;
public bool renderCheck = true;
@@ -432,9 +435,14 @@ namespace ADOFAIRenderer.Core
{
AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip);
}
if (Configuration.RenderAudio)
{
OfflineAudioMixer.InitializeBGM(scrConductor.instance.song.clip, Configuration.AudioChannels, Configuration.AudioSampleRate);
AudioMixer = new OfflineAudioMixer(Configuration.AudioChannels, Configuration.AudioSampleRate);
AudioMixer.InitializeBGM(scrConductor.instance.song.clip);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
SyncCalculator = new AudioSyncCalculator(Configuration.AudioSampleRate, effectiveVideoFps);
}
scrConductor.instance.song.clip = null;
@@ -581,19 +589,17 @@ namespace ADOFAIRenderer.Core
if (Configuration.RenderAudio)
{
int frameIdx = (int)(passCount - 1);
double effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
double frameStartTime = frameIdx / effectiveVideoFps;
double frameEndTime = (frameIdx + 1) / effectiveVideoFps;
var frameIdx = (int)(passCount - 1);
var effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS);
var frameTime = frameIdx / effectiveVideoFps;
int sampleRate = Configuration.AudioSampleRate;
int channels = Configuration.AudioChannels;
int samplesNeeded = (int)(Math.Floor(frameEndTime * sampleRate) - Math.Floor(frameStartTime * sampleRate));
var channels = Configuration.AudioChannels;
var samplesNeeded = SyncCalculator.GetNextFrameSamples();
float[] audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
var audioBuffer = ArrayPool<float>.Shared.Rent(samplesNeeded * channels);
try
{
OfflineAudioMixer.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameStartTime, dspTimeBase);
AudioMixer?.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameTime, dspTimeBase);
videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded);
}
finally
@@ -798,9 +804,10 @@ namespace ADOFAIRenderer.Core
}
_frameReader?.Dispose();
OfflineAudioMixer.Reset();
ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
AudioMixer?.Reset(); AudioMixer = null;
SyncCalculator.Reset();
ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor");
Cursor.visible = true;
}
Patch/Engine/AudioPatches.cs
+19 -14
View File
@@ -1,7 +1,5 @@
using System;
using ADOFAIRenderer.Core;
using ADOFAIRenderer.Util;
using ADOFAIRenderer.Util.Audio;
using HarmonyLib;
using UnityEngine;
using UnityEngine.Audio;
@@ -17,7 +15,7 @@ namespace ADOFAIRenderer.Patch.Engine
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
{
// Cache the clip on the main thread
AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
RenderManager.Instance.AudioMixer?.CacheClip(snd);
}
return true; // Let the original method execute to prevent NullReferenceException
}
@@ -31,7 +29,7 @@ namespace ADOFAIRenderer.Patch.Engine
double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
// Register event to the offline mixer (default pitch = 1.0f) and map it to the AudioSource
OfflineAudioMixer.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
RenderManager.Instance.AudioMixer?.RegisterSoundEvent(__result, snd, relativeStartTime, -1.0, volume, 1.0f);
}
}
}
@@ -44,15 +42,19 @@ namespace ADOFAIRenderer.Patch.Engine
{
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
{
// Cache the clip on the main thread
AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true);
var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null)
{
// Cache the clip on the main thread
mixer.CacheClip(snd);
// Convert absolute DSP times to relative times in seconds from render start
double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
// Convert absolute DSP times to relative times in seconds from render start
double relativeStartTime = time - RenderManager.Instance.dspTimeBase;
double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase;
// Register looping sound event to the offline mixer (default pitch = 1.0f)
OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
// Register looping sound event to the offline mixer (default pitch = 1.0f)
mixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f);
}
}
return true; // Let the original method execute
}
@@ -62,11 +64,13 @@ namespace ADOFAIRenderer.Patch.Engine
public static class AudioSourcePitchPatch
{
[HarmonyPostfix]
// ReSharper disable once InconsistentNaming
public static void Postfix(AudioSource __instance, float value)
{
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
{
if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent))
var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
{
soundEvent.Pitch = value;
}
@@ -84,7 +88,8 @@ namespace ADOFAIRenderer.Patch.Engine
{
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
{
if (OfflineAudioMixer.TryGetSoundEvent(__instance, out var soundEvent))
var mixer = RenderManager.Instance.AudioMixer;
if (mixer != null && mixer.TryGetSoundEvent(__instance, out var soundEvent))
{
soundEvent.EndTime = time - RenderManager.Instance.dspTimeBase;
}
@@ -100,7 +105,7 @@ namespace ADOFAIRenderer.Patch.Engine
{
if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio)
{
OfflineAudioMixer.RemoveSourceMapping(__instance);
RenderManager.Instance.AudioMixer?.RemoveSourceMapping(__instance);
}
}
}
Util/Audio/AudioFilter.cs
+7
View File
@@ -0,0 +1,7 @@
namespace ADOFAIRenderer.Util.Audio
{
public class AudioFilter
{
}
}
Util/Audio/AudioSyncCalculator.cs
+39
View File
@@ -0,0 +1,39 @@
using System;
namespace ADOFAIRenderer.Util.Audio
{
public class AudioSyncCalculator
{
private readonly long _sampleRate;
private readonly (long Numerator, long Denominator) _fpsRational;
private long _currentSample;
private long _remainder;
public AudioSyncCalculator(long sampleRate, double fps)
{
_sampleRate = sampleRate;
_fpsRational = RationalUtil.Double2MaxPrecisionInt(fps);
if (Math.Abs(fps - 29.97) < 0.001) _fpsRational = (30000, 1001);
if (Math.Abs(fps - 59.94) < 0.001) _fpsRational = (60000, 1001);
if (Math.Abs(fps - 23.976) < 0.001 || Math.Abs(fps - 23.98) < 0.001) _fpsRational = (24000, 1001);
}
public void Reset()
{
_currentSample = 0;
_remainder = 0;
}
public int GetNextFrameSamples()
{
var numerator = _sampleRate * _fpsRational.Denominator + _remainder;
var samplesNeeded = numerator / _fpsRational.Numerator;
_remainder = numerator % _fpsRational.Numerator;
_currentSample += samplesNeeded;
return (int)samplesNeeded;
}
}
}
Util/Audio/OfflineAudioMixer.cs
+71 -60
View File
@@ -4,9 +4,9 @@ using UnityEngine;
namespace ADOFAIRenderer.Util.Audio
{
public static class AudioClipCache
public class AudioClipCache
{
private static readonly Dictionary<string, CachedClipData> Cache = new();
private readonly Dictionary<string, CachedClipData> _cache = new();
public class CachedClipData
{
@@ -14,31 +14,31 @@ namespace ADOFAIRenderer.Util.Audio
public double Duration;
}
public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
public CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true)
{
if (string.IsNullOrEmpty(clipName)) return null;
lock (Cache)
lock (_cache)
{
if (Cache.TryGetValue(clipName, out var cached))
if (_cache.TryGetValue(clipName, out var cached))
return cached;
if (!allowLoad) return null;
if (AudioManager.Instance == null) return null;
if (!AudioManager.Instance) return null;
AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
if (clip == null) return null;
var clip = AudioManager.Instance.FindOrLoadAudioClip(clipName);
if (!clip) return null;
try
{
int totalSamples = clip.samples;
int channels = clip.channels;
float[] rawSamples = new float[totalSamples * channels];
var totalSamples = clip.samples;
var channels = clip.channels;
var rawSamples = new float[totalSamples * channels];
clip.GetData(rawSamples, 0);
// Auto-resample during cache generation
float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
var resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate);
var data = new CachedClipData
{
@@ -46,7 +46,7 @@ namespace ADOFAIRenderer.Util.Audio
Duration = (double)resampled.Length / (targetChannels * targetSampleRate)
};
Cache[clipName] = data;
_cache[clipName] = data;
return data;
}
catch (Exception e)
@@ -57,16 +57,16 @@ namespace ADOFAIRenderer.Util.Audio
}
}
public static void Clear()
public void Clear()
{
lock (Cache)
lock (_cache)
{
Cache.Clear();
_cache.Clear();
}
}
}
public static class OfflineAudioMixer
public class OfflineAudioMixer
{
public class SoundEvent
{
@@ -77,26 +77,32 @@ namespace ADOFAIRenderer.Util.Audio
public float Pitch;
}
private static readonly List<SoundEvent> ActiveEvents = new();
private static System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
private static readonly object Lock = new();
private readonly List<SoundEvent> _activeEvents = new();
private System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent> _sourceMappings = new();
private readonly object _lock = new();
// Target Audio Settings
private static int _targetChannels = 2;
private static int _targetSampleRate = 44100;
private static readonly float MasterVolume = 0.2f;
private readonly int _targetChannels;
private readonly int _targetSampleRate;
private const float MasterVolume = 0.05f;
// BGM Cached Samples (Pre-resampled)
private static float[]? _bgmSamples;
private static double _bgmDuration;
private float[]? _bgmSamples;
private double _bgmDuration;
public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate)
// Clip Cache
private readonly AudioClipCache _clipCache = new();
public OfflineAudioMixer(int targetChannels, int targetSampleRate)
{
lock (Lock)
{
_targetChannels = targetChannels;
_targetSampleRate = targetSampleRate;
_targetChannels = targetChannels;
_targetSampleRate = targetSampleRate;
}
public void InitializeBGM(AudioClip? clip)
{
lock (_lock)
{
if (!clip)
{
_bgmSamples = null;
@@ -112,10 +118,10 @@ namespace ADOFAIRenderer.Util.Audio
clip.GetData(rawSamples, 0);
// Pre-resample the BGM to the target format
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate);
_bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate);
_bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, _targetChannels, _targetSampleRate);
_bgmDuration = (double)_bgmSamples.Length / (_targetChannels * _targetSampleRate);
MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}");
MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / _targetChannels}, TargetFreq: {_targetSampleRate}Hz, TargetChannels: {_targetChannels}");
}
catch (Exception e)
{
@@ -126,9 +132,14 @@ namespace ADOFAIRenderer.Util.Audio
}
}
public static void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
public void CacheClip(string clipName)
{
lock (Lock)
_clipCache.GetOrCache(clipName, _targetChannels, _targetSampleRate, allowLoad: true);
}
public void RegisterSoundEvent(AudioSource? source, string clipName, double startTime, double endTime, float volume, float pitch)
{
lock (_lock)
{
var ev = new SoundEvent
{
@@ -138,7 +149,7 @@ namespace ADOFAIRenderer.Util.Audio
Volume = volume,
Pitch = pitch
};
ActiveEvents.Add(ev);
_activeEvents.Add(ev);
if (source != null)
{
_sourceMappings.Remove(source);
@@ -147,50 +158,50 @@ namespace ADOFAIRenderer.Util.Audio
}
}
public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
public void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch)
{
RegisterSoundEvent(null, clipName, startTime, endTime, volume, pitch);
}
public static bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
public bool TryGetSoundEvent(AudioSource? source, out SoundEvent ev)
{
if (source == null)
{
ev = null!;
return false;
}
lock (Lock)
lock (_lock)
{
return _sourceMappings.TryGetValue(source, out ev);
}
}
public static bool RemoveSourceMapping(AudioSource source)
public bool RemoveSourceMapping(AudioSource source)
{
lock (Lock)
lock (_lock)
{
return _sourceMappings.Remove(source);
}
}
public static void Reset()
public void Reset()
{
lock (Lock)
lock (_lock)
{
ActiveEvents.Clear();
_activeEvents.Clear();
_sourceMappings = new System.Runtime.CompilerServices.ConditionalWeakTable<AudioSource, SoundEvent>();
_bgmSamples = null;
_bgmDuration = 0;
AudioClipCache.Clear();
_clipCache.Clear();
}
}
public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
public void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
// mixedBuffer size must be at least samplesNeeded * _targetChannels
Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels);
lock (Lock)
lock (_lock)
{
// 1. Mix BGM
MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase);
@@ -207,7 +218,7 @@ namespace ADOFAIRenderer.Util.Audio
}
}
private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
private void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase)
{
if (_bgmSamples == null || _bgmSamples.Length == 0) return;
@@ -251,24 +262,24 @@ namespace ADOFAIRenderer.Util.Audio
}
}
private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
private void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime)
{
for (int i = ActiveEvents.Count - 1; i >= 0; i--)
for (var i = _activeEvents.Count - 1; i >= 0; i--)
{
var ev = ActiveEvents[i];
var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
var ev = _activeEvents[i];
var clipData = _clipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false);
if (clipData == null)
{
ActiveEvents.RemoveAt(i);
_activeEvents.RemoveAt(i);
continue;
}
bool isSoundFinished = false;
int totalSamples = clipData.Samples.Length / _targetChannels;
var isSoundFinished = false;
var totalSamples = clipData.Samples.Length / _targetChannels;
for (int s = 0; s < samplesNeeded; s++)
for (var s = 0; s < samplesNeeded; s++)
{
double sampleTime = frameStartTime + (double)s / _targetSampleRate;
var sampleTime = frameStartTime + (double)s / _targetSampleRate;
// If it is a looping sound and has reached or exceeded its end time, stop playing.
if (ev.EndTime >= 0 && sampleTime >= ev.EndTime)
@@ -277,10 +288,10 @@ namespace ADOFAIRenderer.Util.Audio
break;
}
double elapsed = sampleTime - ev.StartTime;
var elapsed = sampleTime - ev.StartTime;
if (elapsed < 0) continue; // Sound hasn't started yet
double soundTime = elapsed * ev.Pitch;
var soundTime = elapsed * ev.Pitch;
if (soundTime >= clipData.Duration)
{
@@ -314,7 +325,7 @@ namespace ADOFAIRenderer.Util.Audio
// Clean up finished sound events (both one-shot and looping)
if (isSoundFinished)
{
ActiveEvents.RemoveAt(i);
_activeEvents.RemoveAt(i);
}
}
}