diff --git a/Core/RenderManager.cs b/Core/RenderManager.cs index 811ee44..ed2c52f 100644 --- a/Core/RenderManager.cs +++ b/Core/RenderManager.cs @@ -384,6 +384,11 @@ namespace ADOFAIRenderer.Core private IEnumerator SetupPlay() { + if (Configuration.RenderAudio && GetOutputExtension() != ".mp4") + { + Configuration.AudioSampleRate = 48000; + } + for (; countdown > 0; countdown--) { yield return new WaitForSecondsRealtime(1); @@ -426,6 +431,10 @@ namespace ADOFAIRenderer.Core { AudioSpectrumGenerator.Initialize(scrConductor.instance.song.clip); } + if (Configuration.RenderAudio) + { + OfflineAudioMixer.InitializeBGM(scrConductor.instance.song.clip, Configuration.AudioChannels, Configuration.AudioSampleRate); + } scrConductor.instance.song.clip = null; scrConductor.instance.song.time = -1000; @@ -459,13 +468,7 @@ namespace ADOFAIRenderer.Core var videoFPS = Configuration.VideoFPS; var selectedCodec = Configuration.Codec; - - var ext = selectedCodec switch - { - Codec.AV1 or Codec.VP9 => ".webm", - Codec.VVC => ".mkv", - _ => ".mp4" - }; + var ext = GetOutputExtension(); var startTime = DateTime.Now; var fileName = startTime.ToString("yyyy-MM-dd HH-mm-ss") + ext; @@ -475,6 +478,7 @@ namespace ADOFAIRenderer.Core try { var effectiveVideoFps = Math.Min(videoFPS, renderFPS); + var codecId = GetAudioCodecId(ext); var settings = new FFmpegNativeEncoder.EncoderSettings { Width = Configuration.RenderWidth, @@ -486,7 +490,15 @@ namespace ADOFAIRenderer.Core TargetBitrate = Configuration.EncodingConfig.Bitrate, MaxBitrate = Configuration.EncodingConfig.MaxBitrate, GopSize = Configuration.EncodingConfig.KeyframeInterval, - ForceSoftware = Configuration.EncodingConfig.ForceSoftwareEncoder + ForceSoftware = Configuration.EncodingConfig.ForceSoftwareEncoder, + Audio = Configuration.RenderAudio ? new FFmpegNativeEncoder.AudioSettings + { + SampleRate = Configuration.AudioSampleRate, + Channels = Configuration.AudioChannels, + InputSampleFormat = AVSampleFormat.AV_SAMPLE_FMT_FLT, + AudioCodecId = codecId, + Bitrate = Configuration.AudioBitrate + } : null }; videoFFmpeg = new FFmpegNativeEncoder(settings); @@ -565,6 +577,29 @@ namespace ADOFAIRenderer.Core _frameReader?.RecycleState(frame.ReadbackState); } } + + if (Configuration.RenderAudio) + { + int frameIdx = (int)(passCount - 1); + double effectiveVideoFps = Math.Min(Configuration.VideoFPS, Configuration.RenderFPS); + double frameStartTime = frameIdx / effectiveVideoFps; + double frameEndTime = (frameIdx + 1) / effectiveVideoFps; + + int sampleRate = Configuration.AudioSampleRate; + int channels = Configuration.AudioChannels; + int samplesNeeded = (int)(Math.Floor(frameEndTime * sampleRate) - Math.Floor(frameStartTime * sampleRate)); + + float[] audioBuffer = ArrayPool.Shared.Rent(samplesNeeded * channels); + try + { + OfflineAudioMixer.GenerateAudioForFrame(audioBuffer, samplesNeeded, frameStartTime, dspTimeBase); + videoFFmpeg.WriteAudioSamples(audioBuffer, samplesNeeded); + } + finally + { + ArrayPool.Shared.Return(audioBuffer); + } + } continue; } @@ -762,9 +797,26 @@ namespace ADOFAIRenderer.Core } _frameReader?.Dispose(); + OfflineAudioMixer.Reset(); ADOBase.loader.LoadSceneWithTransition(WipeDirection.StartsFromRight, "scnEditor"); Cursor.visible = true; } + + private string GetOutputExtension() + { + var selectedCodec = Configuration.Codec; + return selectedCodec switch + { + Codec.AV1 or Codec.VP9 => ".webm", + Codec.VVC => ".mkv", + _ => ".mp4" + }; + } + + private AVCodecID GetAudioCodecId(string ext) + { + return ext == ".mp4" ? AVCodecID.AV_CODEC_ID_AAC : AVCodecID.AV_CODEC_ID_OPUS; + } } } \ No newline at end of file diff --git a/Data/RenderConfig.cs b/Data/RenderConfig.cs index 19e12ca..6c8af72 100644 --- a/Data/RenderConfig.cs +++ b/Data/RenderConfig.cs @@ -18,6 +18,10 @@ namespace ADOFAIRenderer.Data public int? RandomSeed { get; set; } = null; public bool ProcessAudio { get; set; } + public bool RenderAudio { get; set; } = true; + public int AudioSampleRate { get; set; } = 44100; + public int AudioChannels { get; set; } = 2; + public int AudioBitrate { get; set; } = 192000; } public class EncodingConfig diff --git a/Patch/Engine/AudioPatches.cs b/Patch/Engine/AudioPatches.cs new file mode 100644 index 0000000..c780e9d --- /dev/null +++ b/Patch/Engine/AudioPatches.cs @@ -0,0 +1,50 @@ +using ADOFAIRenderer.Core; +using ADOFAIRenderer.Util; +using HarmonyLib; +using UnityEngine.Audio; + +namespace ADOFAIRenderer.Patch.Engine +{ + [HarmonyPatch(typeof(AudioManager), "_Play")] + public static class AudioManagerPlayPatch + { + [HarmonyPrefix] + public static bool Prefix(string snd, double time, AudioMixerGroup group, float volume, int priority) + { + if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) + { + // Cache the clip on the main thread + AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true); + + // Convert absolute DSP time to relative time in seconds from render start + double relativeStartTime = time - RenderManager.Instance.dspTimeBase; + + // Register event to the offline mixer (default pitch = 1.0f) + OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, -1.0, volume, 1.0f); + } + return true; // Let the original method execute to prevent NullReferenceException + } + } + + [HarmonyPatch(typeof(scrConductor), "PlayWithEndTime")] + public static class PlayWithEndTimePatch + { + [HarmonyPrefix] + public static bool Prefix(string snd, double time, double endTime, float volume, int priority) + { + if (RenderManager.IsCapturingActive && RenderManager.Instance.Configuration.RenderAudio) + { + // Cache the clip on the main thread + AudioClipCache.GetOrCache(snd, RenderManager.Instance.Configuration.AudioChannels, RenderManager.Instance.Configuration.AudioSampleRate, allowLoad: true); + + // Convert absolute DSP times to relative times in seconds from render start + double relativeStartTime = time - RenderManager.Instance.dspTimeBase; + double relativeEndTime = endTime - RenderManager.Instance.dspTimeBase; + + // Register looping sound event to the offline mixer (default pitch = 1.0f) + OfflineAudioMixer.RegisterSoundEvent(snd, relativeStartTime, relativeEndTime, volume, 1.0f); + } + return true; // Let the original method execute + } + } +} diff --git a/Util/OfflineAudioMixer.cs b/Util/OfflineAudioMixer.cs new file mode 100644 index 0000000..7beda48 --- /dev/null +++ b/Util/OfflineAudioMixer.cs @@ -0,0 +1,379 @@ +using System; +using System.Collections.Generic; +using UnityEngine; + +namespace ADOFAIRenderer.Util +{ + public static class AudioResampler + { + public static float[] Resample(float[] inputSamples, int inputChannels, int inputFrequency, int targetChannels, int targetFrequency) + { + // ── Safety Validation ── + if (inputSamples.Length == 0 || + inputChannels <= 0 || targetChannels <= 0 || + inputFrequency <= 0 || targetFrequency <= 0) + { + return Array.Empty(); + } + + // If format is identical, return original samples + if (inputFrequency == targetFrequency && inputChannels == targetChannels) + return inputSamples; + + int inputLength = inputSamples.Length / inputChannels; + int targetLength = (int)Math.Floor((double)inputLength * targetFrequency / inputFrequency); + float[] outputSamples = new float[targetLength * targetChannels]; + + for (int i = 0; i < targetLength; i++) + { + double srcIndex = (double)i * inputFrequency / targetFrequency; + long idxLow = (long)Math.Floor(srcIndex); + long idxHigh = idxLow + 1; + float t = (float)(srcIndex - idxLow); + + if (idxHigh >= inputLength) idxHigh = inputLength - 1; + if (idxLow >= inputLength) idxLow = inputLength - 1; + + if (targetChannels == 1) // 1. Mono Output + { + float valLow, valHigh; + if (inputChannels == 1) // Mono -> Mono + { + valLow = inputSamples[idxLow]; + valHigh = inputSamples[idxHigh]; + } + else // Multi -> Mono (Downmix: Average all channels) + { + float sumLow = 0f; + float sumHigh = 0f; + for (int c = 0; c < inputChannels; c++) + { + sumLow += inputSamples[idxLow * inputChannels + c]; + sumHigh += inputSamples[idxHigh * inputChannels + c]; + } + valLow = sumLow / inputChannels; + valHigh = sumHigh / inputChannels; + } + outputSamples[i] = valLow + t * (valHigh - valLow); + } + else if (targetChannels == 2) // 2. Stereo Output + { + if (inputChannels == 1) // Mono -> Stereo (Duplicate to Left/Right) + { + float valLow = inputSamples[idxLow]; + float valHigh = inputSamples[idxHigh]; + float val = valLow + t * (valHigh - valLow); + outputSamples[i * 2] = val; + outputSamples[i * 2 + 1] = val; + } + else // Stereo/Multi -> Stereo (Map Left/Right, ignore extra channels) + { + float valLowL = inputSamples[idxLow * inputChannels]; + float valHighL = inputSamples[idxHigh * inputChannels]; + outputSamples[i * 2] = valLowL + t * (valHighL - valLowL); + + float valLowR = inputSamples[idxLow * inputChannels + 1]; + float valHighR = inputSamples[idxHigh * inputChannels + 1]; + outputSamples[i * 2 + 1] = valLowR + t * (valHighR - valLowR); + } + } + else // 3. Multi-channel Output (e.g., 5.1 Surround) + { + for (int c = 0; c < targetChannels; c++) + { + float valLow, valHigh; + if (c < inputChannels) + { + valLow = inputSamples[idxLow * inputChannels + c]; + valHigh = inputSamples[idxHigh * inputChannels + c]; + } + else + { + valLow = inputSamples[idxLow * inputChannels]; + valHigh = inputSamples[idxHigh * inputChannels]; + } + outputSamples[i * targetChannels + c] = valLow + t * (valHigh - valLow); + } + } + } + + return outputSamples; + } + } + + public static class AudioClipCache + { + private static readonly Dictionary Cache = new(); + + public class CachedClipData + { + public float[] Samples = null!; // Pre-resampled to target channels and sample rate + public double Duration; + } + + public static CachedClipData? GetOrCache(string clipName, int targetChannels, int targetSampleRate, bool allowLoad = true) + { + if (string.IsNullOrEmpty(clipName)) return null; + + lock (Cache) + { + if (Cache.TryGetValue(clipName, out var cached)) + return cached; + + if (!allowLoad) return null; + + if (AudioManager.Instance == null) return null; + + AudioClip clip = AudioManager.Instance.FindOrLoadAudioClip(clipName); + if (clip == null) return null; + + try + { + int totalSamples = clip.samples; + int channels = clip.channels; + float[] rawSamples = new float[totalSamples * channels]; + clip.GetData(rawSamples, 0); + + // Auto-resample during cache generation + float[] resampled = AudioResampler.Resample(rawSamples, channels, clip.frequency, targetChannels, targetSampleRate); + + var data = new CachedClipData + { + Samples = resampled, + Duration = (double)resampled.Length / (targetChannels * targetSampleRate) + }; + + Cache[clipName] = data; + return data; + } + catch (Exception e) + { + MelonLoader.MelonLogger.Error($"[AudioClipCache] Failed to cache and resample clip {clipName}: {e}"); + return null; + } + } + } + + public static void Clear() + { + lock (Cache) + { + Cache.Clear(); + } + } + } + + public static class OfflineAudioMixer + { + public struct SoundEvent + { + public string ClipName; + public double StartTime; // In seconds relative to dspTimeBase + public double EndTime; // In seconds relative to dspTimeBase (-1 if one-shot) + public float Volume; + public float Pitch; + } + + private static readonly List ActiveEvents = new(); + private static readonly object Lock = new(); + + // Target Audio Settings + private static int _targetChannels = 2; + private static int _targetSampleRate = 44100; + + // BGM Cached Samples (Pre-resampled) + private static float[]? _bgmSamples; + private static double _bgmDuration; + + public static void InitializeBGM(AudioClip? clip, int targetChannels, int targetSampleRate) + { + lock (Lock) + { + _targetChannels = targetChannels; + _targetSampleRate = targetSampleRate; + + if (!clip) + { + _bgmSamples = null; + _bgmDuration = 0; + return; + } + + try + { + int originalSamples = clip.samples; + int originalChannels = clip.channels; + float[] rawSamples = new float[originalSamples * originalChannels]; + clip.GetData(rawSamples, 0); + + // Pre-resample the BGM to the target format + _bgmSamples = AudioResampler.Resample(rawSamples, originalChannels, clip.frequency, targetChannels, targetSampleRate); + _bgmDuration = (double)_bgmSamples.Length / (targetChannels * targetSampleRate); + + MelonLoader.MelonLogger.Msg($"[OfflineAudioMixer] Initialized resampled BGM: {clip.name}, Samples: {_bgmSamples.Length / targetChannels}, TargetFreq: {targetSampleRate}Hz, TargetChannels: {targetChannels}"); + } + catch (Exception e) + { + MelonLoader.MelonLogger.Error($"[OfflineAudioMixer] Failed to initialize and resample BGM: {e}"); + _bgmSamples = null; + _bgmDuration = 0; + } + } + } + + public static void RegisterSoundEvent(string clipName, double startTime, double endTime, float volume, float pitch) + { + lock (Lock) + { + ActiveEvents.Add(new SoundEvent + { + ClipName = clipName, + StartTime = startTime, + EndTime = endTime, + Volume = volume, + Pitch = pitch + }); + } + } + + public static void Reset() + { + lock (Lock) + { + ActiveEvents.Clear(); + _bgmSamples = null; + _bgmDuration = 0; + AudioClipCache.Clear(); + } + } + + public static void GenerateAudioForFrame(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) + { + // mixedBuffer size must be at least samplesNeeded * _targetChannels + Array.Clear(mixedBuffer, 0, samplesNeeded * _targetChannels); + + lock (Lock) + { + // 1. Mix BGM + MixBGM(mixedBuffer, samplesNeeded, frameStartTime, dspTimeBase); + + // 2. Mix Hitsounds/Effect Sounds + MixEffectSounds(mixedBuffer, samplesNeeded, frameStartTime); + + // 3. Saturation Guard (Clamping) + for (int i = 0; i < samplesNeeded * _targetChannels; i++) + { + if (mixedBuffer[i] > 1.0f) mixedBuffer[i] = 1.0f; + else if (mixedBuffer[i] < -1.0f) mixedBuffer[i] = -1.0f; + } + } + } + + private static void MixBGM(float[] mixedBuffer, int samplesNeeded, double frameStartTime, double dspTimeBase) + { + if (_bgmSamples == null || _bgmSamples.Length == 0) return; + + var conductor = scrConductor.instance; + if (!conductor) return; + + double pitch = conductor.song.pitch; + double dspTimeSong = conductor.dspTimeSong; + double separateCountdownOffset = conductor.separateCountdownTime + ? (conductor.crotchetAtStart / pitch * conductor.adjustedCountdownTicks) + : 0.0; + + // BGM Playback start time relative to dspTimeBase + double time1 = dspTimeSong + separateCountdownOffset; + double relativeTime1 = time1 - dspTimeBase; + + int totalSamples = _bgmSamples.Length / _targetChannels; + + for (int s = 0; s < samplesNeeded; s++) + { + double sampleTime = frameStartTime + (double)s / _targetSampleRate; + double clipTime = (sampleTime - relativeTime1) * pitch; + + if (clipTime >= 0 && clipTime < _bgmDuration) + { + double sampleIdxDouble = clipTime * _targetSampleRate; + long idxLow = (long)Math.Floor(sampleIdxDouble); + long idxHigh = idxLow + 1; + float t = (float)(sampleIdxDouble - idxLow); + + if (idxHigh >= totalSamples) idxHigh = totalSamples - 1; + if (idxLow >= totalSamples) idxLow = totalSamples - 1; + + for (int c = 0; c < _targetChannels; c++) + { + float valLow = _bgmSamples[idxLow * _targetChannels + c]; + float valHigh = _bgmSamples[idxHigh * _targetChannels + c]; + mixedBuffer[s * _targetChannels + c] += valLow + t * (valHigh - valLow); + } + } + } + } + + private static void MixEffectSounds(float[] mixedBuffer, int samplesNeeded, double frameStartTime) + { + for (int i = ActiveEvents.Count - 1; i >= 0; i--) + { + var ev = ActiveEvents[i]; + var clipData = AudioClipCache.GetOrCache(ev.ClipName, _targetChannels, _targetSampleRate, allowLoad: false); + if (clipData == null) continue; + + bool isSoundFinished = false; + int totalSamples = clipData.Samples.Length / _targetChannels; + + for (int s = 0; s < samplesNeeded; s++) + { + double sampleTime = frameStartTime + (double)s / _targetSampleRate; + double elapsed = sampleTime - ev.StartTime; + + if (elapsed < 0) continue; // Sound hasn't started yet + + double soundTime = elapsed * ev.Pitch; + + if (soundTime >= clipData.Duration) + { + if (ev.EndTime < 0) // One-shot + { + isSoundFinished = true; + break; + } + else // Looping + { + if (sampleTime >= ev.EndTime) + { + isSoundFinished = true; + break; + } + soundTime %= clipData.Duration; + } + } + + // Linear interpolation for pitch shifting in the mixer loop + double sampleIdxDouble = soundTime * _targetSampleRate; + long idxLow = (long)Math.Floor(sampleIdxDouble); + long idxHigh = idxLow + 1; + float t = (float)(sampleIdxDouble - idxLow); + + if (idxHigh >= totalSamples) idxHigh = totalSamples - 1; + if (idxLow >= totalSamples) idxLow = totalSamples - 1; + + for (int c = 0; c < _targetChannels; c++) + { + float valLow = clipData.Samples[idxLow * _targetChannels + c]; + float valHigh = clipData.Samples[idxHigh * _targetChannels + c]; + mixedBuffer[s * _targetChannels + c] += (valLow + t * (valHigh - valLow)) * ev.Volume; + } + } + + // Clean up finished one-shot sounds + if (isSoundFinished && ev.EndTime < 0) + { + ActiveEvents.RemoveAt(i); + } + } + } + } +}