Resolve Audio Clips From Level Files

Patch/Engine/AudioPatches.cs

@@ -1,11 +1,31 @@
 using System;
 using ADOFAIRenderer.Core;
+using ADOFAIRenderer.Util.Audio.Clip;
 using HarmonyLib;
 using UnityEngine;
 using UnityEngine.Audio;
 
 namespace ADOFAIRenderer.Patch.Engine;
 
+/// <summary>
+/// Records the file path of each external clip the game loads (always active, also outside rendering,
+/// because levels load their song and PlaySound files before a render starts).
+/// FindOrLoadAudioClipExternal returns the clip already in audioLib when its key exists, so the path is recorded
+/// only when the key is missing and the file is about to be loaded.
+/// </summary>
+[HarmonyPatch(typeof(AudioManager), nameof(AudioManager.FindOrLoadAudioClipExternal))]
+public static class AudioManagerFindOrLoadAudioClipExternalPatch
+{
+    [HarmonyPrefix]
+    // ReSharper disable once InconsistentNaming
+    public static void Prefix(AudioManager __instance, string path)
+    {
+        if (string.IsNullOrEmpty(path)) return;
+        if (__instance.audioLib.ContainsKey(AudioClipPaths.KeyOf(path))) return;
+        AudioClipPaths.Record(path);
+    }
+}
+
 [HarmonyPatch(typeof(AudioManager), "_Play")]
 public static class AudioManagerPlayPatch
 {

Util/Audio/Clip/AudioClipPaths.cs

@@ -0,0 +1,46 @@
+using System.Collections.Generic;
+using System.IO;
+
+namespace ADOFAIRenderer.Util.Audio.Clip;
+
+/// <summary>
+/// File path of every external audio clip the game loads, keyed by the game's audioLib key.
+///
+/// The game loads external clips with AudioManager.FindOrLoadAudioClipExternal(path, ...) and stores them in audioLib under
+/// Path.GetFileName(path) + "*external" (the same string becomes AudioClip.name). Recorded paths:
+/// - Song: Path.Combine(Path.GetDirectoryName(levelPath), levelData.songFilename) (scnGame song loading).
+/// - PlaySound event sound: Path.Combine(Path.GetDirectoryName(levelPath), hitsound) (scnGame.LoadSoundCo);
+///   ffxPlaySound plays it as hitsound + "*external".
+/// <see cref="Record"/> is called only when the game actually loads the file (the key is not in audioLib yet),
+/// so the recorded path is the file of the clip audioLib holds under that key.
+/// Thread-safe.
+/// </summary>
+internal static class AudioClipPaths
+{
+    /// <summary>Suffix of the game's audioLib keys for external clips.</summary>
+    public const string ExternalSuffix = "*external";
+
+    private static readonly Dictionary<string, string> Paths = new();
+
+    /// <summary>audioLib key of an external clip loaded from <paramref name="path"/>.</summary>
+    public static string KeyOf(string path) => Path.GetFileName(path) + ExternalSuffix;
+
+    /// <summary>Records <paramref name="path"/> under <see cref="KeyOf"/>(path), replacing an earlier path of the same key.</summary>
+    public static void Record(string path)
+    {
+        if (string.IsNullOrEmpty(path)) return;
+        lock (Paths)
+        {
+            Paths[KeyOf(path)] = path;
+        }
+    }
+
+    /// <summary>Recorded path for an audioLib key (or AudioClip.name) of an external clip.</summary>
+    public static bool TryGetPath(string key, out string path)
+    {
+        lock (Paths)
+        {
+            return Paths.TryGetValue(key, out path!);
+        }
+    }
+}

Util/Audio/Clip/AudioClipResolver.cs

@@ -0,0 +1,60 @@
+using System;
+using System.IO;
+using MelonLoader;
+using UnityEngine;
+
+namespace ADOFAIRenderer.Util.Audio.Clip;
+
+/// <summary>
+/// Opens the <see cref="AudioClipSource"/> for a clip the game plays, keyed by the game's audioLib key (also AudioClip.name):
+/// - Key recorded in <see cref="AudioClipPaths"/> and the file exists: decoded from the file with <see cref="FFmpegPCMReader"/>
+///   (mono MP3 through <see cref="MP3MonoLayoutReader"/>)
+///   (song files and PlaySound files of custom levels).
+/// - Otherwise, or when FFmpeg cannot open the file: the Unity clip is copied with <see cref="UnityClipLoader"/>
+///   (built-in hit sounds from Resources, official/DLC level songs from Addressables).
+/// Main thread only (the Unity copy calls AudioClip.GetData).
+/// </summary>
+internal static class AudioClipResolver
+{
+    /// <param name="store">Store of the current render.</param>
+    /// <param name="key">audioLib key of the clip (for example "song.ogg*external" or "sndKick").</param>
+    /// <param name="clip">Loaded Unity clip for the key, used when the clip has no file; may be null when the key has a file.</param>
+    /// <returns>The source, or null when neither the file nor the clip can be read.</returns>
+    public static AudioClipSource? GetOrAdd(AudioClipStore store, string key, AudioClip? clip)
+    {
+        if (store.TryGet(key, out var existing)) return existing;
+
+        if (AudioClipPaths.TryGetPath(key, out var path) && File.Exists(path))
+        {
+            IPCMReader? fileReader = null;
+            try
+            {
+                fileReader = OpenFile(path);
+                if (fileReader is MP3MonoLayoutReader)
+                    Melon<RenderMod>.Logger.Warning($"[AudioClipResolver] Mono MP3 '{path}' is rendered like the game: half speed, one octave lower.");
+            }
+            catch (Exception e)
+            {
+                Melon<RenderMod>.Logger.Warning($"[AudioClipResolver] FFmpeg could not open '{path}' for '{key}', using the Unity clip: {e.Message}");
+            }
+
+            if (fileReader != null)
+            {
+                var source = store.GetOrAdd(key, () => fileReader, () => OpenFile(path));
+                if (source == null) fileReader.Dispose();
+                return source;
+            }
+        }
+
+        if (!clip) return store.GetOrAdd(key, () => null);
+        return store.GetOrAdd(key, () => UnityClipLoader.Load(clip!, store));
+    }
+
+    /// <summary>Opens <paramref name="path"/> with FFmpeg; mono MP3 files are wrapped in <see cref="MP3MonoLayoutReader"/> (game layout).</summary>
+    private static IPCMReader OpenFile(string path)
+    {
+        var reader = new FFmpegPCMReader(path);
+        if (!reader.IsMP3 || reader.Channels != 1) return reader;
+        return new MP3MonoLayoutReader(reader);
+    }
+}

Util/Audio/Clip/AudioClipSource.cs

@@ -545,6 +545,11 @@ internal sealed class AudioClipSource
             try
             {
                 reader = _openReader();
+                if (reader is { IsForwardOnly: true })
+                {
+                    reader.Dispose();
+                    reader = null;
+                }
                 if (reader != null)
                 {
                     var stream = new DecodeStream(reader, Frequency);

Util/Audio/Clip/AudioClipStore.cs

@@ -26,6 +26,9 @@ internal sealed class AudioClipStore : IDisposable
     /// <summary>Longest wait of a consumer between checks when no chunk is loaded, in milliseconds.</summary>
     private const int ConsumerWaitMs = 50;
 
+    /// <summary>Longest wait for the decoder thread to stop in <see cref="Dispose"/>, in milliseconds.</summary>
+    private const int DecoderStopTimeoutMs = 3000;
+
     private readonly Dictionary<string, AudioClipSource?> _sources = new();
     private readonly List<AudioClipSource> _sourceList = new();
     private readonly Stack<float[]> _chunkPool = new();
@@ -128,10 +131,10 @@ internal sealed class AudioClipStore : IDisposable
     /// <summary>
     /// Returns the source registered under <paramref name="key"/>, creating it with <paramref name="createReader"/> on first use.
     /// A key whose reader could not be created is remembered as null and not retried.
-    /// For readers that can seek, <paramref name="createReader"/> is kept and called again on the decoder thread
-    /// to open additional decode streams (see <see cref="AudioClipSource.MaxDecodeStreams"/>).
+    /// For readers that can seek, additional decode streams (see <see cref="AudioClipSource.MaxDecodeStreams"/>) are opened on the
+    /// decoder thread with <paramref name="openAdditionalReader"/>, or with <paramref name="createReader"/> when it is null.
     /// </summary>
-    public AudioClipSource? GetOrAdd(string key, Func<IPCMReader?> createReader)
+    public AudioClipSource? GetOrAdd(string key, Func<IPCMReader?> createReader, Func<IPCMReader?>? openAdditionalReader = null)
     {
         lock (_sources)
         {
@@ -147,7 +150,7 @@ internal sealed class AudioClipStore : IDisposable
             {
                 try
                 {
-                    source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : createReader, OutputRate);
+                    source = new AudioClipSource(this, key, reader, reader.IsForwardOnly ? null : openAdditionalReader ?? createReader, OutputRate);
                 }
                 catch
                 {
@@ -315,7 +318,12 @@ internal sealed class AudioClipStore : IDisposable
         _stopRequested = true;
         _serviceRequested.Set();
         NotifyChunkLoaded();
-        _decoderThread?.Join();
+
+        // A decoder thread blocked inside a read (for example a file on an unresponsive drive) is left running:
+        // it is a background thread, and its sources keep their readers so the blocked call does not touch freed FFmpeg contexts.
+        var decoderStopped = _decoderThread == null || _decoderThread.Join(DecoderStopTimeoutMs);
+        if (!decoderStopped)
+            Melon<RenderMod>.Logger.Warning($"[AudioClipStore] Decoder thread did not stop within {DecoderStopTimeoutMs} ms; its readers are left open.");
 
         LogMemoryUsage();
 
@@ -325,8 +333,11 @@ internal sealed class AudioClipStore : IDisposable
             _sources.Clear();
         }
 
-        foreach (var source in sources)
-            source.DisposeInternal();
+        if (decoderStopped)
+        {
+            foreach (var source in sources)
+                source.DisposeInternal();
+        }
 
         lock (_poolLock)
         {