chore: add a tool for finding Music tracks whose files have gone

Lidarr renames an artist or album folder, music-mirror prunes the old path and
encodes the new one, and every entry in Apple Music pointing at the old path
dies. Music offers no view of those, and the exclamation mark only appears once
a track is touched.

Reads the XML from Music's own Export Library rather than asking Music itself.
A broken track makes AppleScript's `location` raise instead of returning a
value, so a bulk query dies on the first one with error -1728 and a per-track
loop costs an Apple event apiece.

Checked against a single directory walk rather than a test per file. On a
fifty-thousand-track library that is around seven thousand directory reads
instead of fifty thousand stat calls, and over SMB each of those stats is a
network round trip -- which is the difference between seconds and minutes.

Two comparisons have to be loosened or most of the library reads as missing.
macOS stores filenames decomposed while the share composes them, so the umlauts
in Motley Crue are two different byte strings depending on which side wrote the
name; both are normalised to NFC. And the share is very likely
case-insensitive, so a file is not missing because someone capitalised it
differently.

The test stage now copies tools/ as well, since the suite covers this and the
runtime image deliberately does not carry it.

Dockerfile

@@ -28,5 +28,8 @@ FROM runtime AS test
 
 RUN pip install --no-cache-dir pytest
 COPY pytest.ini ./
+# The host-side tools are not part of the runtime image, but their tests are
+# part of the suite, so they have to be present for it.
+COPY tools ./tools
 COPY tests ./tests
 RUN python -m pytest

README.md

@@ -377,6 +377,43 @@ stubbed. No network, no credentials, no rate limit. On a Nix machine:
 nix shell nixpkgs#python3Packages.pytest -c pytest
 ```
 
+## Tools
+
+Host-side scripts under `tools/`. Not part of the container image; run them
+wherever they are needed.
+
+### `find_missing_tracks.py`
+
+Reports tracks in an Apple Music library whose files are no longer on disk —
+which happens whenever Lidarr renames an artist or album folder and
+music-mirror prunes the old path.
+
+```sh
+# Music: File > Library > Export Library... then, with the share mounted:
+python3 tools/find_missing_tracks.py Library.xml --root /Volumes/music-mp3
+```
+
+Reading the exported XML rather than asking Music itself is deliberate. A
+broken track makes AppleScript's `location` raise instead of returning a value,
+so a bulk query dies on the first one with `-1728` and a per-track loop costs an
+Apple event apiece.
+
+It checks against a **single directory walk**, not a test per file. On a
+50,000-track library that is ~7,000 directory reads instead of 50,000 stat
+calls, and over SMB every one of those stats is a network round trip.
+
+Two comparisons that have to be loosened, or most of the library reads as
+missing:
+
+- **Unicode.** macOS stores filenames decomposed; the share composes them.
+  `Mötley Crüe` is two different byte strings depending on which side wrote it.
+  Both sides are normalised to NFC.
+- **Case.** The share is very likely case-insensitive. A file is not missing
+  because someone capitalised it differently.
+
+Pass `--root` if the library holds anything outside the mirror: the root it
+otherwise derives is the common parent of every track, which can be `/`.
+
 ## Where this is going
 
 | Stage | Status |

tests/test_find_missing_tracks.py

@@ -0,0 +1,168 @@
+import os
+import plistlib
+import sys
+import unicodedata
+import urllib.parse
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import find_missing_tracks  # noqa: E402
+
+
+def write_library(path, tracks):
+    """Write a Library.xml in the shape Music exports."""
+    path.write_bytes(plistlib.dumps({"Tracks": {str(i): t for i, t in enumerate(tracks)}}))
+    return path
+
+
+def track(root, name, filename=None):
+    """A local track entry. Omit filename for a streaming entry with no file."""
+    entry = {"Name": name, "Artist": "An Artist", "Album": "An Album"}
+    if filename is not None:
+        entry["Location"] = "file://" + urllib.parse.quote(str(root / filename))
+    return entry
+
+
+def missing_from(library, roots):
+    with open(library, "rb") as handle:
+        loaded = plistlib.load(handle)
+    present = find_missing_tracks.existing_under(roots)
+    gone = []
+    for entry in loaded["Tracks"].values():
+        path = find_missing_tracks.location_of(entry)
+        if path is not None and find_missing_tracks.key_for(path) not in present:
+            gone.append(entry["Name"])
+    return sorted(gone)
+
+
+def test_a_deleted_file_is_reported(tmp_path):
+    (tmp_path / "kept.mp3").write_bytes(b"x")
+    library = write_library(
+        tmp_path / "Library.xml",
+        [track(tmp_path, "Kept", "kept.mp3"), track(tmp_path, "Gone", "gone.mp3")],
+    )
+
+    assert missing_from(library, [tmp_path]) == ["Gone"]
+
+
+def test_a_decomposed_filename_is_not_reported_missing(tmp_path):
+    """macOS stores filenames decomposed and most everything else composes them,
+    so the umlauts in Motley Crue are two different byte strings depending on
+    which side wrote the name."""
+    composed = unicodedata.normalize("NFC", "Mötley Crüe.mp3")
+    (tmp_path / composed).write_bytes(b"x")
+    library = write_library(
+        tmp_path / "Library.xml",
+        [track(tmp_path, "Umlauts", unicodedata.normalize("NFD", "Mötley Crüe.mp3"))],
+    )
+
+    assert missing_from(library, [tmp_path]) == []
+
+
+def test_a_case_difference_is_not_reported_missing(tmp_path):
+    """The share is very likely case-insensitive, and a file is not missing
+    because someone capitalised it differently."""
+    (tmp_path / "Hells Bells.mp3").write_bytes(b"x")
+    library = write_library(
+        tmp_path / "Library.xml", [track(tmp_path, "Bells", "hells bells.MP3")]
+    )
+
+    assert missing_from(library, [tmp_path]) == []
+
+
+def test_a_track_with_no_file_is_not_a_missing_file(tmp_path):
+    """A streaming entry has never had a file to lose."""
+    library = write_library(tmp_path / "Library.xml", [track(tmp_path, "Streamed", None)])
+
+    assert missing_from(library, [tmp_path]) == []
+
+
+def test_the_walk_finds_files_nested_below_the_root(tmp_path):
+    nested = tmp_path / "Artist" / "Album"
+    nested.mkdir(parents=True)
+    (nested / "deep.mp3").write_bytes(b"x")
+
+    found = find_missing_tracks.existing_under([tmp_path])
+
+    assert find_missing_tracks.key_for(nested / "deep.mp3") in found
+
+
+def test_the_root_is_derived_from_the_tracks(tmp_path):
+    paths = [tmp_path / "a" / "one.mp3", tmp_path / "b" / "two.mp3"]
+
+    assert find_missing_tracks.roots_of(paths, None) == [tmp_path]
+
+
+def test_an_explicit_root_overrides_the_derived_one(tmp_path):
+    """Worth using: tracks spread beyond the mirror can derive a common parent
+    of / and send the walk across the whole disk."""
+    paths = [tmp_path / "a" / "one.mp3"]
+
+    assert find_missing_tracks.roots_of(paths, ["/Volumes/music-mp3"]) == [
+        Path("/Volumes/music-mp3")
+    ]
+
+
+def test_the_whole_library_missing_is_called_out(tmp_path, capsys):
+    """Almost always an unmounted share rather than an empty library."""
+    library = write_library(
+        tmp_path / "Library.xml", [track(tmp_path / "elsewhere", "Gone", "gone.mp3")]
+    )
+
+    find_missing_tracks.main([str(library), "--root", str(tmp_path)])
+
+    assert "not mounted" in capsys.readouterr().err
+
+
+def test_a_healthy_library_says_nothing_alarming(tmp_path, capsys):
+    (tmp_path / "kept.mp3").write_bytes(b"x")
+    library = write_library(tmp_path / "Library.xml", [track(tmp_path, "Kept", "kept.mp3")])
+
+    find_missing_tracks.main([str(library), "--root", str(tmp_path)])
+
+    captured = capsys.readouterr()
+    assert "0 of 1 local tracks are missing" in captured.err
+    assert "not mounted" not in captured.err
+    assert captured.out == ""
+
+
+@pytest.mark.parametrize(
+    ("location", "expected"),
+    [
+        ("file:///music/a%20b.mp3", Path("/music/a b.mp3")),
+        ("file:///music/plain.mp3", Path("/music/plain.mp3")),
+        ("https://example.invalid/stream", None),
+        ("", None),
+    ],
+)
+def test_locations_are_decoded(location, expected):
+    assert find_missing_tracks.location_of({"Location": location} if location else {}) == expected
+
+
+def test_the_walk_costs_one_read_per_directory_not_one_per_file(tmp_path):
+    """The whole point. Over SMB a per-file check is a round trip per track."""
+    for album in range(20):
+        directory = tmp_path / f"Album {album}"
+        directory.mkdir()
+        for index in range(25):
+            (directory / f"{index}.mp3").write_bytes(b"x")
+
+    reads = {"n": 0}
+    real_walk = os.walk
+
+    def counting_walk(*args, **kwargs):
+        for entry in real_walk(*args, **kwargs):
+            reads["n"] += 1
+            yield entry
+
+    find_missing_tracks.os.walk = counting_walk
+    try:
+        found = find_missing_tracks.existing_under([tmp_path])
+    finally:
+        find_missing_tracks.os.walk = real_walk
+
+    assert len(found) == 500
+    assert reads["n"] == 21  # the root and its twenty albums, not 500 files

tools/find_missing_tracks.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Report tracks in a Music library whose files are no longer on disk.
+
+Reads the XML from Music's File > Library > Export Library. Asking Music itself
+does not work: a broken track makes AppleScript's `location` raise rather than
+return a value, so a bulk query dies on the first one with error -1728, and a
+per-track loop costs an Apple event apiece.
+
+The library is checked against a single directory walk rather than by testing
+each file. Over SMB a per-file test is one network round trip per track --
+fifty thousand of them -- where a walk reads each directory once and gets every
+name in it back at once.
+"""
+
+import argparse
+import os
+import plistlib
+import sys
+import unicodedata
+import urllib.parse
+from pathlib import Path
+
+
+def location_of(track):
+    """Return the filesystem path a track points at, or None if it has none."""
+    location = track.get("Location")
+    if not location or not location.startswith("file://"):
+        return None
+    return Path(urllib.parse.unquote(urllib.parse.urlparse(location).path))
+
+
+def key_for(path):
+    """Return a comparison key for a path.
+
+    Normalised to NFC because macOS stores filenames decomposed and most
+    everything else composes them, so "Motley Crue" with its umlauts is two
+    different byte strings depending on which side wrote it. Casefolded because
+    the share is very likely case-insensitive and a file is not missing merely
+    because someone capitalised it differently.
+    """
+    return unicodedata.normalize("NFC", str(path)).casefold()
+
+
+def existing_under(roots):
+    """Return every file below the given roots, keyed for comparison."""
+    found = set()
+    for root in roots:
+        for base, _, names in os.walk(root):
+            for name in names:
+                found.add(key_for(os.path.join(base, name)))
+    return found
+
+
+def roots_of(paths, given):
+    """Return the directories worth walking."""
+    if given:
+        return [Path(root) for root in given]
+    try:
+        return [Path(os.path.commonpath([str(path) for path in paths]))]
+    except ValueError:
+        # Tracks spread across separate volumes have no common parent.
+        return sorted({path.parents[-2] for path in paths if len(path.parents) > 1})
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("library", help="Library.xml exported from Music")
+    parser.add_argument("--root", action="append", help="directory to scan; repeatable")
+    parser.add_argument("--limit", type=int, default=0, help="show at most this many")
+    args = parser.parse_args(argv)
+
+    with open(args.library, "rb") as handle:
+        library = plistlib.load(handle)
+
+    tracks = [
+        (track, location_of(track)) for track in library.get("Tracks", {}).values()
+    ]
+    local = [(track, path) for track, path in tracks if path is not None]
+    if not local:
+        print("no local files in this library", file=sys.stderr)
+        return 0
+
+    roots = roots_of([path for _, path in local], args.root)
+    print(f"scanning {', '.join(str(root) for root in roots)}", file=sys.stderr)
+    present = existing_under(roots)
+
+    gone = [(track, path) for track, path in local if key_for(path) not in present]
+    for track, path in gone[: args.limit or None]:
+        print(
+            "\t".join(
+                (
+                    track.get("Artist", ""),
+                    track.get("Album", ""),
+                    track.get("Name", ""),
+                    str(path),
+                )
+            )
+        )
+
+    print(
+        f"\n{len(gone)} of {len(local)} local tracks are missing their file"
+        f" ({len(tracks) - len(local)} have no file at all)",
+        file=sys.stderr,
+    )
+    if gone and len(gone) == len(local):
+        print(
+            "Every single one is missing, which almost certainly means the share"
+            " is not mounted rather than that the library is empty.",
+            file=sys.stderr,
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())