diff --git a/Dockerfile b/Dockerfile index c9e6af0..c3da9c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,5 +28,8 @@ FROM runtime AS test RUN pip install --no-cache-dir pytest COPY pytest.ini ./ +# The host-side tools are not part of the runtime image, but their tests are +# part of the suite, so they have to be present for it. +COPY tools ./tools COPY tests ./tests RUN python -m pytest diff --git a/README.md b/README.md index 3ae5d07..fa308b9 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,43 @@ stubbed. No network, no credentials, no rate limit. On a Nix machine: nix shell nixpkgs#python3Packages.pytest -c pytest ``` +## Tools + +Host-side scripts under `tools/`. Not part of the container image; run them +wherever they are needed. + +### `find_missing_tracks.py` + +Reports tracks in an Apple Music library whose files are no longer on disk — +which happens whenever Lidarr renames an artist or album folder and +music-mirror prunes the old path. + +```sh +# Music: File > Library > Export Library... then, with the share mounted: +python3 tools/find_missing_tracks.py Library.xml --root /Volumes/music-mp3 +``` + +Reading the exported XML rather than asking Music itself is deliberate. A +broken track makes AppleScript's `location` raise instead of returning a value, +so a bulk query dies on the first one with `-1728` and a per-track loop costs an +Apple event apiece. + +It checks against a **single directory walk**, not a test per file. On a +50,000-track library that is ~7,000 directory reads instead of 50,000 stat +calls, and over SMB every one of those stats is a network round trip. + +Two comparisons that have to be loosened, or most of the library reads as +missing: + +- **Unicode.** macOS stores filenames decomposed; the share composes them. + `Mötley Crüe` is two different byte strings depending on which side wrote it. + Both sides are normalised to NFC. +- **Case.** The share is very likely case-insensitive. A file is not missing + because someone capitalised it differently. + +Pass `--root` if the library holds anything outside the mirror: the root it +otherwise derives is the common parent of every track, which can be `/`. + ## Where this is going | Stage | Status | diff --git a/tests/test_find_missing_tracks.py b/tests/test_find_missing_tracks.py new file mode 100644 index 0000000..17d463c --- /dev/null +++ b/tests/test_find_missing_tracks.py @@ -0,0 +1,168 @@ +import os +import plistlib +import sys +import unicodedata +import urllib.parse +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools")) + +import find_missing_tracks # noqa: E402 + + +def write_library(path, tracks): + """Write a Library.xml in the shape Music exports.""" + path.write_bytes(plistlib.dumps({"Tracks": {str(i): t for i, t in enumerate(tracks)}})) + return path + + +def track(root, name, filename=None): + """A local track entry. Omit filename for a streaming entry with no file.""" + entry = {"Name": name, "Artist": "An Artist", "Album": "An Album"} + if filename is not None: + entry["Location"] = "file://" + urllib.parse.quote(str(root / filename)) + return entry + + +def missing_from(library, roots): + with open(library, "rb") as handle: + loaded = plistlib.load(handle) + present = find_missing_tracks.existing_under(roots) + gone = [] + for entry in loaded["Tracks"].values(): + path = find_missing_tracks.location_of(entry) + if path is not None and find_missing_tracks.key_for(path) not in present: + gone.append(entry["Name"]) + return sorted(gone) + + +def test_a_deleted_file_is_reported(tmp_path): + (tmp_path / "kept.mp3").write_bytes(b"x") + library = write_library( + tmp_path / "Library.xml", + [track(tmp_path, "Kept", "kept.mp3"), track(tmp_path, "Gone", "gone.mp3")], + ) + + assert missing_from(library, [tmp_path]) == ["Gone"] + + +def test_a_decomposed_filename_is_not_reported_missing(tmp_path): + """macOS stores filenames decomposed and most everything else composes them, + so the umlauts in Motley Crue are two different byte strings depending on + which side wrote the name.""" + composed = unicodedata.normalize("NFC", "Mötley Crüe.mp3") + (tmp_path / composed).write_bytes(b"x") + library = write_library( + tmp_path / "Library.xml", + [track(tmp_path, "Umlauts", unicodedata.normalize("NFD", "Mötley Crüe.mp3"))], + ) + + assert missing_from(library, [tmp_path]) == [] + + +def test_a_case_difference_is_not_reported_missing(tmp_path): + """The share is very likely case-insensitive, and a file is not missing + because someone capitalised it differently.""" + (tmp_path / "Hells Bells.mp3").write_bytes(b"x") + library = write_library( + tmp_path / "Library.xml", [track(tmp_path, "Bells", "hells bells.MP3")] + ) + + assert missing_from(library, [tmp_path]) == [] + + +def test_a_track_with_no_file_is_not_a_missing_file(tmp_path): + """A streaming entry has never had a file to lose.""" + library = write_library(tmp_path / "Library.xml", [track(tmp_path, "Streamed", None)]) + + assert missing_from(library, [tmp_path]) == [] + + +def test_the_walk_finds_files_nested_below_the_root(tmp_path): + nested = tmp_path / "Artist" / "Album" + nested.mkdir(parents=True) + (nested / "deep.mp3").write_bytes(b"x") + + found = find_missing_tracks.existing_under([tmp_path]) + + assert find_missing_tracks.key_for(nested / "deep.mp3") in found + + +def test_the_root_is_derived_from_the_tracks(tmp_path): + paths = [tmp_path / "a" / "one.mp3", tmp_path / "b" / "two.mp3"] + + assert find_missing_tracks.roots_of(paths, None) == [tmp_path] + + +def test_an_explicit_root_overrides_the_derived_one(tmp_path): + """Worth using: tracks spread beyond the mirror can derive a common parent + of / and send the walk across the whole disk.""" + paths = [tmp_path / "a" / "one.mp3"] + + assert find_missing_tracks.roots_of(paths, ["/Volumes/music-mp3"]) == [ + Path("/Volumes/music-mp3") + ] + + +def test_the_whole_library_missing_is_called_out(tmp_path, capsys): + """Almost always an unmounted share rather than an empty library.""" + library = write_library( + tmp_path / "Library.xml", [track(tmp_path / "elsewhere", "Gone", "gone.mp3")] + ) + + find_missing_tracks.main([str(library), "--root", str(tmp_path)]) + + assert "not mounted" in capsys.readouterr().err + + +def test_a_healthy_library_says_nothing_alarming(tmp_path, capsys): + (tmp_path / "kept.mp3").write_bytes(b"x") + library = write_library(tmp_path / "Library.xml", [track(tmp_path, "Kept", "kept.mp3")]) + + find_missing_tracks.main([str(library), "--root", str(tmp_path)]) + + captured = capsys.readouterr() + assert "0 of 1 local tracks are missing" in captured.err + assert "not mounted" not in captured.err + assert captured.out == "" + + +@pytest.mark.parametrize( + ("location", "expected"), + [ + ("file:///music/a%20b.mp3", Path("/music/a b.mp3")), + ("file:///music/plain.mp3", Path("/music/plain.mp3")), + ("https://example.invalid/stream", None), + ("", None), + ], +) +def test_locations_are_decoded(location, expected): + assert find_missing_tracks.location_of({"Location": location} if location else {}) == expected + + +def test_the_walk_costs_one_read_per_directory_not_one_per_file(tmp_path): + """The whole point. Over SMB a per-file check is a round trip per track.""" + for album in range(20): + directory = tmp_path / f"Album {album}" + directory.mkdir() + for index in range(25): + (directory / f"{index}.mp3").write_bytes(b"x") + + reads = {"n": 0} + real_walk = os.walk + + def counting_walk(*args, **kwargs): + for entry in real_walk(*args, **kwargs): + reads["n"] += 1 + yield entry + + find_missing_tracks.os.walk = counting_walk + try: + found = find_missing_tracks.existing_under([tmp_path]) + finally: + find_missing_tracks.os.walk = real_walk + + assert len(found) == 500 + assert reads["n"] == 21 # the root and its twenty albums, not 500 files diff --git a/tools/find_missing_tracks.py b/tools/find_missing_tracks.py new file mode 100755 index 0000000..97372a8 --- /dev/null +++ b/tools/find_missing_tracks.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Report tracks in a Music library whose files are no longer on disk. + +Reads the XML from Music's File > Library > Export Library. Asking Music itself +does not work: a broken track makes AppleScript's `location` raise rather than +return a value, so a bulk query dies on the first one with error -1728, and a +per-track loop costs an Apple event apiece. + +The library is checked against a single directory walk rather than by testing +each file. Over SMB a per-file test is one network round trip per track -- +fifty thousand of them -- where a walk reads each directory once and gets every +name in it back at once. +""" + +import argparse +import os +import plistlib +import sys +import unicodedata +import urllib.parse +from pathlib import Path + + +def location_of(track): + """Return the filesystem path a track points at, or None if it has none.""" + location = track.get("Location") + if not location or not location.startswith("file://"): + return None + return Path(urllib.parse.unquote(urllib.parse.urlparse(location).path)) + + +def key_for(path): + """Return a comparison key for a path. + + Normalised to NFC because macOS stores filenames decomposed and most + everything else composes them, so "Motley Crue" with its umlauts is two + different byte strings depending on which side wrote it. Casefolded because + the share is very likely case-insensitive and a file is not missing merely + because someone capitalised it differently. + """ + return unicodedata.normalize("NFC", str(path)).casefold() + + +def existing_under(roots): + """Return every file below the given roots, keyed for comparison.""" + found = set() + for root in roots: + for base, _, names in os.walk(root): + for name in names: + found.add(key_for(os.path.join(base, name))) + return found + + +def roots_of(paths, given): + """Return the directories worth walking.""" + if given: + return [Path(root) for root in given] + try: + return [Path(os.path.commonpath([str(path) for path in paths]))] + except ValueError: + # Tracks spread across separate volumes have no common parent. + return sorted({path.parents[-2] for path in paths if len(path.parents) > 1}) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("library", help="Library.xml exported from Music") + parser.add_argument("--root", action="append", help="directory to scan; repeatable") + parser.add_argument("--limit", type=int, default=0, help="show at most this many") + args = parser.parse_args(argv) + + with open(args.library, "rb") as handle: + library = plistlib.load(handle) + + tracks = [ + (track, location_of(track)) for track in library.get("Tracks", {}).values() + ] + local = [(track, path) for track, path in tracks if path is not None] + if not local: + print("no local files in this library", file=sys.stderr) + return 0 + + roots = roots_of([path for _, path in local], args.root) + print(f"scanning {', '.join(str(root) for root in roots)}", file=sys.stderr) + present = existing_under(roots) + + gone = [(track, path) for track, path in local if key_for(path) not in present] + for track, path in gone[: args.limit or None]: + print( + "\t".join( + ( + track.get("Artist", ""), + track.get("Album", ""), + track.get("Name", ""), + str(path), + ) + ) + ) + + print( + f"\n{len(gone)} of {len(local)} local tracks are missing their file" + f" ({len(tracks) - len(local)} have no file at all)", + file=sys.stderr, + ) + if gone and len(gone) == len(local): + print( + "Every single one is missing, which almost certainly means the share" + " is not mounted rather than that the library is empty.", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())