diff --git a/README.md b/README.md index b2f8c52..9e20d89 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,44 @@ is what a percentage and an estimate that mean something cost — and renders th rest itself. Piped to a log it prints a plain line every thirty seconds instead, with no carriage returns, and a summary at the end either way. +### The Rockbox database + +Point `MUSIC_MIRROR_DATABASE_TOOL` at Rockbox's host-side builder and the sync +rebuilds the database itself, so it never has to happen on the device. + +```sh +git clone --depth 1 https://github.com/Rockbox/rockbox.git +cd rockbox && mkdir build-db && cd build-db +../tools/configure --target=ipodvideo --type=d && make -j$(nproc) +``` + +It needs a native compiler and SDL2 development headers, not the ARM +cross-toolchain, and `tools/configure` detects `__aarch64__` correctly. On a +distribution without `/usr/bin/perl` or `gcc-ar` — NixOS, say — patch the +shebangs in `tools/*.pl` and pass `AR=ar`. + +Building it here rather than on the device is not merely faster. The on-device +commit sorts the whole index in whatever memory `core_alloc_maximum()` can +scrape together; on a fifty-thousand-track library it runs for hours or aborts +outright. + +The scan runs against a scratch root — a real `.rockbox` beside a symlink +standing in for wherever the music lands on the device — so the paths recorded +are the ones Rockbox will look up, while the **bytes are read from the mirror +rather than over USB**. Only the dozen `.tcd` files cross to the card. + +Cost, measured: the parser makes about 49 reads and 43 seeks per file, probing +the head for ID3v2 and the tail for ID3v1. On a local disk that is 2,000 files +in half a second. Over SMB, readahead absorbs most of the reads but the opens +and the head/tail split are real round trips, so a first full scan is minutes +rather than seconds. It is a one-time cost: the builder is incremental, and the +scratch root is kept between runs, so a later pass over unchanged files does no +metadata reads at all. + +If minutes is still too many, run the builder where the mirror is local — on +the NAS — and copy the `.tcd` files across. There is nothing to parallelise: +the tool is single-threaded, and two instances cannot produce one database. + ### If the sync is interrupted No partially copied track is ever left under a name Rockbox would play. rsync diff --git a/tests/test_sync_to_ipod.py b/tests/test_sync_to_ipod.py index 1813068..c0b4e82 100644 --- a/tests/test_sync_to_ipod.py +++ b/tests/test_sync_to_ipod.py @@ -228,3 +228,46 @@ def test_the_flush_and_unmount_happen_on_every_exit_path(): # so one cannot drift from the other. assert script.count("finish\n") >= 2 assert "--partial" not in script, "rsync must delete partial files, not keep them" + + +def test_the_database_step_is_skipped_without_a_tool(mirror, tmp_path, monkeypatch): + """Opt-in, like the scrobbler: absent configuration is not an error.""" + destination = tmp_path / "dest" + destination.mkdir() + monkeypatch.delenv("MUSIC_MIRROR_DATABASE_TOOL", raising=False) + + result = run("-f", "-S", "-U", str(mirror), str(destination)) + + assert result.returncode == 0, result.stderr + assert "no database tool configured" in result.stderr + + +def test_a_missing_database_tool_is_refused(mirror, tmp_path, monkeypatch): + destination = tmp_path / "dest" + destination.mkdir() + monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent")) + + result = run("-f", "-S", "-U", str(mirror), str(destination)) + + assert result.returncode == 1 + assert "not executable" in result.stderr + + +def test_the_database_step_can_be_skipped(mirror, tmp_path, monkeypatch): + destination = tmp_path / "dest" + destination.mkdir() + monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent")) + + result = run("-f", "-S", "-U", "-B", str(mirror), str(destination)) + + assert result.returncode == 0, result.stderr + assert "not executable" not in result.stderr + + +def test_the_scan_reads_from_the_mirror_not_the_device(): + """The whole point: tags come off the mirror, only the .tcd files go over + USB. Reading 49,600 files through an iPod's USB bridge is the slow path.""" + script = SCRIPT.read_text() + + assert 'ln -s "$mirror"' in script + assert 'cd "$scratch"' in script diff --git a/tools/sync-to-ipod.sh b/tools/sync-to-ipod.sh index ed119d4..9b7a0fc 100755 --- a/tools/sync-to-ipod.sh +++ b/tools/sync-to-ipod.sh @@ -26,8 +26,16 @@ usage: sync-to-ipod.sh [options] of the source tree, which over SMB is the expensive part -f copy even if the FAT32 check finds unacceptable paths -S skip submitting the Rockbox scrobbler log to Last.fm + -B skip rebuilding the Rockbox database -U leave the destination mounted afterwards +Rebuilding the database needs MUSIC_MIRROR_DATABASE_TOOL pointing at Rockbox's +host-side builder (tools/database, built with ./tools/configure --type=d). It +is skipped with a note when unset. The scan reads tags from the mirror rather +than from the device, so it costs seconds rather than the hours an on-device +commit takes -- and on a large library the on-device commit may not finish at +all. + Submitting scrobbles needs LASTFM_API_KEY and LASTFM_API_SECRET; it is skipped with a note when they are unset. Scrobbling is a write method and needs the secret, unlike the read-only calls elsewhere in these projects. @@ -64,15 +72,17 @@ quick=false force=false unmount=true scrobble=true +database=true for argument in "$@"; do [ "$argument" = "--help" ] && usage help done -while getopts ":nQfSUh" option; do +while getopts ":nQfSBUh" option; do case "$option" in n) dry_run=true ;; Q) quick=true ;; f) force=true ;; S) scrobble=false ;; + B) database=false ;; U) unmount=false ;; h) usage help ;; *) usage ;; @@ -241,4 +251,53 @@ rsync "${options[@]}" --out-format='%l %n' "$mirror/" "$destination/" | status=${PIPESTATUS[0]} [ "$status" -eq 0 ] || die "rsync exited $status" +# Rockbox reads its database from .tcd files in .rockbox. Building them here +# rather than on the device is not just faster: the on-device commit sorts the +# whole index in whatever memory it can scrape together, and on a large library +# it runs for hours or dies outright. +# +# The scan reads tags through a scratch root -- a real .rockbox beside a symlink +# standing in for where the music lands on the device -- so the paths recorded +# match what Rockbox will look up, while the bytes are read from the mirror +# instead of over USB. The scratch is kept between runs because the builder is +# incremental: a second pass over unchanged files does no work at all. +rebuild_database() { + local tool=${MUSIC_MIRROR_DATABASE_TOOL:-} + if [ -z "$tool" ]; then + printf 'sync-to-ipod: no database tool configured, skipping the database\n' >&2 + return 0 + fi + [ -x "$tool" ] || die "$tool is not executable" + + local device_rockbox="$mounted_on/.rockbox" + if [ ! -d "$device_rockbox" ]; then + printf 'sync-to-ipod: no .rockbox on the device, skipping the database\n' >&2 + return 0 + fi + + local scratch="${XDG_CACHE_HOME:-$HOME/.cache}/music-mirror/database" + mkdir -p "$scratch/.rockbox" + + # Rebuild the symlink layout each time; the mirror path or the device + # prefix may have changed since the last run. + find "$scratch" -maxdepth 1 -type l -delete + if [ "$device_prefix" = "/" ]; then + ln -s "$mirror"/* "$scratch/" 2>/dev/null || true + else + local under=${device_prefix#/} + rm -rf "${scratch:?}/${under%%/*}" + mkdir -p "$scratch/$(dirname "$under")" + ln -s "$mirror" "$scratch/$under" + fi + + printf 'sync-to-ipod: building the database from the mirror...\n' >&2 + ( cd "$scratch" && "$tool" ) >/dev/null || die "the database build failed" + + cp -- "$scratch"/.rockbox/*.tcd "$device_rockbox/" || + die "could not copy the database onto the device" + printf 'sync-to-ipod: database copied to %s\n' "$device_rockbox" >&2 +} + +$database && rebuild_database + finish