README.md

@@ -185,6 +185,44 @@ is what a percentage and an estimate that mean something cost — and renders th
 rest itself. Piped to a log it prints a plain line every thirty seconds
 instead, with no carriage returns, and a summary at the end either way.
 
+### The Rockbox database
+
+Point `MUSIC_MIRROR_DATABASE_TOOL` at Rockbox's host-side builder and the sync
+rebuilds the database itself, so it never has to happen on the device.
+
+```sh
+git clone --depth 1 https://github.com/Rockbox/rockbox.git
+cd rockbox && mkdir build-db && cd build-db
+../tools/configure --target=ipodvideo --type=d && make -j$(nproc)
+```
+
+It needs a native compiler and SDL2 development headers, not the ARM
+cross-toolchain, and `tools/configure` detects `__aarch64__` correctly. On a
+distribution without `/usr/bin/perl` or `gcc-ar` — NixOS, say — patch the
+shebangs in `tools/*.pl` and pass `AR=ar`.
+
+Building it here rather than on the device is not merely faster. The on-device
+commit sorts the whole index in whatever memory `core_alloc_maximum()` can
+scrape together; on a fifty-thousand-track library it runs for hours or aborts
+outright.
+
+The scan runs against a scratch root — a real `.rockbox` beside a symlink
+standing in for wherever the music lands on the device — so the paths recorded
+are the ones Rockbox will look up, while the **bytes are read from the mirror
+rather than over USB**. Only the dozen `.tcd` files cross to the card.
+
+Cost, measured: the parser makes about 49 reads and 43 seeks per file, probing
+the head for ID3v2 and the tail for ID3v1. On a local disk that is 2,000 files
+in half a second. Over SMB, readahead absorbs most of the reads but the opens
+and the head/tail split are real round trips, so a first full scan is minutes
+rather than seconds. It is a one-time cost: the builder is incremental, and the
+scratch root is kept between runs, so a later pass over unchanged files does no
+metadata reads at all.
+
+If minutes is still too many, run the builder where the mirror is local — on
+the NAS — and copy the `.tcd` files across. There is nothing to parallelise:
+the tool is single-threaded, and two instances cannot produce one database.
+
 ### If the sync is interrupted
 
 No partially copied track is ever left under a name Rockbox would play. rsync

tests/test_sync_to_ipod.py

@@ -228,3 +228,46 @@ def test_the_flush_and_unmount_happen_on_every_exit_path():
     # so one cannot drift from the other.
     assert script.count("finish\n") >= 2
     assert "--partial" not in script, "rsync must delete partial files, not keep them"
+
+
+def test_the_database_step_is_skipped_without_a_tool(mirror, tmp_path, monkeypatch):
+    """Opt-in, like the scrobbler: absent configuration is not an error."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    monkeypatch.delenv("MUSIC_MIRROR_DATABASE_TOOL", raising=False)
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "no database tool configured" in result.stderr
+
+
+def test_a_missing_database_tool_is_refused(mirror, tmp_path, monkeypatch):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 1
+    assert "not executable" in result.stderr
+
+
+def test_the_database_step_can_be_skipped(mirror, tmp_path, monkeypatch):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
+
+    result = run("-f", "-S", "-U", "-B", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "not executable" not in result.stderr
+
+
+def test_the_scan_reads_from_the_mirror_not_the_device():
+    """The whole point: tags come off the mirror, only the .tcd files go over
+    USB. Reading 49,600 files through an iPod's USB bridge is the slow path."""
+    script = SCRIPT.read_text()
+
+    assert 'ln -s "$mirror"' in script
+    assert 'cd "$scratch"' in script

tools/sync-to-ipod.sh

@@ -26,8 +26,16 @@ usage: sync-to-ipod.sh [options] <mirror> <destination>
       of the source tree, which over SMB is the expensive part
   -f  copy even if the FAT32 check finds unacceptable paths
   -S  skip submitting the Rockbox scrobbler log to Last.fm
+  -B  skip rebuilding the Rockbox database
   -U  leave the destination mounted afterwards
 
+Rebuilding the database needs MUSIC_MIRROR_DATABASE_TOOL pointing at Rockbox's
+host-side builder (tools/database, built with ./tools/configure --type=d). It
+is skipped with a note when unset. The scan reads tags from the mirror rather
+than from the device, so it costs seconds rather than the hours an on-device
+commit takes -- and on a large library the on-device commit may not finish at
+all.
+
 Submitting scrobbles needs LASTFM_API_KEY and LASTFM_API_SECRET; it is skipped
 with a note when they are unset. Scrobbling is a write method and needs the
 secret, unlike the read-only calls elsewhere in these projects.
@@ -64,15 +72,17 @@ quick=false
 force=false
 unmount=true
 scrobble=true
+database=true
 for argument in "$@"; do
     [ "$argument" = "--help" ] && usage help
 done
-while getopts ":nQfSUh" option; do
+while getopts ":nQfSBUh" option; do
     case "$option" in
         n) dry_run=true ;;
         Q) quick=true ;;
         f) force=true ;;
         S) scrobble=false ;;
+        B) database=false ;;
         U) unmount=false ;;
         h) usage help ;;
         *) usage ;;
@@ -241,4 +251,53 @@ rsync "${options[@]}" --out-format='%l %n' "$mirror/" "$destination/" |
 status=${PIPESTATUS[0]}
 [ "$status" -eq 0 ] || die "rsync exited $status"
 
+# Rockbox reads its database from .tcd files in .rockbox. Building them here
+# rather than on the device is not just faster: the on-device commit sorts the
+# whole index in whatever memory it can scrape together, and on a large library
+# it runs for hours or dies outright.
+#
+# The scan reads tags through a scratch root -- a real .rockbox beside a symlink
+# standing in for where the music lands on the device -- so the paths recorded
+# match what Rockbox will look up, while the bytes are read from the mirror
+# instead of over USB. The scratch is kept between runs because the builder is
+# incremental: a second pass over unchanged files does no work at all.
+rebuild_database() {
+    local tool=${MUSIC_MIRROR_DATABASE_TOOL:-}
+    if [ -z "$tool" ]; then
+        printf 'sync-to-ipod: no database tool configured, skipping the database\n' >&2
+        return 0
+    fi
+    [ -x "$tool" ] || die "$tool is not executable"
+
+    local device_rockbox="$mounted_on/.rockbox"
+    if [ ! -d "$device_rockbox" ]; then
+        printf 'sync-to-ipod: no .rockbox on the device, skipping the database\n' >&2
+        return 0
+    fi
+
+    local scratch="${XDG_CACHE_HOME:-$HOME/.cache}/music-mirror/database"
+    mkdir -p "$scratch/.rockbox"
+
+    # Rebuild the symlink layout each time; the mirror path or the device
+    # prefix may have changed since the last run.
+    find "$scratch" -maxdepth 1 -type l -delete
+    if [ "$device_prefix" = "/" ]; then
+        ln -s "$mirror"/* "$scratch/" 2>/dev/null || true
+    else
+        local under=${device_prefix#/}
+        rm -rf "${scratch:?}/${under%%/*}"
+        mkdir -p "$scratch/$(dirname "$under")"
+        ln -s "$mirror" "$scratch/$under"
+    fi
+
+    printf 'sync-to-ipod: building the database from the mirror...\n' >&2
+    ( cd "$scratch" && "$tool" ) >/dev/null || die "the database build failed"
+
+    cp -- "$scratch"/.rockbox/*.tcd "$device_rockbox/" ||
+        die "could not copy the database onto the device"
+    printf 'sync-to-ipod: database copied to %s\n' "$device_rockbox" >&2
+}
+
+$database && rebuild_database
+
 finish