From c99b423b72cb345231b5d3709a2b817d2966ba30 Mon Sep 17 00:00:00 2001 From: Emma Thorpe Date: Wed, 26 Aug 2026 13:20:20 +0100 Subject: [PATCH] feat: rebuild the Rockbox database during the sync, off the mirror The on-device database commit does not work at this library size. It sorts the whole index in whatever memory core_alloc_maximum() can scrape together, and on fifty thousand tracks it runs for hours or aborts with a data abort -- observed across several builds including stable. Rockbox ships a host-side builder for exactly this, and the sync is the moment the library changes, so it belongs here. MUSIC_MIRROR_DATABASE_TOOL points at it; the step is skipped with a note when unset, as the scrobbler step is. The scan runs against a scratch root -- a real .rockbox beside a symlink standing in for wherever the music lands on the device -- so the paths recorded are the ones Rockbox will look up, while the bytes are read from the mirror rather than over USB. Only the dozen .tcd files cross to the card. Verified: scanning through the symlink records /Music/... paths while reading from somewhere else entirely. The scratch root is kept between runs because the builder is incremental. A second pass over unchanged files performs no metadata reads and finishes in a fraction of a second, so only the first build pays the full cost. That cost, measured rather than guessed: about 49 reads and 43 seeks per file, the parser probing the head for ID3v2 and the tail for ID3v1. Two thousand files in half a second on local disk. Over SMB the opens and the head/tail split are real round trips, making a first full scan minutes rather than seconds -- still preferable to an on-device commit that does not finish. There is nothing to parallelise: the tool is single-threaded and two instances cannot produce one database. --- README.md | 38 ++++++++++++++++++++++++ tests/test_sync_to_ipod.py | 43 +++++++++++++++++++++++++++ tools/sync-to-ipod.sh | 61 +++++++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b2f8c52..9e20d89 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,44 @@ is what a percentage and an estimate that mean something cost — and renders th rest itself. Piped to a log it prints a plain line every thirty seconds instead, with no carriage returns, and a summary at the end either way. +### The Rockbox database + +Point `MUSIC_MIRROR_DATABASE_TOOL` at Rockbox's host-side builder and the sync +rebuilds the database itself, so it never has to happen on the device. + +```sh +git clone --depth 1 https://github.com/Rockbox/rockbox.git +cd rockbox && mkdir build-db && cd build-db +../tools/configure --target=ipodvideo --type=d && make -j$(nproc) +``` + +It needs a native compiler and SDL2 development headers, not the ARM +cross-toolchain, and `tools/configure` detects `__aarch64__` correctly. On a +distribution without `/usr/bin/perl` or `gcc-ar` — NixOS, say — patch the +shebangs in `tools/*.pl` and pass `AR=ar`. + +Building it here rather than on the device is not merely faster. The on-device +commit sorts the whole index in whatever memory `core_alloc_maximum()` can +scrape together; on a fifty-thousand-track library it runs for hours or aborts +outright. + +The scan runs against a scratch root — a real `.rockbox` beside a symlink +standing in for wherever the music lands on the device — so the paths recorded +are the ones Rockbox will look up, while the **bytes are read from the mirror +rather than over USB**. Only the dozen `.tcd` files cross to the card. + +Cost, measured: the parser makes about 49 reads and 43 seeks per file, probing +the head for ID3v2 and the tail for ID3v1. On a local disk that is 2,000 files +in half a second. Over SMB, readahead absorbs most of the reads but the opens +and the head/tail split are real round trips, so a first full scan is minutes +rather than seconds. It is a one-time cost: the builder is incremental, and the +scratch root is kept between runs, so a later pass over unchanged files does no +metadata reads at all. + +If minutes is still too many, run the builder where the mirror is local — on +the NAS — and copy the `.tcd` files across. There is nothing to parallelise: +the tool is single-threaded, and two instances cannot produce one database. + ### If the sync is interrupted No partially copied track is ever left under a name Rockbox would play. rsync diff --git a/tests/test_sync_to_ipod.py b/tests/test_sync_to_ipod.py index 1813068..c0b4e82 100644 --- a/tests/test_sync_to_ipod.py +++ b/tests/test_sync_to_ipod.py @@ -228,3 +228,46 @@ def test_the_flush_and_unmount_happen_on_every_exit_path(): # so one cannot drift from the other. assert script.count("finish\n") >= 2 assert "--partial" not in script, "rsync must delete partial files, not keep them" + + +def test_the_database_step_is_skipped_without_a_tool(mirror, tmp_path, monkeypatch): + """Opt-in, like the scrobbler: absent configuration is not an error.""" + destination = tmp_path / "dest" + destination.mkdir() + monkeypatch.delenv("MUSIC_MIRROR_DATABASE_TOOL", raising=False) + + result = run("-f", "-S", "-U", str(mirror), str(destination)) + + assert result.returncode == 0, result.stderr + assert "no database tool configured" in result.stderr + + +def test_a_missing_database_tool_is_refused(mirror, tmp_path, monkeypatch): + destination = tmp_path / "dest" + destination.mkdir() + monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent")) + + result = run("-f", "-S", "-U", str(mirror), str(destination)) + + assert result.returncode == 1 + assert "not executable" in result.stderr + + +def test_the_database_step_can_be_skipped(mirror, tmp_path, monkeypatch): + destination = tmp_path / "dest" + destination.mkdir() + monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent")) + + result = run("-f", "-S", "-U", "-B", str(mirror), str(destination)) + + assert result.returncode == 0, result.stderr + assert "not executable" not in result.stderr + + +def test_the_scan_reads_from_the_mirror_not_the_device(): + """The whole point: tags come off the mirror, only the .tcd files go over + USB. Reading 49,600 files through an iPod's USB bridge is the slow path.""" + script = SCRIPT.read_text() + + assert 'ln -s "$mirror"' in script + assert 'cd "$scratch"' in script diff --git a/tools/sync-to-ipod.sh b/tools/sync-to-ipod.sh index ed119d4..9b7a0fc 100755 --- a/tools/sync-to-ipod.sh +++ b/tools/sync-to-ipod.sh @@ -26,8 +26,16 @@ usage: sync-to-ipod.sh [options] of the source tree, which over SMB is the expensive part -f copy even if the FAT32 check finds unacceptable paths -S skip submitting the Rockbox scrobbler log to Last.fm + -B skip rebuilding the Rockbox database -U leave the destination mounted afterwards +Rebuilding the database needs MUSIC_MIRROR_DATABASE_TOOL pointing at Rockbox's +host-side builder (tools/database, built with ./tools/configure --type=d). It +is skipped with a note when unset. The scan reads tags from the mirror rather +than from the device, so it costs seconds rather than the hours an on-device +commit takes -- and on a large library the on-device commit may not finish at +all. + Submitting scrobbles needs LASTFM_API_KEY and LASTFM_API_SECRET; it is skipped with a note when they are unset. Scrobbling is a write method and needs the secret, unlike the read-only calls elsewhere in these projects. @@ -64,15 +72,17 @@ quick=false force=false unmount=true scrobble=true +database=true for argument in "$@"; do [ "$argument" = "--help" ] && usage help done -while getopts ":nQfSUh" option; do +while getopts ":nQfSBUh" option; do case "$option" in n) dry_run=true ;; Q) quick=true ;; f) force=true ;; S) scrobble=false ;; + B) database=false ;; U) unmount=false ;; h) usage help ;; *) usage ;; @@ -241,4 +251,53 @@ rsync "${options[@]}" --out-format='%l %n' "$mirror/" "$destination/" | status=${PIPESTATUS[0]} [ "$status" -eq 0 ] || die "rsync exited $status" +# Rockbox reads its database from .tcd files in .rockbox. Building them here +# rather than on the device is not just faster: the on-device commit sorts the +# whole index in whatever memory it can scrape together, and on a large library +# it runs for hours or dies outright. +# +# The scan reads tags through a scratch root -- a real .rockbox beside a symlink +# standing in for where the music lands on the device -- so the paths recorded +# match what Rockbox will look up, while the bytes are read from the mirror +# instead of over USB. The scratch is kept between runs because the builder is +# incremental: a second pass over unchanged files does no work at all. +rebuild_database() { + local tool=${MUSIC_MIRROR_DATABASE_TOOL:-} + if [ -z "$tool" ]; then + printf 'sync-to-ipod: no database tool configured, skipping the database\n' >&2 + return 0 + fi + [ -x "$tool" ] || die "$tool is not executable" + + local device_rockbox="$mounted_on/.rockbox" + if [ ! -d "$device_rockbox" ]; then + printf 'sync-to-ipod: no .rockbox on the device, skipping the database\n' >&2 + return 0 + fi + + local scratch="${XDG_CACHE_HOME:-$HOME/.cache}/music-mirror/database" + mkdir -p "$scratch/.rockbox" + + # Rebuild the symlink layout each time; the mirror path or the device + # prefix may have changed since the last run. + find "$scratch" -maxdepth 1 -type l -delete + if [ "$device_prefix" = "/" ]; then + ln -s "$mirror"/* "$scratch/" 2>/dev/null || true + else + local under=${device_prefix#/} + rm -rf "${scratch:?}/${under%%/*}" + mkdir -p "$scratch/$(dirname "$under")" + ln -s "$mirror" "$scratch/$under" + fi + + printf 'sync-to-ipod: building the database from the mirror...\n' >&2 + ( cd "$scratch" && "$tool" ) >/dev/null || die "the database build failed" + + cp -- "$scratch"/.rockbox/*.tcd "$device_rockbox/" || + die "could not copy the database onto the device" + printf 'sync-to-ipod: database copied to %s\n' "$device_rockbox" >&2 +} + +$database && rebuild_database + finish