feat: rebuild the Rockbox database during the sync, off the mirror
Build and publish container / build (pull_request) Successful in 6m36s
Build and publish container / build (pull_request) Successful in 6m36s
The on-device database commit does not work at this library size. It sorts the whole index in whatever memory core_alloc_maximum() can scrape together, and on fifty thousand tracks it runs for hours or aborts with a data abort -- observed across several builds including stable. Rockbox ships a host-side builder for exactly this, and the sync is the moment the library changes, so it belongs here. MUSIC_MIRROR_DATABASE_TOOL points at it; the step is skipped with a note when unset, as the scrobbler step is. The scan runs against a scratch root -- a real .rockbox beside a symlink standing in for wherever the music lands on the device -- so the paths recorded are the ones Rockbox will look up, while the bytes are read from the mirror rather than over USB. Only the dozen .tcd files cross to the card. Verified: scanning through the symlink records /Music/... paths while reading from somewhere else entirely. The scratch root is kept between runs because the builder is incremental. A second pass over unchanged files performs no metadata reads and finishes in a fraction of a second, so only the first build pays the full cost. That cost, measured rather than guessed: about 49 reads and 43 seeks per file, the parser probing the head for ID3v2 and the tail for ID3v1. Two thousand files in half a second on local disk. Over SMB the opens and the head/tail split are real round trips, making a first full scan minutes rather than seconds -- still preferable to an on-device commit that does not finish. There is nothing to parallelise: the tool is single-threaded and two instances cannot produce one database.
This commit is contained in:
@@ -228,3 +228,46 @@ def test_the_flush_and_unmount_happen_on_every_exit_path():
|
||||
# so one cannot drift from the other.
|
||||
assert script.count("finish\n") >= 2
|
||||
assert "--partial" not in script, "rsync must delete partial files, not keep them"
|
||||
|
||||
|
||||
def test_the_database_step_is_skipped_without_a_tool(mirror, tmp_path, monkeypatch):
|
||||
"""Opt-in, like the scrobbler: absent configuration is not an error."""
|
||||
destination = tmp_path / "dest"
|
||||
destination.mkdir()
|
||||
monkeypatch.delenv("MUSIC_MIRROR_DATABASE_TOOL", raising=False)
|
||||
|
||||
result = run("-f", "-S", "-U", str(mirror), str(destination))
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "no database tool configured" in result.stderr
|
||||
|
||||
|
||||
def test_a_missing_database_tool_is_refused(mirror, tmp_path, monkeypatch):
|
||||
destination = tmp_path / "dest"
|
||||
destination.mkdir()
|
||||
monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
|
||||
|
||||
result = run("-f", "-S", "-U", str(mirror), str(destination))
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "not executable" in result.stderr
|
||||
|
||||
|
||||
def test_the_database_step_can_be_skipped(mirror, tmp_path, monkeypatch):
|
||||
destination = tmp_path / "dest"
|
||||
destination.mkdir()
|
||||
monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
|
||||
|
||||
result = run("-f", "-S", "-U", "-B", str(mirror), str(destination))
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "not executable" not in result.stderr
|
||||
|
||||
|
||||
def test_the_scan_reads_from_the_mirror_not_the_device():
|
||||
"""The whole point: tags come off the mirror, only the .tcd files go over
|
||||
USB. Reading 49,600 files through an iPod's USB bridge is the slow path."""
|
||||
script = SCRIPT.read_text()
|
||||
|
||||
assert 'ln -s "$mirror"' in script
|
||||
assert 'cd "$scratch"' in script
|
||||
|
||||
Reference in New Issue
Block a user