Emma Thorpe
c99b423b72
Build and publish container / build (pull_request) Successful in 6m36s
The on-device database commit does not work at this library size. It sorts the whole index in whatever memory core_alloc_maximum() can scrape together, and on fifty thousand tracks it runs for hours or aborts with a data abort -- observed across several builds including stable. Rockbox ships a host-side builder for exactly this, and the sync is the moment the library changes, so it belongs here. MUSIC_MIRROR_DATABASE_TOOL points at it; the step is skipped with a note when unset, as the scrobbler step is. The scan runs against a scratch root -- a real .rockbox beside a symlink standing in for wherever the music lands on the device -- so the paths recorded are the ones Rockbox will look up, while the bytes are read from the mirror rather than over USB. Only the dozen .tcd files cross to the card. Verified: scanning through the symlink records /Music/... paths while reading from somewhere else entirely. The scratch root is kept between runs because the builder is incremental. A second pass over unchanged files performs no metadata reads and finishes in a fraction of a second, so only the first build pays the full cost. That cost, measured rather than guessed: about 49 reads and 43 seeks per file, the parser probing the head for ID3v2 and the tail for ID3v1. Two thousand files in half a second on local disk. Over SMB the opens and the head/tail split are real round trips, making a first full scan minutes rather than seconds -- still preferable to an on-device commit that does not finish. There is nothing to parallelise: the tool is single-threaded and two instances cannot produce one database.
274 lines
8.9 KiB
Python
274 lines
8.9 KiB
Python
"""The guards on sync-to-ipod.sh, which are the substance of the script.
|
|
|
|
rsync --delete is being aimed at a whole filesystem, so every refusal here is
|
|
protecting against emptying the wrong directory -- a mistake that does not
|
|
announce itself.
|
|
"""
|
|
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
SCRIPT = Path(__file__).resolve().parent.parent / "tools" / "sync-to-ipod.sh"
|
|
|
|
# Skipped rather than failed where the tools are absent: this is a host-side
|
|
# script, and a machine without rsync is not a machine that would run it.
|
|
REQUIRED = ("bash", "rsync", "findmnt")
|
|
pytestmark = pytest.mark.skipif(
|
|
not all(shutil.which(tool) for tool in REQUIRED),
|
|
reason=f"needs {', '.join(REQUIRED)} on PATH",
|
|
)
|
|
|
|
|
|
def run(*arguments):
|
|
return subprocess.run(
|
|
["bash", str(SCRIPT), *arguments], capture_output=True, text=True
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mirror(tmp_path):
|
|
source = tmp_path / "mirror"
|
|
(source / "Album").mkdir(parents=True)
|
|
(source / "Album" / "track.mp3").write_bytes(b"x")
|
|
return source
|
|
|
|
|
|
def test_the_host_root_is_refused(mirror):
|
|
"""Stripping the trailing slash from "/" leaves an empty string, and an
|
|
earlier version then reported it as "not a directory" instead."""
|
|
result = run(str(mirror), "/")
|
|
|
|
assert result.returncode == 1
|
|
assert "refusing to sync onto /" in result.stderr
|
|
|
|
|
|
def test_an_empty_mirror_is_refused(tmp_path):
|
|
"""Mirroring nothing onto the device would delete everything on it."""
|
|
empty = tmp_path / "empty"
|
|
empty.mkdir()
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
|
|
result = run(str(empty), str(destination))
|
|
|
|
assert result.returncode == 1
|
|
assert "refusing to mirror nothing" in result.stderr
|
|
|
|
|
|
def test_syncing_a_directory_onto_itself_is_refused(mirror):
|
|
result = run(str(mirror), str(mirror))
|
|
|
|
assert result.returncode == 1
|
|
assert "same directory" in result.stderr
|
|
|
|
|
|
def test_a_non_fat_destination_is_refused(mirror, tmp_path):
|
|
"""Which is also how an unmounted device is caught: /media/IPOD/Music then
|
|
resolves to the host's own root filesystem."""
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
|
|
result = run(str(mirror), str(destination))
|
|
|
|
assert result.returncode == 1
|
|
assert "not FAT" in result.stderr
|
|
assert "Is the device mounted?" in result.stderr
|
|
|
|
|
|
def test_a_missing_destination_is_refused(mirror, tmp_path):
|
|
result = run(str(mirror), str(tmp_path / "nowhere"))
|
|
|
|
assert result.returncode == 1
|
|
assert "not a directory" in result.stderr
|
|
|
|
|
|
def test_a_subdirectory_of_the_device_is_a_valid_target(mirror, tmp_path):
|
|
"""The better target, in fact: --delete is confined to it."""
|
|
destination = tmp_path / "dest" / "Music"
|
|
destination.mkdir(parents=True)
|
|
|
|
result = run("-f", "-n", str(mirror), str(destination))
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "dry run, nothing was written" in result.stderr
|
|
|
|
|
|
def test_the_device_prefix_is_derived_from_the_destination(mirror, tmp_path):
|
|
"""Derived rather than configured, so it cannot disagree with where the
|
|
files are actually going -- and the device's path limit applies to it."""
|
|
destination = tmp_path / "dest" / "Music"
|
|
destination.mkdir(parents=True)
|
|
|
|
result = run("-f", "-n", str(mirror), str(destination))
|
|
|
|
assert "the device will see this as /" in result.stderr
|
|
|
|
|
|
def test_a_dry_run_writes_nothing(mirror, tmp_path):
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
|
|
run("-f", "-n", str(mirror), str(destination))
|
|
|
|
assert list(destination.iterdir()) == []
|
|
|
|
|
|
def test_rockbox_is_never_deleted(mirror, tmp_path):
|
|
"""A sync to the card root would otherwise remove the Rockbox install,
|
|
since the mirror does not contain it."""
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
(destination / ".rockbox").mkdir()
|
|
(destination / ".rockbox" / "rockbox.ipod").write_bytes(b"firmware")
|
|
(destination / ".scrobbler.log").write_bytes(b"#AUDIOSCROBBLER/1.1\n")
|
|
(destination / "Stale.mp3").write_bytes(b"old")
|
|
|
|
result = run("-f", "-S", "-U", str(mirror), str(destination))
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (destination / ".rockbox" / "rockbox.ipod").is_file()
|
|
assert (destination / ".scrobbler.log").is_file()
|
|
# But a track whose source has gone is still removed. That is the point.
|
|
assert not (destination / "Stale.mp3").exists()
|
|
assert (destination / "Album" / "track.mp3").is_file()
|
|
|
|
|
|
def test_help_goes_to_stdout_and_exits_clean():
|
|
"""Asking for help is not an error; getting the arguments wrong is."""
|
|
result = run("--help")
|
|
|
|
assert result.returncode == 0
|
|
assert result.stdout.startswith("usage:")
|
|
assert result.stderr == ""
|
|
|
|
|
|
def test_short_help_behaves_the_same():
|
|
result = run("-h")
|
|
|
|
assert result.returncode == 0
|
|
assert result.stdout.startswith("usage:")
|
|
|
|
|
|
def test_misuse_goes_to_stderr_and_does_not():
|
|
result = run("only-one-argument")
|
|
|
|
assert result.returncode == 2
|
|
assert result.stderr.startswith("usage:")
|
|
assert result.stdout == ""
|
|
|
|
|
|
def test_the_help_explains_what_the_destination_should_be():
|
|
"""The question this script actually gets asked."""
|
|
help_text = run("--help").stdout
|
|
|
|
assert "/media/IPOD/Music" in help_text
|
|
assert "artist folders" in help_text
|
|
assert ".rockbox" in help_text
|
|
|
|
|
|
def test_the_help_says_how_to_reach_and_leave_disk_mode():
|
|
help_text = run("--help").stdout
|
|
|
|
assert "Menu+Select" in help_text
|
|
assert "holding Play" in help_text
|
|
|
|
|
|
def test_quick_mode_skips_the_counting_pass(mirror, tmp_path):
|
|
"""Over SMB the walk is the expensive part, and doing it twice for a
|
|
percentage is not always the trade you want."""
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
|
|
result = run("-f", "-S", "-U", "-Q", str(mirror), str(destination))
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "skipping the count" in result.stderr
|
|
assert "files to copy" not in result.stderr
|
|
assert (destination / "Album" / "track.mp3").is_file()
|
|
|
|
|
|
def test_the_delta_algorithm_is_disabled(mirror, tmp_path):
|
|
"""It would read every destination file back over USB to checksum it, to
|
|
avoid resending an MP3 that has changed in its entirety anyway."""
|
|
script = SCRIPT.read_text()
|
|
|
|
assert "--whole-file" in script
|
|
|
|
|
|
def test_directory_timestamps_are_not_set(mirror, tmp_path):
|
|
"""One setattr round trip per directory, across six thousand albums, to set
|
|
timestamps nothing reads."""
|
|
script = SCRIPT.read_text()
|
|
|
|
assert "--omit-dir-times" in script
|
|
|
|
|
|
def test_an_interrupt_is_trapped_so_the_filesystem_is_flushed():
|
|
"""Ctrl-C during a transfer would otherwise skip the sync and the unmount,
|
|
leaving a journal-less FAT filesystem with dirty buffers -- which is the
|
|
corruption this script exists to prevent.
|
|
|
|
Structural rather than timed: reproducing a mid-transfer signal needs a
|
|
payload large enough to be slow, and a test that depends on losing a race
|
|
is a test that fails in CI for no reason.
|
|
"""
|
|
script = SCRIPT.read_text()
|
|
|
|
assert "trap interrupted INT TERM" in script
|
|
assert "exit 130" in script
|
|
|
|
|
|
def test_the_flush_and_unmount_happen_on_every_exit_path():
|
|
script = SCRIPT.read_text()
|
|
|
|
# Both the normal path and the interrupt path go through the same function,
|
|
# so one cannot drift from the other.
|
|
assert script.count("finish\n") >= 2
|
|
assert "--partial" not in script, "rsync must delete partial files, not keep them"
|
|
|
|
|
|
def test_the_database_step_is_skipped_without_a_tool(mirror, tmp_path, monkeypatch):
|
|
"""Opt-in, like the scrobbler: absent configuration is not an error."""
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
monkeypatch.delenv("MUSIC_MIRROR_DATABASE_TOOL", raising=False)
|
|
|
|
result = run("-f", "-S", "-U", str(mirror), str(destination))
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "no database tool configured" in result.stderr
|
|
|
|
|
|
def test_a_missing_database_tool_is_refused(mirror, tmp_path, monkeypatch):
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
|
|
|
|
result = run("-f", "-S", "-U", str(mirror), str(destination))
|
|
|
|
assert result.returncode == 1
|
|
assert "not executable" in result.stderr
|
|
|
|
|
|
def test_the_database_step_can_be_skipped(mirror, tmp_path, monkeypatch):
|
|
destination = tmp_path / "dest"
|
|
destination.mkdir()
|
|
monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
|
|
|
|
result = run("-f", "-S", "-U", "-B", str(mirror), str(destination))
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "not executable" not in result.stderr
|
|
|
|
|
|
def test_the_scan_reads_from_the_mirror_not_the_device():
|
|
"""The whole point: tags come off the mirror, only the .tcd files go over
|
|
USB. Reading 49,600 files through an iPod's USB bridge is the slow path."""
|
|
script = SCRIPT.read_text()
|
|
|
|
assert 'ln -s "$mirror"' in script
|
|
assert 'cd "$scratch"' in script
|