README.md

@@ -163,6 +163,19 @@ and the various filesystem metadata directories from deletion — the mirror doe
 not contain them, and without the exclusion a sync to the card root would
 remove the Rockbox install.
 
+Progress is a single line that rewrites itself:
+
+```
+[ 12,345 / 49,600  24%] King Gizzard & the Lizard Wizard / PetroDragonic Apoc…
+```
+
+rsync says nothing at all while it builds its file list, which on fifty
+thousand files over USB is minutes of apparent hang, and its own `progress2`
+percentage is computed against a list it has not finished discovering. So the
+script counts first — a second pass over the tree, which is what a percentage
+that means something costs — and renders the rest itself. Piped to a log it
+prints a plain line every thirty seconds instead, with no carriage returns.
+
 The unmount is the point of doing this in a script. FAT32 has no journal and
 the device is reached through disk mode, so an interrupted write is corruption
 that needs `fsck.vfat` from another machine.

tests/test_rsync_progress.py

@@ -0,0 +1,104 @@
+import io
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import rsync_progress  # noqa: E402
+
+
+class NotATerminal(io.StringIO):
+    def isatty(self):
+        return False
+
+
+class Terminal(io.StringIO):
+    def isatty(self):
+        return True
+
+
+def run(lines, total=0, out=None):
+    out = out or NotATerminal()
+    rsync_progress.main(["--total", str(total)], stream=io.StringIO(lines), out=out)
+    return out.getvalue()
+
+
+def test_the_artist_and_album_are_pulled_from_the_path():
+    assert rsync_progress.album_of("Pendulum/Immersion/01 - Watercolour.mp3") == (
+        "Pendulum / Immersion"
+    )
+
+
+def test_a_shallower_path_degrades_rather_than_failing():
+    assert rsync_progress.album_of("Pendulum/loose.mp3") == "Pendulum"
+    assert rsync_progress.album_of("loose.mp3") == ""
+
+
+def test_directories_are_not_counted():
+    """rsync reports them too, and counting them puts the percentage past 100."""
+    output = run("Artist/\nArtist/Album/\nArtist/Album/track.mp3\n", total=1)
+
+    assert "1 / 1" in output
+    assert "100%" in output
+
+
+def test_the_percentage_tracks_the_total():
+    output = run("".join(f"A/B/{i}.mp3\n" for i in range(5)), total=10)
+
+    assert "5 / 10" in output
+    assert " 50%" in output
+
+
+def test_without_a_total_it_counts_instead_of_guessing():
+    output = run("A/B/one.mp3\nA/B/two.mp3\n")
+
+    assert "2 files" in output
+    assert "%" not in output
+
+
+def test_a_final_line_is_always_printed():
+    """Otherwise the last state of a rewriting line is whatever it happened to
+    be when the interval last elapsed."""
+    output = run("A/B/one.mp3\n", total=1)
+
+    assert output.endswith("\n")
+    assert "1 / 1" in output
+
+
+def test_nothing_transferred_still_reports():
+    output = run("", total=0)
+
+    assert "0 files" in output
+
+
+def test_a_log_gets_no_carriage_returns():
+    """A non-terminal filling with \\r and escape codes is unreadable."""
+    output = run("".join(f"A/B/{i}.mp3\n" for i in range(50)), total=50)
+
+    assert "\r" not in output
+    assert "\033" not in output
+
+
+def test_a_terminal_rewrites_one_line():
+    output = run("".join(f"A/B/{i}.mp3\n" for i in range(50)), total=50, out=Terminal())
+
+    assert "\r\033[2K" in output
+
+
+@pytest.mark.parametrize(
+    ("text", "width", "expected"),
+    [
+        ("short", 20, "short"),
+        ("King Gizzard / PetroDragonic Apocalypse", 20, "…agonic Apocalypse"),
+    ],
+)
+def test_long_labels_are_trimmed_from_the_left(text, width, expected):
+    """The album is the informative end, so the artist is what gets cut."""
+    trimmed = rsync_progress.fit(text, width)
+
+    assert len(trimmed) <= width
+    if len(text) > width:
+        assert trimmed.startswith("…")
+        assert text.endswith(trimmed.lstrip("…"))

tools/rsync_progress.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Render rsync's per-file output as a single updating status line.
+
+Fed the paths rsync reports with --out-format='%n', one per line. Prints one
+line that rewrites itself, showing how far through the transfer is and which
+album is currently going across, rather than either scrolling fifty thousand
+filenames past or -- as rsync does while it builds its file list -- saying
+nothing at all for several minutes.
+
+Falls back to periodic plain lines when stderr is not a terminal, so a log does
+not fill up with carriage returns.
+"""
+
+import argparse
+import os
+import shutil
+import sys
+import time
+
+
+def album_of(path):
+    """Return "Artist / Album" for a mirror-relative path."""
+    parts = [part for part in path.strip("/").split("/") if part]
+    if len(parts) >= 3:
+        return f"{parts[0]} / {parts[1]}"
+    if len(parts) == 2:
+        return parts[0]
+    return ""
+
+
+def fit(text, width):
+    """Trim to the terminal, from the left: the album matters more than the artist."""
+    if width <= 1 or len(text) <= width:
+        return text
+    return "…" + text[-(width - 1) :]
+
+
+def render(done, total, label, width):
+    if total > 0:
+        share = min(100, done * 100 // total)
+        head = f"[{done:>6,} / {total:<6,} {share:>3}%] "
+    else:
+        head = f"[{done:>6,} files] "
+    return head + fit(label, max(0, width - len(head)))
+
+
+def main(argv=None, stream=None, out=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--total", type=int, default=0, help="files expected")
+    parser.add_argument("--interval", type=float, default=0.1, help="seconds between redraws")
+    args = parser.parse_args(argv)
+
+    stream = stream or sys.stdin
+    out = out or sys.stderr
+    interactive = out.isatty()
+    width = shutil.get_terminal_size((100, 24)).columns - 1
+
+    done = 0
+    last_drawn = 0.0
+    label = ""
+    for line in stream:
+        path = line.rstrip("\n")
+        # rsync reports directories too, with a trailing slash. They are not
+        # files and counting them would put the percentage past a hundred.
+        if not path or path.endswith("/"):
+            continue
+        done += 1
+        label = album_of(path) or os.path.basename(path)
+
+        now = time.monotonic()
+        if interactive:
+            if now - last_drawn >= args.interval:
+                out.write("\r\033[2K" + render(done, args.total, label, width))
+                out.flush()
+                last_drawn = now
+        elif now - last_drawn >= 30:
+            out.write(render(done, args.total, label, width) + "\n")
+            out.flush()
+            last_drawn = now
+
+    if interactive:
+        out.write("\r\033[2K")
+    out.write(render(done, args.total, label, width).rstrip() + "\n")
+    out.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/sync-to-ipod.sh

@@ -45,6 +45,11 @@ The destination must be on a mounted FAT filesystem. That check is also what
 catches an unmounted device: /media/IPOD/Music then resolves to the host's own
 root filesystem, and this refuses to empty that.
 
+Progress is one line that rewrites itself, showing the album currently going
+across and how far through the transfer is. Working the total out first means a
+second pass over the tree, which is the price of a percentage that means
+something; rsync's own is computed against a file list it is still building.
+
 Reach the device with the Apple firmware's disk mode: Menu+Select to reboot,
 then immediately Select+Play. Power off afterwards by holding Play.
 USAGE
@@ -146,26 +151,39 @@ fi
 # asking for them produces a screenful of errors and a non-zero exit.
 # --modify-window=2 because FAT stores mtimes to two-second resolution, without
 # which every file looks changed and the whole library is copied every time.
+options=(--recursive --times --delete --modify-window=2)
 # --delete removes tracks whose source has gone, which is the point. It would
 # also remove everything on the device that the mirror does not contain -- and
 # if the destination is the card root that means /.rockbox, the Rockbox install
 # itself. Excluded paths are not deleted unless --delete-excluded is given,
 # which it never is here.
-options=(--recursive --times --delete --modify-window=2 --human-readable --info=progress2)
 for owned in "/.rockbox" "/.scrobbler.log" "/.scrobbler.log.*" "/.playlist_control" \
     "/System Volume Information" "/.Spotlight-V100" "/.Trashes" "/.fseventsd"; do
     options+=(--exclude "$owned")
 done
-$dry_run && options+=(--dry-run --verbose)
 
 printf 'sync-to-ipod: %s -> %s\n' "$mirror" "$destination" >&2
-rsync "${options[@]}" "$mirror/" "$destination/"
 
 if $dry_run; then
+    rsync "${options[@]}" --dry-run --verbose "$mirror/" "$destination/"
     printf 'sync-to-ipod: dry run, nothing was written\n' >&2
     exit 0
 fi
 
+# rsync says nothing at all while it builds its file list, which on fifty
+# thousand files over USB is minutes of apparent hang. Counting first costs a
+# second pass over the tree but means the transfer can show a real percentage
+# rather than a number that grows as rsync discovers more work.
+printf 'sync-to-ipod: working out what needs copying...\n' >&2
+total=$(rsync "${options[@]}" --dry-run --out-format='%n' "$mirror/" "$destination/" |
+    grep -cve '/$' || true)
+printf 'sync-to-ipod: %s files to copy\n' "$total" >&2
+
+rsync "${options[@]}" --out-format='%n' "$mirror/" "$destination/" |
+    python3 "$here/rsync_progress.py" --total "$total"
+status=${PIPESTATUS[0]}
+[ "$status" -eq 0 ] || die "rsync exited $status"
+
 sync
 if $unmount; then
     device=$(findmnt -no SOURCE --target "$destination")