chore(release): v0.5.0

Reviewed-on: #12

Dockerfile

@@ -7,8 +7,10 @@ FROM python:3.13-alpine AS runtime
 
 ENV PYTHONUNBUFFERED=1
 
-# ffmpeg does the encoding; the application itself has no Python dependencies.
-RUN apk add --no-cache ffmpeg
+# ffmpeg does the encoding and rsgain the volume levelling; the application
+# itself has no Python dependencies. rsgain lives in the community repository,
+# which the official Python images already have enabled.
+RUN apk add --no-cache ffmpeg rsgain
 
 WORKDIR /app
 COPY pyproject.toml README.md ./
@@ -26,6 +28,9 @@ ENTRYPOINT ["music-mirror"]
 FROM runtime AS test
 
 RUN pip install --no-cache-dir pytest
+# sync-to-ipod.sh and its tests need these; the runtime image deliberately does
+# not carry them, and neither does the base.
+RUN apk add --no-cache bash rsync findmnt
 COPY pytest.ini ./
 # Host-side tools; not in the runtime image, but the suite covers them.
 COPY tools ./tools

README.md

@@ -22,6 +22,7 @@ remembering to do anything.
 | Mirror up to date                | skip                                     |
 | Source is already MP3            | copy verbatim                            |
 | Source gone                      | delete the mirror file, prune empty dirs |
+| An album gained or lost a track  | re-measure its ReplayGain, tags only     |
 
 Freshness is modification time: an encoded file is stamped with its source's
 mtime, so a file is stale exactly when the two differ. There is no database to
@@ -94,6 +95,7 @@ music-mirror --source /music --mirror /music-mp3 --subdir "Artist/Album"
 | `--interval` | `MUSIC_MIRROR_INTERVAL` | unset     | Repeat forever, e.g. `45m`, `6h`, `1d`         |
 | `--subdir`   | —                       | unset     | Limit the pass to one directory; skips pruning |
 | `--fat32-safe` | `MUSIC_MIRROR_FAT32_SAFE` | off     | Name files so a FAT32 device accepts them      |
+| `--no-replaygain` | `MUSIC_MIRROR_REPLAYGAIN` | on   | Write ReplayGain tags; set the variable to `0` to skip |
 | `--no-prune` | —                       | off       | Keep mirror files whose source has gone        |
 | `--dry-run`  | —                       | off       | Report what would change, write nothing        |
 
@@ -114,7 +116,9 @@ The first full pass is the expensive one; after that only new and changed files
 are touched. Lower `MUSIC_MIRROR_JOBS` if you would rather the NAS stayed
 responsive than finished sooner.
 
-Requires `ffmpeg` and `ffprobe` on `PATH`. The container image provides both.
+Requires `ffmpeg` and `ffprobe` on `PATH`, and `rsgain` for volume levelling.
+The container image provides all three. A missing `rsgain` is reported once per
+pass and costs the ReplayGain tags; everything else still runs.
 
 ## Running it on TrueNAS Scale
 
@@ -150,19 +154,164 @@ tools/sync-to-ipod.sh /mnt/tank/media/music-mp3 /media/IPOD/Music
 tools/sync-to-ipod.sh -n /mnt/tank/media/music-mp3 /media/IPOD/Music   # dry run
 ```
 
-It refuses to start unless the destination is a mounted FAT filesystem that is
-its own mount point, because `--delete` aimed at the wrong directory empties it
-and does not announce itself. It also excludes `/.rockbox`, the scrobbler logs
+The destination is where the artist folders should end up — normally a
+subdirectory such as `/media/IPOD/Music`, not the card root. A subdirectory is
+the better target: `--delete` is confined to it, and the device path budget is
+derived from it rather than configured, so the two cannot disagree.
+
+It refuses to start unless the destination is on a mounted FAT filesystem. That
+check is also what catches an unmounted device — `/media/IPOD/Music` then
+resolves to the host's own root filesystem, and this refuses to empty that.
+`--help` says all of it. It also excludes `/.rockbox`, the scrobbler logs
 and the various filesystem metadata directories from deletion — the mirror does
 not contain them, and without the exclusion a sync to the card root would
 remove the Rockbox install.
 
+Progress is a single line that rewrites itself:
+
+```
+[ 24%] 12,345/49,600  3.2 GiB/13.1 GiB  4.4 MiB/s  ETA 38m12s  King Gizzard / Petro…
+```
+
+The estimate comes from rsync's `%l`, which gives each file's size as it
+completes. Bytes done over time elapsed is the same arithmetic rsync would do,
+and needs nothing it does not already print. The rate is measured over a
+trailing thirty seconds rather than the whole run, so it follows a device that
+slows down instead of averaging the slowdown away — and it is suppressed
+entirely for the first two seconds, where the window is microseconds wide and
+would report gigabytes per second.
+
+rsync says nothing at all while it builds its file list, which on fifty
+thousand files is minutes of apparent hang, and its own `progress2` percentage
+is computed against a list it has not finished discovering. So the script
+renders its own.
+
+**The percentage and the estimate are opt-in, via `-P`.** They need a total,
+the total needs a counting pass, and that pass walks and compares both trees in
+full exactly as the transfer does. Measured on a real card: read from the
+source at 35 MB/s and write to the card at 21 MB/s, yet the sync crawled —
+because the traversal, not the data, was the cost, and it was being paid twice.
+Without `-P` the line still shows the running count, the rate and the album in
+flight; only the two figures that needed the second walk are missing.
+
+Piped to a log it prints a plain line every thirty seconds instead, with no
+carriage returns, and a summary at the end either way.
+
+### Albums that changed are copied whole
+
+After the main pass, any album that gained or lost a track has the rest of its
+tracks copied again with `--ignore-times`. Their ReplayGain tags were rewritten
+in place when the album was re-levelled, which changes neither their size nor
+their mtime — the only two things rsync compares — so nothing else would ever
+send them. `touched_albums.py` works out the list from rsync's own report of
+what it moved, so this costs no extra traversal, and tracks the main pass has
+already copied are left out of it. See [Volume levelling](#volume-levelling).
+
+### The Rockbox database
+
+Point `MUSIC_MIRROR_DATABASE_TOOL` at Rockbox's host-side builder and the sync
+rebuilds the database itself, so it never has to happen on the device.
+
+```sh
+git clone --depth 1 https://github.com/Rockbox/rockbox.git
+cd rockbox && mkdir build-db && cd build-db
+../tools/configure --target=ipodvideo --type=d && make -j$(nproc)
+```
+
+It needs a native compiler and SDL2 development headers, not the ARM
+cross-toolchain, and `tools/configure` detects `__aarch64__` correctly. On a
+distribution without `/usr/bin/perl` or `gcc-ar` — NixOS, say — patch the
+shebangs in `tools/*.pl` and pass `AR=ar`.
+
+Building it here rather than on the device is not merely faster. The on-device
+commit sorts the whole index in whatever memory `core_alloc_maximum()` can
+scrape together; on a fifty-thousand-track library it runs for hours or aborts
+outright.
+
+The scan runs against a scratch root — a real `.rockbox` beside a symlink
+standing in for wherever the music lands on the device — so the paths recorded
+are the ones Rockbox will look up, while the **bytes are read from the mirror
+rather than over USB**. Only the dozen `.tcd` files cross to the card.
+
+Cost, measured: the parser makes about 49 reads and 43 seeks per file, probing
+the head for ID3v2 and the tail for ID3v1. On a local disk that is 2,000 files
+in half a second. Over SMB, readahead absorbs most of the reads but the opens
+and the head/tail split are real round trips, so a first full scan is minutes
+rather than seconds. It is a one-time cost: the builder is incremental, and the
+scratch root is kept between runs, so a later pass over unchanged files does no
+metadata reads at all.
+
+If minutes is still too many, run the builder where the mirror is local — on
+the NAS — and copy the `.tcd` files across. There is nothing to parallelise:
+the tool is single-threaded, and two instances cannot produce one database.
+
+### If the sync is interrupted
+
+No partially copied track is ever left under a name Rockbox would play. rsync
+writes to a hidden temporary file and only renames it into place once the file
+is complete, and *"by default, rsync will delete any partially transferred file
+if the transfer is interrupted"*. `--partial` is deliberately not used, and
+there is a test asserting it never will be.
+
+After an unclean kill or a power cut a hidden `.track.mp3.XXXXXX` can survive.
+It is not playable, it is not in the source, and the next run's `--delete`
+removes it.
+
+Interrupting does not, by itself, endanger the filesystem. The kernel flushes
+dirty pages within `dirty_expire_centisecs` — thirty seconds by default — and
+`umount` always syncs before it returns. Losing data needs you to interrupt,
+*and* pull the card inside that window, *and* skip the unmount.
+
+The script still traps `INT` and `TERM` and flushes and unmounts on the way
+out, exiting 130. Not because a Ctrl-C is dangerous, but because it removes the
+manual step and makes the exit deterministic — you get the same "safe to
+disconnect" either way, rather than having to remember which path you took.
+Both paths call the same function, so they cannot drift apart.
+
+What genuinely does lose data is pulling the cable or the card without
+unmounting at all, interrupted or not. FAT32 has no journal. Wait for the
+unmount line.
+
+### Making it faster over a network mount
+
+The transfer is metadata-bound, not throughput-bound: 49,600 files means 49,600
+round trips, and the counting pass doubles that. In rough order of what it is
+worth doing:
+
+| Lever | Why |
+| ----- | --- |
+| Mount the source with `actimeo=60,cache=loose` | SMB defaults to a **one second** attribute cache, so nearly every `stat` goes to the wire — twice, once per pass. This is the single biggest change and it is a mount option, not an rsync flag. |
+| Put the card in a reader for the first load | USB 2.0 through an iPod in disk mode is the floor for the destination. No amount of source tuning gets past it. |
+| Counting is off by default | The percentage costs a second full traversal of both trees. On a FAT card of fifty thousand files that is slower than the transfer. `-P` asks for it. |
+| `--whole-file`, `--omit-dir-times` | Already set. The first stops rsync checksumming destination files it is about to overwrite whole; the second drops a setattr per directory, 6,150 of them. |
+
+**NFS instead of SMB** is worth trying but is not the big win it looks like.
+Its attribute caching defaults are far more generous than SMB's — `acregmax` of
+sixty seconds against `actimeo=1` — which is precisely the gap that
+`actimeo=60` closes on the mount you already have. Bulk read throughput between
+the two is much of a muchness on a gigabit link. Try the mount option first; it
+is one line and needs no change on the NAS.
+
+And if the destination is the iPod rather than a card reader, none of this
+matters much: the source can feed data faster than USB 2.0 through an iPod will
+take it either way.
+
 The unmount is the point of doing this in a script. FAT32 has no journal and
 the device is reached through disk mode, so an interrupted write is corruption
 that needs `fsck.vfat` from another machine.
 
-`submit_scrobbles.py` sends the Rockbox scrobbler log to Last.fm and sets it
-aside. Rockbox writes `/.scrobbler.log` in AUDIOSCROBBLER 1.1 format, one
+`submit_scrobbles.py` sends what was played to Last.fm and sets the logs aside.
+
+It reads **Rockbox's own `playback.log`**, which core Rockbox writes whenever
+"play log" is enabled, with no plugin running. Each line is
+`timestamp:elapsed_ms:length_ms:path` — a path and nothing else, which is why
+the on-device scrobbler plugin exists at all: reading tags back off the player
+is slow. Off the mirror it is free, so `--mirror` lets the conversion happen
+here and the plugin never has to be run. A play counts as listened at half the
+track's length, the same fraction the plugin uses, so the two cannot disagree
+about what a play was.
+
+It still reads a `.scrobbler.log` if the plugin has been run and left one. Rockbox writes `/.scrobbler.log` in AUDIOSCROBBLER 1.1 format, one
 tab-separated line per track rated `L` for listened or `S` for skipped; only
 the listened ones are sent. It runs **before** the copy, since the plays
 already happened and a failed transfer is no reason to lose them.
@@ -174,8 +323,29 @@ real-time clock Rockbox writes `/.scrobbler-timeless.log` with every timestamp
 set to zero; those are counted and reported but never submitted, because
 scrobbling them would mean inventing when they happened.
 
-The log is renamed rather than deleted once accepted. If Last.fm quietly
-dropped something, the evidence is still on the device.
+### Nothing played is thrown away
+
+Two separate obligations, because a play that happened and never reached
+Last.fm is gone for good.
+
+**The original is renamed, never deleted.** If Last.fm quietly dropped
+something, the evidence is still on the device as `playback.log.<ts>.submitted`.
+
+**Anything not submitted is written back** into a live log for the next run:
+
+| Outcome | What happens to it |
+| ------------------------------ | ----------------------------------------- |
+| Accepted by Last.fm | dropped from the live log |
+| Not in the mirror yet | written back, tried again next run |
+| In a batch that failed | written back, tried again next run |
+| Nothing accepted at all | logs left completely untouched |
+| A skip, or no usable timestamp | not retained — neither can ever be submitted, and the original still has it |
+
+The batch boundary matters: submission is recorded as each batch is accepted,
+so a failure partway through knows exactly what got through and writes back
+only the remainder. No duplicates, no losses.
+
+A file `ffprobe` cannot read costs one unidentified play, not the run.
 
 `check_fat32.py` reports paths a FAT32 device will not accept — reserved
 characters, trailing dots and spaces, over-long components and paths, and names
@@ -191,6 +361,8 @@ docker build --target test .   # what CI runs
 pytest                         # needs ffmpeg and pytest on PATH
 ```
 
+The ReplayGain tests need `rsgain` as well and skip without it.
+
 The suite runs real ffmpeg encodes rather than mocking them. The interesting
 failures are in what ffmpeg actually does with tags, cover art and container
 formats, and a mock cannot fail that way — which is also why CI runs the tests
@@ -202,7 +374,7 @@ Run them directly instead if you prefer; they skip when ffmpeg is absent. On a
 Nix machine:
 
 ```sh
-nix shell nixpkgs#python3Packages.pytest nixpkgs#ffmpeg -c pytest
+nix shell nixpkgs#python3Packages.pytest nixpkgs#ffmpeg nixpkgs#rsgain -c pytest
 ```
 
 ## FAT32 and Rockbox
@@ -292,6 +464,64 @@ firmware reads the embedded one; Rockbox reads the file. Both are satisfied.
 A cover left behind in a directory whose tracks have all gone is pruned, or the
 directory would never look empty and never be removed.
 
+### Volume levelling
+
+Rockbox can level the volume between tracks, but only from tags. It applies the
+offset a ReplayGain tag carries and has no loudness analysis of its own, so a
+mirror without those tags plays every album at whatever level it was mastered
+to — and a 2008 remaster next to a 1972 pressing is a reach for the volume
+wheel on every track change.
+
+The tags are therefore written here, with `rsgain`, once an album's tracks are
+in place. Both album gain and track gain are measured: album gain preserves the
+quiet track that a record is supposed to have, track gain is the one that makes
+sense on shuffle, and which of them is used is the device's decision, not this
+one.
+
+Turn it on at the player end under **Settings → Playback Settings →
+Replaygain**:
+
+| Setting          | Suggested                    | Why                                                        |
+| ---------------- | ---------------------------- | ---------------------------------------------------------- |
+| Replaygain type  | `Track Gain if Shuffling`    | Album gain while playing a record, track gain once shuffle is on — the only setting that is right in both cases |
+| Prevent clipping | `Yes`                        | Backs the gain off using the peak tags rather than distorting |
+| Pre-amp          | `0 dB`                       | The reference is already −18 LUFS; raise it only if everything ends up too quiet |
+
+`Track Gain if Shuffling` is not a compromise between the other two — Rockbox
+reads the shuffle setting and picks whole-hog album or track gain from it
+(`apps/misc.c`, `replaygain_setting_mode`). It is also Rockbox's default, so on
+a fresh install there may be nothing to change but `Prevent clipping`.
+
+Measuring costs a full decode of every track, so the first pass after enabling
+it takes roughly as long as the original encode did. After that only albums
+that gained, lost or replaced a track are re-measured. An album is re-measured
+as a whole, because album gain is a property of all of its tracks and one new
+track makes the value stored on every sibling wrong.
+
+Tagging rewrites the file, and staleness here is an mtime comparison, so
+`rsgain` is run with `--preserve-mtimes`. Without it every levelled track would
+look newer than its source and the next pass would re-encode the entire
+library.
+
+That has a consequence for the sync, and it is not obvious. The first time a
+track is levelled its tag grows by about a kilobyte, so its size changes and
+rsync copies it — the whole library goes across once, and there is no way
+around that; the tag sits at the head of the file and every byte after it
+moves. But `rsgain` leaves padding behind, so a *later* re-level fits inside it
+and changes neither the size nor the mtime:
+
+```
+before re-level: 277757 bytes, mtime 1577880000, album gain 3.75 dB
+after  re-level: 277757 bytes, mtime 1577880000, album gain 6.25 dB
+```
+
+Those are the two things rsync's quick check compares, so it would see nothing
+to do and the device would keep the old gains. `sync-to-ipod.sh` handles it: the
+track that arrived or left is always visible, so any album the main pass
+touched has the rest of its tracks copied again with `--ignore-times`. Albums
+whose file set has not changed are not touched, which is what keeps this from
+being a full re-copy.
+
 ## Getting the result onto an iPod
 
 The mirror is just a directory of MP3s, so any client will do:
@@ -331,6 +561,6 @@ The mirror is just a directory of MP3s, so any client will do:
 Neither client transcodes at sync time; they copy finished MP3s.
 
 Two device-side details worth knowing: the iPod reads cover art from the file's
-tags and ignores `folder.jpg`, which is why art is embedded here; and volume
-levelling on the device uses iTunes' Soundcheck tag, not ReplayGain, so
-ReplayGain tags in the source are not carried over as such.
+tags and ignores `folder.jpg`, which is why art is embedded here; and the Apple
+firmware levels volume from iTunes' Soundcheck tag rather than ReplayGain, so
+the tags written here do nothing until the iPod is running Rockbox.

compose.yaml

@@ -31,6 +31,11 @@ services:
       # enabling it is quick -- but it is a one-way change to every such path,
       # so decide before running it rather than after.
       MUSIC_MIRROR_FAT32_SAFE: "true"
+      # ReplayGain tags, so Rockbox can level the volume between albums. On by
+      # default; set to 0 to skip the measuring pass. The first pass after
+      # enabling it levels the whole library, which costs a decode of every
+      # track, so expect it to take about as long as the original encode.
+      # MUSIC_MIRROR_REPLAYGAIN: "0"
     volumes:
       - /mnt/tank/media/music:/music:ro
       - /mnt/tank/media/music-mp3:/mirror

music_mirror.py

@@ -11,6 +11,11 @@ writes to the source library.
 Staleness is tracked by modification time: an encoded file is given its
 source's mtime, so a file is out of date exactly when the two differ. That
 makes runs idempotent without a database to keep in step.
+
+Finished albums are levelled with rsgain, which writes ReplayGain tags into the
+mirror. Rockbox applies the offset those tags carry but has no loudness
+analysis of its own, so without them every album plays at whatever level it was
+mastered to.
 """
 
 import argparse
@@ -81,6 +86,21 @@ MIRROR_COVER = "cover.jpg"
 # Files the mirror is allowed to contain, and therefore allowed to delete.
 MIRROR_SUFFIX = ".mp3"
 
+# Measures loudness and writes the ReplayGain tags. Not a hard requirement: a
+# pass without it still produces a correct mirror, only one the player cannot
+# level, so a missing binary is a warning rather than a failure.
+REPLAYGAIN_TOOL = "rsgain"
+
+# Looked for in a file's ID3v2 tag to tell a levelled track from an unlevelled
+# one. Album gain rather than track gain because the album value is the one
+# this writes for; a file carrying only track gain came from somewhere else and
+# should be rescanned.
+REPLAYGAIN_TAG = b"replaygain_album_gain"
+
+# Enough of a TXXX frame body to hold the encoding byte and the description.
+# The value after it says what the gain is, which is not the question here.
+TXXX_DESCRIPTION_BYTES = 128
+
 # Filesystems disagree about mtime precision; SMB in particular rounds.
 MTIME_TOLERANCE_SECONDS = 2
 
@@ -541,8 +561,13 @@ def prune(mirror_root, expected, dry_run):
     Driven by the set of paths the pass expects to exist rather than by
     probing the source tree for names, which would disagree with it over
     letter case and over any extension the walker does not collect.
+
+    Returns the number of files removed and the directories they came out of.
+    Losing a track changes an album's loudness, so those directories need
+    levelling again even though nothing was written into them.
     """
     removed = 0
+    emptied = set()
 
     for mirror in sorted(mirror_root.rglob(f"*{MIRROR_SUFFIX}")):
         if mirror in expected:
@@ -553,6 +578,7 @@ def prune(mirror_root, expected, dry_run):
             continue
         logger.info("removing orphan %s", mirror)
         mirror.unlink(missing_ok=True)
+        emptied.add(mirror.parent)
 
     if not dry_run:
         # A cover copied for an album whose tracks have all gone is an orphan
@@ -567,7 +593,164 @@ def prune(mirror_root, expected, dry_run):
             if directory.is_dir() and not any(directory.iterdir()):
                 directory.rmdir()
 
-    return removed
+    return removed, emptied
+
+
+def syncsafe(data):
+    """Return the integer held in syncsafe bytes: seven bits of each."""
+    value = 0
+    for byte in data:
+        value = (value << 7) | (byte & 0x7F)
+    return value
+
+
+def has_replaygain(path):
+    """Return whether an MP3 already carries ReplayGain tags.
+
+    Walks the ID3v2 frame headers and seeks over the bodies rather than reading
+    the tag whole. Every file in this mirror has its cover art embedded, so the
+    tag is routinely half a megabyte; reading all of it for every track on
+    every pass would turn an idle pass into a full read of the library.
+    """
+    try:
+        with open(path, "rb") as handle:
+            header = handle.read(10)
+            if len(header) < 10 or header[:3] != b"ID3" or header[3] not in (3, 4):
+                return False
+            remaining = syncsafe(header[6:10])
+
+            # Unsynchronisation shifts every offset in the tag, and the two
+            # versions describe an extended header differently. Nothing that
+            # writes this mirror emits either, so reading the tag whole is a
+            # cheaper answer than the code to walk one that does.
+            if header[5] & 0xC0:
+                return REPLAYGAIN_TAG in handle.read(remaining).lower()
+
+            while remaining >= 10:
+                frame = handle.read(10)
+                remaining -= 10
+                # Frame ids are upper-case letters and digits, so anything else
+                # is the padding that follows the last frame.
+                if len(frame) < 10 or not frame[:4].isalnum():
+                    return False
+                # 2.3 sizes count all eight bits per byte; 2.4 made them
+                # syncsafe like the tag length above.
+                length = (
+                    int.from_bytes(frame[4:8], "big")
+                    if header[3] == 3
+                    else syncsafe(frame[4:8])
+                )
+                if length <= 0 or length > remaining:
+                    return False
+                if frame[:4] == b"TXXX":
+                    body = handle.read(min(length, TXXX_DESCRIPTION_BYTES))
+                    handle.seek(length - len(body), os.SEEK_CUR)
+                    if REPLAYGAIN_TAG in body.lower():
+                        return True
+                else:
+                    handle.seek(length, os.SEEK_CUR)
+                remaining -= length
+    except OSError:
+        return False
+    return False
+
+
+def replaygain_albums(expected, written):
+    """Return the album directories needing a scan, each with its tracks.
+
+    A directory is scanned when this pass changed what is in it, because album
+    gain is a property of the whole album: one track added, replaced or removed
+    makes the value stored on every one of its siblings wrong. It is also
+    scanned when a track in it has never been levelled, which is what backfills
+    a mirror built before any of this existed.
+    """
+    albums = {}
+    for mirror in expected:
+        albums.setdefault(mirror.parent, []).append(mirror)
+
+    needed = {}
+    for directory, tracks in sorted(albums.items()):
+        # A dry run reaches here before anything has been encoded, so the
+        # tracks a changed album is going to hold do not exist yet.
+        present = sorted(track for track in tracks if track.is_file())
+        if directory in written:
+            needed[directory] = present
+        elif present and not all(map(has_replaygain, present)):
+            needed[directory] = present
+    return needed
+
+
+def replaygain_command(tracks):
+    """Return the rsgain command that levels one album directory."""
+    return [
+        REPLAYGAIN_TOOL,
+        "custom",
+        # Album mode writes the per-track tags as well as the album ones, so
+        # the device is left to choose between them -- Rockbox can apply track
+        # gain when shuffling and album gain otherwise, and only if both are
+        # present.
+        "--album",
+        "--tagmode=i",
+        # The mirror is ID3v2.3 for the iPod firmware's sake. rsgain would
+        # otherwise keep whatever version it found, and "whatever it found" is
+        # not a guarantee.
+        "--id3v2-version=3",
+        # Staleness here is an mtime comparison and tagging rewrites the file.
+        # Without this every levelled track would look newer than its source
+        # and the next pass would re-encode the entire library, forever.
+        "--preserve-mtimes",
+        "--quiet",
+        *[str(track) for track in tracks],
+    ]
+
+
+def scan_album(directory, tracks):
+    """Write ReplayGain tags across one album. Returns whether it worked."""
+    completed = subprocess.run(replaygain_command(tracks), capture_output=True, text=True)
+    if completed.returncode != 0:
+        lines = completed.stderr.strip().splitlines()
+        logger.warning("could not level %s: %s", directory, lines[-1] if lines else "rsgain failed")
+        return False
+    logger.info("levelled %s", directory)
+    return True
+
+
+def replaygain(expected, written, jobs, dry_run):
+    """Write ReplayGain tags into the albums that need them. Returns how many.
+
+    A failure here is reported and then left alone. The mirror is still correct
+    audio in the right place; it just plays at the level it was mastered to,
+    which is what every pass before this one produced.
+    """
+    albums = {
+        directory: tracks
+        for directory, tracks in replaygain_albums(expected, written).items()
+        if tracks or dry_run
+    }
+    if not albums:
+        return 0
+
+    if dry_run:
+        logger.info("would level %d album%s", len(albums), "" if len(albums) == 1 else "s")
+        return len(albums)
+
+    if shutil.which(REPLAYGAIN_TOOL) is None:
+        logger.warning(
+            "%s is not on PATH; %d albums are left without ReplayGain tags",
+            REPLAYGAIN_TOOL,
+            len(albums),
+        )
+        return 0
+
+    levelled = 0
+    with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
+        futures = [
+            pool.submit(scan_album, directory, tracks) for directory, tracks in albums.items()
+        ]
+        for future in concurrent.futures.as_completed(futures):
+            if future.result():
+                levelled += 1
+    return levelled
 
 
 def run_once(
@@ -580,6 +763,7 @@ def run_once(
     do_prune,
     safe=False,
     budget=0,
+    do_replaygain=True,
 ):
     """Run a single pass. Returns the number of failures.
 
@@ -590,6 +774,7 @@ def run_once(
     logger.info("pass starting with %d concurrent encoders", jobs)
     counts = {"encoded": 0, "copied": 0, "renamed": 0, "skipped": 0, "failed": 0}
     failures = []
+    written = set()
 
     work = plan(scan_root, source_root, mirror_root, safe, budget)
 
@@ -617,6 +802,8 @@ def run_once(
             counts[result.action] += 1
             if result.action == "failed":
                 failures.append(result)
+            elif result.action != "skipped":
+                written.add(result.path.parent)
 
     expected = set(work)
     if safe and dry_run:
@@ -627,20 +814,25 @@ def run_once(
         for source in work.values():
             expected.add(mirror_path_for(source, source_root, mirror_root))
             expected.add(mirror_path_for(source, source_root, mirror_root, True))
-    removed = prune(mirror_root, expected, dry_run) if do_prune else 0
+    removed, emptied = prune(mirror_root, expected, dry_run) if do_prune else (0, set())
+
+    # After pruning, so an album is not measured with a track in it that is
+    # about to be deleted.
+    levelled = replaygain(expected, written | emptied, jobs, dry_run) if do_replaygain else 0
 
     for failure in failures:
         logger.error("failed: %s: %s", failure.path, failure.error)
 
     logger.info(
         "pass complete in %.1fs: %d encoded, %d copied, %d renamed, %d up to date,"
-        " %d removed, %d failed",
+        " %d removed, %d levelled, %d failed",
         time.monotonic() - started,
         counts["encoded"],
         counts["copied"],
         counts["renamed"],
         counts["skipped"],
         removed,
+        levelled,
         counts["failed"],
     )
     return counts["failed"]
@@ -740,6 +932,13 @@ def build_parser():
         action="store_true",
         help="keep mirror files whose source has been deleted",
     )
+    parser.add_argument(
+        "--no-replaygain",
+        action="store_true",
+        default=os.getenv("MUSIC_MIRROR_REPLAYGAIN", "").lower() in ("0", "false", "no"),
+        help="do not write ReplayGain tags; skips the rsgain pass over changed"
+        " albums (env MUSIC_MIRROR_REPLAYGAIN=0)",
+    )
     parser.add_argument(
         "--dry-run",
         action="store_true",
@@ -831,6 +1030,7 @@ def main(argv=None):
                 do_prune,
                 args.fat32_safe,
                 budget,
+                not args.no_replaygain,
             )
             if interval is None or stopping:
                 return 1 if failures else 0

pyproject.toml

@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "music-mirror"
-version = "0.3.0"
+version = "0.5.0"
 description = "Maintain a lossy MP3 mirror of a lossless music library"
 readme = "README.md"
 requires-python = ">=3.11"
-# No runtime Python dependencies: the work is done by ffmpeg, which must be on
-# PATH.
+# No runtime Python dependencies: the work is done by ffmpeg and rsgain, which
+# must be on PATH. A missing rsgain costs the ReplayGain tags and nothing else.
 dependencies = []
 
 [project.scripts]

tests/conftest.py

@@ -19,6 +19,13 @@ def require_ffmpeg():
             pytest.skip(f"{tool} is not on PATH", allow_module_level=True)
 
 
+@pytest.fixture
+def require_rsgain():
+    """Skip a test that measures loudness for real rather than faking it."""
+    if shutil.which("rsgain") is None:
+        pytest.skip("rsgain is not on PATH")
+
+
 @pytest.fixture
 def tight_umask():
     """Run a test under a umask that would otherwise make the mirror private."""
@@ -46,7 +53,14 @@ def owner_hostile_umask():
 def make_flac():
     """Return a factory writing a short tagged FLAC file."""
 
-    def factory(path, title="Test Title", artist="Test Artist", album="Test Album", seconds=1):
+    def factory(
+        path,
+        title="Test Title",
+        artist="Test Artist",
+        album="Test Album",
+        seconds=1,
+        gain=0,
+    ):
         path.parent.mkdir(parents=True, exist_ok=True)
         subprocess.run(
             [
@@ -60,6 +74,9 @@ def make_flac():
                 "lavfi",
                 "-i",
                 f"sine=frequency=440:duration={seconds}",
+                # Quieter or louder than the default, for tests that need two
+                # tracks at different levels.
+                *(["-af", f"volume={gain}dB"] if gain else []),
                 "-metadata",
                 f"title={title}",
                 "-metadata",

tests/test_music_mirror.py

@@ -14,6 +14,16 @@ def run(source, mirror, *extra):
     return music_mirror.main(["--source", str(source), "--mirror", str(mirror), *extra])
 
 
+def audio_frames(path):
+    """Return a hash of a file's audio frames, ignoring its tags."""
+    return subprocess.run(
+        ["ffmpeg", "-v", "error", "-i", str(path), "-map", "0:a", "-c", "copy", "-f", "md5", "-"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+
 def test_parse_quality_accepts_vbr_and_cbr():
     assert music_mirror.parse_quality("V0") == ["-q:a", "0"]
     assert music_mirror.parse_quality("v2") == ["-q:a", "2"]
@@ -61,8 +71,10 @@ def test_output_is_mp3(tmp_path, make_flac):
             "a:0",
             "-show_entries",
             "stream=codec_name",
+            # Not csv: a levelled file carries ReplayGain side data, which the
+            # csv writer renders as a trailing empty field.
             "-of",
-            "csv=p=0",
+            "default=noprint_wrappers=1:nokey=1",
             str(mirror / "a.mp3"),
         ],
         check=True,
@@ -140,6 +152,9 @@ def test_no_prune_keeps_orphans(tmp_path, make_flac):
 
 
 def test_existing_mp3_is_copied_not_re_encoded(tmp_path, make_flac):
+    """Compared by the audio frames rather than the whole file: the copy is
+    tagged with its ReplayGain values afterwards, so the two differ in the
+    container while carrying identical audio."""
     source = tmp_path / "src"
     mirror = tmp_path / "dst"
     flac = make_flac(source / "a.flac")
@@ -152,7 +167,7 @@ def test_existing_mp3_is_copied_not_re_encoded(tmp_path, make_flac):
 
     run(source, mirror)
 
-    assert (mirror / "b.mp3").read_bytes() == (source / "b.mp3").read_bytes()
+    assert audio_frames(mirror / "b.mp3") == audio_frames(source / "b.mp3")
 
 
 def test_interrupted_copy_leaves_nothing_behind(tmp_path, make_flac, monkeypatch):
@@ -786,3 +801,181 @@ def test_the_budget_is_reported_so_it_can_be_checked(tmp_path, make_flac, caplog
         run(source, mirror, "--fat32-safe")
 
     assert "limited to 253 characters" in caplog.text
+
+
+# Rockbox applies the offset a ReplayGain tag carries but never measures
+# loudness itself, so an untagged mirror plays each album at whatever level it
+# was mastered to. The tags have to be written here or nowhere.
+
+
+def test_replaygain_tags_are_written(tmp_path, make_flac, probe_tag, require_rsgain):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+
+    run(source, mirror)
+
+    track = mirror / "Album" / "a.mp3"
+    assert probe_tag(track, "REPLAYGAIN_TRACK_GAIN").endswith("dB")
+    assert probe_tag(track, "REPLAYGAIN_ALBUM_GAIN").endswith("dB")
+    assert probe_tag(track, "REPLAYGAIN_TRACK_PEAK")
+
+
+def test_the_album_shares_one_gain_and_the_tracks_keep_their_own(
+    tmp_path, make_flac, probe_tag, require_rsgain
+):
+    """Album gain is what keeps a quiet track quiet within a record it belongs
+    to. Track gain is written alongside it so the device can pick the other
+    behaviour when shuffling."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "loud.flac")
+    make_flac(source / "Album" / "quiet.flac", gain=-12)
+
+    run(source, mirror)
+
+    loud = mirror / "Album" / "loud.mp3"
+    quiet = mirror / "Album" / "quiet.mp3"
+    assert probe_tag(loud, "REPLAYGAIN_ALBUM_GAIN") == probe_tag(quiet, "REPLAYGAIN_ALBUM_GAIN")
+    assert probe_tag(loud, "REPLAYGAIN_TRACK_GAIN") != probe_tag(quiet, "REPLAYGAIN_TRACK_GAIN")
+
+
+def test_levelling_does_not_make_the_next_pass_re_encode(
+    tmp_path, make_flac, caplog, require_rsgain
+):
+    """Writing tags rewrites the file, and staleness here is an mtime
+    comparison. Without --preserve-mtimes every pass would re-encode the whole
+    library and then level it again, forever."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+
+    run(source, mirror)
+    before = (mirror / "Album" / "a.mp3").stat().st_mtime
+
+    with caplog.at_level("INFO"):
+        run(source, mirror)
+
+    assert "0 encoded" in caplog.text
+    assert (mirror / "Album" / "a.mp3").stat().st_mtime == before
+
+
+def test_an_already_levelled_album_is_not_measured_again(
+    tmp_path, make_flac, caplog, require_rsgain
+):
+    """Measuring costs a decode of every track. A pass that changed nothing
+    must not pay it."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    run(source, mirror)
+
+    with caplog.at_level("INFO"):
+        run(source, mirror)
+
+    assert "0 levelled" in caplog.text
+
+
+def test_a_new_track_relevels_the_album_around_it(
+    tmp_path, make_flac, probe_tag, require_rsgain
+):
+    """Album gain is a property of the whole album, so a track arriving late
+    makes the value stored on every one of its siblings wrong.
+
+    -10 dB rather than something more dramatic: R128 gates quiet passages out
+    of the measurement, and a track far enough below the rest of the record is
+    excluded from it entirely."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    run(source, mirror)
+    first = probe_tag(mirror / "Album" / "a.mp3", "REPLAYGAIN_ALBUM_GAIN")
+
+    make_flac(source / "Album" / "b.flac", gain=-10)
+    run(source, mirror)
+
+    assert probe_tag(mirror / "Album" / "a.mp3", "REPLAYGAIN_ALBUM_GAIN") != first
+
+
+def test_a_removed_track_relevels_the_album_behind_it(
+    tmp_path, make_flac, caplog, require_rsgain
+):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    make_flac(source / "Album" / "b.flac", gain=-20)
+    run(source, mirror)
+
+    (source / "Album" / "b.flac").unlink()
+    with caplog.at_level("INFO"):
+        run(source, mirror)
+
+    assert "1 levelled" in caplog.text
+
+
+def test_embedded_cover_art_does_not_hide_the_tags(
+    tmp_path, make_flac, make_cover, require_rsgain
+):
+    """The check walks ID3v2 frame headers and seeks over the bodies. Cover art
+    sits between the text frames and the ones rsgain appends, so a check that
+    only read the start of the tag would never reach them -- and would measure
+    every album with a cover on every pass."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    make_cover(source / "Album" / "cover.jpg")
+
+    run(source, mirror)
+
+    assert music_mirror.has_replaygain(mirror / "Album" / "a.mp3")
+
+
+def test_no_replaygain_leaves_the_tags_off(tmp_path, make_flac, probe_tag):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+
+    run(source, mirror, "--no-replaygain")
+
+    assert not probe_tag(mirror / "Album" / "a.mp3", "REPLAYGAIN_ALBUM_GAIN")
+    assert not music_mirror.has_replaygain(mirror / "Album" / "a.mp3")
+
+
+def test_an_unlevelled_mirror_is_backfilled(tmp_path, make_flac, probe_tag, require_rsgain):
+    """A mirror built before any of this existed has correct mtimes, so no pass
+    would ever revisit those files on its own."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    run(source, mirror, "--no-replaygain")
+
+    run(source, mirror)
+
+    assert probe_tag(mirror / "Album" / "a.mp3", "REPLAYGAIN_ALBUM_GAIN").endswith("dB")
+
+
+def test_a_missing_scanner_is_reported_and_not_fatal(tmp_path, make_flac, caplog, monkeypatch):
+    """The mirror is still correct audio in the right place. It just plays at
+    the level it was mastered to."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    monkeypatch.setattr(music_mirror.shutil, "which", lambda name: None)
+
+    with caplog.at_level("INFO"):
+        assert run(source, mirror) == 0
+
+    assert "rsgain is not on PATH" in caplog.text
+    assert (mirror / "Album" / "a.mp3").is_file()
+
+
+def test_a_dry_run_measures_nothing(tmp_path, make_flac, caplog, require_rsgain):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+
+    with caplog.at_level("INFO"):
+        run(source, mirror, "--dry-run")
+
+    assert "would level 1 album" in caplog.text
+    assert not mirror.joinpath("Album").exists()

tests/test_rsync_progress.py

@@ -0,0 +1,180 @@
+import io
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import rsync_progress  # noqa: E402
+
+
+class NotATerminal(io.StringIO):
+    def isatty(self):
+        return False
+
+
+class Terminal(io.StringIO):
+    def isatty(self):
+        return True
+
+
+def run(lines, total=0, out=None, bytes_expected=0):
+    out = out or NotATerminal()
+    rsync_progress.main(
+        ["--total", str(total), "--bytes", str(bytes_expected)],
+        stream=io.StringIO(lines),
+        out=out,
+    )
+    return out.getvalue()
+
+
+def test_the_artist_and_album_are_pulled_from_the_path():
+    assert rsync_progress.album_of("Pendulum/Immersion/01 - Watercolour.mp3") == (
+        "Pendulum / Immersion"
+    )
+
+
+def test_a_shallower_path_degrades_rather_than_failing():
+    assert rsync_progress.album_of("Pendulum/loose.mp3") == "Pendulum"
+    assert rsync_progress.album_of("loose.mp3") == ""
+
+
+def test_directories_are_not_counted():
+    """rsync reports them too, and counting them puts the percentage past 100."""
+    output = run("Artist/\nArtist/Album/\nArtist/Album/track.mp3\n", total=1)
+
+    assert "1/1" in output
+    assert "100%" in output
+
+
+def test_the_percentage_tracks_the_total():
+    output = run("".join(f"A/B/{i}.mp3\n" for i in range(5)), total=10)
+
+    assert "5/10" in output
+    assert "50%" in output
+
+
+def test_without_a_total_it_counts_instead_of_guessing():
+    output = run("A/B/one.mp3\nA/B/two.mp3\n")
+
+    assert "2 files" in output
+    assert "%" not in output
+
+
+def test_a_final_line_is_always_printed():
+    """Otherwise the last state of a rewriting line is whatever it happened to
+    be when the interval last elapsed."""
+    output = run("A/B/one.mp3\n", total=1)
+
+    assert output.endswith("\n")
+    assert "1/1" in output
+
+
+def test_nothing_transferred_still_reports():
+    output = run("", total=0)
+
+    assert "0 files" in output
+
+
+def test_a_log_gets_no_carriage_returns():
+    """A non-terminal filling with \\r and escape codes is unreadable."""
+    output = run("".join(f"A/B/{i}.mp3\n" for i in range(50)), total=50)
+
+    assert "\r" not in output
+    assert "\033" not in output
+
+
+def test_a_terminal_rewrites_one_line():
+    output = run("".join(f"A/B/{i}.mp3\n" for i in range(50)), total=50, out=Terminal())
+
+    assert "\r\033[2K" in output
+
+
+@pytest.mark.parametrize(
+    ("text", "width", "expected"),
+    [
+        ("short", 20, "short"),
+        ("King Gizzard / PetroDragonic Apocalypse", 20, "…agonic Apocalypse"),
+    ],
+)
+def test_long_labels_are_trimmed_from_the_left(text, width, expected):
+    """The album is the informative end, so the artist is what gets cut."""
+    trimmed = rsync_progress.fit(text, width)
+
+    assert len(trimmed) <= width
+    if len(text) > width:
+        assert trimmed.startswith("…")
+        assert text.endswith(trimmed.lstrip("…"))
+
+
+def test_the_size_and_path_are_parsed():
+    assert rsync_progress.parse("5000 Artist/Album/Track.mp3\n") == (
+        5000,
+        "Artist/Album/Track.mp3",
+    )
+
+
+def test_a_filename_containing_spaces_survives():
+    """Splitting on every space would lose most of the library."""
+    assert rsync_progress.parse("1234 Artist/An Album/A Track With Spaces.mp3") == (
+        1234,
+        "Artist/An Album/A Track With Spaces.mp3",
+    )
+
+
+def test_a_bare_path_is_tolerated():
+    """In case this is fed --out-format='%n' by something older."""
+    assert rsync_progress.parse("Artist/Album/Track.mp3") == (0, "Artist/Album/Track.mp3")
+
+
+def test_directory_sizes_do_not_inflate_the_total():
+    """rsync reports directories with a 4096 inode size, which is several
+    megabytes of nothing across six thousand albums."""
+    output = run("4096 Artist/\n4096 Artist/Album/\n5000 Artist/Album/t.mp3\n", total=1)
+
+    assert "4.9 KiB" in output
+    assert "12" not in output.split("Artist")[0]
+
+
+def test_a_rate_is_not_reported_until_it_means_something():
+    """The first files arrive microseconds apart and would give a rate in the
+    gigabytes per second and an ETA of zero."""
+    rate = rsync_progress.Rate()
+    rate.add(100.0, 0)
+    rate.add(100.5, 5_000_000)
+
+    assert rate.per_second() == 0.0
+
+
+def test_a_rate_over_a_long_enough_window_is_reported():
+    rate = rsync_progress.Rate()
+    rate.add(100.0, 0)
+    rate.add(110.0, 10_000_000)
+
+    assert rate.per_second() == pytest.approx(1_000_000)
+
+
+def test_the_window_forgets_the_distant_past():
+    """So the estimate follows a device that slows down rather than averaging
+    the slowdown away."""
+    rate = rsync_progress.Rate(window=30.0)
+    for second in range(0, 100, 10):
+        rate.add(float(second), second * 1_000_000)
+    rate.add(200.0, 100_000_000)
+
+    assert rate.samples[0][0] >= 90.0
+
+
+@pytest.mark.parametrize(
+    ("seconds", "expected"),
+    [(0, "0s"), (45, "45s"), (60, "1m00s"), (1092, "18m12s"), (7500, "2h05m")],
+)
+def test_durations_read_without_arithmetic(seconds, expected):
+    assert rsync_progress.human_duration(seconds) == expected
+
+
+def test_a_summary_is_printed_at_the_end():
+    output = run("5000000 A/B/one.mp3\n", total=1, bytes_expected=5000000)
+
+    assert "copied 4.8 MiB in" in output

tests/test_submit_scrobbles.py

@@ -1,4 +1,5 @@
 import json
+import subprocess
 import sys
 from pathlib import Path
 
@@ -187,3 +188,250 @@ def test_write_credentials_are_required(tmp_path, capsys, monkeypatch):
 
     assert code == 2
     assert "LASTFM_API_SECRET" in capsys.readouterr().err
+
+
+PLAYBACK_LOG = """1700000300:180000:245000:/Music/Pendulum/Immersion/01.mp3
+1700000200:9000:180000:/Music/Green Day/Dookie/07.mp3
+0:180000:245000:/Music/No/Clock/track.mp3
+malformed line without colons
+"""
+
+
+def tags_of(artist="Pendulum", title="Watercolour", album="Immersion", track="1/11"):
+    def runner(path):
+        return json.dumps(
+            {"format": {"tags": {"ARTIST": artist, "TITLE": title,
+                                 "ALBUM": album, "track": track}}}
+        )
+
+    return runner
+
+
+def test_the_playback_log_format_is_four_fields():
+    """timestamp:elapsed_ms:length_ms:path, written by Rockbox core."""
+    plays = submit_scrobbles.parse_playback_log(PLAYBACK_LOG)
+
+    assert len(plays) == 3  # the malformed line is dropped
+    assert plays[0] == (1700000300, 180000, 245000, "/Music/Pendulum/Immersion/01.mp3")
+
+
+def test_a_device_path_maps_onto_the_mirror():
+    assert submit_scrobbles.device_to_local(
+        "/Music/Pendulum/Immersion/01.mp3", "/Music", "/mnt/mirror"
+    ) == Path("/mnt/mirror/Pendulum/Immersion/01.mp3")
+
+
+def test_a_path_outside_the_prefix_is_not_mapped():
+    """Something played from elsewhere on the card is not in the mirror."""
+    assert submit_scrobbles.device_to_local(
+        "/Podcasts/episode.mp3", "/Music", "/mnt/mirror"
+    ) is None
+
+
+def test_a_path_at_the_card_root_maps_straight_across():
+    assert submit_scrobbles.device_to_local(
+        "/Pendulum/Immersion/01.mp3", "/", "/mnt/mirror"
+    ) == Path("/mnt/mirror/Pendulum/Immersion/01.mp3")
+
+
+def test_a_short_play_is_a_skip_not_a_scrobble(tmp_path, monkeypatch):
+    """Nine seconds of a three minute track. The on-device plugin uses the same
+    fraction, so the two never disagree about what counted as a play."""
+    monkeypatch.setattr(Path, "is_file", lambda self: True)
+
+    result = submit_scrobbles.plays_from_playback_log(
+        PLAYBACK_LOG, "/Music", "/mnt/mirror", runner=tags_of()
+    )
+
+    assert result.skipped == 1
+    assert [entry["timestamp"] for entry in result.played] == ["1700000300"]
+
+
+def test_a_zero_timestamp_is_refused(tmp_path, monkeypatch):
+    """Without a real-time clock Rockbox logs ticks, not dates. Scrobbling
+    those would mean inventing when they happened."""
+    monkeypatch.setattr(Path, "is_file", lambda self: True)
+
+    result = submit_scrobbles.plays_from_playback_log(
+        PLAYBACK_LOG, "/Music", "/mnt/mirror", runner=tags_of()
+    )
+
+    assert result.timeless == 1
+
+
+def test_tags_come_from_the_mirror(tmp_path, monkeypatch):
+    """The log carries only a path -- which is precisely why the on-device
+    plugin exists. Off the mirror the tags are free."""
+    monkeypatch.setattr(Path, "is_file", lambda self: True)
+
+    played = submit_scrobbles.plays_from_playback_log(
+        PLAYBACK_LOG, "/Music", "/mnt/mirror", runner=tags_of()
+    ).played
+
+    assert played[0]["artist"] == "Pendulum"
+    assert played[0]["album"] == "Immersion"
+    assert played[0]["trackNumber"] == "1"       # "1/11" -> "1"
+    assert played[0]["duration"] == "245"        # milliseconds -> seconds
+
+
+def test_a_track_missing_from_the_mirror_is_counted_not_guessed(monkeypatch):
+    monkeypatch.setattr(Path, "is_file", lambda self: False)
+
+    result = submit_scrobbles.plays_from_playback_log(
+        PLAYBACK_LOG, "/Music", "/mnt/mirror", runner=tags_of()
+    )
+
+    assert result.played == []
+    # One: the skip and the clockless entry are filtered before the file is
+    # looked for, since neither would be submitted either way.
+    assert result.unresolved == 1
+    # And that one play is kept, so a later run can try it again.
+    assert len(result.retain) == 1
+
+
+def test_playback_logs_are_found_including_rotations(tmp_path):
+    """Rockbox rotates the log once it passes half a megabyte."""
+    rockbox = tmp_path / ".rockbox"
+    rockbox.mkdir()
+    for name in ("playback.log", "playback_0001.log", "playback_0002.log"):
+        (rockbox / name).write_text("1700000000:1:1:/Music/a.mp3\n")
+    (rockbox / "empty.log").write_text("")
+
+    found = submit_scrobbles.find_playback_logs(tmp_path)
+
+    assert [path.name for path in found] == [
+        "playback.log", "playback_0001.log", "playback_0002.log"
+    ]
+
+
+def test_without_a_mirror_it_says_what_is_needed(tmp_path, capsys):
+    device = tmp_path / "IPOD"
+    (device / ".rockbox").mkdir(parents=True)
+    (device / ".rockbox" / "playback.log").write_text("1700000000:1:1:/Music/a.mp3\n")
+
+    submit_scrobbles.main([str(device)], transport=fake_transport([]))
+
+    assert "pass --mirror" in capsys.readouterr().err
+
+
+MIRROR = "/mnt/mirror"
+
+
+def stub_ffprobe(monkeypatch, **tags):
+    """Answer for anything under the mirror without invoking ffprobe."""
+    monkeypatch.setattr(submit_scrobbles, "_ffprobe", tags_of(**tags))
+
+
+def only_mirror_files_exist(monkeypatch, resolvable=None):
+    """Make the mirror's files appear to exist, and nothing else.
+
+    Patching is_file wholesale makes the device directory look like a log file,
+    which sends main() down the .scrobbler.log path instead.
+    """
+    real = Path.is_file
+
+    def patched(self):
+        text = str(self)
+        if text.startswith(MIRROR):
+            return resolvable is None or text == resolvable
+        return real(self)
+
+    monkeypatch.setattr(Path, "is_file", patched)
+
+
+def playback_device(tmp_path, log=PLAYBACK_LOG):
+    device = tmp_path / "IPOD"
+    (device / ".rockbox").mkdir(parents=True)
+    (device / ".rockbox" / "playback.log").write_text(log)
+    return device
+
+
+def test_a_failed_submission_keeps_every_log(tmp_path, monkeypatch):
+    """Nothing got through, so nothing may be set aside."""
+    only_mirror_files_exist(monkeypatch)
+    stub_ffprobe(monkeypatch)
+    monkeypatch.setattr(submit_scrobbles, "load_session", lambda: "sk")
+    device = playback_device(tmp_path)
+    transport = fake_transport([{"error": 29, "message": "Rate limit"}])
+
+    code = submit_scrobbles.main(
+        [str(device), "--mirror", MIRROR, "--api-key", "k", "--api-secret", "s"],
+        transport=transport,
+    )
+
+    assert code == 1
+    assert (device / ".rockbox" / "playback.log").is_file()
+    assert not list((device / ".rockbox").glob("*.submitted"))
+
+
+def test_an_unmatched_play_is_written_back_not_lost(tmp_path, monkeypatch, capsys):
+    """A track the mirror does not yet hold is still a play that happened. It
+    is kept so a later run, after the file has been copied, can submit it."""
+    monkeypatch.setattr(submit_scrobbles, "load_session", lambda: "sk")
+    # Only the first track resolves; the rest are absent from the mirror.
+    only_mirror_files_exist(monkeypatch, f"{MIRROR}/Pendulum/Immersion/01.mp3")
+    stub_ffprobe(monkeypatch)
+    log = (
+        "1700000300:180000:245000:/Music/Pendulum/Immersion/01.mp3\n"
+        "1700000400:180000:245000:/Music/Missing/Album/09.mp3\n"
+    )
+    device = playback_device(tmp_path, log)
+    transport = fake_transport([{"scrobbles": {"@attr": {"accepted": 1, "ignored": 0}}}])
+
+    code = submit_scrobbles.main(
+        [str(device), "--mirror", MIRROR, "--api-key", "k", "--api-secret", "s"],
+        transport=transport,
+    )
+
+    assert code == 0
+    rockbox = device / ".rockbox"
+    # The original is preserved untouched...
+    assert list(rockbox.glob("playback.log.*.submitted"))
+    # ...and the unmatched play is back in a live log for the next attempt.
+    written = (rockbox / "playback.log").read_text()
+    assert "Missing/Album/09.mp3" in written
+    assert "Pendulum/Immersion/01.mp3" not in written
+
+
+def test_the_original_is_renamed_rather_than_deleted(tmp_path, monkeypatch):
+    """If Last.fm quietly dropped something, the evidence stays on the device."""
+    only_mirror_files_exist(monkeypatch)
+    stub_ffprobe(monkeypatch)
+    monkeypatch.setattr(submit_scrobbles, "load_session", lambda: "sk")
+    device = playback_device(tmp_path)
+    transport = fake_transport([{"scrobbles": {"@attr": {"accepted": 1, "ignored": 0}}}])
+
+    submit_scrobbles.main(
+        [str(device), "--mirror", MIRROR, "--api-key", "k", "--api-secret", "s"],
+        transport=transport,
+    )
+
+    aside = list((device / ".rockbox").glob("playback.log.*.submitted"))
+    assert len(aside) == 1
+    assert PLAYBACK_LOG.splitlines()[0] in aside[0].read_text()
+
+
+def test_keep_leaves_everything_alone(tmp_path, monkeypatch):
+    only_mirror_files_exist(monkeypatch)
+    stub_ffprobe(monkeypatch)
+    monkeypatch.setattr(submit_scrobbles, "load_session", lambda: "sk")
+    device = playback_device(tmp_path)
+    transport = fake_transport([{"scrobbles": {"@attr": {"accepted": 1, "ignored": 0}}}])
+
+    submit_scrobbles.main(
+        [str(device), "--mirror", MIRROR, "--keep",
+         "--api-key", "k", "--api-secret", "s"],
+        transport=transport,
+    )
+
+    assert (device / ".rockbox" / "playback.log").read_text() == PLAYBACK_LOG
+    assert not list((device / ".rockbox").glob("*.submitted"))
+
+
+def test_a_file_ffprobe_cannot_read_does_not_abandon_the_rest(monkeypatch):
+    """CalledProcessError is not an OSError, so one bad file used to take the
+    whole submission with it."""
+    def broken(path):
+        raise subprocess.CalledProcessError(1, "ffprobe")
+
+    assert submit_scrobbles.read_tags(Path("/mnt/mirror/x.mp3"), runner=broken) == {}

tests/test_sync_to_ipod.py

@@ -0,0 +1,434 @@
+"""The guards on sync-to-ipod.sh, which are the substance of the script.
+
+rsync --delete is being aimed at a whole filesystem, so every refusal here is
+protecting against emptying the wrong directory -- a mistake that does not
+announce itself.
+"""
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+SCRIPT = Path(__file__).resolve().parent.parent / "tools" / "sync-to-ipod.sh"
+
+# Skipped rather than failed where the tools are absent: this is a host-side
+# script, and a machine without rsync is not a machine that would run it.
+REQUIRED = ("bash", "rsync", "findmnt")
+pytestmark = pytest.mark.skipif(
+    not all(shutil.which(tool) for tool in REQUIRED),
+    reason=f"needs {', '.join(REQUIRED)} on PATH",
+)
+
+
+def run(*arguments):
+    return subprocess.run(
+        ["bash", str(SCRIPT), *arguments], capture_output=True, text=True
+    )
+
+
+@pytest.fixture
+def mirror(tmp_path):
+    source = tmp_path / "mirror"
+    (source / "Album").mkdir(parents=True)
+    (source / "Album" / "track.mp3").write_bytes(b"x")
+    return source
+
+
+def test_the_host_root_is_refused(mirror):
+    """Stripping the trailing slash from "/" leaves an empty string, and an
+    earlier version then reported it as "not a directory" instead."""
+    result = run(str(mirror), "/")
+
+    assert result.returncode == 1
+    assert "refusing to sync onto /" in result.stderr
+
+
+def test_an_empty_mirror_is_refused(tmp_path):
+    """Mirroring nothing onto the device would delete everything on it."""
+    empty = tmp_path / "empty"
+    empty.mkdir()
+    destination = tmp_path / "dest"
+    destination.mkdir()
+
+    result = run(str(empty), str(destination))
+
+    assert result.returncode == 1
+    assert "refusing to mirror nothing" in result.stderr
+
+
+def test_syncing_a_directory_onto_itself_is_refused(mirror):
+    result = run(str(mirror), str(mirror))
+
+    assert result.returncode == 1
+    assert "same directory" in result.stderr
+
+
+def test_a_non_fat_destination_is_refused(mirror, tmp_path):
+    """Which is also how an unmounted device is caught: /media/IPOD/Music then
+    resolves to the host's own root filesystem."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+
+    result = run(str(mirror), str(destination))
+
+    assert result.returncode == 1
+    assert "not FAT" in result.stderr
+    assert "Is the device mounted?" in result.stderr
+
+
+def test_a_missing_destination_is_refused(mirror, tmp_path):
+    result = run(str(mirror), str(tmp_path / "nowhere"))
+
+    assert result.returncode == 1
+    assert "not a directory" in result.stderr
+
+
+def test_a_subdirectory_of_the_device_is_a_valid_target(mirror, tmp_path):
+    """The better target, in fact: --delete is confined to it."""
+    destination = tmp_path / "dest" / "Music"
+    destination.mkdir(parents=True)
+
+    result = run("-f", "-n", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "dry run, nothing was written" in result.stderr
+
+
+def test_the_device_prefix_is_derived_from_the_destination(mirror, tmp_path):
+    """Derived rather than configured, so it cannot disagree with where the
+    files are actually going -- and the device's path limit applies to it."""
+    destination = tmp_path / "dest" / "Music"
+    destination.mkdir(parents=True)
+
+    result = run("-f", "-n", str(mirror), str(destination))
+
+    assert "the device will see this as /" in result.stderr
+
+
+def test_a_dry_run_writes_nothing(mirror, tmp_path):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+
+    run("-f", "-n", str(mirror), str(destination))
+
+    assert list(destination.iterdir()) == []
+
+
+def test_rockbox_is_never_deleted(mirror, tmp_path):
+    """A sync to the card root would otherwise remove the Rockbox install,
+    since the mirror does not contain it."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    (destination / ".rockbox").mkdir()
+    (destination / ".rockbox" / "rockbox.ipod").write_bytes(b"firmware")
+    (destination / ".scrobbler.log").write_bytes(b"#AUDIOSCROBBLER/1.1\n")
+    (destination / "Stale.mp3").write_bytes(b"old")
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert (destination / ".rockbox" / "rockbox.ipod").is_file()
+    assert (destination / ".scrobbler.log").is_file()
+    # But a track whose source has gone is still removed. That is the point.
+    assert not (destination / "Stale.mp3").exists()
+    assert (destination / "Album" / "track.mp3").is_file()
+
+
+def test_help_goes_to_stdout_and_exits_clean():
+    """Asking for help is not an error; getting the arguments wrong is."""
+    result = run("--help")
+
+    assert result.returncode == 0
+    assert result.stdout.startswith("usage:")
+    assert result.stderr == ""
+
+
+def test_short_help_behaves_the_same():
+    result = run("-h")
+
+    assert result.returncode == 0
+    assert result.stdout.startswith("usage:")
+
+
+def test_misuse_goes_to_stderr_and_does_not():
+    result = run("only-one-argument")
+
+    assert result.returncode == 2
+    assert result.stderr.startswith("usage:")
+    assert result.stdout == ""
+
+
+def test_the_help_explains_what_the_destination_should_be():
+    """The question this script actually gets asked."""
+    help_text = run("--help").stdout
+
+    assert "/media/IPOD/Music" in help_text
+    assert "artist folders" in help_text
+    assert ".rockbox" in help_text
+
+
+def test_the_help_says_how_to_reach_and_leave_disk_mode():
+    help_text = run("--help").stdout
+
+    assert "Menu+Select" in help_text
+    assert "holding Play" in help_text
+
+
+def test_counting_is_off_by_default(mirror, tmp_path):
+    """The counting pass walks and compares both trees exactly as the transfer
+    does. On a FAT card of fifty thousand files that costs more than moving the
+    data, so the percentage has to be asked for."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "files to copy" not in result.stderr
+    assert (destination / "Album" / "track.mp3").is_file()
+
+
+def test_the_delta_algorithm_is_disabled(mirror, tmp_path):
+    """It would read every destination file back over USB to checksum it, to
+    avoid resending an MP3 that has changed in its entirety anyway."""
+    script = SCRIPT.read_text()
+
+    assert "--whole-file" in script
+
+
+def test_directory_timestamps_are_not_set(mirror, tmp_path):
+    """One setattr round trip per directory, across six thousand albums, to set
+    timestamps nothing reads."""
+    script = SCRIPT.read_text()
+
+    assert "--omit-dir-times" in script
+
+
+def test_an_interrupt_is_trapped_so_the_filesystem_is_flushed():
+    """Ctrl-C during a transfer would otherwise skip the sync and the unmount,
+    leaving a journal-less FAT filesystem with dirty buffers -- which is the
+    corruption this script exists to prevent.
+
+    Structural rather than timed: reproducing a mid-transfer signal needs a
+    payload large enough to be slow, and a test that depends on losing a race
+    is a test that fails in CI for no reason.
+    """
+    script = SCRIPT.read_text()
+
+    assert "trap interrupted INT TERM" in script
+    assert "exit 130" in script
+
+
+def test_the_flush_and_unmount_happen_on_every_exit_path():
+    script = SCRIPT.read_text()
+
+    # Both the normal path and the interrupt path go through the same function,
+    # so one cannot drift from the other.
+    assert script.count("finish\n") >= 2
+    assert "--partial" not in script, "rsync must delete partial files, not keep them"
+
+
+def test_the_database_step_is_skipped_without_a_tool(mirror, tmp_path, monkeypatch):
+    """Opt-in, like the scrobbler: absent configuration is not an error."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    monkeypatch.delenv("MUSIC_MIRROR_DATABASE_TOOL", raising=False)
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "no database tool configured" in result.stderr
+
+
+def test_a_missing_database_tool_is_refused(mirror, tmp_path, monkeypatch):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 1
+    assert "not executable" in result.stderr
+
+
+def test_the_database_step_can_be_skipped(mirror, tmp_path, monkeypatch):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    monkeypatch.setenv("MUSIC_MIRROR_DATABASE_TOOL", str(tmp_path / "nonexistent"))
+
+    result = run("-f", "-S", "-U", "-B", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "not executable" not in result.stderr
+
+
+def test_the_scan_reads_from_the_mirror_not_the_device():
+    """The whole point: tags come off the mirror, only the .tcd files go over
+    USB. Reading 49,600 files through an iPod's USB bridge is the slow path."""
+    script = SCRIPT.read_text()
+
+    assert 'ln -s "$mirror"' in script
+    assert 'cd "$scratch"' in script
+
+
+def can_bind_mount():
+    """User namespaces let an unprivileged process bind mount. Not everywhere,
+    notably not inside some containers, so the test that needs it skips."""
+    return (
+        subprocess.run(
+            ["unshare", "-Umr", "true"], capture_output=True, check=False
+        ).returncode
+        == 0
+    )
+
+
+@pytest.mark.skipif(not can_bind_mount(), reason="needs unprivileged user namespaces")
+def test_the_database_lands_at_the_device_root_not_the_music_folder(tmp_path):
+    """The two tools disagree about where the root is. rsync copies artist
+    folders into <device>/Music; the database tool must run one level up, where
+    .rockbox lives, and must record /Music/... paths while reading the bytes
+    from the mirror. device_prefix is what reconciles them.
+    """
+    mirror = tmp_path / "mirror" / "Pendulum" / "Immersion"
+    mirror.mkdir(parents=True)
+    (mirror / "01.mp3").write_bytes(b"not really an mp3")
+    card = tmp_path / "card"
+    (card / ".rockbox").mkdir(parents=True)
+    (card / "Music").mkdir()
+    device = tmp_path / "device"
+    device.mkdir()
+
+    tool = tmp_path / "fake-database"
+    # Records where it was run and what it could see, which is the whole
+    # question; producing a real database needs Rockbox's builder.
+    tool.write_text(
+        "#!/bin/sh\n"
+        "printf '%s\\n' \"$PWD\" > .rockbox/where.txt\n"
+        "ls Music/ > .rockbox/saw.txt\n"
+        "echo db > .rockbox/database_0.tcd\n"
+    )
+    tool.chmod(0o755)
+
+    script = (
+        f"mount --bind {card} {device} && "
+        f"XDG_CACHE_HOME={tmp_path / 'cache'} MUSIC_MIRROR_DATABASE_TOOL={tool} "
+        f"bash {SCRIPT} -f -S -U {tmp_path / 'mirror'} {device / 'Music'}"
+    )
+    result = subprocess.run(
+        ["unshare", "-Umr", "sh", "-c", script], capture_output=True, text=True
+    )
+
+    assert result.returncode == 0, result.stderr
+    # The database lands beside the device root, not inside Music.
+    assert (card / ".rockbox" / "database_0.tcd").is_file()
+    # Only *.tcd is copied across, so the markers stay in the scratch root --
+    # which is itself the point: nothing else is written to the device.
+    scratch = tmp_path / "cache" / "music-mirror" / "database" / ".rockbox"
+    assert not (card / ".rockbox" / "where.txt").exists()
+
+    # It ran in the scratch root, not on the card.
+    where = (scratch / "where.txt").read_text().strip()
+    assert where.endswith("music-mirror/database"), where
+    # ...and could walk into the mirror through a symlink named for the device
+    # prefix, which is how the paths come out as /Music/... while the bytes are
+    # read from somewhere else entirely.
+    assert "Pendulum" in (scratch / "saw.txt").read_text()
+
+
+def test_counting_can_be_asked_for(mirror, tmp_path):
+    """When the destination is cheap to traverse, the percentage is worth it."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+
+    result = run("-f", "-S", "-U", "-P", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "files to copy" in result.stderr
+
+
+# A ReplayGain re-level rewrites a track's tags in the padding the previous
+# write left behind, so neither the size nor the mtime changes -- and those are
+# the two things rsync's quick check compares.
+
+
+def stale_copy(mirror, destination, relative, current, previous):
+    """Put a file on the device that differs only in content from the mirror's."""
+    source = mirror / relative
+    source.parent.mkdir(parents=True, exist_ok=True)
+    source.write_bytes(current)
+    device = destination / relative
+    device.parent.mkdir(parents=True, exist_ok=True)
+    device.write_bytes(previous)
+    os.utime(device, (source.stat().st_atime, source.stat().st_mtime))
+    return device
+
+
+def test_a_tag_only_change_reaches_the_device_with_the_track_that_caused_it(
+    mirror, tmp_path
+):
+    """The new track is visible to rsync; its re-levelled sibling is not, and
+    would otherwise keep the old album gain on the device forever."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    sibling = stale_copy(mirror, destination, "Album/sibling.mp3", b"NEW", b"OLD")
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert sibling.read_bytes() == b"NEW"
+    assert (destination / "Album" / "track.mp3").is_file()
+
+
+def test_a_deletion_also_relevels_what_is_left_behind(mirror, tmp_path):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    sibling = stale_copy(mirror, destination, "Album/sibling.mp3", b"NEW", b"OLD")
+    (destination / "Album" / "gone.mp3").write_bytes(b"old")
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert sibling.read_bytes() == b"NEW"
+    assert not (destination / "Album" / "gone.mp3").exists()
+
+
+def test_an_untouched_album_is_not_copied_again(mirror, tmp_path):
+    """The second pass is scoped to albums that changed. An album whose file
+    set is the same is left where it is, which is the whole point of not
+    running --ignore-times over the library."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    quiet = stale_copy(mirror, destination, "Quiet/only.mp3", b"NEW", b"OLD")
+    (destination / "Album").mkdir()
+    shutil.copy2(mirror / "Album" / "track.mp3", destination / "Album" / "track.mp3")
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert quiet.read_bytes() == b"OLD"
+
+
+def test_a_dry_run_says_how_many_extra_tracks_are_involved(mirror, tmp_path):
+    destination = tmp_path / "dest"
+    destination.mkdir()
+    stale_copy(mirror, destination, "Album/sibling.mp3", b"NEW", b"OLD")
+
+    result = run("-f", "-S", "-U", "-n", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "and 1 more in those albums" in result.stderr
+
+
+def test_a_first_sync_does_not_copy_anything_twice(mirror, tmp_path):
+    """Everything is transferred by the main pass, so there is nothing left for
+    the second one and it must not announce itself."""
+    destination = tmp_path / "dest"
+    destination.mkdir()
+
+    result = run("-f", "-S", "-U", str(mirror), str(destination))
+
+    assert result.returncode == 0, result.stderr
+    assert "re-levelled" not in result.stderr

tests/test_touched_albums.py

@@ -0,0 +1,128 @@
+"""A ReplayGain re-level rewrites a track's tags without changing its size or
+its mtime, which is precisely the pair rsync's quick check compares. These
+cover the list that is fed back to rsync to copy those tracks anyway."""
+
+import io
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import touched_albums  # noqa: E402
+
+
+def album(root, name, *tracks):
+    directory = root / name
+    directory.mkdir(parents=True, exist_ok=True)
+    for track in tracks:
+        (directory / track).write_bytes(b"x")
+    return directory
+
+
+def listing(mirror, *lines):
+    return touched_albums.remaining(mirror, *touched_albums.touched(list(lines)))
+
+
+def test_the_siblings_of_a_new_track_are_listed(tmp_path):
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3", "03.mp3")
+
+    assert listing(tmp_path, "4096 Artist/Album/03.mp3") == [
+        "Artist/Album/01.mp3",
+        "Artist/Album/02.mp3",
+    ]
+
+
+def test_a_track_rsync_just_copied_is_not_copied_twice(tmp_path):
+    """A quality upgrade replaces every track on the record. Listing them again
+    would send the album across twice."""
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3")
+
+    assert listing(tmp_path, "4096 Artist/Album/01.mp3", "4096 Artist/Album/02.mp3") == []
+
+
+def test_a_deletion_relevels_what_is_left(tmp_path):
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3")
+
+    assert listing(tmp_path, "deleting Artist/Album/03.mp3") == [
+        "Artist/Album/01.mp3",
+        "Artist/Album/02.mp3",
+    ]
+
+
+def test_an_album_deleted_outright_lists_nothing(tmp_path):
+    assert listing(tmp_path, "deleting Artist/Gone/01.mp3") == []
+
+
+def test_untouched_albums_are_left_alone(tmp_path):
+    album(tmp_path, "Artist/Changed", "01.mp3", "02.mp3")
+    album(tmp_path, "Artist/Quiet", "01.mp3", "02.mp3")
+
+    assert listing(tmp_path, "4096 Artist/Changed/01.mp3") == ["Artist/Changed/02.mp3"]
+
+
+def test_a_replaced_cover_is_not_an_album_change(tmp_path):
+    """Only a track can change an album's gains, and covers are replaced often
+    enough that treating one as a re-level would copy records for nothing."""
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3")
+
+    assert listing(tmp_path, "17408 Artist/Album/cover.jpg") == []
+
+
+def test_directories_are_not_mistaken_for_tracks(tmp_path):
+    album(tmp_path, "Artist/Album", "01.mp3")
+
+    assert listing(tmp_path, "4096 Artist/Album/", "deleting Artist/Old/") == []
+
+
+def test_rsync_talking_to_the_operator_is_not_a_path(tmp_path):
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3")
+
+    assert (
+        listing(
+            tmp_path,
+            "sending incremental file list",
+            "",
+            "sent 1,234 bytes  received 56 bytes  2,580.00 bytes/sec",
+            "total size is 7,890  speedup is 6.12",
+        )
+        == []
+    )
+
+
+def test_a_track_at_the_mirror_root_does_not_pull_in_the_whole_tree(tmp_path):
+    """Nothing writes a mirror this way, but the directory of a root-level file
+    is the root, and recursing from there would be the whole library."""
+    (tmp_path / "loose.mp3").write_bytes(b"x")
+    (tmp_path / "other.mp3").write_bytes(b"x")
+    album(tmp_path, "Artist/Album", "01.mp3")
+
+    assert listing(tmp_path, "4096 loose.mp3") == ["other.mp3"]
+
+
+def test_the_paths_are_written_one_per_line(tmp_path):
+    """They are fed straight back to rsync as --files-from."""
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3")
+    out = io.StringIO()
+
+    touched_albums.main(
+        ["--mirror", str(tmp_path)],
+        stream=["4096 Artist/Album/01.mp3"],
+        out=out,
+    )
+
+    assert out.getvalue() == "Artist/Album/02.mp3\n"
+
+
+def test_it_runs_as_a_script(tmp_path):
+    album(tmp_path, "Artist/Album", "01.mp3", "02.mp3")
+
+    completed = subprocess.run(
+        [sys.executable, touched_albums.__file__, "--mirror", str(tmp_path)],
+        input="4096 Artist/Album/01.mp3\n",
+        capture_output=True,
+        text=True,
+    )
+
+    assert completed.returncode == 0
+    assert completed.stdout == "Artist/Album/02.mp3\n"

tools/rsync_progress.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""Render rsync's per-file output as a single updating status line.
+
+Fed the size and path rsync reports with --out-format='%l %n', one per line.
+The size is what makes an estimate possible: rsync's own rate is not exposed
+per file, but bytes completed over time elapsed is the same arithmetic and
+needs nothing rsync does not already print. Prints one
+line that rewrites itself, showing how far through the transfer is and which
+album is currently going across, rather than either scrolling fifty thousand
+filenames past or -- as rsync does while it builds its file list -- saying
+nothing at all for several minutes.
+
+Falls back to periodic plain lines when stderr is not a terminal, so a log does
+not fill up with carriage returns.
+"""
+
+import argparse
+import collections
+import os
+import shutil
+import sys
+import time
+
+# The estimate is taken over a trailing window rather than the whole run, so it
+# follows a device that slows down instead of averaging the slowdown away.
+RATE_WINDOW_SECONDS = 30.0
+
+# Below this the window is too narrow to divide by: the first few files arrive
+# in microseconds and produce a rate in the gigabytes per second, and an ETA of
+# nothing at all. Better to show neither until the figure means something.
+RATE_MINIMUM_SPAN_SECONDS = 2.0
+
+
+def parse(line):
+    """Return (bytes, path) for one line of rsync output.
+
+    Tolerates a bare path, in case someone runs this against --out-format='%n'.
+    """
+    line = line.rstrip("\n")
+    size, separator, path = line.partition(" ")
+    if separator and size.isdigit():
+        return int(size), path
+    return 0, line
+
+
+def human_bytes(count):
+    size = float(count)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if size < 1024 or unit == "TiB":
+            return f"{size:.1f} {unit}"
+        size /= 1024
+
+
+def human_duration(seconds):
+    """Return a duration nobody has to do arithmetic on."""
+    seconds = int(seconds)
+    if seconds < 60:
+        return f"{seconds}s"
+    if seconds < 3600:
+        return f"{seconds // 60}m{seconds % 60:02d}s"
+    return f"{seconds // 3600}h{(seconds % 3600) // 60:02d}m"
+
+
+class Rate:
+    """Bytes per second over a trailing window."""
+
+    def __init__(self, window=RATE_WINDOW_SECONDS):
+        self.window = window
+        self.samples = collections.deque()
+
+    def add(self, when, total_bytes):
+        self.samples.append((when, total_bytes))
+        while len(self.samples) > 2 and when - self.samples[0][0] > self.window:
+            self.samples.popleft()
+
+    def per_second(self):
+        if len(self.samples) < 2:
+            return 0.0
+        (first_time, first_bytes), (last_time, last_bytes) = (
+            self.samples[0],
+            self.samples[-1],
+        )
+        elapsed = last_time - first_time
+        if elapsed < RATE_MINIMUM_SPAN_SECONDS:
+            return 0.0
+        return (last_bytes - first_bytes) / elapsed
+
+
+def album_of(path):
+    """Return "Artist / Album" for a mirror-relative path."""
+    parts = [part for part in path.strip("/").split("/") if part]
+    if len(parts) >= 3:
+        return f"{parts[0]} / {parts[1]}"
+    if len(parts) == 2:
+        return parts[0]
+    return ""
+
+
+def fit(text, width):
+    """Trim to the terminal, from the left: the album matters more than the artist."""
+    if width <= 1 or len(text) <= width:
+        return text
+    return "…" + text[-(width - 1) :]
+
+
+def render(done, total, copied, expected, rate, label, width):
+    """Build the status line, giving whatever room is left to the album."""
+    if total > 0:
+        share = min(100, done * 100 // total)
+        head = f"[{share:>3}%] {done:,}/{total:,}"
+    else:
+        head = f"[{done:,} files]"
+
+    if expected > 0:
+        head += f"  {human_bytes(copied)}/{human_bytes(expected)}"
+    if rate > 0:
+        head += f"  {human_bytes(rate)}/s"
+        remaining = expected - copied
+        if remaining > 0:
+            head += f"  ETA {human_duration(remaining / rate)}"
+    head += "  "
+    return head + fit(label, max(0, width - len(head)))
+
+
+def main(argv=None, stream=None, out=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--total", type=int, default=0, help="files expected")
+    parser.add_argument("--bytes", type=int, default=0, help="bytes expected")
+    parser.add_argument("--interval", type=float, default=0.1, help="seconds between redraws")
+    args = parser.parse_args(argv)
+
+    stream = stream or sys.stdin
+    out = out or sys.stderr
+    interactive = out.isatty()
+    width = shutil.get_terminal_size((100, 24)).columns - 1
+
+    done = 0
+    copied = 0
+    rate = Rate()
+    started = time.monotonic()
+    rate.add(started, 0)
+    last_drawn = 0.0
+    label = ""
+
+    for line in stream:
+        size, path = parse(line)
+        # rsync reports directories too, with a trailing slash and an inode
+        # size. Counting them puts the percentage past a hundred and the byte
+        # total well over what will actually be transferred.
+        if not path or path.endswith("/"):
+            continue
+        done += 1
+        copied += size
+        label = album_of(path) or os.path.basename(path)
+
+        now = time.monotonic()
+        rate.add(now, copied)
+        if interactive:
+            if now - last_drawn >= args.interval:
+                out.write(
+                    "\r\033[2K"
+                    + render(done, args.total, copied, args.bytes, rate.per_second(),
+                             label, width)
+                )
+                out.flush()
+                last_drawn = now
+        elif now - last_drawn >= 30:
+            out.write(
+                render(done, args.total, copied, args.bytes, rate.per_second(), label, width)
+                + "\n"
+            )
+            out.flush()
+            last_drawn = now
+
+    elapsed = max(1e-9, time.monotonic() - started)
+    if interactive:
+        out.write("\r\033[2K")
+    summary = render(done, args.total, copied, args.bytes, 0, label, width).rstrip()
+    out.write(f"{summary}\n")
+    if copied:
+        out.write(
+            f"copied {human_bytes(copied)} in {human_duration(elapsed)}"
+            f" at {human_bytes(copied / elapsed)}/s\n"
+        )
+    out.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/submit_scrobbles.py

@@ -14,11 +14,13 @@ import argparse
 import hashlib
 import json
 import os
+import subprocess
 import sys
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
+from dataclasses import dataclass
 from pathlib import Path
 
 API_ROOT = "https://ws.audioscrobbler.com/2.0/"
@@ -31,6 +33,21 @@ BATCH = 50
 # scrobble.
 LOG_NAMES = (".scrobbler.log", ".scrobbler-timeless.log")
 
+# Rockbox core writes this whenever "play log" is on, with no plugin running:
+#     timestamp:elapsed_ms:length_ms:/Music/Artist/Album/Track.mp3
+# It is rotated once it grows past half a megabyte. Converting it needs tags,
+# which is why the on-device plugin exists -- reading them back off the player
+# is slow. Off the mirror it is free, so the plugin can be skipped entirely.
+PLAYBACK_LOG_NAMES = ("playback.log", "playback_*.log")
+
+# The plugin counts a track as listened at savepct of its length, defaulting to
+# fifty. Same rule here, or the two disagree about what a play is.
+LISTENED_FRACTION = 0.5
+
+# Below this a timestamp is not a wall-clock time. Without a real-time clock
+# Rockbox logs ticks in milliseconds instead, which is not a date.
+EARLIEST_PLAUSIBLE = 1_000_000_000
+
 SESSION_FILE = Path(
     os.getenv("XDG_CONFIG_HOME", Path.home() / ".config")
 ) / "music-mirror" / "lastfm.json"
@@ -83,6 +100,132 @@ def parse_log(text):
     return played, skipped, timeless
 
 
+def parse_playback_log(text):
+    """Return (timestamp, elapsed_ms, length_ms, path) for each logged play."""
+    plays = []
+    for line in text.splitlines():
+        fields = line.strip().split(":", 3)
+        if len(fields) != 4:
+            continue
+        stamp, elapsed, length, path = fields
+        try:
+            plays.append((int(stamp), int(elapsed), int(length), path))
+        except ValueError:
+            continue
+    return plays
+
+
+def device_to_local(path, device_prefix, mirror):
+    """Map a path as the player sees it onto the mirror it was copied from."""
+    prefix = "/" + device_prefix.strip("/")
+    if prefix != "/":
+        if not path.startswith(prefix + "/"):
+            return None
+        path = path[len(prefix) :]
+    return Path(mirror) / path.lstrip("/")
+
+
+def read_tags(path, runner=None):
+    """Return the tags of a local file, via ffprobe. Empty if it cannot be read.
+
+    A file ffprobe chokes on is one play left unidentified, not a reason to
+    abandon the rest -- and CalledProcessError is not an OSError, so catching
+    the obvious things is not enough.
+    """
+    runner = runner or _ffprobe
+    try:
+        payload = json.loads(runner(path))
+    except (OSError, ValueError, subprocess.SubprocessError):
+        return {}
+    return {
+        key.lower(): value
+        for key, value in (payload.get("format", {}).get("tags") or {}).items()
+    }
+
+
+def _ffprobe(path):
+    return subprocess.run(
+        ["ffprobe", "-v", "error", "-show_entries", "format_tags",
+         "-of", "json", str(path)],
+        capture_output=True, text=True, check=True,
+    ).stdout
+
+
+@dataclass
+class Conversion:
+    """What a playback log turned into, and what must not be thrown away.
+
+    `retain` holds the raw lines of plays that were real but could not be
+    submitted -- a track absent from the mirror, usually because the sync had
+    not copied it yet. Those are written back so a later run can try again.
+    Skips and clockless entries are not retained: neither can ever be
+    submitted, and the untouched original is set aside regardless.
+    """
+
+    played: list
+    skipped: int = 0
+    unresolved: int = 0
+    timeless: int = 0
+    retain: list = None
+
+    def __post_init__(self):
+        if self.retain is None:
+            self.retain = []
+
+
+def plays_from_playback_log(text, device_prefix, mirror, runner=None):
+    """Return the conversion of a playback log.
+
+    Skips are decided by the same fraction the on-device plugin uses, so the
+    two never disagree about what counted as a play.
+    """
+    result = Conversion(played=[])
+    for line in text.splitlines():
+        parsed = parse_playback_log(line)
+        if not parsed:
+            continue
+        stamp, elapsed, length, device_path = parsed[0]
+        if stamp < EARLIEST_PLAUSIBLE:
+            result.timeless += 1
+            continue
+        if length > 0 and elapsed < length * LISTENED_FRACTION:
+            result.skipped += 1
+            continue
+        local = device_to_local(device_path, device_prefix, mirror)
+        tags = read_tags(local, runner) if local and local.is_file() else {}
+        artist = tags.get("artist") or tags.get("album_artist") or ""
+        title = tags.get("title") or ""
+        if not artist or not title:
+            # A real play of a track this run could not identify. Kept, so a
+            # later run -- after the file has been copied, or the tags fixed --
+            # can submit it rather than the play being lost.
+            result.unresolved += 1
+            result.retain.append(line)
+            continue
+        result.played.append(
+            {
+                "artist": artist,
+                "track": title,
+                "album": tags.get("album", ""),
+                "trackNumber": (tags.get("track") or "").split("/")[0],
+                "duration": str(length // 1000) if length > 0 else "",
+                "timestamp": str(stamp),
+                "mbid": tags.get("musicbrainz_trackid", ""),
+                "line": line,
+            }
+        )
+    result.played.sort(key=lambda entry: int(entry["timestamp"]))
+    return result
+
+
+def find_playback_logs(device):
+    """Return every playback log on a device, oldest first."""
+    found = []
+    for pattern in PLAYBACK_LOG_NAMES:
+        found.extend(sorted(Path(device).glob(f".rockbox/{pattern}")))
+    return [path for path in found if path.is_file() and path.stat().st_size]
+
+
 def sign(params, secret):
     """Return Last.fm's method signature for a set of parameters.
 
@@ -152,7 +295,7 @@ def batch_params(entries):
     return params
 
 
-def submit(entries, key, secret, session, transport, delay=1.0):
+def submit(entries, key, secret, session, transport, delay=1.0, on_sent=None):
     """Submit every entry. Returns how many the service accepted.
 
     Batches are counted as they succeed rather than at the end, so a failure
@@ -168,6 +311,8 @@ def submit(entries, key, secret, session, transport, delay=1.0):
         block = payload.get("scrobbles", {})
         summary = block.get("@attr", block)
         accepted += int(summary.get("accepted", len(chunk)))
+        if on_sent is not None:
+            on_sent(chunk)
         ignored = int(summary.get("ignored", 0))
         if ignored:
             print(f"  {ignored} of {len(chunk)} ignored by Last.fm", file=sys.stderr)
@@ -193,6 +338,18 @@ def find_log(device):
 def main(argv=None, transport=http_post):
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("device", help="the mounted device, or a scrobbler log file")
+    parser.add_argument(
+        "--mirror",
+        help="the mirror the device was copied from. Given this, Rockbox's own"
+        " playback.log is converted here rather than needing the on-device"
+        " plugin run first",
+    )
+    parser.add_argument(
+        "--device-prefix",
+        default="/Music",
+        help="where the music sits on the device, stripped when mapping a logged"
+        " path back onto the mirror",
+    )
     parser.add_argument("--api-key", default=os.getenv("LASTFM_API_KEY"))
     parser.add_argument("--api-secret", default=os.getenv("LASTFM_API_SECRET"))
     parser.add_argument("--dry-run", action="store_true", help="parse and report only")
@@ -203,12 +360,51 @@ def main(argv=None, transport=http_post):
 
     target = Path(args.device)
     log = target if target.is_file() else find_log(target)
-    if log is None:
-        print("no scrobbler log to submit", file=sys.stderr)
+    logs = []
+
+    conversion = None
+    if log is not None:
+        played, skipped, timeless = parse_log(
+            log.read_text(encoding="utf-8", errors="replace")
+        )
+        unresolved = 0
+        logs = [log]
+        print(f"{log}: {len(played)} listened, {skipped} skipped", file=sys.stderr)
+    elif args.mirror:
+        # No plugin has been run, but the core log is there. Tags come off the
+        # mirror, which is the only reason the plugin was needed at all.
+        logs = find_playback_logs(target)
+        if not logs:
+            print("no scrobbler log to submit", file=sys.stderr)
+            return 0
+        text = "\n".join(
+            path.read_text(encoding="utf-8", errors="replace") for path in logs
+        )
+        conversion = plays_from_playback_log(text, args.device_prefix, args.mirror)
+        played = conversion.played
+        skipped, unresolved, timeless = (
+            conversion.skipped,
+            conversion.unresolved,
+            conversion.timeless,
+        )
+        print(
+            f"{len(logs)} playback log(s): {len(played)} listened, {skipped} skipped",
+            file=sys.stderr,
+        )
+        if unresolved:
+            print(
+                f"  {unresolved} could not be matched to a file in the mirror."
+                " Those plays are kept for a later run rather than discarded.",
+                file=sys.stderr,
+            )
+    else:
+        print(
+            "no scrobbler log to submit. Rockbox's own playback.log can be used"
+            " instead -- pass --mirror so tags can be read from it.",
+            file=sys.stderr,
+        )
         return 0
 
-    played, skipped, timeless = parse_log(log.read_text(encoding="utf-8", errors="replace"))
-    print(f"{log}: {len(played)} listened, {skipped} skipped", file=sys.stderr)
     if timeless:
         print(
             f"  {timeless} entries have no timestamp, so this target has no clock."
@@ -235,21 +431,66 @@ def main(argv=None, transport=http_post):
         session = authorise(args.api_key, args.api_secret, transport)
         save_session(session)
 
+    # Recorded as each batch is accepted, so a failure partway through knows
+    # exactly what got through and what did not.
+    sent = []
     try:
-        accepted = submit(played, args.api_key, args.api_secret, session, transport)
+        accepted = submit(
+            played, args.api_key, args.api_secret, session, transport,
+            on_sent=sent.extend,
+        )
     except LastfmError as error:
-        print(f"submission failed: {error}", file=sys.stderr)
+        print(f"submission failed after {len(sent)} scrobbles: {error}", file=sys.stderr)
+        keep_history(logs, played, sent, conversion, args.keep)
         return 1
 
     print(f"{accepted} scrobbles accepted", file=sys.stderr)
-    if not args.keep and accepted:
-        # Renamed rather than deleted: if Last.fm quietly dropped something,
-        # the evidence is still on the device.
-        aside = log.with_name(f"{log.name}.{played[-1]['timestamp']}.submitted")
-        log.rename(aside)
-        print(f"log moved to {aside.name}", file=sys.stderr)
+    keep_history(logs, played, sent, conversion, args.keep)
     return 0
 
 
+def keep_history(logs, played, sent, conversion, keep):
+    """Set the logs aside, writing back anything still owed a submission.
+
+    Two separate obligations. The original is preserved untouched, renamed
+    rather than deleted, so a play is never lost to a mistake here. And any
+    play that was not submitted -- unmatched, or in a batch that failed -- is
+    written back into a live log, so the next run tries it again instead of it
+    quietly vanishing with the rest.
+    """
+    if keep or not logs:
+        if keep:
+            print("logs left in place", file=sys.stderr)
+        return
+
+    submitted = {id(entry) for entry in sent}
+    # A .scrobbler.log was converted by the on-device plugin and carries no
+    # per-line record, so there is nothing to write back for it -- only the
+    # rename below, which loses nothing.
+    pending = list(conversion.retain) if conversion is not None else []
+    pending += [
+        entry["line"] for entry in played
+        if "line" in entry and id(entry) not in submitted
+    ]
+
+    if not sent:
+        print("nothing was accepted; logs left untouched", file=sys.stderr)
+        return
+
+    stamp = played[-1]["timestamp"] if played else "0"
+    for path in logs:
+        path.rename(path.with_name(f"{path.name}.{stamp}.submitted"))
+
+    if pending:
+        live = logs[0].with_name("playback.log")
+        live.write_text("\n".join(pending) + "\n", encoding="utf-8")
+        print(
+            f"{len(pending)} plays not submitted were written back to"
+            f" {live.name} for the next run",
+            file=sys.stderr,
+        )
+    print(f"{len(logs)} log(s) set aside as .submitted", file=sys.stderr)
+
+
 if __name__ == "__main__":
     sys.exit(main())

tools/sync-to-ipod.sh

@@ -12,46 +12,103 @@
 set -euo pipefail
 
 usage() {
-    cat >&2 <<'USAGE'
+    # Help goes to stdout and exits clean; misuse goes to stderr and does not.
+    local stream=2 code=2
+    if [ "${1:-}" = "help" ]; then
+        stream=1
+        code=0
+    fi
+    cat >&"$stream" <<'USAGE'
 usage: sync-to-ipod.sh [options] <mirror> <destination>
 
   -n  dry run; show what would change and touch nothing
+  -P  count what needs copying first, so progress can show a percentage and
+      an estimate. Costs a second full traversal of both trees, which on a
+      FAT card of fifty thousand files is slower than the transfer itself
   -f  copy even if the FAT32 check finds unacceptable paths
   -S  skip submitting the Rockbox scrobbler log to Last.fm
+  -B  skip rebuilding the Rockbox database
   -U  leave the destination mounted afterwards
 
+Rebuilding the database needs MUSIC_MIRROR_DATABASE_TOOL pointing at Rockbox's
+host-side builder (tools/database, built with ./tools/configure --type=d). It
+is skipped with a note when unset. The scan reads tags from the mirror rather
+than from the device, so it costs seconds rather than the hours an on-device
+commit takes -- and on a large library the on-device commit may not finish at
+all.
+
 Submitting scrobbles needs LASTFM_API_KEY and LASTFM_API_SECRET; it is skipped
 with a note when they are unset. Scrobbling is a write method and needs the
 secret, unlike the read-only calls elsewhere in these projects.
 
-The destination must be a mounted FAT filesystem. Reach it with the Apple
-firmware's disk mode: Menu+Select to reboot, then immediately Select+Play.
+The mirror is the directory holding the artist folders. The destination is
+where those folders should end up on the device -- not the card root, unless
+that is genuinely where you want them:
+
+  sync-to-ipod.sh /mnt/tank/media/music-mp3 /media/IPOD/Music
+
+A subdirectory is the better target: --delete is confined to it, and the path
+budget is derived from it, since the device's 260-character limit counts the
+whole path as the device sees it. /.rockbox and the scrobbler logs are never
+deleted wherever you point this.
+
+The destination must be on a mounted FAT filesystem. That check is also what
+catches an unmounted device: /media/IPOD/Music then resolves to the host's own
+root filesystem, and this refuses to empty that.
+
+An album that gained or lost a track is copied again in full afterwards. Its
+surviving tracks have had their ReplayGain tags rewritten in place, which
+changes neither their size nor their mtime, so the main pass cannot see them.
+
+Progress is one line that rewrites itself, showing the album currently going
+across, how far through the transfer is, the rate, and an estimate of what is
+left. Working the totals out first means a second pass over the tree, which is
+the price of figures that mean something; rsync's own percentage is computed
+against a file list it is still building.
+
+Reach the device with the Apple firmware's disk mode: Menu+Select to reboot,
+then immediately Select+Play. Power off afterwards by holding Play.
 USAGE
-    exit 2
+    exit "$code"
 }
 
 dry_run=false
+counting=false
 force=false
 unmount=true
 scrobble=true
-while getopts ":nfSUh" option; do
+database=true
+for argument in "$@"; do
+    [ "$argument" = "--help" ] && usage help
+done
+while getopts ":nPfSBUh" option; do
     case "$option" in
         n) dry_run=true ;;
+        P) counting=true ;;
         f) force=true ;;
         S) scrobble=false ;;
+        B) database=false ;;
         U) unmount=false ;;
+        h) usage help ;;
         *) usage ;;
     esac
 done
 shift $((OPTIND - 1))
 [ $# -eq 2 ] || usage
 
+# Trailing slashes are stripped for tidiness, but stripping one from "/" leaves
+# an empty string, and the guard below would then never see the root it is
+# there to refuse.
 mirror=${1%/}
+mirror=${mirror:-/}
 destination=${2%/}
+destination=${destination:-/}
 here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
 
 die() {
-    printf 'sync-to-ipod: %s\n' "$1" >&2
+    # Every argument, not just the first: the second half of a message is
+    # usually the half that says what to do about it.
+    printf 'sync-to-ipod: %s\n' "$*" >&2
     exit 1
 }
 
@@ -59,28 +116,41 @@ die() {
 [ -n "$(ls -A "$mirror")" ] || die "mirror $mirror is empty; refusing to mirror nothing"
 [ -d "$destination" ] || die "destination $destination is not a directory"
 
-# --delete makes every one of these load-bearing. A destination that is not its
-# own mount point means the path is wrong, and emptying the wrong directory is
-# not a mistake that announces itself.
+# --delete makes every one of these load-bearing. Emptying the wrong directory
+# is not a mistake that announces itself.
 case "$destination" in
     "" | "/" | "$HOME") die "refusing to sync onto $destination" ;;
 esac
 [ "$(readlink -f "$mirror")" != "$(readlink -f "$destination")" ] ||
     die "mirror and destination are the same directory"
-mountpoint -q -- "$destination" || die "$destination is not a mount point"
 
+# The filesystem the destination sits on, which is the check that matters: a
+# subdirectory of the card is a perfectly good target, and is the better one,
+# because --delete is then confined to it. Being FAT is also what proves the
+# card is mounted at all -- an unmounted /media/IPOD/Music resolves to the
+# host's own root filesystem, and this refuses to empty that.
 filesystem=$(findmnt -no FSTYPE --target "$destination")
+mounted_on=$(findmnt -no TARGET --target "$destination")
 case "$filesystem" in
     vfat | exfat) ;;
     *)
-        $force || die "$destination is $filesystem, not FAT; pass -f if that is deliberate"
+        $force ||
+            die "$destination is on a $filesystem filesystem, not FAT." \
+                "Is the device mounted? Pass -f if this is deliberate."
         printf 'sync-to-ipod: destination is %s, not FAT\n' "$filesystem" >&2
         ;;
 esac
 
+# What the device will call this directory, which is what its path limit
+# applies to. Derived rather than configured, so it cannot disagree with where
+# the files are actually going.
+device_prefix=${destination#"$mounted_on"}
+device_prefix="/${device_prefix#/}"
+printf 'sync-to-ipod: the device will see this as %s\n' "$device_prefix" >&2
+
 if $force; then
     printf 'sync-to-ipod: skipping the FAT32 check\n' >&2
-elif ! python3 "$here/check_fat32.py" "$mirror"; then
+elif ! python3 "$here/check_fat32.py" --device-prefix "$device_prefix" "$mirror"; then
     die "the mirror holds paths FAT32 will not take; run music-mirror with --fat32-safe"
 fi
 
@@ -92,7 +162,11 @@ if $scrobble; then
     else
         scrobble_options=()
         $dry_run && scrobble_options+=(--dry-run)
-        python3 "$here/submit_scrobbles.py" "${scrobble_options[@]}" "$destination" ||
+        # --mirror lets it convert Rockbox's own playback.log, so the on-device
+        # scrobbler plugin never has to be run. The device root, not the music
+        # directory: the logs live in .rockbox.
+        python3 "$here/submit_scrobbles.py" "${scrobble_options[@]}" \
+            --mirror "$mirror" --device-prefix "$device_prefix" "$mounted_on" ||
             die "submitting scrobbles failed; nothing has been copied"
     fi
 fi
@@ -101,36 +175,180 @@ fi
 # asking for them produces a screenful of errors and a non-zero exit.
 # --modify-window=2 because FAT stores mtimes to two-second resolution, without
 # which every file looks changed and the whole library is copied every time.
+# --whole-file is already the default when both ends are local paths, and an
+# SMB or FAT mount counts as one, but stating it documents that the delta
+# algorithm is deliberately not wanted: it would read every destination file
+# back over USB to compute a checksum, to save sending an MP3 that has changed
+# entirely anyway.
+#
+# --omit-dir-times drops a setattr round trip per directory. Across six
+# thousand album folders on a FAT card that is six thousand operations to set
+# timestamps nothing reads.
+options=(--recursive --times --delete --modify-window=2 --whole-file --omit-dir-times)
 # --delete removes tracks whose source has gone, which is the point. It would
 # also remove everything on the device that the mirror does not contain -- and
 # if the destination is the card root that means /.rockbox, the Rockbox install
 # itself. Excluded paths are not deleted unless --delete-excluded is given,
 # which it never is here.
-options=(--recursive --times --delete --modify-window=2 --human-readable --info=progress2)
 for owned in "/.rockbox" "/.scrobbler.log" "/.scrobbler.log.*" "/.playlist_control" \
     "/System Volume Information" "/.Spotlight-V100" "/.Trashes" "/.fseventsd"; do
     options+=(--exclude "$owned")
 done
-$dry_run && options+=(--dry-run --verbose)
 
 printf 'sync-to-ipod: %s -> %s\n' "$mirror" "$destination" >&2
-rsync "${options[@]}" "$mirror/" "$destination/"
+
+# Both passes below feed their file list through touched_albums.py, which reads
+# rsync's own report of what it moved. --files-from separates paths by newline,
+# so a filename containing one would be read as two -- which is a path FAT32
+# will not take either, and check_fat32.py above has already refused the run
+# unless -f was given to skip it.
+changed=$(mktemp)
+relevelled=$(mktemp)
+trap 'rm -f "$changed" "$relevelled"' EXIT
 
 if $dry_run; then
+    rsync "${options[@]}" --dry-run --verbose --out-format='%l %n' \
+        "$mirror/" "$destination/" | tee "$changed"
+    also=$(python3 "$here/touched_albums.py" --mirror "$mirror" <"$changed" | wc -l)
+    if [ "$also" -gt 0 ]; then
+        printf 'sync-to-ipod: and %s more in those albums, whose ReplayGain tags\n' "$also" >&2
+        printf 'sync-to-ipod: change without changing their size or their mtime\n' >&2
+    fi
     printf 'sync-to-ipod: dry run, nothing was written\n' >&2
     exit 0
 fi
 
-sync
-if $unmount; then
-    device=$(findmnt -no SOURCE --target "$destination")
-    printf 'sync-to-ipod: unmounting %s\n' "$device" >&2
-    if command -v udisksctl >/dev/null 2>&1; then
-        udisksctl unmount -b "$device"
-    else
-        umount -- "$destination"
-    fi
-    printf 'sync-to-ipod: safe to disconnect\n' >&2
-else
-    printf 'sync-to-ipod: still mounted; unmount before disconnecting\n' >&2
+# rsync says nothing at all while it builds its file list, which on fifty
+# thousand files over USB is minutes of apparent hang. Counting first costs a
+# second pass over the tree but means the transfer can show a real percentage
+# rather than a number that grows as rsync discovers more work.
+# Counting is opt-in because it is not cheap. It walks and compares both trees
+# in full, exactly as the transfer does, and on a FAT card holding fifty
+# thousand files that traversal costs more than moving the data. Without it the
+# progress line still shows the running count, the rate and the album in
+# flight; only the percentage and the estimate are lost, and those were the
+# least useful part of it.
+total=0
+total_bytes=0
+if $counting; then
+    printf 'sync-to-ipod: working out what needs copying...\n' >&2
+    # %l is the file's size, which is what makes an estimate possible.
+    # Directories are dropped: rsync reports those too, with an inode size that
+    # would inflate the total by several megabytes of nothing.
+    counted=$(rsync "${options[@]}" --dry-run --out-format='%l %n' "$mirror/" "$destination/" |
+        awk '!/\/$/ { files++; bytes += $1 } END { print files + 0, bytes + 0 }')
+    total=${counted% *}
+    total_bytes=${counted#* }
+    printf 'sync-to-ipod: %s files to copy\n' "$total" >&2
 fi
+
+# Flushing and unmounting is the whole reason this is a script, so it has to
+# happen on the way out whichever way that is. Ctrl-C during a transfer would
+# otherwise leave a FAT filesystem with dirty buffers and no journal, which is
+# the corruption this exists to avoid.
+finish() {
+    sync
+    if $unmount; then
+        device=$(findmnt -no SOURCE --target "$destination" 2>/dev/null || true)
+        if [ -n "$device" ]; then
+            printf 'sync-to-ipod: unmounting %s\n' "$device" >&2
+            if command -v udisksctl >/dev/null 2>&1; then
+                udisksctl unmount -b "$device" || umount -- "$destination" || true
+            else
+                umount -- "$destination" || true
+            fi
+            printf 'sync-to-ipod: safe to disconnect\n' >&2
+        fi
+    else
+        printf 'sync-to-ipod: still mounted; unmount before disconnecting\n' >&2
+    fi
+}
+
+interrupted() {
+    trap - INT TERM
+    printf '\nsync-to-ipod: interrupted -- rsync leaves no partial files, but the\n' >&2
+    printf 'sync-to-ipod: filesystem still needs flushing before you pull anything\n' >&2
+    finish
+    exit 130
+}
+
+trap interrupted INT TERM
+
+rsync "${options[@]}" --out-format='%l %n' "$mirror/" "$destination/" |
+    tee "$changed" |
+    python3 "$here/rsync_progress.py" --total "$total" --bytes "$total_bytes"
+status=${PIPESTATUS[0]}
+[ "$status" -eq 0 ] || die "rsync exited $status"
+
+# A ReplayGain album gain belongs to the whole record, so a track arriving or
+# leaving rewrites the tags on all of its siblings. rsgain fits the new values
+# into the padding its previous write left behind, which changes neither the
+# size nor the mtime -- the only two things the pass above compares. Those
+# tracks are invisible to it, and the device would keep the old gains.
+#
+# The track that arrived or left is visible, though. So every album the pass
+# touched has the rest of its tracks copied again, with --ignore-times to
+# defeat the same quick check. No --delete: the pass above has already settled
+# what should be on the device, and --delete aimed at an explicit file list
+# does not mean what it looks like it means.
+python3 "$here/touched_albums.py" --mirror "$mirror" <"$changed" >"$relevelled"
+if [ -s "$relevelled" ]; then
+    also=$(wc -l <"$relevelled")
+    printf 'sync-to-ipod: re-copying %s tracks whose album was re-levelled\n' "$also" >&2
+    rsync --times --modify-window=2 --whole-file --omit-dir-times --ignore-times \
+        --files-from="$relevelled" --out-format='%l %n' "$mirror/" "$destination/" |
+        python3 "$here/rsync_progress.py" --total "$also"
+    status=${PIPESTATUS[0]}
+    [ "$status" -eq 0 ] || die "the re-levelled tracks failed to copy: rsync exited $status"
+fi
+
+# Rockbox reads its database from .tcd files in .rockbox. Building them here
+# rather than on the device is not just faster: the on-device commit sorts the
+# whole index in whatever memory it can scrape together, and on a large library
+# it runs for hours or dies outright.
+#
+# The scan reads tags through a scratch root -- a real .rockbox beside a symlink
+# standing in for where the music lands on the device -- so the paths recorded
+# match what Rockbox will look up, while the bytes are read from the mirror
+# instead of over USB. The scratch is kept between runs because the builder is
+# incremental: a second pass over unchanged files does no work at all.
+rebuild_database() {
+    local tool=${MUSIC_MIRROR_DATABASE_TOOL:-}
+    if [ -z "$tool" ]; then
+        printf 'sync-to-ipod: no database tool configured, skipping the database\n' >&2
+        return 0
+    fi
+    [ -x "$tool" ] || die "$tool is not executable"
+
+    local device_rockbox="$mounted_on/.rockbox"
+    if [ ! -d "$device_rockbox" ]; then
+        printf 'sync-to-ipod: no .rockbox on the device, skipping the database\n' >&2
+        return 0
+    fi
+
+    local scratch="${XDG_CACHE_HOME:-$HOME/.cache}/music-mirror/database"
+    mkdir -p "$scratch/.rockbox"
+
+    # Rebuild the symlink layout each time; the mirror path or the device
+    # prefix may have changed since the last run.
+    find "$scratch" -maxdepth 1 -type l -delete
+    if [ "$device_prefix" = "/" ]; then
+        ln -s "$mirror"/* "$scratch/" 2>/dev/null || true
+    else
+        local under=${device_prefix#/}
+        rm -rf "${scratch:?}/${under%%/*}"
+        mkdir -p "$scratch/$(dirname "$under")"
+        ln -s "$mirror" "$scratch/$under"
+    fi
+
+    printf 'sync-to-ipod: building the database from the mirror...\n' >&2
+    ( cd "$scratch" && "$tool" ) >/dev/null || die "the database build failed"
+
+    cp -- "$scratch"/.rockbox/*.tcd "$device_rockbox/" ||
+        die "could not copy the database onto the device"
+    printf 'sync-to-ipod: database copied to %s\n' "$device_rockbox" >&2
+}
+
+$database && rebuild_database
+
+finish

tools/touched_albums.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""List the tracks a sync has to copy again because their album was re-levelled.
+
+Reads rsync's `--out-format='%l %n'` output on stdin and writes mirror-relative
+paths on stdout, one per line, for feeding straight back to rsync as
+`--files-from`.
+
+The problem it solves: a ReplayGain album gain is a property of every track on
+the record, so one track arriving or leaving changes the tags on all of its
+siblings. rsgain writes the new values into the padding its previous write left
+behind, which leaves both the file's size and its mtime untouched -- and size
+and mtime are exactly what rsync's quick check compares. It sees nothing to do,
+and the device keeps the old gains.
+
+What rsync always can see is the track that arrived or left. So any album it
+touched has the rest of its tracks copied again, and nothing else does.
+
+Tracks rsync has already dealt with are left out of the list. After a quality
+upgrade that replaces every track on a record, listing them again would send
+the whole album twice.
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+# Under --out-format='%l %n' a transfer is reported as the size, a space and
+# the path. A removal is reported as "deleting <path>" whatever the format is.
+# Everything else on the stream is rsync talking to the operator -- the file
+# list preamble, the byte totals -- and is not a path.
+TRANSFER = re.compile(r"^(\d+) (.+)$")
+DELETION = re.compile(r"^deleting (.+)$")
+
+# What the mirror is made of, and so the only thing worth copying again. A
+# cover is not rewritten by a re-level.
+MIRROR_SUFFIX = ".mp3"
+
+
+def touched(lines):
+    """Return the paths rsync transferred and the directories it changed."""
+    transferred = set()
+    directories = set()
+
+    for line in lines:
+        line = line.rstrip("\n")
+        deletion = DELETION.match(line)
+        transfer = None if deletion else TRANSFER.match(line)
+        if deletion:
+            path = deletion.group(1)
+        elif transfer:
+            path = transfer.group(2)
+        else:
+            continue
+        # Only a track can change an album's gains. rsync reports the
+        # directories it creates and removes too, and a cover replaced on its
+        # own is no reason to send the record again.
+        if not path.endswith(MIRROR_SUFFIX):
+            continue
+        if transfer:
+            transferred.add(path)
+        directories.add(path.rpartition("/")[0])
+
+    return transferred, directories
+
+
+def remaining(mirror, transferred, directories):
+    """Return the tracks in those directories that rsync has not just copied."""
+    paths = []
+    for directory in sorted(directories):
+        album = mirror / directory if directory else mirror
+        if not album.is_dir():
+            # Removed along with the last of its tracks. Nothing to copy.
+            continue
+        for track in sorted(album.glob(f"*{MIRROR_SUFFIX}")):
+            relative = f"{directory}/{track.name}" if directory else track.name
+            if relative not in transferred:
+                paths.append(relative)
+    return paths
+
+
+def main(argv=None, stream=None, out=None):
+    parser = argparse.ArgumentParser(
+        prog="touched_albums.py",
+        description="List the tracks to copy again after an album was re-levelled.",
+    )
+    parser.add_argument("--mirror", required=True, type=Path, help="root of the MP3 mirror")
+    args = parser.parse_args(argv)
+
+    transferred, directories = touched(stream if stream is not None else sys.stdin)
+    for path in remaining(args.mirror, transferred, directories):
+        print(path, file=out if out is not None else sys.stdout)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())