chore(release): v0.3.0

Reviewed-on: #8

.gitea/workflows/build-and-publish.yaml

@@ -45,7 +45,8 @@ jobs:
 
       # The suite runs inside the image, against the ffmpeg that ships, rather
       # than against whatever the runner happens to provide. A failing test
-      # fails the build. Layers are shared with the push build below.
+      # fails the build. The runtime stage below is built from the same daemon
+      # afterwards, so its layers are already in cache.
       - name: Run the test suite inside the image
         run: docker build --target test -t music-mirror:test .
 
@@ -124,9 +125,6 @@ jobs:
           echo "release=${release}" >> "$GITHUB_OUTPUT"
           echo "Computed bump=${bump}, release=${release}, base=${base}"
 
-      - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
-
       - name: Log in to the Gitea container registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
@@ -135,21 +133,34 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.PACKAGES_TOKEN }}
 
-      - name: Build and push
+      # Plain `docker build` rather than buildx. buildx boots its own buildkit
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
+      # in a container with a cache of its own, so it shared nothing with the
-        with:
+      # test build above and rebuilt the image from the base up -- installing
-          context: .
+      # ffmpeg and the package a second time, for nothing. It earns that cost
-          # Without this the last stage in the Dockerfile -- the test stage --
+      # when building for several platforms; this only ever targets the amd64
-          # would be what gets published.
+      # NAS, so it does not.
-          target: runtime
+      #
-          # The NAS is the only host this runs on. Building arm64 as well would
+      # `--target runtime` is a strict prefix of the test stage, so every layer
-          # mean emulating it under QEMU for no consumer.
+      # is already in the daemon's cache and this resolves in seconds.
-          platforms: linux/amd64
+      - name: Build the runtime image
-          push: ${{ github.event_name != 'pull_request' }}
+        run: |
-          tags: ${{ steps.version.outputs.tags }}
+          set -euo pipefail
-          labels: |
+          tags=()
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+          while IFS= read -r tag; do
-            org.opencontainers.image.revision=${{ github.sha }}
+            [ -n "$tag" ] && tags+=(-t "$tag")
+          done <<< "${{ steps.version.outputs.tags }}"
+          docker build --target runtime \
+            --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \
+            --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+            "${tags[@]}" .
+
+      - name: Push
+        if: github.event_name != 'pull_request'
+        run: |
+          set -euo pipefail
+          while IFS= read -r tag; do
+            [ -n "$tag" ] && docker push "$tag"
+          done <<< "${{ steps.version.outputs.tags }}"
 
       # Record the release: write the computed version into pyproject.toml, then
       # commit and tag it, so the packaging metadata always matches the release

Dockerfile

@@ -27,5 +27,7 @@ FROM runtime AS test
 
 RUN pip install --no-cache-dir pytest
 COPY pytest.ini ./
+# Host-side tools; not in the runtime image, but the suite covers them.
+COPY tools ./tools
 COPY tests ./tests
 RUN python -m pytest

README.md

@@ -51,9 +51,30 @@ The mtime is read _before_ encoding rather than after. A file still being
 written when the pass reaches it would otherwise be stamped with its final
 mtime while holding truncated audio, and never be revisited.
 
-Encodes are written to a temporary file and renamed into place, so an
+Both encodes and copies are written to a temporary file and renamed into place,
-interrupted run cannot leave a truncated MP3 that the next run mistakes for
+so an interrupted run cannot leave a truncated MP3 that the next run mistakes
-finished work. A lock file in the mirror root stops two passes overlapping.
+for finished work. Copies need it as much as encodes do: the mtime comes across
+with the bytes, so a half-written copy would look current for ever. A lock file
+in the mirror root stops two passes overlapping.
+
+### Permissions
+
+Everything written into the mirror is made group-readable, and its directories
+group-traversable, so the mirror can be read back by whatever serves it. Neither
+writer does that unaided: the temporary file an encode renames into place is
+created `0600` regardless of the umask, and a straight copy of an existing MP3
+inherits the mode of a source file in a library this tool does not own.
+
+Directories are handled by clearing the owner and group read/execute bits from
+the process umask, once, at startup. Owner as well as group, because a umask
+carrying `0400` produces directories of mode `0300` — writable and enterable,
+unreadable to the very run that created them. The `other` bits are left where
+the umask puts them: whether the mirror is world-readable is a genuine policy
+question, and so is its ownership.
+
+Mirror files written before this existed are topped up on the next pass. Their
+mtimes are correct, so nothing else would revisit them — and they are not
+re-encoded, only chmod'ed.
 
 ## Usage
 
@@ -72,6 +93,7 @@ music-mirror --source /music --mirror /music-mp3 --subdir "Artist/Album"
 | `--jobs`     | `MUSIC_MIRROR_JOBS`     | CPU count | Concurrent encodes                             |
 | `--interval` | `MUSIC_MIRROR_INTERVAL` | unset     | Repeat forever, e.g. `45m`, `6h`, `1d`         |
 | `--subdir`   | —                       | unset     | Limit the pass to one directory; skips pruning |
+| `--fat32-safe` | `MUSIC_MIRROR_FAT32_SAFE` | off     | Name files so a FAT32 device accepts them      |
 | `--no-prune` | —                       | off       | Keep mirror files whose source has gone        |
 | `--dry-run`  | —                       | off       | Report what would change, write nothing        |
 
@@ -116,6 +138,52 @@ New Lidarr imports are picked up on the next pass. With `MUSIC_MIRROR_INTERVAL`
 at `6h` that is the worst case; run `--subdir` by hand if you want an album
 immediately.
 
+## Tools
+
+Host-side scripts under `tools/`, not part of the container image.
+
+`sync-to-ipod.sh` does a whole transfer: submits the scrobbler log, checks the
+mirror, rsyncs, syncs and unmounts.
+
+```sh
+tools/sync-to-ipod.sh /mnt/tank/media/music-mp3 /media/IPOD/Music
+tools/sync-to-ipod.sh -n /mnt/tank/media/music-mp3 /media/IPOD/Music   # dry run
+```
+
+It refuses to start unless the destination is a mounted FAT filesystem that is
+its own mount point, because `--delete` aimed at the wrong directory empties it
+and does not announce itself. It also excludes `/.rockbox`, the scrobbler logs
+and the various filesystem metadata directories from deletion — the mirror does
+not contain them, and without the exclusion a sync to the card root would
+remove the Rockbox install.
+
+The unmount is the point of doing this in a script. FAT32 has no journal and
+the device is reached through disk mode, so an interrupted write is corruption
+that needs `fsck.vfat` from another machine.
+
+`submit_scrobbles.py` sends the Rockbox scrobbler log to Last.fm and sets it
+aside. Rockbox writes `/.scrobbler.log` in AUDIOSCROBBLER 1.1 format, one
+tab-separated line per track rated `L` for listened or `S` for skipped; only
+the listened ones are sent. It runs **before** the copy, since the plays
+already happened and a failed transfer is no reason to lose them.
+
+Two things are unlike every other Last.fm call in these projects. Scrobbling is
+a *write* method, so it needs `LASTFM_API_SECRET` and a session key obtained
+once through the browser, not just the read-only key. And on a target with no
+real-time clock Rockbox writes `/.scrobbler-timeless.log` with every timestamp
+set to zero; those are counted and reported but never submitted, because
+scrobbling them would mean inventing when they happened.
+
+The log is renamed rather than deleted once accepted. If Last.fm quietly
+dropped something, the evidence is still on the device.
+
+`check_fat32.py` reports paths a FAT32 device will not accept — reserved
+characters, trailing dots and spaces, over-long components and paths, and names
+colliding case-insensitively. Run it against the mirror **before** an rsync:
+rsync reports the failures too, but scattered through fifty thousand files where
+they are easy to miss. Exits non-zero when it finds anything, so it can gate a
+script.
+
 ## Tests
 
 ```sh
@@ -137,6 +205,93 @@ Nix machine:
 nix shell nixpkgs#python3Packages.pytest nixpkgs#ffmpeg -c pytest
 ```
 
+## FAT32 and Rockbox
+
+`--fat32-safe` names mirror files so a FAT32 device will accept them. Off by
+default, because turning it on renames files and that should be a decision
+rather than a surprise.
+
+What it handles, per path component:
+
+| Problem | Treatment |
+| ------------------------------- | ------------------------------ |
+| `< > : " \ \| ? *` and control characters | replaced with `_` |
+| trailing dots and spaces | stripped — FAT eats them silently, so the name round-trips as a different name |
+| a component left empty | becomes `_` |
+| names differing only in case | detected and reported; one wins, as with any other collision |
+
+`Dada Life - Kick Out the Epic Motherf**ker` is a real example from a real
+library. Without this it simply never arrives on the device.
+
+**Turning it on does not re-encode anything.** Every track whose name held a
+reserved character changes path, and re-encoding those would be hours of work
+producing files that already exist byte for byte. The run moves them instead,
+and says so. Prune then finds nothing to remove because nothing was left
+behind.
+
+Renames are counted apart from encodes in the pass summary, and `--dry-run`
+reports `would rename` rather than `would encode` — the difference between the
+two is a minute against an afternoon, so a preview that conflated them would be
+worse than no preview. A dry run also does not list the pre-rename files as
+orphans: nothing was moved, so they are still there, but they are what a real
+run would move rather than what it would delete.
+
+### Path length
+
+Rockbox's `MAX_PATH` is 260, from `firmware/include/fs_defines.h`, and it bounds
+the path *as the device sees it*. The directory the mirror is copied into comes
+out of the same budget, so `--device-prefix` (default `/Music`) is subtracted
+from `--max-path` to get what a mirror-relative path may spend:
+
+| Destination on the device | Mirror-relative budget |
+| ------------------------- | ---------------------- |
+| `/Music/` | 253 |
+| the card root | 259 |
+
+Worth being exact about, because a checker that measures mirror-relative paths
+against the flat 260 quietly passes everything from 253 to 260 — and those are
+the paths most likely to be near the edge in the first place.
+
+Over-budget paths are shortened from the **deepest component outward**: the
+track name carries the least navigational value and the artist directory the
+most, so the filename goes first and the artist is touched only if nothing else
+will do.
+
+A component is cut **from the middle**, not the end, because of how these names
+are built. Lidarr writes `Artist - Album - 07 - Flamethrower.mp3` inside a
+directory already named for that artist and album, so a long album title
+appears three times in one path and the informative part — the track number and
+title — is at the very end. Cutting from the end throws exactly that away:
+
+```
+before  King Gizzard & the Lizard Wizard - PetroDragonic Apocalypse; or, Dawn of Eternal
+        Night - An Annihilation of Planet Earth and the Beginning of Merciless
+        Damnation - 07 - Flamethrower.mp3
+after   King Gizzard & the Lizard~c526~ginning of Merciless Damnation - 07 - Flamethrower.mp3
+```
+
+Two thirds of the remaining room goes to the tail, since the head is usually a
+restatement of the directory it sits in. A shortened component gains four hex
+digits of the original name: two names sharing both a head and a tail would
+otherwise produce the same string, and a silent collision between two tracks is
+worse than an ugly filename.
+
+The result is stable: the same source always produces the same shortened name,
+so a pass does not rename what the previous pass wrote. A path too deeply
+nested to fit without reducing every component to nonsense is left alone and
+reported instead.
+
+### Album art
+
+Rockbox looks for cover art **on the filesystem** — `cover.jpg`, `folder.jpg`
+and friends beside the track or in its parent — and that search never touches
+the picture embedded in the tag. So a JPEG cover found beside the source is now
+copied into the mirror as `cover.jpg`, in addition to being embedded. The iPod
+firmware reads the embedded one; Rockbox reads the file. Both are satisfied.
+
+A cover left behind in a directory whose tracks have all gone is pruned, or the
+directory would never look empty and never be removed.
+
 ## Getting the result onto an iPod
 
 The mirror is just a directory of MP3s, so any client will do:
@@ -148,6 +303,30 @@ The mirror is just a directory of MP3s, so any client will do:
 - **Linux.** Rhythmbox links `libgpod` and handles iPod sync. An iPod Video
   (5th generation) predates the models whose database has to be signed, so no
   firmware-hash trickery is needed.
+- **Rockbox.** No database to write at all — it reads a plain directory tree
+  and builds its own index from tags. Copy the mirror across with rsync:
+
+  ```sh
+  python3 tools/check_fat32.py /mnt/tank/media/music-mp3   # before, not during
+  rsync -rtv --delete --modify-window=2 \
+      /mnt/tank/media/music-mp3/ /media/IPOD/Music/
+  ```
+
+  `--modify-window=2` because FAT stores modification times to two-second
+  resolution; without it rsync re-copies the entire library on every run. `-rt`
+  rather than `-a` because owners, groups and permissions mean nothing on FAT
+  and asking for them only produces errors.
+
+  Do not route this through Rhythmbox. Its `rb_ipod_helpers_is_ipod()` reads
+  `access-protocols` from media-player-info first and returns true without
+  looking at the filesystem at all, so an iPod in disk mode is identified by its
+  USB id and managed as an iPod — writing a database Rockbox does not want.
+  Deleting `iPod_Control` does not change this. Either use rsync, or untick
+  Preferences → Plugins → Portable Players - iPod.
+
+  Faster still for the first bulk copy: take the card out of the iFlash adapter
+  and use a card reader. Fifty thousand files over USB 2.0 through an iPod is a
+  long evening, and it avoids Rockbox's USB stack entirely.
 
 Neither client transcodes at sync time; they copy finished MP3s.
 

compose.yaml

@@ -22,6 +22,15 @@ services:
       # Concurrent encodes; defaults to the CPU count. Lower it to leave the
       # NAS responsive during the first full pass.
       # MUSIC_MIRROR_JOBS: "4"
+      # Name files so a FAT32 device will take them, for copying to a Rockbox
+      # player. Quoted, because an unquoted yes or true is a YAML boolean and
+      # compose rejects a boolean here. Accepts 1, true or yes.
+      #
+      # Turning this on renames every file whose name held a reserved
+      # character. They are moved, not re-encoded, so the first pass after
+      # enabling it is quick -- but it is a one-way change to every such path,
+      # so decide before running it rather than after.
+      MUSIC_MIRROR_FAT32_SAFE: "true"
     volumes:
       - /mnt/tank/media/music:/music:ro
       - /mnt/tank/media/music-mp3:/mirror

music_mirror.py

@@ -17,6 +17,7 @@ import argparse
 import concurrent.futures
 import fcntl
 import functools
+import hashlib
 import logging
 import os
 import re
@@ -71,12 +72,48 @@ SOURCE_PRIORITY = [
 # Looked for in the source directory when a file has no embedded picture.
 COVER_NAMES = ("cover.jpg", "folder.jpg", "front.jpg", "cover.png", "folder.png")
 
+# What a copied cover is called in the mirror. Rockbox looks for album art on
+# the filesystem -- cover.jpg, folder.jpg and friends beside the track -- and
+# its search never touches the picture embedded in the tag, so a mirror that
+# only embeds art shows none of it on the device.
+MIRROR_COVER = "cover.jpg"
+
 # Files the mirror is allowed to contain, and therefore allowed to delete.
 MIRROR_SUFFIX = ".mp3"
 
 # Filesystems disagree about mtime precision; SMB in particular rounds.
 MTIME_TOLERANCE_SECONDS = 2
 
+# What FAT32 refuses in a filename, plus the control characters. The mirror is
+# copied onto a FAT32 device, and one of these in a path means a track that
+# never arrives -- "Kick Out the Epic Motherf**ker" is a real example.
+FAT32_RESERVED = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
+
+# Rockbox's MAX_PATH, from firmware/include/fs_defines.h. It bounds the whole
+# path as the device sees it, so the budget for a mirror-relative path is this
+# less whatever directory the mirror is copied into.
+MAX_PATH = 260
+DEVICE_PREFIX = "/Music"
+
+# A component cut below this is no longer recognisable, and a path that cannot
+# be brought under the limit without going there is better reported than
+# mangled.
+MIN_COMPONENT = 12
+
+# The mirror exists to be read back by something else -- an SMB share, another
+# account on the box -- so everything written into it has to be group-readable.
+# Neither writer manages that unaided: tempfile.mkstemp forces 0600 whatever the
+# umask, and shutil.copy2 carries the source file's mode across from a library
+# that may be tighter still. Directories need the execute bit too, or the group
+# cannot enter them to reach the readable files inside.
+GROUP_READ = 0o040
+
+# Cleared from the umask so directories this run creates can be listed and
+# entered. Owner as well as group: a umask carrying 0400 -- which is unusual but
+# not ours to assume away -- otherwise produces a mirror tree that not even the
+# process that built it can read back.
+DIRECTORY_ACCESS = 0o550
+
 
 @dataclass
 class Result:
@@ -112,9 +149,102 @@ def parse_interval(interval):
     return value * {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400}[match.group(2)]
 
 
-def mirror_path_for(source, source_root, mirror_root):
+def fat32_safe(component):
+    """Return a single path component FAT32 will accept.
+
+    The mirror exists to be copied onto a FAT32 device, and a name that device
+    will not take is a track that silently does not arrive. Cheaper to produce
+    an acceptable name here than to discover the problem partway through
+    copying fifty thousand files.
+
+    Handled: the reserved characters, control characters, and the trailing dots
+    and spaces that FAT quietly eats -- a name ending in one round-trips as a
+    different name, which is worse than being rejected outright.
+    """
+    cleaned = FAT32_RESERVED.sub("_", component).rstrip(". ")
+    # Stripping can empty a component outright: a directory named "..." is
+    # legal on ext4 and nothing at all on FAT.
+    return cleaned or "_"
+
+
+def device_prefix_length(prefix):
+    """Return the on-device prefix as it will actually appear, with slashes.
+
+    "/Music" costs seven characters -- the leading slash, the name, and the
+    separator before the mirror's own path -- while an empty prefix costs one.
+    Approximating that loses a character at the root, which is precisely where
+    the longest paths are.
+    """
+    cleaned = prefix.strip("/")
+    return f"/{cleaned}/" if cleaned else "/"
+
+
+def shorten_component(component, budget):
+    """Return a component of at most `budget` characters, cut from the middle.
+
+    From the middle, not the end, because of how these names are built. Lidarr
+    writes "Artist - Album - 07 - Flamethrower.mp3" inside a directory already
+    named for that artist and album, so the informative part -- the track
+    number and title -- is at the very end. Cutting from the end discards it
+    and leaves every track on the record with the same name.
+
+    The four hex digits are of the original component. Two names sharing both a
+    head and a tail would otherwise produce the same string, and a silent
+    collision between two tracks is worse than an ugly filename.
+    """
+    stem, dot, extension = component.rpartition(".")
+    if not dot or len(extension) > 4:
+        stem, extension = component, ""
+    else:
+        extension = dot + extension
+
+    digest = hashlib.blake2s(component.encode("utf-8"), digest_size=2).hexdigest()
+    marker = f"~{digest}~"
+    room = max(2, budget - len(extension) - len(marker))
+    if room >= len(stem):
+        return stem + extension
+
+    # Two thirds to the tail: the head is usually a restatement of the
+    # directory it sits in, and the tail is what tells two tracks apart.
+    keep_end = min(len(stem), room * 2 // 3)
+    keep_start = max(1, room - keep_end)
+    return stem[:keep_start].rstrip(". ") + marker + stem[len(stem) - keep_end :] + extension
+
+
+def fit_path(relative, budget):
+    """Return a relative path within `budget` characters, or the best available.
+
+    Shortened from the deepest component outward. The filename carries the least
+    navigational value and the artist directory the most, so the track name is
+    sacrificed before the album and the album before the artist.
+    """
+    parts = list(relative.parts)
+    for index in reversed(range(len(parts))):
+        overage = len(str(Path(*parts))) - budget
+        if overage <= 0:
+            break
+        allowed = max(MIN_COMPONENT, len(parts[index]) - overage)
+        if allowed < len(parts[index]):
+            parts[index] = shorten_component(parts[index], allowed)
+    fitted = Path(*parts)
+    if len(str(fitted)) > budget:
+        logger.warning(
+            "%s is still %d characters over the limit after shortening; it is too"
+            " deeply nested to fit",
+            relative,
+            len(str(fitted)) - budget,
+        )
+    return fitted
+
+
+def mirror_path_for(source, source_root, mirror_root, safe=False, budget=0):
     """Return the mirror path corresponding to a source file."""
-    return (mirror_root / source.relative_to(source_root)).with_suffix(MIRROR_SUFFIX)
+    relative = source.relative_to(source_root).with_suffix(MIRROR_SUFFIX)
+    if safe:
+        relative = Path(*(fat32_safe(part) for part in relative.parts))
+        if budget > 0 and len(str(relative)) > budget:
+            relative = fit_path(relative, budget)
+    return mirror_root / relative
 
 
 def is_current(source, mirror):
@@ -124,6 +254,37 @@ def is_current(source, mirror):
     return abs(source.stat().st_mtime - mirror.stat().st_mtime) <= MTIME_TOLERANCE_SECONDS
 
 
+@functools.lru_cache(maxsize=4096)
+def mirror_cover(source_directory, mirror_directory):
+    """Put a copy of the album's cover beside its tracks in the mirror.
+
+    Cached per directory: an album's tracks all ask for the same file, and the
+    answer cannot change within a pass.
+    """
+    cover = find_cover(source_directory)
+    if cover is None or cover.suffix.lower() not in (".jpg", ".jpeg"):
+        # Only JPEG is copied. Rockbox will read a BMP too, but converting a
+        # PNG is ffmpeg work for a file nothing else in the pass needs.
+        return None
+    destination = mirror_directory / MIRROR_COVER
+    try:
+        if destination.is_file() and destination.stat().st_size == cover.stat().st_size:
+            return destination
+        shutil.copy2(cover, destination)
+        make_group_readable(destination)
+    except OSError as error:
+        logger.warning("could not copy cover for %s: %s", source_directory, error)
+        return None
+    return destination
+
+
+def make_group_readable(path):
+    """Add the group-read bit to a mirror file, leaving the rest of the mode alone."""
+    mode = path.stat().st_mode
+    if not mode & GROUP_READ:
+        path.chmod(mode | GROUP_READ)
+
+
 @functools.lru_cache(maxsize=4096)
 def find_cover(directory):
     """Return an external cover image for a directory, if one is present.
@@ -207,6 +368,7 @@ def encode(source, mirror, quality_args, dry_run):
         return Result("encoded", mirror)
 
     mirror.parent.mkdir(parents=True, exist_ok=True)
+    mirror_cover(source.parent, mirror.parent)
     # Probing costs an ffprobe process per file, so only ask when the answer
     # can change the command. With no cover file beside the track, `-map
     # 0:v:0?` carries embedded art if there is any and shrugs if there is not.
@@ -234,6 +396,9 @@ def encode(source, mirror, quality_args, dry_run):
             lines = completed.stderr.strip().splitlines()
             return Result("failed", source, lines[-1] if lines else "ffmpeg failed")
         os.utime(temporary, (stat.st_atime, stat.st_mtime))
+        # Before the rename, so the file is never visible in the mirror without
+        # the bit.
+        make_group_readable(temporary)
         os.replace(temporary, mirror)
     except Exception as error:  # noqa: BLE001 - reported per file, run continues
         return Result("failed", source, str(error))
@@ -245,24 +410,79 @@ def encode(source, mirror, quality_args, dry_run):
 
 
 def copy(source, mirror, dry_run):
-    """Copy an already-MP3 source into the mirror."""
+    """Copy an already-MP3 source into the mirror, atomically."""
     if dry_run:
         logger.info("would copy %s", source)
         return Result("copied", mirror)
 
     mirror.parent.mkdir(parents=True, exist_ok=True)
+    mirror_cover(source.parent, mirror.parent)
+
+    # Through a temporary file and a rename, for the same reason encodes go
+    # that way, and a sharper one: copy2 reproduces the source's mtime as well
+    # as its bytes, so a copy cut short by a full disk or a killed container
+    # would leave a truncated MP3 that every later pass reads as current.
+    handle, temporary = tempfile.mkstemp(dir=mirror.parent, suffix=".mp3.part")
+    os.close(handle)
+    temporary = Path(temporary)
+
     try:
-        shutil.copy2(source, mirror)
+        shutil.copy2(source, temporary)
+        # copy2 brings the source's mode with it, and the source library is not
+        # ours to have permissions opinions about.
+        make_group_readable(temporary)
+        os.replace(temporary, mirror)
     except OSError as error:
         return Result("failed", source, str(error))
+    finally:
+        temporary.unlink(missing_ok=True)
 
     logger.info("copied %s", source)
     return Result("copied", mirror)
 
 
-def process(source, mirror, quality_args, dry_run):
+def adopt_existing(source, mirror, candidates, dry_run=False):
+    """Move an already-encoded file to its new name. Returns whether it moved.
+
+    Turning on FAT32-safe naming changes the path of every track whose name
+    held a reserved character. Without this the run would encode them all again
+    and then prune the originals -- hours of work to produce files that already
+    exist, byte for byte, under the old name.
+
+    Several candidates are tried because there is more than one previous
+    naming: the original, and the sanitised-but-not-yet-shortened form left by
+    an earlier version.
+    """
+    for previous in candidates:
+        if previous == mirror or not previous.is_file() or not is_current(source, previous):
+            continue
+        if dry_run:
+            logger.info("would rename %s -> %s", previous.name, mirror.name)
+            return True
+        mirror.parent.mkdir(parents=True, exist_ok=True)
+        os.replace(previous, mirror)
+        logger.info("renamed %s -> %s", previous.name, mirror.name)
+        return True
+    return False
+
+
+def process(source, mirror, quality_args, dry_run, previous=None):
     """Bring one source file's mirror entry up to date."""
+    # Counted separately from an encode, and reported in a dry run, because the
+    # difference between moving a file and re-encoding it is the difference
+    # between a minute and an afternoon.
+    if previous is not None and not mirror.exists():
+        if adopt_existing(source, mirror, previous, dry_run):
+            return Result("renamed", mirror)
     if is_current(source, mirror):
+        # A mirror written before this bit was set has a correct mtime, so
+        # nothing else in the pass would ever revisit it. Top it up here
+        # instead: one stat per file, and no chmod at all once it is right.
+        if not dry_run:
+            try:
+                make_group_readable(mirror)
+            except OSError as error:
+                return Result("failed", mirror, str(error))
         return Result("skipped", mirror)
     if source.suffix.lower() in COPY_EXTENSIONS:
         return copy(source, mirror, dry_run)
@@ -277,7 +497,7 @@ def find_sources(root):
             yield path
 
 
-def plan(scan_root, source_root, mirror_root):
+def plan(scan_root, source_root, mirror_root, safe=False, budget=0):
     """Map each mirror path to the one source that should produce it.
 
     Two sources can want the same mirror path -- `01 Song.flac` alongside a
@@ -288,12 +508,20 @@ def plan(scan_root, source_root, mirror_root):
     the outcome stable and predictable instead.
     """
     chosen = {}
+    # Keyed case-insensitively when the target is FAT32, because two names
+    # differing only in case are two files here and one file there. Detecting
+    # that now beats discovering it as a silent overwrite during the copy.
+    seen = {}
     for source in find_sources(scan_root):
-        mirror = mirror_path_for(source, source_root, mirror_root)
+        mirror = mirror_path_for(source, source_root, mirror_root, safe, budget)
-        rival = chosen.get(mirror)
+        key = str(mirror).casefold() if safe else str(mirror)
+        rival_path = seen.get(key)
+        rival = chosen.get(rival_path) if rival_path else None
         if rival is None:
+            seen[key] = mirror
             chosen[mirror] = source
             continue
+        mirror = rival_path
         winner, loser = sorted((source, rival), key=source_rank)
         logger.warning("%s and %s both map to %s; using %s", rival, source, mirror, winner)
         chosen[mirror] = winner
@@ -327,6 +555,13 @@ def prune(mirror_root, expected, dry_run):
         mirror.unlink(missing_ok=True)
 
     if not dry_run:
+        # A cover copied for an album whose tracks have all gone is an orphan
+        # too, and while it sits there the directory never looks empty.
+        for cover in sorted(mirror_root.rglob(MIRROR_COVER)):
+            if not any(cover.parent.glob(f"*{MIRROR_SUFFIX}")):
+                logger.info("removing orphan %s", cover)
+                cover.unlink(missing_ok=True)
+
         # Deepest first, so a directory emptied by the loop above is caught.
         for directory in sorted(mirror_root.rglob("*"), reverse=True):
             if directory.is_dir() and not any(directory.iterdir()):
@@ -335,7 +570,17 @@ def prune(mirror_root, expected, dry_run):
     return removed
 
 
-def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, do_prune):
+def run_once(
+    scan_root,
+    source_root,
+    mirror_root,
+    quality_args,
+    jobs,
+    dry_run,
+    do_prune,
+    safe=False,
+    budget=0,
+):
     """Run a single pass. Returns the number of failures.
 
     `scan_root` is what gets walked and `source_root` is what mirror paths are
@@ -343,14 +588,28 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d
     """
     started = time.monotonic()
     logger.info("pass starting with %d concurrent encoders", jobs)
-    counts = {"encoded": 0, "copied": 0, "skipped": 0, "failed": 0}
+    counts = {"encoded": 0, "copied": 0, "renamed": 0, "skipped": 0, "failed": 0}
     failures = []
 
-    work = plan(scan_root, source_root, mirror_root)
+    work = plan(scan_root, source_root, mirror_root, safe, budget)
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
         futures = [
-            pool.submit(process, source, mirror, quality_args, dry_run)
+            pool.submit(
+                process,
+                source,
+                mirror,
+                quality_args,
+                dry_run,
+                (
+                    [
+                        mirror_path_for(source, source_root, mirror_root),
+                        mirror_path_for(source, source_root, mirror_root, True),
+                    ]
+                    if safe
+                    else None
+                ),
+            )
             for mirror, source in work.items()
         ]
         for future in concurrent.futures.as_completed(futures):
@@ -359,16 +618,27 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d
             if result.action == "failed":
                 failures.append(result)
 
-    removed = prune(mirror_root, set(work), dry_run) if do_prune else 0
+    expected = set(work)
+    if safe and dry_run:
+        # Nothing was actually renamed, so the pre-sanitisation files are still
+        # on disk. They are not orphans -- they are the files a real run would
+        # move -- and reporting them for deletion would misrepresent the pass
+        # twice over.
+        for source in work.values():
+            expected.add(mirror_path_for(source, source_root, mirror_root))
+            expected.add(mirror_path_for(source, source_root, mirror_root, True))
+    removed = prune(mirror_root, expected, dry_run) if do_prune else 0
 
     for failure in failures:
         logger.error("failed: %s: %s", failure.path, failure.error)
 
     logger.info(
-        "pass complete in %.1fs: %d encoded, %d copied, %d up to date, %d removed, %d failed",
+        "pass complete in %.1fs: %d encoded, %d copied, %d renamed, %d up to date,"
+        " %d removed, %d failed",
         time.monotonic() - started,
         counts["encoded"],
         counts["copied"],
+        counts["renamed"],
         counts["skipped"],
         removed,
         counts["failed"],
@@ -445,6 +715,26 @@ def build_parser():
         default=None,
         help="limit the pass to one directory below --source; skips pruning",
     )
+    parser.add_argument(
+        "--fat32-safe",
+        action="store_true",
+        default=os.getenv("MUSIC_MIRROR_FAT32_SAFE", "").lower() in ("1", "true", "yes"),
+        help="name mirror files so a FAT32 device will accept them"
+        " (env MUSIC_MIRROR_FAT32_SAFE)",
+    )
+    parser.add_argument(
+        "--max-path",
+        type=int,
+        default=int(os.getenv("MUSIC_MIRROR_MAX_PATH", str(MAX_PATH))),
+        help=f"longest path the device will take, counted from its root; Rockbox's"
+        f" MAX_PATH is {MAX_PATH} (env MUSIC_MIRROR_MAX_PATH)",
+    )
+    parser.add_argument(
+        "--device-prefix",
+        default=os.getenv("MUSIC_MIRROR_DEVICE_PREFIX", DEVICE_PREFIX),
+        help="directory the mirror is copied into on the device, whose length comes"
+        " out of the path budget (env MUSIC_MIRROR_DEVICE_PREFIX)",
+    )
     parser.add_argument(
         "--no-prune",
         action="store_true",
@@ -463,6 +753,15 @@ def main(argv=None):
     logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO)
     args = build_parser().parse_args(argv)
 
+    # Directories are created with 0o777 masked by the umask, so clear the bits
+    # that matter from it once here rather than chmod'ing every directory the
+    # walk creates. The `other` bits are left alone, since whether the mirror is
+    # world-readable is a real policy question; owner and group access is not.
+    # Files cannot be handled this way -- mkstemp and copy2 both set a mode
+    # outright, ignoring the umask -- so they get an explicit chmod instead.
+    inherited = os.umask(0o077)
+    os.umask(inherited & ~DIRECTORY_ACCESS)
+
     if not args.source or not args.mirror:
         logger.error("both --source and --mirror are required")
         return 2
@@ -494,6 +793,17 @@ def main(argv=None):
         # A partial pass cannot tell an orphan from a file outside its scope.
         do_prune = False
 
+    # The device's limit covers the whole path it will see, so what the mirror
+    # may spend is that less the directory it gets copied into.
+    budget = max(0, args.max_path - len(device_prefix_length(args.device_prefix)))
+    if args.fat32_safe:
+        logger.info(
+            "paths are limited to %d characters, from --max-path %d less the %r prefix",
+            budget,
+            args.max_path,
+            args.device_prefix,
+        )
+
     lock = acquire_lock(mirror_root)
     if lock is None:
         logger.error("another pass is already running over %s", mirror_root)
@@ -519,6 +829,8 @@ def main(argv=None):
                 args.jobs,
                 args.dry_run,
                 do_prune,
+                args.fat32_safe,
+                budget,
             )
             if interval is None or stopping:
                 return 1 if failures else 0

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "music-mirror"
-version = "0.1.1"
+version = "0.3.0"
 description = "Maintain a lossy MP3 mirror of a lossless music library"
 readme = "README.md"
 requires-python = ">=3.11"

tests/conftest.py

@@ -19,6 +19,29 @@ def require_ffmpeg():
             pytest.skip(f"{tool} is not on PATH", allow_module_level=True)
 
 
+@pytest.fixture
+def tight_umask():
+    """Run a test under a umask that would otherwise make the mirror private."""
+    previous = os.umask(0o077)
+    yield
+    os.umask(previous)
+
+
+@pytest.fixture
+def owner_hostile_umask():
+    """Return a callable applying a umask that masks off the owner's read bit.
+
+    Unusual, but it is what produces a mirror tree of mode 0300 -- writable and
+    enterable, unreadable to the very process that built it. Applied on demand
+    rather than for the whole test, because the source library is built by
+    something else entirely and the same umask would make the test's own
+    fixtures unreadable before the run under test even started.
+    """
+    previous = os.umask(0o022)
+    yield lambda: os.umask(0o477)
+    os.umask(previous)
+
+
 @pytest.fixture
 def make_flac():
     """Return a factory writing a short tagged FLAC file."""
@@ -55,6 +78,23 @@ def make_flac():
     return factory
 
 
+@pytest.fixture
+def make_cover():
+    """Return a factory writing a small JPEG beside an album's tracks."""
+
+    def factory(path):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        subprocess.run(
+            ["ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y",
+             "-f", "lavfi", "-i", "color=c=red:s=64x64:d=1", "-frames:v", "1", str(path)],
+            check=True,
+            capture_output=True,
+        )
+        return path
+
+    return factory
+
+
 @pytest.fixture
 def probe_tag():
     """Return a helper reading a single metadata tag from a file."""

tests/test_check_fat32.py

@@ -0,0 +1,64 @@
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import check_fat32  # noqa: E402
+
+
+def problems(name):
+    return check_fat32.problems_with(Path(name))
+
+
+def test_a_reserved_character_is_a_problem():
+    assert any("reserved" in p for p in problems("Album/Motherf**ker.mp3"))
+
+
+def test_a_trailing_dot_or_space_is_a_problem():
+    """FAT eats them silently, so the name round-trips as a different name."""
+    assert any("trailing" in p for p in problems("Trailing Dot./x.mp3"))
+    assert any("trailing" in p for p in problems("Space /x.mp3"))
+
+
+def test_an_ordinary_name_is_fine():
+    assert problems("Artist/Album/01 Fine Track.mp3") == []
+
+
+def test_accents_are_fine():
+    assert problems("Mötley Crüe/Album/Track.mp3") == []
+
+
+def test_an_over_long_path_is_a_problem():
+    deep = "/".join("d" * 40 for _ in range(10)) + "/track.mp3"
+    assert any("path of" in p for p in problems(deep))
+
+
+def test_case_collisions_are_reported(tmp_path):
+    album = tmp_path / "Album"
+    album.mkdir()
+    (album / "Song.mp3").write_bytes(b"x")
+    (album / "SONG.mp3").write_bytes(b"x")
+
+    completed = subprocess.run(
+        [sys.executable, str(Path(check_fat32.__file__)), str(tmp_path)],
+        capture_output=True,
+        text=True,
+    )
+
+    assert completed.returncode == 1
+    assert "collides case-insensitively" in completed.stdout
+
+
+def test_a_clean_tree_exits_zero(tmp_path):
+    (tmp_path / "Album").mkdir()
+    (tmp_path / "Album" / "Fine.mp3").write_bytes(b"x")
+
+    completed = subprocess.run(
+        [sys.executable, str(Path(check_fat32.__file__)), str(tmp_path)],
+        capture_output=True,
+        text=True,
+    )
+
+    assert completed.returncode == 0
+    assert "0 problems" in completed.stderr

tests/test_music_mirror.py

@@ -1,7 +1,9 @@
 import os
 import shutil
+import stat
 import subprocess
 import time
+from pathlib import Path
 
 import pytest
 
@@ -153,6 +155,126 @@ def test_existing_mp3_is_copied_not_re_encoded(tmp_path, make_flac):
     assert (mirror / "b.mp3").read_bytes() == (source / "b.mp3").read_bytes()
 
 
+def test_interrupted_copy_leaves_nothing_behind(tmp_path, make_flac, monkeypatch):
+    """copy2 reproduces the source mtime, so a truncated copy left in the mirror
+    would be read as current by every later pass."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    flac = make_flac(source / "a.flac")
+    subprocess.run(
+        ["ffmpeg", "-loglevel", "error", "-y", "-i", str(flac), str(source / "b.mp3")],
+        check=True,
+        capture_output=True,
+    )
+    flac.unlink()
+
+    def truncated(src, destination, **kwargs):
+        Path(destination).write_bytes(Path(src).read_bytes()[:64])
+        raise OSError("no space left on device")
+
+    monkeypatch.setattr(music_mirror.shutil, "copy2", truncated)
+
+    assert run(source, mirror) == 1
+    assert not (mirror / "b.mp3").exists()
+    assert list(mirror.rglob("*.part")) == []
+
+
+def test_encoded_file_is_group_readable(tmp_path, make_flac, tight_umask):
+    """mkstemp creates 0600 whatever the umask, so the bit has to be added."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    run(source, mirror)
+
+    assert (mirror / "a.mp3").stat().st_mode & stat.S_IRGRP
+
+
+def test_copied_file_is_group_readable(tmp_path, make_flac, tight_umask):
+    """copy2 carries the source's mode across, and the source may be private."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    flac = make_flac(source / "a.flac")
+    subprocess.run(
+        ["ffmpeg", "-loglevel", "error", "-y", "-i", str(flac), str(source / "b.mp3")],
+        check=True,
+        capture_output=True,
+    )
+    flac.unlink()
+    (source / "b.mp3").chmod(0o600)
+
+    run(source, mirror)
+
+    assert (mirror / "b.mp3").stat().st_mode & stat.S_IRGRP
+
+
+def test_mirror_directories_are_group_traversable(tmp_path, make_flac, tight_umask):
+    """A readable file is unreachable if the group cannot enter its directory."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Artist" / "Album" / "a.flac")
+
+    run(source, mirror)
+
+    for directory in (mirror, mirror / "Artist", mirror / "Artist" / "Album"):
+        mode = directory.stat().st_mode
+        assert mode & stat.S_IRGRP, directory
+        assert mode & stat.S_IXGRP, directory
+
+
+def test_mirror_directories_survive_an_owner_hostile_umask(
+    tmp_path, make_flac, owner_hostile_umask
+):
+    """A umask carrying 0400 otherwise builds a tree the run cannot read back."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Artist" / "Album" / "a.flac")
+
+    # Applied only now: the library already exists, and the umask under test is
+    # the one the container starts this run with.
+    owner_hostile_umask()
+    run(source, mirror)
+
+    for directory in (mirror, mirror / "Artist", mirror / "Artist" / "Album"):
+        mode = directory.stat().st_mode
+        assert mode & stat.S_IRUSR, directory
+        assert mode & stat.S_IXUSR, directory
+        assert mode & stat.S_IRGRP, directory
+        assert mode & stat.S_IXGRP, directory
+
+
+def test_private_mirror_file_is_repaired_without_re_encoding(tmp_path, make_flac):
+    """A mirror written by an older version has a correct mtime, so nothing
+    else in the pass would revisit it."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    run(source, mirror)
+    output = mirror / "a.mp3"
+    output.chmod(output.stat().st_mode & ~stat.S_IRGRP)
+    before = output.stat().st_mtime_ns
+
+    run(source, mirror)
+
+    assert output.stat().st_mode & stat.S_IRGRP
+    assert output.stat().st_mtime_ns == before
+
+
+def test_dry_run_does_not_change_permissions(tmp_path, make_flac):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    run(source, mirror)
+    output = mirror / "a.mp3"
+    output.chmod(output.stat().st_mode & ~stat.S_IRGRP)
+
+    run(source, mirror, "--dry-run")
+
+    assert not output.stat().st_mode & stat.S_IRGRP
+
+
 def test_format_upgrade_replaces_rather_than_duplicating(tmp_path, make_flac):
     """Lidarr replacing an MP3 with a FLAC must not leave two mirror files."""
     source = tmp_path / "src"
@@ -321,3 +443,346 @@ def test_mirror_inside_source_is_refused(tmp_path, make_flac):
 
 def test_missing_source_is_refused(tmp_path):
     assert run(tmp_path / "nope", tmp_path / "dst") == 2
+
+
+@pytest.mark.parametrize(
+    ("name", "expected"),
+    [
+        ("Kick Out the Epic Motherf**ker", "Kick Out the Epic Motherf__ker"),
+        ("Where Are You?", "Where Are You_"),
+        ("Song: Part 2", "Song_ Part 2"),
+        ('"Heroes"', "_Heroes_"),
+        ("trailing dot.", "trailing dot"),
+        ("trailing space ", "trailing space"),
+        ("...", "_"),
+        ("Mötley Crüe", "Mötley Crüe"),
+        ("perfectly ordinary", "perfectly ordinary"),
+    ],
+)
+def test_fat32_safe_names(name, expected):
+    """A name FAT32 will not take is a track that silently does not arrive."""
+    assert music_mirror.fat32_safe(name) == expected
+
+
+def test_a_reserved_character_is_replaced_in_the_mirror_path(tmp_path, make_flac):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Dada Life" / "Album" / "Kick Out the Epic Motherf**ker.flac")
+
+    run(source, mirror, "--fat32-safe")
+
+    assert (mirror / "Dada Life" / "Album" / "Kick Out the Epic Motherf__ker.mp3").is_file()
+
+
+def test_without_the_flag_the_name_is_left_alone(tmp_path, make_flac):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Where Are You?.flac")
+
+    run(source, mirror)
+
+    assert (mirror / "Album" / "Where Are You?.mp3").is_file()
+
+
+def test_an_existing_mirror_is_renamed_not_re_encoded(tmp_path, make_flac):
+    """Turning the flag on changes the path of every track holding a reserved
+    character. Re-encoding those would be hours of work to produce files that
+    already exist byte for byte."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Where Are You?.flac")
+
+    run(source, mirror)
+    before = mirror / "Album" / "Where Are You?.mp3"
+    contents = before.read_bytes()
+    stamp = before.stat().st_mtime_ns
+
+    run(source, mirror, "--fat32-safe")
+
+    after = mirror / "Album" / "Where Are You_.mp3"
+    assert after.is_file()
+    assert not before.exists()
+    assert after.read_bytes() == contents, "it was re-encoded rather than moved"
+    assert after.stat().st_mtime_ns == stamp
+
+
+def test_names_colliding_only_by_case_are_caught(tmp_path, make_flac):
+    """Two files here, one file on FAT32. Better found now than as a silent
+    overwrite partway through the copy."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Song.flac")
+    make_flac(source / "Album" / "SONG.flac")
+
+    run(source, mirror, "--fat32-safe")
+
+    written = sorted(p.name for p in (mirror / "Album").glob("*.mp3"))
+    assert len(written) == 1, written
+
+
+def test_the_album_cover_is_copied_beside_the_tracks(tmp_path, make_flac, make_cover):
+    """Rockbox looks for art on the filesystem; its search never touches the
+    picture embedded in the tag."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    make_cover(source / "Album" / "cover.jpg")
+
+    run(source, mirror)
+
+    assert (mirror / "Album" / "cover.jpg").is_file()
+
+
+def test_a_copied_cover_is_group_readable(tmp_path, make_flac, make_cover, tight_umask):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "a.flac")
+    make_cover(source / "Album" / "cover.jpg")
+
+    run(source, mirror)
+
+    assert (mirror / "Album" / "cover.jpg").stat().st_mode & stat.S_IRGRP
+
+
+def test_a_cover_left_without_tracks_is_pruned(tmp_path, make_flac, make_cover):
+    """Otherwise the directory never looks empty and never goes."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Gone" / "a.flac")
+    make_cover(source / "Gone" / "cover.jpg")
+
+    run(source, mirror)
+    assert (mirror / "Gone" / "cover.jpg").is_file()
+
+    shutil.rmtree(source / "Gone")
+    run(source, mirror)
+
+    assert not (mirror / "Gone").exists()
+
+
+def test_a_dry_run_reports_a_rename_not_an_encode(tmp_path, make_flac, caplog):
+    """The difference between moving a file and re-encoding it is the
+    difference between a minute and an afternoon, so a dry run must not
+    describe the first as the second."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Where Are You?.flac")
+    run(source, mirror)
+
+    with caplog.at_level("INFO"):
+        run(source, mirror, "--fat32-safe", "--dry-run")
+
+    assert "would rename" in caplog.text
+    assert "would encode" not in caplog.text
+
+
+def test_a_dry_run_does_not_call_the_old_paths_orphans(tmp_path, make_flac, caplog):
+    """Nothing was renamed, so they are still there -- but they are the files a
+    real run would move, not files it would delete."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Where Are You?.flac")
+    run(source, mirror)
+
+    with caplog.at_level("INFO"):
+        run(source, mirror, "--fat32-safe", "--dry-run")
+
+    assert "would remove orphan" not in caplog.text
+
+
+def test_a_dry_run_moves_nothing(tmp_path, make_flac):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Where Are You?.flac")
+    run(source, mirror)
+
+    run(source, mirror, "--fat32-safe", "--dry-run")
+
+    assert (mirror / "Album" / "Where Are You?.mp3").is_file()
+    assert not (mirror / "Album" / "Where Are You_.mp3").exists()
+
+
+def test_renames_are_counted_separately_from_encodes(tmp_path, make_flac, caplog):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / "Where Are You?.flac")
+    run(source, mirror)
+
+    with caplog.at_level("INFO"):
+        run(source, mirror, "--fat32-safe")
+
+    assert "1 renamed" in caplog.text
+    assert "0 encoded" in caplog.text
+
+
+def test_a_long_path_is_shortened_from_the_deepest_component(tmp_path, make_flac):
+    """The track name carries the least navigational value and the artist the
+    most, so the filename is sacrificed before the album."""
+    artist = "A" * 60
+    album = "B" * 60
+    title = "C" * 150
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / artist / album / f"{title}.flac")
+
+    run(source, mirror, "--fat32-safe", "--max-path", "160", "--device-prefix", "/Music")
+
+    written = list(mirror.rglob("*.mp3"))
+    assert len(written) == 1
+    relative = written[0].relative_to(mirror)
+    assert relative.parts[0] == artist, "the artist directory should be untouched"
+    assert relative.parts[1] == album, "the album directory should be untouched"
+    assert len(str(relative)) <= 160 - len("Music") - 2
+
+
+def test_shortening_is_stable_across_passes(tmp_path, make_flac):
+    """An unstable name would rename every file on every pass, for ever."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / ("D" * 80) / ("E" * 80) / f"{'F' * 120}.flac")
+
+    run(source, mirror, "--fat32-safe", "--max-path", "180")
+    first = sorted(str(p.relative_to(mirror)) for p in mirror.rglob("*.mp3"))
+    stamp = next(mirror.rglob("*.mp3")).stat().st_mtime_ns
+
+    run(source, mirror, "--fat32-safe", "--max-path", "180")
+
+    assert sorted(str(p.relative_to(mirror)) for p in mirror.rglob("*.mp3")) == first
+    assert next(mirror.rglob("*.mp3")).stat().st_mtime_ns == stamp
+
+
+def test_two_long_names_do_not_collide_after_shortening(tmp_path, make_flac):
+    """They share a prefix and cut to the same string; the hash is what keeps
+    them apart."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    shared = "G" * 140
+    make_flac(source / "Album" / f"{shared}one.flac")
+    make_flac(source / "Album" / f"{shared}two.flac")
+
+    run(source, mirror, "--fat32-safe", "--max-path", "120")
+
+    assert len(list(mirror.rglob("*.mp3"))) == 2
+
+
+def test_a_sanitised_mirror_is_renamed_rather_than_re_encoded_when_shortening(
+    tmp_path, make_flac
+):
+    """The previous naming is sanitised-but-not-shortened, not the original."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Album" / f"Where Are You? {'H' * 140}.flac")
+
+    run(source, mirror, "--fat32-safe", "--max-path", "400")
+    before = next(mirror.rglob("*.mp3"))
+    contents = before.read_bytes()
+
+    run(source, mirror, "--fat32-safe", "--max-path", "120")
+
+    after = next(mirror.rglob("*.mp3"))
+    assert after != before
+    assert after.read_bytes() == contents, "it was re-encoded rather than moved"
+
+
+def test_shortening_only_applies_when_over_budget(tmp_path, make_flac):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Artist" / "Album" / "Short Name.flac")
+
+    run(source, mirror, "--fat32-safe")
+
+    assert (mirror / "Artist" / "Album" / "Short Name.mp3").is_file()
+
+
+def test_a_path_that_cannot_be_made_to_fit_is_reported(tmp_path, make_flac, caplog):
+    """Too deeply nested to shorten without making every component unreadable."""
+    deep = Path(*["I" * 20 for _ in range(10)])
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / deep / "track.flac")
+
+    with caplog.at_level("WARNING"):
+        run(source, mirror, "--fat32-safe", "--max-path", "80")
+
+    assert "too" in caplog.text and "nested" in caplog.text
+
+
+# Lidarr writes "Artist - Album - NN - Title.mp3" inside a directory already
+# named for that artist and album, so a long album title appears three times in
+# one path. These are real paths from a real library.
+GIZZARD_ALBUM = (
+    "PetroDragonic Apocalypse; or, Dawn of Eternal Night - An Annihilation of"
+    " Planet Earth and the Beginning of Merciless Damnation"
+)
+GIZZARD_TRACKS = (
+    "01 - Motor Spirit",
+    "02 - Supercell",
+    "03 - Converge",
+    "04 - Witchcraft",
+    "05 - Gila Monster",
+    "06 - Dragon",
+    "07 - Flamethrower",
+)
+
+
+def gizzard_path(track):
+    return Path(
+        "King Gizzard & the Lizard Wizard",
+        f"{GIZZARD_ALBUM} (2023)",
+        f"King Gizzard & the Lizard Wizard - {GIZZARD_ALBUM} - {track}.mp3",
+    )
+
+
+def test_shortening_keeps_the_part_that_tells_tracks_apart():
+    """Cutting from the end discards the track number and title, which is all
+    that distinguishes one track on the record from another."""
+    for track in GIZZARD_TRACKS:
+        fitted = music_mirror.fit_path(gizzard_path(track), 253)
+        assert len(str(fitted)) <= 253
+        assert fitted.name.endswith(f"{track}.mp3"), fitted.name
+
+
+def test_every_track_on_a_long_album_keeps_a_distinct_name():
+    fitted = {music_mirror.fit_path(gizzard_path(t), 253).name for t in GIZZARD_TRACKS}
+
+    assert len(fitted) == len(GIZZARD_TRACKS)
+
+
+def test_a_long_title_keeps_both_ends():
+    """The Beatles' 'The Long One' is one track whose title is the long part."""
+    path = Path(
+        "The Beatles",
+        "Abbey Road (1969)",
+        "Digital Media 03",
+        "The Beatles - Abbey Road - 09 - The Long One - You Never Give Me Your Money"
+        " + Sun King + Mean Mr Mustard + Her Majesty + Polythene Pam + She Came In"
+        " Through the Bathroom Window+ Golden Slumbers + Carry That Weight + The End.mp3",
+    )
+
+    fitted = music_mirror.fit_path(path, 253)
+
+    assert len(str(fitted)) <= 253
+    assert fitted.name.startswith("The Beatles - Abbey Road - 09 - The Long One")
+    assert fitted.name.endswith("The End.mp3")
+
+
+@pytest.mark.parametrize(
+    ("prefix", "expected"),
+    [("/Music", 253), ("Music", 253), ("/Music/", 253), ("", 259), ("/", 259)],
+)
+def test_the_device_prefix_is_costed_exactly(prefix, expected):
+    """A mirror-relative path of 253 characters becomes 260 on the device once
+    /Music/ is in front of it, which is the whole of the limit. Approximating
+    the prefix loses a character at the root, where the longest paths are."""
+    assert 260 - len(music_mirror.device_prefix_length(prefix)) == expected
+
+
+def test_the_budget_is_reported_so_it_can_be_checked(tmp_path, make_flac, caplog):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    with caplog.at_level("INFO"):
+        run(source, mirror, "--fat32-safe")
+
+    assert "limited to 253 characters" in caplog.text

tests/test_submit_scrobbles.py

@@ -0,0 +1,189 @@
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools"))
+
+import submit_scrobbles  # noqa: E402
+
+LOG = """#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox ipodvideo 4.0
+#ARTIST\t#ALBUM\t#TITLE\t#TRACKNUM\t#LENGTH\t#RATING\t#TIMESTAMP\t#MUSICBRAINZ_TRACKID
+Pendulum\tImmersion\tWatercolour\t3\t245\tL\t1700000300\t
+Green Day\tDookie\tBasket Case\t7\t180\tS\t1700000200\t
+Mötley Crüe\tDr. Feelgood\tKickstart My Heart\t2\t283\tL\t1700000100\tmb-1
+"""
+
+
+def test_only_listened_tracks_are_submitted():
+    """A skip is not a play."""
+    played, skipped, timeless = submit_scrobbles.parse_log(LOG)
+
+    assert [entry["track"] for entry in played] == ["Kickstart My Heart", "Watercolour"]
+    assert skipped == 1
+    assert timeless == 0
+
+
+def test_entries_come_back_oldest_first():
+    played, _, _ = submit_scrobbles.parse_log(LOG)
+
+    assert [entry["timestamp"] for entry in played] == ["1700000100", "1700000300"]
+
+
+def test_a_timeless_log_is_counted_and_not_submitted():
+    """Without a real-time clock Rockbox writes every timestamp as zero. Those
+    cannot be scrobbled without inventing when they happened."""
+    log = LOG + "Band\tAlbum\tTrack\t-1\t100\tL\t0\t\n"
+
+    played, _, timeless = submit_scrobbles.parse_log(log)
+
+    assert timeless == 1
+    assert all(int(entry["timestamp"]) > 0 for entry in played)
+
+
+def test_absent_optional_fields_are_dropped():
+    played, _, _ = submit_scrobbles.parse_log(LOG)
+    params = submit_scrobbles.batch_params(played)
+
+    assert "mbid[0]" in params  # Kickstart My Heart has one
+    assert "mbid[1]" not in params  # Watercolour does not
+    assert params["trackNumber[0]"] == "2"
+
+
+def test_a_track_number_of_minus_one_is_not_sent():
+    """Rockbox writes -1 when it does not know, which is not a track number."""
+    played, _, _ = submit_scrobbles.parse_log(
+        "Band\tAlbum\tTrack\t-1\t100\tL\t1700000000\t\n"
+    )
+
+    assert submit_scrobbles.batch_params(played).get("trackNumber[0]") is None
+
+
+def test_the_signature_sorts_names_by_ascii_not_by_number():
+    """Last.fm sorts parameter names as strings, so artist[10] precedes
+    artist[1]. Sorting numerically produces an invalid signature and nothing
+    else."""
+    params = {"artist[1]": "b", "artist[10]": "a", "api_key": "k"}
+
+    expected = submit_scrobbles.hashlib.md5(
+        ("api_keyk" + "artist[1]b" + "artist[10]a" + "s").encode()
+    ).hexdigest()
+    assert submit_scrobbles.sign(params, "s") != expected
+
+    correct = submit_scrobbles.hashlib.md5(
+        ("api_keyk" + "artist[10]a" + "artist[1]b" + "s").encode()
+    ).hexdigest()
+    assert submit_scrobbles.sign(params, "s") == correct
+
+
+def fake_transport(responses):
+    """Return a transport serving canned responses and recording requests."""
+    sent = []
+
+    def transport(request, timeout=None):
+        sent.append(dict(submit_scrobbles.urllib.parse.parse_qsl(request.data.decode())))
+        return json.dumps(responses[len(sent) - 1])
+
+    transport.sent = sent
+    return transport
+
+
+def test_scrobbles_are_sent_in_batches_of_fifty():
+    entries = [
+        {"artist": "A", "track": f"T{i}", "timestamp": str(1700000000 + i)}
+        for i in range(120)
+    ]
+    accepted = {"scrobbles": {"@attr": {"accepted": 50, "ignored": 0}}}
+    transport = fake_transport([accepted, accepted, accepted])
+
+    submit_scrobbles.submit(entries, "k", "s", "sk", transport, delay=0)
+
+    assert len(transport.sent) == 3
+    assert transport.sent[0]["method"] == "track.scrobble"
+    assert "artist[49]" in transport.sent[0]
+    assert "artist[50]" not in transport.sent[0]
+
+
+def test_every_request_carries_a_signature_and_session():
+    entries = [{"artist": "A", "track": "T", "timestamp": "1700000000"}]
+    transport = fake_transport([{"scrobbles": {"@attr": {"accepted": 1, "ignored": 0}}}])
+
+    submit_scrobbles.submit(entries, "k", "s", "session-key", transport, delay=0)
+
+    assert transport.sent[0]["sk"] == "session-key"
+    assert len(transport.sent[0]["api_sig"]) == 32
+
+
+def test_a_service_error_is_raised_not_swallowed():
+    entries = [{"artist": "A", "track": "T", "timestamp": "1700000000"}]
+    transport = fake_transport([{"error": 9, "message": "Invalid session key"}])
+
+    with pytest.raises(submit_scrobbles.LastfmError, match="error 9"):
+        submit_scrobbles.submit(entries, "k", "s", "sk", transport, delay=0)
+
+
+def test_the_log_is_set_aside_after_a_successful_submission(tmp_path, monkeypatch):
+    device = tmp_path / "IPOD"
+    device.mkdir()
+    (device / ".scrobbler.log").write_text(LOG, encoding="utf-8")
+    transport = fake_transport([{"scrobbles": {"@attr": {"accepted": 2, "ignored": 0}}}])
+    monkeypatch.setattr(submit_scrobbles, "load_session", lambda: "sk")
+
+    submit_scrobbles.main(
+        [str(device), "--api-key", "k", "--api-secret", "s"], transport=transport
+    )
+
+    assert not (device / ".scrobbler.log").exists()
+    # Renamed, not deleted: if Last.fm quietly dropped one, the evidence remains.
+    assert list(device.glob(".scrobbler.log.*.submitted"))
+
+
+def test_a_failed_submission_leaves_the_log_alone(tmp_path, monkeypatch):
+    device = tmp_path / "IPOD"
+    device.mkdir()
+    (device / ".scrobbler.log").write_text(LOG, encoding="utf-8")
+    transport = fake_transport([{"error": 29, "message": "Rate limit"}])
+    monkeypatch.setattr(submit_scrobbles, "load_session", lambda: "sk")
+
+    code = submit_scrobbles.main(
+        [str(device), "--api-key", "k", "--api-secret", "s"], transport=transport
+    )
+
+    assert code == 1
+    assert (device / ".scrobbler.log").is_file()
+
+
+def test_a_dry_run_submits_nothing(tmp_path):
+    device = tmp_path / "IPOD"
+    device.mkdir()
+    (device / ".scrobbler.log").write_text(LOG, encoding="utf-8")
+    transport = fake_transport([])
+
+    submit_scrobbles.main([str(device), "--dry-run"], transport=transport)
+
+    assert transport.sent == []
+    assert (device / ".scrobbler.log").is_file()
+
+
+def test_no_log_is_not_an_error(tmp_path):
+    device = tmp_path / "IPOD"
+    device.mkdir()
+
+    assert submit_scrobbles.main([str(device)], transport=fake_transport([])) == 0
+
+
+def test_write_credentials_are_required(tmp_path, capsys, monkeypatch):
+    """The read-only key used elsewhere is not enough for a write method."""
+    monkeypatch.delenv("LASTFM_API_KEY", raising=False)
+    monkeypatch.delenv("LASTFM_API_SECRET", raising=False)
+    device = tmp_path / "IPOD"
+    device.mkdir()
+    (device / ".scrobbler.log").write_text(LOG, encoding="utf-8")
+
+    code = submit_scrobbles.main([str(device)], transport=fake_transport([]))
+
+    assert code == 2
+    assert "LASTFM_API_SECRET" in capsys.readouterr().err

tools/check_fat32.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Report paths a FAT32 device will not accept, before copying rather than during.
+
+Run this against the mirror before an rsync to a Rockbox iPod. rsync will
+report the failures too, but scattered through a run of fifty thousand files,
+where they are easy to lose.
+
+Checks the four ways a name fails on FAT32: reserved characters, trailing dots
+or spaces that FAT silently eats, components longer than 255 characters, and
+names that differ only in case -- two files here, one file there, and the
+second silently overwrites the first.
+"""
+
+import argparse
+import os
+import re
+import sys
+import unicodedata
+from collections import defaultdict
+from pathlib import Path
+
+RESERVED = re.compile(r'[<>:"\\|?*\x00-\x1f]')
+COMPONENT_LIMIT = 255
+# Rockbox's MAX_PATH, from firmware/include/fs_defines.h. It bounds the path as
+# the device sees it, so the directory the mirror is copied into comes out of
+# the same budget.
+PATH_LIMIT = 260
+DEVICE_PREFIX = "/Music"
+
+
+def device_prefix_length(prefix):
+    """Return the on-device prefix as it will actually appear, with slashes.
+
+    "/Music" costs seven characters -- the leading slash, the name, and the
+    separator before the mirror's own path -- while an empty prefix costs one.
+    Approximating that loses a character at the root, which is precisely where
+    the longest paths are.
+    """
+    cleaned = prefix.strip("/")
+    return f"/{cleaned}/" if cleaned else "/"
+
+
+def problems_with(relative, budget=PATH_LIMIT):
+    """Return every reason this relative path is unfit for FAT32."""
+    found = []
+    for part in relative.parts:
+        if RESERVED.search(part):
+            found.append(f"reserved character in {part!r}")
+        if part != part.rstrip(". "):
+            found.append(f"trailing dot or space in {part!r}")
+        if len(part) > COMPONENT_LIMIT:
+            found.append(f"component of {len(part)} characters")
+    if len(str(relative)) > budget:
+        found.append(f"path of {len(str(relative))} characters, over a budget of {budget}")
+    return found
+
+
+def walk(root):
+    """Yield every file below a root, as a path relative to it."""
+    for base, _, names in os.walk(root):
+        for name in names:
+            yield Path(os.path.join(base, name)).relative_to(root)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("root", help="directory to check, e.g. the mirror")
+    parser.add_argument("--limit", type=int, default=0, help="show at most this many")
+    parser.add_argument(
+        "--max-path",
+        type=int,
+        default=PATH_LIMIT,
+        help=f"longest path the device will take, from its root (default {PATH_LIMIT},"
+        " Rockbox's MAX_PATH)",
+    )
+    parser.add_argument(
+        "--device-prefix",
+        default=DEVICE_PREFIX,
+        help="directory the mirror is copied into on the device; its length comes out"
+        f" of the budget (default {DEVICE_PREFIX})",
+    )
+    args = parser.parse_args(argv)
+    budget = max(0, args.max_path - len(device_prefix_length(args.device_prefix)))
+
+    root = Path(args.root)
+    if not root.is_dir():
+        print(f"{root} is not a directory", file=sys.stderr)
+        return 2
+
+    faults = []
+    by_case = defaultdict(list)
+    total = 0
+    for relative in walk(root):
+        total += 1
+        # NFC first: the same name written by two systems is otherwise two
+        # different strings, and the collision check would miss it.
+        key = unicodedata.normalize("NFC", str(relative)).casefold()
+        by_case[key].append(relative)
+        for problem in problems_with(relative, budget):
+            faults.append((relative, problem))
+
+    for relative, group in sorted(by_case.items()):
+        if len(group) > 1:
+            names = ", ".join(str(path) for path in sorted(group))
+            faults.append((group[0], f"collides case-insensitively with: {names}"))
+
+    for relative, problem in faults[: args.limit or None]:
+        print(f"{relative}\t{problem}")
+
+    print(f"\n{len(faults)} problems across {total} files", file=sys.stderr)
+    if faults:
+        print(
+            "Run music-mirror with --fat32-safe to have the mirror named acceptably"
+            " in the first place; it shortens over-long paths as well.",
+            file=sys.stderr,
+        )
+    return 1 if faults else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/submit_scrobbles.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""Submit a Rockbox scrobbler log to Last.fm, then set it aside.
+
+Rockbox writes /.scrobbler.log on the device in AUDIOSCROBBLER 1.1 format: one
+tab-separated line per track, rated `L` for listened or `S` for skipped. Only
+the listened ones are submitted; a skip is not a play.
+
+Scrobbling is a write method, so unlike everything else here it needs the API
+secret and a session key, obtained once through the browser. Read-only calls
+elsewhere in these projects need neither.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+API_ROOT = "https://ws.audioscrobbler.com/2.0/"
+
+# Last.fm's documented ceiling for one track.scrobble call.
+BATCH = 50
+
+# Rockbox names the log for whether the target has a real-time clock. Without
+# one every timestamp it writes is zero, which is not a time anything can
+# scrobble.
+LOG_NAMES = (".scrobbler.log", ".scrobbler-timeless.log")
+
+SESSION_FILE = Path(
+    os.getenv("XDG_CONFIG_HOME", Path.home() / ".config")
+) / "music-mirror" / "lastfm.json"
+
+
+class LastfmError(Exception):
+    """A Last.fm request that failed."""
+
+
+def parse_log(text):
+    """Return the listened tracks in an AUDIOSCROBBLER log, oldest first.
+
+    Fields are artist, album, title, track number, length, rating, timestamp
+    and MusicBrainz id. Rockbox converts any tab inside a field to a space
+    before writing, so splitting on tabs is safe.
+    """
+    played, skipped, timeless = [], 0, 0
+    for line in text.splitlines():
+        if not line or line.startswith("#"):
+            continue
+        fields = line.split("\t")
+        if len(fields) < 7:
+            continue
+        artist, album, title, number, length, rating, timestamp = fields[:7]
+        mbid = fields[7] if len(fields) > 7 else ""
+        if rating.strip().upper() != "L":
+            skipped += 1
+            continue
+        try:
+            when = int(timestamp)
+        except ValueError:
+            continue
+        if when <= 0:
+            timeless += 1
+            continue
+        if not artist or not title:
+            continue
+        played.append(
+            {
+                "artist": artist,
+                "album": album,
+                "track": title,
+                "trackNumber": number if number not in ("", "-1") else "",
+                "duration": length if length.isdigit() and int(length) > 0 else "",
+                "timestamp": str(when),
+                "mbid": mbid,
+            }
+        )
+    played.sort(key=lambda entry: int(entry["timestamp"]))
+    return played, skipped, timeless
+
+
+def sign(params, secret):
+    """Return Last.fm's method signature for a set of parameters.
+
+    Names are sorted by the ASCII table rather than numerically, which is why
+    `artist[10]` comes before `artist[1]`. Getting that wrong produces an
+    invalid signature and nothing else.
+    """
+    joined = "".join(f"{name}{params[name]}" for name in sorted(params))
+    return hashlib.md5((joined + secret).encode("utf-8")).hexdigest()  # noqa: S324
+
+
+def post(params, transport):
+    """Sign, post, and return the decoded response."""
+    body = urllib.parse.urlencode(params).encode("utf-8")
+    request = urllib.request.Request(API_ROOT, data=body)
+    try:
+        payload = json.loads(transport(request))
+    except urllib.error.HTTPError as error:
+        detail = error.read().decode("utf-8", "replace")[:300]
+        raise LastfmError(f"HTTP {error.code}: {detail}") from error
+    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error:
+        raise LastfmError(str(error)) from error
+    if payload.get("error"):
+        raise LastfmError(f"error {payload['error']}: {payload.get('message', '')}")
+    return payload
+
+
+def call(method, params, key, secret, session, transport):
+    """Make one signed, authenticated call."""
+    full = {**params, "method": method, "api_key": key}
+    if session:
+        full["sk"] = session
+    full["api_sig"] = sign(full, secret)
+    full["format"] = "json"
+    return post(full, transport)
+
+
+def authorise(key, secret, transport, opener=print):
+    """Walk the one-time browser authorisation and return a session key."""
+    token = call("auth.getToken", {}, key, secret, None, transport)["token"]
+    url = f"https://www.last.fm/api/auth/?api_key={key}&token={token}"
+    opener(f"Open this, approve the application, then press Enter:\n\n  {url}\n")
+    input()
+    session = call("auth.getSession", {"token": token}, key, secret, None, transport)
+    return session["session"]["key"]
+
+
+def load_session():
+    if SESSION_FILE.is_file():
+        return json.loads(SESSION_FILE.read_text(encoding="utf-8")).get("session")
+    return None
+
+
+def save_session(session):
+    SESSION_FILE.parent.mkdir(parents=True, exist_ok=True)
+    SESSION_FILE.write_text(json.dumps({"session": session}), encoding="utf-8")
+    SESSION_FILE.chmod(0o600)
+
+
+def batch_params(entries):
+    """Return the indexed parameters for one track.scrobble call."""
+    params = {}
+    for index, entry in enumerate(entries):
+        for name in ("artist", "track", "timestamp", "album", "trackNumber", "duration", "mbid"):
+            if entry.get(name):
+                params[f"{name}[{index}]"] = entry[name]
+    return params
+
+
+def submit(entries, key, secret, session, transport, delay=1.0):
+    """Submit every entry. Returns how many the service accepted.
+
+    Batches are counted as they succeed rather than at the end, so a failure
+    partway through leaves an honest number and the caller can keep the rest of
+    the log instead of losing it.
+    """
+    accepted = 0
+    for start in range(0, len(entries), BATCH):
+        chunk = entries[start : start + BATCH]
+        payload = call(
+            "track.scrobble", batch_params(chunk), key, secret, session, transport
+        )
+        block = payload.get("scrobbles", {})
+        summary = block.get("@attr", block)
+        accepted += int(summary.get("accepted", len(chunk)))
+        ignored = int(summary.get("ignored", 0))
+        if ignored:
+            print(f"  {ignored} of {len(chunk)} ignored by Last.fm", file=sys.stderr)
+        if start + BATCH < len(entries):
+            time.sleep(delay)
+    return accepted
+
+
+def http_post(request, timeout=30):
+    with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+        return response.read().decode("utf-8")
+
+
+def find_log(device):
+    """Return the scrobbler log on a mounted device, or None."""
+    for name in LOG_NAMES:
+        candidate = Path(device) / name
+        if candidate.is_file() and candidate.stat().st_size:
+            return candidate
+    return None
+
+
+def main(argv=None, transport=http_post):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("device", help="the mounted device, or a scrobbler log file")
+    parser.add_argument("--api-key", default=os.getenv("LASTFM_API_KEY"))
+    parser.add_argument("--api-secret", default=os.getenv("LASTFM_API_SECRET"))
+    parser.add_argument("--dry-run", action="store_true", help="parse and report only")
+    parser.add_argument(
+        "--keep", action="store_true", help="do not set the log aside afterwards"
+    )
+    args = parser.parse_args(argv)
+
+    target = Path(args.device)
+    log = target if target.is_file() else find_log(target)
+    if log is None:
+        print("no scrobbler log to submit", file=sys.stderr)
+        return 0
+
+    played, skipped, timeless = parse_log(log.read_text(encoding="utf-8", errors="replace"))
+    print(f"{log}: {len(played)} listened, {skipped} skipped", file=sys.stderr)
+    if timeless:
+        print(
+            f"  {timeless} entries have no timestamp, so this target has no clock."
+            " They cannot be scrobbled without inventing when they happened.",
+            file=sys.stderr,
+        )
+    if not played:
+        return 0
+    if args.dry_run:
+        for entry in played[:20]:
+            print(f"{entry['timestamp']}\t{entry['artist']}\t{entry['track']}")
+        return 0
+
+    if not args.api_key or not args.api_secret:
+        print(
+            "scrobbling is a write method: it needs LASTFM_API_KEY and"
+            " LASTFM_API_SECRET, not just the read-only key",
+            file=sys.stderr,
+        )
+        return 2
+
+    session = load_session()
+    if not session:
+        session = authorise(args.api_key, args.api_secret, transport)
+        save_session(session)
+
+    try:
+        accepted = submit(played, args.api_key, args.api_secret, session, transport)
+    except LastfmError as error:
+        print(f"submission failed: {error}", file=sys.stderr)
+        return 1
+
+    print(f"{accepted} scrobbles accepted", file=sys.stderr)
+    if not args.keep and accepted:
+        # Renamed rather than deleted: if Last.fm quietly dropped something,
+        # the evidence is still on the device.
+        aside = log.with_name(f"{log.name}.{played[-1]['timestamp']}.submitted")
+        log.rename(aside)
+        print(f"log moved to {aside.name}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/sync-to-ipod.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# Copy the mirror onto a Rockbox device, then unmount it cleanly.
+#
+# The device is FAT32 with no journal, reached through the Apple firmware's
+# disk mode because Rockbox's own mass storage is unreliable on an iFlash. An
+# interrupted write is corruption that needs fsck.vfat from another machine, so
+# this syncs and unmounts rather than leaving that to whoever pulls the cable.
+#
+# rsync --delete is pointed at a whole filesystem, so the checks below are the
+# point of the script rather than decoration.
+
+set -euo pipefail
+
+usage() {
+    cat >&2 <<'USAGE'
+usage: sync-to-ipod.sh [options] <mirror> <destination>
+
+  -n  dry run; show what would change and touch nothing
+  -f  copy even if the FAT32 check finds unacceptable paths
+  -S  skip submitting the Rockbox scrobbler log to Last.fm
+  -U  leave the destination mounted afterwards
+
+Submitting scrobbles needs LASTFM_API_KEY and LASTFM_API_SECRET; it is skipped
+with a note when they are unset. Scrobbling is a write method and needs the
+secret, unlike the read-only calls elsewhere in these projects.
+
+The destination must be a mounted FAT filesystem. Reach it with the Apple
+firmware's disk mode: Menu+Select to reboot, then immediately Select+Play.
+USAGE
+    exit 2
+}
+
+dry_run=false
+force=false
+unmount=true
+scrobble=true
+while getopts ":nfSUh" option; do
+    case "$option" in
+        n) dry_run=true ;;
+        f) force=true ;;
+        S) scrobble=false ;;
+        U) unmount=false ;;
+        *) usage ;;
+    esac
+done
+shift $((OPTIND - 1))
+[ $# -eq 2 ] || usage
+
+mirror=${1%/}
+destination=${2%/}
+here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+
+die() {
+    printf 'sync-to-ipod: %s\n' "$1" >&2
+    exit 1
+}
+
+[ -d "$mirror" ] || die "mirror $mirror is not a directory"
+[ -n "$(ls -A "$mirror")" ] || die "mirror $mirror is empty; refusing to mirror nothing"
+[ -d "$destination" ] || die "destination $destination is not a directory"
+
+# --delete makes every one of these load-bearing. A destination that is not its
+# own mount point means the path is wrong, and emptying the wrong directory is
+# not a mistake that announces itself.
+case "$destination" in
+    "" | "/" | "$HOME") die "refusing to sync onto $destination" ;;
+esac
+[ "$(readlink -f "$mirror")" != "$(readlink -f "$destination")" ] ||
+    die "mirror and destination are the same directory"
+mountpoint -q -- "$destination" || die "$destination is not a mount point"
+
+filesystem=$(findmnt -no FSTYPE --target "$destination")
+case "$filesystem" in
+    vfat | exfat) ;;
+    *)
+        $force || die "$destination is $filesystem, not FAT; pass -f if that is deliberate"
+        printf 'sync-to-ipod: destination is %s, not FAT\n' "$filesystem" >&2
+        ;;
+esac
+
+if $force; then
+    printf 'sync-to-ipod: skipping the FAT32 check\n' >&2
+elif ! python3 "$here/check_fat32.py" "$mirror"; then
+    die "the mirror holds paths FAT32 will not take; run music-mirror with --fat32-safe"
+fi
+
+# Before the copy, not after: the plays already happened, and if the transfer
+# then fails there is no reason to have lost them too.
+if $scrobble; then
+    if [ -z "${LASTFM_API_KEY:-}" ] || [ -z "${LASTFM_API_SECRET:-}" ]; then
+        printf 'sync-to-ipod: no Last.fm credentials, skipping the scrobbler log\n' >&2
+    else
+        scrobble_options=()
+        $dry_run && scrobble_options+=(--dry-run)
+        python3 "$here/submit_scrobbles.py" "${scrobble_options[@]}" "$destination" ||
+            die "submitting scrobbles failed; nothing has been copied"
+    fi
+fi
+
+# -rt rather than -a: owners, groups and permissions mean nothing on FAT, and
+# asking for them produces a screenful of errors and a non-zero exit.
+# --modify-window=2 because FAT stores mtimes to two-second resolution, without
+# which every file looks changed and the whole library is copied every time.
+# --delete removes tracks whose source has gone, which is the point. It would
+# also remove everything on the device that the mirror does not contain -- and
+# if the destination is the card root that means /.rockbox, the Rockbox install
+# itself. Excluded paths are not deleted unless --delete-excluded is given,
+# which it never is here.
+options=(--recursive --times --delete --modify-window=2 --human-readable --info=progress2)
+for owned in "/.rockbox" "/.scrobbler.log" "/.scrobbler.log.*" "/.playlist_control" \
+    "/System Volume Information" "/.Spotlight-V100" "/.Trashes" "/.fseventsd"; do
+    options+=(--exclude "$owned")
+done
+$dry_run && options+=(--dry-run --verbose)
+
+printf 'sync-to-ipod: %s -> %s\n' "$mirror" "$destination" >&2
+rsync "${options[@]}" "$mirror/" "$destination/"
+
+if $dry_run; then
+    printf 'sync-to-ipod: dry run, nothing was written\n' >&2
+    exit 0
+fi
+
+sync
+if $unmount; then
+    device=$(findmnt -no SOURCE --target "$destination")
+    printf 'sync-to-ipod: unmounting %s\n' "$device" >&2
+    if command -v udisksctl >/dev/null 2>&1; then
+        udisksctl unmount -b "$device"
+    else
+        umount -- "$destination"
+    fi
+    printf 'sync-to-ipod: safe to disconnect\n' >&2
+else
+    printf 'sync-to-ipod: still mounted; unmount before disconnecting\n' >&2
+fi