music-mirror/tools/sync-to-ipod.sh

#!/usr/bin/env bash
# Copy the mirror onto a Rockbox device, then unmount it cleanly.
#
# The device is FAT32 with no journal, reached through the Apple firmware's
# disk mode because Rockbox's own mass storage is unreliable on an iFlash. An
# interrupted write is corruption that needs fsck.vfat from another machine, so
# this syncs and unmounts rather than leaving that to whoever pulls the cable.
#
# rsync --delete is pointed at a whole filesystem, so the checks below are the
# point of the script rather than decoration.

set -euo pipefail

usage() {
    # Help goes to stdout and exits clean; misuse goes to stderr and does not.
    local stream=2 code=2
    if [ "${1:-}" = "help" ]; then
        stream=1
        code=0
    fi
    cat >&"$stream" <<'USAGE'
usage: sync-to-ipod.sh [options] <mirror> <destination>

  -n  dry run; show what would change and touch nothing
  -P  count what needs copying first, so progress can show a percentage and
      an estimate. Costs a second full traversal of both trees, which on a
      FAT card of fifty thousand files is slower than the transfer itself
  -f  copy even if the FAT32 check finds unacceptable paths
  -S  skip submitting the Rockbox scrobbler log to Last.fm
  -B  skip rebuilding the Rockbox database
  -U  leave the destination mounted afterwards

Rebuilding the database needs MUSIC_MIRROR_DATABASE_TOOL pointing at Rockbox's
host-side builder (tools/database, built with ./tools/configure --type=d). It
is skipped with a note when unset. The scan reads tags from the mirror rather
than from the device, so it costs seconds rather than the hours an on-device
commit takes -- and on a large library the on-device commit may not finish at
all.

Submitting scrobbles needs LASTFM_API_KEY and LASTFM_API_SECRET; it is skipped
with a note when they are unset. Scrobbling is a write method and needs the
secret, unlike the read-only calls elsewhere in these projects.

Rockbox has no notion of a timezone: its clock holds local wall time and its
logs record that, not UTC. Set ROCKBOX_TIMEZONE to the zone the player's clock
is keeping (an IANA name, such as Europe/London) if it differs from this
machine's, which is otherwise assumed. Getting it wrong shifts every scrobble
by the difference.

The mirror is the directory holding the artist folders. The destination is
where those folders should end up on the device -- not the card root, unless
that is genuinely where you want them:

  sync-to-ipod.sh /mnt/tank/media/music-mp3 /media/IPOD/Music

A subdirectory is the better target: --delete is confined to it, and the path
budget is derived from it, since the device's 260-character limit counts the
whole path as the device sees it. /.rockbox and the scrobbler logs are never
deleted wherever you point this.

The destination must be on a mounted FAT filesystem. That check is also what
catches an unmounted device: /media/IPOD/Music then resolves to the host's own
root filesystem, and this refuses to empty that.

Progress is one line that rewrites itself, showing the album currently going
across, how far through the transfer is, the rate, and an estimate of what is
left. Working the totals out first means a second pass over the tree, which is
the price of figures that mean something; rsync's own percentage is computed
against a file list it is still building.

Reach the device with the Apple firmware's disk mode: Menu+Select to reboot,
then immediately Select+Play. Power off afterwards by holding Play.
USAGE
    exit "$code"
}

dry_run=false
counting=false
force=false
unmount=true
scrobble=true
database=true
for argument in "$@"; do
    [ "$argument" = "--help" ] && usage help
done
while getopts ":nPfSBUh" option; do
    case "$option" in
        n) dry_run=true ;;
        P) counting=true ;;
        f) force=true ;;
        S) scrobble=false ;;
        B) database=false ;;
        U) unmount=false ;;
        h) usage help ;;
        *) usage ;;
    esac
done
shift $((OPTIND - 1))
[ $# -eq 2 ] || usage

# Trailing slashes are stripped for tidiness, but stripping one from "/" leaves
# an empty string, and the guard below would then never see the root it is
# there to refuse.
mirror=${1%/}
mirror=${mirror:-/}
destination=${2%/}
destination=${destination:-/}
here=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)

die() {
    # Every argument, not just the first: the second half of a message is
    # usually the half that says what to do about it.
    printf 'sync-to-ipod: %s\n' "$*" >&2
    exit 1
}

[ -d "$mirror" ] || die "mirror $mirror is not a directory"
[ -n "$(ls -A "$mirror")" ] || die "mirror $mirror is empty; refusing to mirror nothing"
[ -d "$destination" ] || die "destination $destination is not a directory"

# --delete makes every one of these load-bearing. Emptying the wrong directory
# is not a mistake that announces itself.
case "$destination" in
    "" | "/" | "$HOME") die "refusing to sync onto $destination" ;;
esac
[ "$(readlink -f "$mirror")" != "$(readlink -f "$destination")" ] ||
    die "mirror and destination are the same directory"

# The filesystem the destination sits on, which is the check that matters: a
# subdirectory of the card is a perfectly good target, and is the better one,
# because --delete is then confined to it. Being FAT is also what proves the
# card is mounted at all -- an unmounted /media/IPOD/Music resolves to the
# host's own root filesystem, and this refuses to empty that.
filesystem=$(findmnt -no FSTYPE --target "$destination")
mounted_on=$(findmnt -no TARGET --target "$destination")
case "$filesystem" in
    vfat | exfat) ;;
    *)
        $force ||
            die "$destination is on a $filesystem filesystem, not FAT." \
                "Is the device mounted? Pass -f if this is deliberate."
        printf 'sync-to-ipod: destination is %s, not FAT\n' "$filesystem" >&2
        ;;
esac

# What the device will call this directory, which is what its path limit
# applies to. Derived rather than configured, so it cannot disagree with where
# the files are actually going.
device_prefix=${destination#"$mounted_on"}
device_prefix="/${device_prefix#/}"
printf 'sync-to-ipod: the device will see this as %s\n' "$device_prefix" >&2

if $force; then
    printf 'sync-to-ipod: skipping the FAT32 check\n' >&2
elif ! python3 "$here/check_fat32.py" --device-prefix "$device_prefix" "$mirror"; then
    die "the mirror holds paths FAT32 will not take; run music-mirror with --fat32-safe"
fi

# Before the copy, not after: the plays already happened, and if the transfer
# then fails there is no reason to have lost them too.
if $scrobble; then
    if [ -z "${LASTFM_API_KEY:-}" ] || [ -z "${LASTFM_API_SECRET:-}" ]; then
        printf 'sync-to-ipod: no Last.fm credentials, skipping the scrobbler log\n' >&2
    else
        scrobble_options=()
        $dry_run && scrobble_options+=(--dry-run)
        # Rockbox keeps local wall time with no notion of a zone, so its
        # timestamps are not the UTC Last.fm expects. Naming the zone the
        # player's clock is set to lets them be corrected.
        [ -n "${ROCKBOX_TIMEZONE:-}" ] &&
            scrobble_options+=(--device-timezone "$ROCKBOX_TIMEZONE")
        # --mirror lets it convert Rockbox's own playback.log, so the on-device
        # scrobbler plugin never has to be run. The device root, not the music
        # directory: the logs live in .rockbox.
        python3 "$here/submit_scrobbles.py" "${scrobble_options[@]}" \
            --mirror "$mirror" --device-prefix "$device_prefix" "$mounted_on" ||
            die "submitting scrobbles failed; nothing has been copied"
    fi
fi

# -rt rather than -a: owners, groups and permissions mean nothing on FAT, and
# asking for them produces a screenful of errors and a non-zero exit.
# --modify-window=2 because FAT stores mtimes to two-second resolution, without
# which every file looks changed and the whole library is copied every time.
# --whole-file is already the default when both ends are local paths, and an
# SMB or FAT mount counts as one, but stating it documents that the delta
# algorithm is deliberately not wanted: it would read every destination file
# back over USB to compute a checksum, to save sending an MP3 that has changed
# entirely anyway.
#
# --omit-dir-times drops a setattr round trip per directory. Across six
# thousand album folders on a FAT card that is six thousand operations to set
# timestamps nothing reads.
options=(--recursive --times --delete --modify-window=2 --whole-file --omit-dir-times)
# --delete removes tracks whose source has gone, which is the point. It would
# also remove everything on the device that the mirror does not contain -- and
# if the destination is the card root that means /.rockbox, the Rockbox install
# itself. Excluded paths are not deleted unless --delete-excluded is given,
# which it never is here.
for owned in "/.rockbox" "/.scrobbler.log" "/.scrobbler.log.*" "/.playlist_control" \
    "/System Volume Information" "/.Spotlight-V100" "/.Trashes" "/.fseventsd"; do
    options+=(--exclude "$owned")
done

printf 'sync-to-ipod: %s -> %s\n' "$mirror" "$destination" >&2

if $dry_run; then
    rsync "${options[@]}" --dry-run --verbose "$mirror/" "$destination/"
    printf 'sync-to-ipod: dry run, nothing was written\n' >&2
    exit 0
fi

# rsync says nothing at all while it builds its file list, which on fifty
# thousand files over USB is minutes of apparent hang. Counting first costs a
# second pass over the tree but means the transfer can show a real percentage
# rather than a number that grows as rsync discovers more work.
# Counting is opt-in because it is not cheap. It walks and compares both trees
# in full, exactly as the transfer does, and on a FAT card holding fifty
# thousand files that traversal costs more than moving the data. Without it the
# progress line still shows the running count, the rate and the album in
# flight; only the percentage and the estimate are lost, and those were the
# least useful part of it.
total=0
total_bytes=0
if $counting; then
    printf 'sync-to-ipod: working out what needs copying...\n' >&2
    # %l is the file's size, which is what makes an estimate possible.
    # Directories are dropped: rsync reports those too, with an inode size that
    # would inflate the total by several megabytes of nothing.
    counted=$(rsync "${options[@]}" --dry-run --out-format='%l %n' "$mirror/" "$destination/" |
        awk '!/\/$/ { files++; bytes += $1 } END { print files + 0, bytes + 0 }')
    total=${counted% *}
    total_bytes=${counted#* }
    printf 'sync-to-ipod: %s files to copy\n' "$total" >&2
fi

# Flushing and unmounting is the whole reason this is a script, so it has to
# happen on the way out whichever way that is. Ctrl-C during a transfer would
# otherwise leave a FAT filesystem with dirty buffers and no journal, which is
# the corruption this exists to avoid.
finish() {
    sync
    if $unmount; then
        device=$(findmnt -no SOURCE --target "$destination" 2>/dev/null || true)
        if [ -n "$device" ]; then
            printf 'sync-to-ipod: unmounting %s\n' "$device" >&2
            if command -v udisksctl >/dev/null 2>&1; then
                udisksctl unmount -b "$device" || umount -- "$destination" || true
            else
                umount -- "$destination" || true
            fi
            printf 'sync-to-ipod: safe to disconnect\n' >&2
        fi
    else
        printf 'sync-to-ipod: still mounted; unmount before disconnecting\n' >&2
    fi
}

interrupted() {
    trap - INT TERM
    printf '\nsync-to-ipod: interrupted -- rsync leaves no partial files, but the\n' >&2
    printf 'sync-to-ipod: filesystem still needs flushing before you pull anything\n' >&2
    finish
    exit 130
}

trap interrupted INT TERM

rsync "${options[@]}" --out-format='%l %n' "$mirror/" "$destination/" |
    python3 "$here/rsync_progress.py" --total "$total" --bytes "$total_bytes"
status=${PIPESTATUS[0]}
[ "$status" -eq 0 ] || die "rsync exited $status"

# Rockbox reads its database from .tcd files in .rockbox. Building them here
# rather than on the device is not just faster: the on-device commit sorts the
# whole index in whatever memory it can scrape together, and on a large library
# it runs for hours or dies outright.
#
# The scan reads tags through a scratch root -- a real .rockbox beside a symlink
# standing in for where the music lands on the device -- so the paths recorded
# match what Rockbox will look up, while the bytes are read from the mirror
# instead of over USB. The scratch is kept between runs because the builder is
# incremental: a second pass over unchanged files does no work at all.
rebuild_database() {
    local tool=${MUSIC_MIRROR_DATABASE_TOOL:-}
    if [ -z "$tool" ]; then
        printf 'sync-to-ipod: no database tool configured, skipping the database\n' >&2
        return 0
    fi
    [ -x "$tool" ] || die "$tool is not executable"

    local device_rockbox="$mounted_on/.rockbox"
    if [ ! -d "$device_rockbox" ]; then
        printf 'sync-to-ipod: no .rockbox on the device, skipping the database\n' >&2
        return 0
    fi

    local scratch="${XDG_CACHE_HOME:-$HOME/.cache}/music-mirror/database"
    mkdir -p "$scratch/.rockbox"

    # Rebuild the symlink layout each time; the mirror path or the device
    # prefix may have changed since the last run.
    find "$scratch" -maxdepth 1 -type l -delete
    if [ "$device_prefix" = "/" ]; then
        ln -s "$mirror"/* "$scratch/" 2>/dev/null || true
    else
        local under=${device_prefix#/}
        rm -rf "${scratch:?}/${under%%/*}"
        mkdir -p "$scratch/$(dirname "$under")"
        ln -s "$mirror" "$scratch/$under"
    fi

    printf 'sync-to-ipod: building the database from the mirror...\n' >&2
    ( cd "$scratch" && "$tool" ) >/dev/null || die "the database build failed"

    cp -- "$scratch"/.rockbox/*.tcd "$device_rockbox/" ||
        die "could not copy the database onto the device"
    printf 'sync-to-ipod: database copied to %s\n' "$device_rockbox" >&2
}

$database && rebuild_database

finish