music-mirror/tools/submit_scrobbles.py

#!/usr/bin/env python3
"""Submit a Rockbox scrobbler log to Last.fm, then set it aside.

Rockbox writes /.scrobbler.log on the device in AUDIOSCROBBLER 1.1 format: one
tab-separated line per track, rated `L` for listened or `S` for skipped. Only
the listened ones are submitted; a skip is not a play.

Scrobbling is a write method, so unlike everything else here it needs the API
secret and a session key, obtained once through the browser. Read-only calls
elsewhere in these projects need neither.
"""

import argparse
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError

API_ROOT = "https://ws.audioscrobbler.com/2.0/"

# Last.fm's documented ceiling for one track.scrobble call.
BATCH = 50

# Rockbox names the log for whether the target has a real-time clock. Without
# one every timestamp it writes is zero, which is not a time anything can
# scrobble.
LOG_NAMES = (".scrobbler.log", ".scrobbler-timeless.log")

# Rockbox core writes this whenever "play log" is on, with no plugin running:
#     timestamp:elapsed_ms:length_ms:/Music/Artist/Album/Track.mp3
# It is rotated once it grows past half a megabyte. Converting it needs tags,
# which is why the on-device plugin exists -- reading them back off the player
# is slow. Off the mirror it is free, so the plugin can be skipped entirely.
PLAYBACK_LOG_NAMES = ("playback.log", "playback_*.log")

# The plugin counts a track as listened at savepct of its length, defaulting to
# fifty. Same rule here, or the two disagree about what a play is.
LISTENED_FRACTION = 0.5

# Below this a timestamp is not a wall-clock time. Without a real-time clock
# Rockbox logs ticks in milliseconds instead, which is not a date.
EARLIEST_PLAUSIBLE = 1_000_000_000

# How far ahead of now a converted play may sit before it is reported. Some
# slack absorbs a device clock drifting by a minute or two; an hour out means
# the zone is wrong or the clock was never put forward.
FUTURE_TOLERANCE_SECONDS = 300

SESSION_FILE = Path(
    os.getenv("XDG_CONFIG_HOME", Path.home() / ".config")
) / "music-mirror" / "lastfm.json"


class LastfmError(Exception):
    """A Last.fm request that failed."""


def system_zone_name():
    """Return this machine's IANA zone name, or "" if nothing states it.

    The name has to come out whole. /etc/localtime is a symlink into the
    tzdata tree, so the part after `zoneinfo/` is the name -- taking only the
    last component yields "London", which no database has, and falls back to a
    fixed offset that would then be wrong for half the year.
    """
    try:
        named = Path("/etc/timezone").read_text(encoding="utf-8").strip()
        if named:
            return named
    except OSError:
        pass
    try:
        parts = Path("/etc/localtime").resolve().parts
    except OSError:
        return ""
    if "zoneinfo" in parts:
        return "/".join(parts[len(parts) - parts[::-1].index("zoneinfo"):])
    return ""


def device_zone(name=None):
    """Return the zone the device's clock is keeping.

    Rockbox has no concept of a timezone, so its clock is set to local wall
    time and the zone has to be supplied from outside. Defaulting to this
    machine's zone is right whenever the player and the laptop are in the same
    place, which for a device synced by cable they are.
    """
    if name:
        return ZoneInfo(name)
    for candidate in (os.getenv("TZ"), system_zone_name()):
        if not candidate:
            continue
        try:
            return ZoneInfo(candidate)
        except (ZoneInfoNotFoundError, ValueError):
            continue
    # Nothing named the zone, so the offset cannot be resolved per play: this
    # is today's offset applied to every timestamp, which is wrong either side
    # of a daylight saving change. Still better than pretending it logged UTC.
    print(
        "warning: no IANA timezone found for this machine; using its current"
        " offset for every play. Pass --device-timezone to fix older plays"
        " across a daylight saving change.",
        file=sys.stderr,
    )
    return datetime.now().astimezone().tzinfo


def device_time_to_utc(stamp, zone):
    """Return the true UTC epoch of a timestamp Rockbox wrote.

    Rockbox builds its timestamps with `mktime(get_time())`, and its mktime
    (firmware/libc/mktime.c) is plain calendar arithmetic with no zone applied.
    Fed the RTC's local fields it yields local-wall-clock-as-if-UTC, so the
    number is ahead of real UTC by whatever the offset was. Its own scrobbler
    plugin admits this by writing `#TZ/UNKNOWN`, leaving the correction here.

    Decoding the number back into those fields and reinterpreting them in the
    device's zone recovers the instant, and does so per play, so a log
    straddling a daylight-saving change converts each side by its own offset.
    """
    fields = datetime.fromtimestamp(stamp, timezone.utc).replace(tzinfo=zone)
    return int(fields.timestamp())


def future_plays(played, now=None):
    """Return the plays timestamped later than now, which cannot have happened.

    A device clock left on the wrong offset, or never adjusted across a
    daylight-saving change, shows up here. Last.fm has no way to tell such a
    scrobble from a real one, so it is worth saying out loud.
    """
    now = time.time() if now is None else now
    return [
        entry for entry in played
        if int(entry["timestamp"]) > now + FUTURE_TOLERANCE_SECONDS
    ]


def declares_utc(text):
    """Whether an AUDIOSCROBBLER log says its timestamps are already UTC.

    The format's header carries `#TZ/UTC` or `#TZ/UNKNOWN`, and its spec is
    explicit that a device may only claim UTC if it converted. Rockbox writes
    UNKNOWN, meaning the times are local wall clock and want correcting; a log
    from anything that claims UTC must be left alone.
    """
    for line in text.splitlines():
        if not line.startswith("#"):
            break
        if line.strip().upper().startswith("#TZ/"):
            return line.strip().upper() == "#TZ/UTC"
    return False


def parse_log(text, zone=None):
    """Return the listened tracks in an AUDIOSCROBBLER log, oldest first.

    Fields are artist, album, title, track number, length, rating, timestamp
    and MusicBrainz id. Rockbox converts any tab inside a field to a space
    before writing, so splitting on tabs is safe.

    Timestamps are corrected from the device's local wall clock to UTC unless
    the log's own header claims it did that already.
    """
    played, skipped, timeless = [], 0, 0
    convert = zone is not None and not declares_utc(text)
    for line in text.splitlines():
        if not line or line.startswith("#"):
            continue
        fields = line.split("\t")
        if len(fields) < 7:
            continue
        artist, album, title, number, length, rating, timestamp = fields[:7]
        mbid = fields[7] if len(fields) > 7 else ""
        if rating.strip().upper() != "L":
            skipped += 1
            continue
        try:
            when = int(timestamp)
        except ValueError:
            continue
        if when <= 0:
            timeless += 1
            continue
        if not artist or not title:
            continue
        if convert:
            when = device_time_to_utc(when, zone)
        played.append(
            {
                "artist": artist,
                "album": album,
                "track": title,
                "trackNumber": number if number not in ("", "-1") else "",
                "duration": length if length.isdigit() and int(length) > 0 else "",
                "timestamp": str(when),
                "mbid": mbid,
            }
        )
    played.sort(key=lambda entry: int(entry["timestamp"]))
    return played, skipped, timeless


def parse_playback_log(text):
    """Return (timestamp, elapsed_ms, length_ms, path) for each logged play."""
    plays = []
    for line in text.splitlines():
        fields = line.strip().split(":", 3)
        if len(fields) != 4:
            continue
        stamp, elapsed, length, path = fields
        try:
            plays.append((int(stamp), int(elapsed), int(length), path))
        except ValueError:
            continue
    return plays


def device_to_local(path, device_prefix, mirror):
    """Map a path as the player sees it onto the mirror it was copied from."""
    prefix = "/" + device_prefix.strip("/")
    if prefix != "/":
        if not path.startswith(prefix + "/"):
            return None
        path = path[len(prefix) :]
    return Path(mirror) / path.lstrip("/")


def read_tags(path, runner=None):
    """Return the tags of a local file, via ffprobe. Empty if it cannot be read.

    A file ffprobe chokes on is one play left unidentified, not a reason to
    abandon the rest -- and CalledProcessError is not an OSError, so catching
    the obvious things is not enough.
    """
    runner = runner or _ffprobe
    try:
        payload = json.loads(runner(path))
    except (OSError, ValueError, subprocess.SubprocessError):
        return {}
    return {
        key.lower(): value
        for key, value in (payload.get("format", {}).get("tags") or {}).items()
    }


def _ffprobe(path):
    return subprocess.run(
        ["ffprobe", "-v", "error", "-show_entries", "format_tags",
         "-of", "json", str(path)],
        capture_output=True, text=True, check=True,
    ).stdout


@dataclass
class Conversion:
    """What a playback log turned into, and what must not be thrown away.

    `retain` holds the raw lines of plays that were real but could not be
    submitted -- a track absent from the mirror, usually because the sync had
    not copied it yet. Those are written back so a later run can try again.
    Skips and clockless entries are not retained: neither can ever be
    submitted, and the untouched original is set aside regardless.
    """

    played: list
    skipped: int = 0
    unresolved: int = 0
    timeless: int = 0
    retain: list = None

    def __post_init__(self):
        if self.retain is None:
            self.retain = []


def plays_from_playback_log(text, device_prefix, mirror, runner=None, zone=None):
    """Return the conversion of a playback log.

    Skips are decided by the same fraction the on-device plugin uses, so the
    two never disagree about what counted as a play. Timestamps are corrected
    from the device's wall clock to UTC; the core log has no header to say so,
    but it is written the same way the plugin's UNKNOWN times are.
    """
    result = Conversion(played=[])
    for line in text.splitlines():
        parsed = parse_playback_log(line)
        if not parsed:
            continue
        stamp, elapsed, length, device_path = parsed[0]
        if stamp < EARLIEST_PLAUSIBLE:
            result.timeless += 1
            continue
        if length > 0 and elapsed < length * LISTENED_FRACTION:
            result.skipped += 1
            continue
        local = device_to_local(device_path, device_prefix, mirror)
        tags = read_tags(local, runner) if local and local.is_file() else {}
        artist = tags.get("artist") or tags.get("album_artist") or ""
        title = tags.get("title") or ""
        if not artist or not title:
            # A real play of a track this run could not identify. Kept, so a
            # later run -- after the file has been copied, or the tags fixed --
            # can submit it rather than the play being lost.
            result.unresolved += 1
            result.retain.append(line)
            continue
        result.played.append(
            {
                "artist": artist,
                "track": title,
                "album": tags.get("album", ""),
                "trackNumber": (tags.get("track") or "").split("/")[0],
                "duration": str(length // 1000) if length > 0 else "",
                "timestamp": str(
                    device_time_to_utc(stamp, zone) if zone is not None else stamp
                ),
                "mbid": tags.get("musicbrainz_trackid", ""),
                "line": line,
            }
        )
    result.played.sort(key=lambda entry: int(entry["timestamp"]))
    return result


def find_playback_logs(device):
    """Return every playback log on a device, oldest first."""
    found = []
    for pattern in PLAYBACK_LOG_NAMES:
        found.extend(sorted(Path(device).glob(f".rockbox/{pattern}")))
    return [path for path in found if path.is_file() and path.stat().st_size]


def sign(params, secret):
    """Return Last.fm's method signature for a set of parameters.

    Names are sorted by the ASCII table rather than numerically, which is why
    `artist[10]` comes before `artist[1]`. Getting that wrong produces an
    invalid signature and nothing else.
    """
    joined = "".join(f"{name}{params[name]}" for name in sorted(params))
    return hashlib.md5((joined + secret).encode("utf-8")).hexdigest()  # noqa: S324


def post(params, transport):
    """Sign, post, and return the decoded response."""
    body = urllib.parse.urlencode(params).encode("utf-8")
    request = urllib.request.Request(API_ROOT, data=body)
    try:
        payload = json.loads(transport(request))
    except urllib.error.HTTPError as error:
        detail = error.read().decode("utf-8", "replace")[:300]
        raise LastfmError(f"HTTP {error.code}: {detail}") from error
    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error:
        raise LastfmError(str(error)) from error
    if payload.get("error"):
        raise LastfmError(f"error {payload['error']}: {payload.get('message', '')}")
    return payload


def call(method, params, key, secret, session, transport):
    """Make one signed, authenticated call."""
    full = {**params, "method": method, "api_key": key}
    if session:
        full["sk"] = session
    full["api_sig"] = sign(full, secret)
    full["format"] = "json"
    return post(full, transport)


def authorise(key, secret, transport, opener=print):
    """Walk the one-time browser authorisation and return a session key."""
    token = call("auth.getToken", {}, key, secret, None, transport)["token"]
    url = f"https://www.last.fm/api/auth/?api_key={key}&token={token}"
    opener(f"Open this, approve the application, then press Enter:\n\n  {url}\n")
    input()
    session = call("auth.getSession", {"token": token}, key, secret, None, transport)
    return session["session"]["key"]


def load_session():
    if SESSION_FILE.is_file():
        return json.loads(SESSION_FILE.read_text(encoding="utf-8")).get("session")
    return None


def save_session(session):
    SESSION_FILE.parent.mkdir(parents=True, exist_ok=True)
    SESSION_FILE.write_text(json.dumps({"session": session}), encoding="utf-8")
    SESSION_FILE.chmod(0o600)


def batch_params(entries):
    """Return the indexed parameters for one track.scrobble call."""
    params = {}
    for index, entry in enumerate(entries):
        for name in ("artist", "track", "timestamp", "album", "trackNumber", "duration", "mbid"):
            if entry.get(name):
                params[f"{name}[{index}]"] = entry[name]
    return params


def submit(entries, key, secret, session, transport, delay=1.0, on_sent=None):
    """Submit every entry. Returns how many the service accepted.

    Batches are counted as they succeed rather than at the end, so a failure
    partway through leaves an honest number and the caller can keep the rest of
    the log instead of losing it.
    """
    accepted = 0
    for start in range(0, len(entries), BATCH):
        chunk = entries[start : start + BATCH]
        payload = call(
            "track.scrobble", batch_params(chunk), key, secret, session, transport
        )
        block = payload.get("scrobbles", {})
        summary = block.get("@attr", block)
        accepted += int(summary.get("accepted", len(chunk)))
        if on_sent is not None:
            on_sent(chunk)
        ignored = int(summary.get("ignored", 0))
        if ignored:
            print(f"  {ignored} of {len(chunk)} ignored by Last.fm", file=sys.stderr)
        if start + BATCH < len(entries):
            time.sleep(delay)
    return accepted


def http_post(request, timeout=30):
    with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
        return response.read().decode("utf-8")


def find_log(device):
    """Return the scrobbler log on a mounted device, or None."""
    for name in LOG_NAMES:
        candidate = Path(device) / name
        if candidate.is_file() and candidate.stat().st_size:
            return candidate
    return None


def main(argv=None, transport=http_post):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("device", help="the mounted device, or a scrobbler log file")
    parser.add_argument(
        "--mirror",
        help="the mirror the device was copied from. Given this, Rockbox's own"
        " playback.log is converted here rather than needing the on-device"
        " plugin run first",
    )
    parser.add_argument(
        "--device-prefix",
        default="/Music",
        help="where the music sits on the device, stripped when mapping a logged"
        " path back onto the mirror",
    )
    parser.add_argument(
        "--device-timezone",
        default=os.getenv("ROCKBOX_TIMEZONE"),
        help="the zone the player's clock is set to, as an IANA name such as"
        " Europe/London. Rockbox keeps local wall time and cannot record an"
        " offset, so its timestamps need this to become the UTC Last.fm wants."
        " Defaults to this machine's zone",
    )
    parser.add_argument("--api-key", default=os.getenv("LASTFM_API_KEY"))
    parser.add_argument("--api-secret", default=os.getenv("LASTFM_API_SECRET"))
    parser.add_argument("--dry-run", action="store_true", help="parse and report only")
    parser.add_argument(
        "--keep", action="store_true", help="do not set the log aside afterwards"
    )
    args = parser.parse_args(argv)

    try:
        zone = device_zone(args.device_timezone)
    except (ZoneInfoNotFoundError, ValueError) as error:
        print(f"unknown timezone {args.device_timezone!r}: {error}", file=sys.stderr)
        return 2

    target = Path(args.device)
    log = target if target.is_file() else find_log(target)
    logs = []

    conversion = None
    if log is not None:
        played, skipped, timeless = parse_log(
            log.read_text(encoding="utf-8", errors="replace"), zone
        )
        unresolved = 0
        logs = [log]
        print(f"{log}: {len(played)} listened, {skipped} skipped", file=sys.stderr)
    elif args.mirror:
        # No plugin has been run, but the core log is there. Tags come off the
        # mirror, which is the only reason the plugin was needed at all.
        logs = find_playback_logs(target)
        if not logs:
            print("no scrobbler log to submit", file=sys.stderr)
            return 0
        text = "\n".join(
            path.read_text(encoding="utf-8", errors="replace") for path in logs
        )
        conversion = plays_from_playback_log(
            text, args.device_prefix, args.mirror, zone=zone
        )
        played = conversion.played
        skipped, unresolved, timeless = (
            conversion.skipped,
            conversion.unresolved,
            conversion.timeless,
        )
        print(
            f"{len(logs)} playback log(s): {len(played)} listened, {skipped} skipped",
            file=sys.stderr,
        )
        if unresolved:
            print(
                f"  {unresolved} could not be matched to a file in the mirror."
                " Those plays are kept for a later run rather than discarded.",
                file=sys.stderr,
            )
    else:
        print(
            "no scrobbler log to submit. Rockbox's own playback.log can be used"
            " instead -- pass --mirror so tags can be read from it.",
            file=sys.stderr,
        )
        return 0

    if timeless:
        print(
            f"  {timeless} entries have no timestamp, so this target has no clock."
            " They cannot be scrobbled without inventing when they happened.",
            file=sys.stderr,
        )
    if not played:
        return 0

    ahead = future_plays(played)
    if ahead:
        newest = int(ahead[-1]["timestamp"]) - int(time.time())
        print(
            f"  {len(ahead)} plays are timestamped up to {newest // 60} minutes in"
            f" the future, converting from {zone}. Either the player's clock is"
            " wrong or that is not the zone it is set to.",
            file=sys.stderr,
        )

    if args.dry_run:
        for entry in played[:20]:
            when = datetime.fromtimestamp(int(entry["timestamp"]), zone)
            print(
                f"{entry['timestamp']}\t{when:%Y-%m-%d %H:%M %Z}"
                f"\t{entry['artist']}\t{entry['track']}"
            )
        return 0

    if not args.api_key or not args.api_secret:
        print(
            "scrobbling is a write method: it needs LASTFM_API_KEY and"
            " LASTFM_API_SECRET, not just the read-only key",
            file=sys.stderr,
        )
        return 2

    session = load_session()
    if not session:
        session = authorise(args.api_key, args.api_secret, transport)
        save_session(session)

    # Recorded as each batch is accepted, so a failure partway through knows
    # exactly what got through and what did not.
    sent = []
    try:
        accepted = submit(
            played, args.api_key, args.api_secret, session, transport,
            on_sent=sent.extend,
        )
    except LastfmError as error:
        print(f"submission failed after {len(sent)} scrobbles: {error}", file=sys.stderr)
        keep_history(logs, played, sent, conversion, args.keep)
        return 1

    print(f"{accepted} scrobbles accepted", file=sys.stderr)
    keep_history(logs, played, sent, conversion, args.keep)
    return 0


def keep_history(logs, played, sent, conversion, keep):
    """Set the logs aside, writing back anything still owed a submission.

    Two separate obligations. The original is preserved untouched, renamed
    rather than deleted, so a play is never lost to a mistake here. And any
    play that was not submitted -- unmatched, or in a batch that failed -- is
    written back into a live log, so the next run tries it again instead of it
    quietly vanishing with the rest.
    """
    if keep or not logs:
        if keep:
            print("logs left in place", file=sys.stderr)
        return

    submitted = {id(entry) for entry in sent}
    # A .scrobbler.log was converted by the on-device plugin and carries no
    # per-line record, so there is nothing to write back for it -- only the
    # rename below, which loses nothing.
    pending = list(conversion.retain) if conversion is not None else []
    pending += [
        entry["line"] for entry in played
        if "line" in entry and id(entry) not in submitted
    ]

    if not sent:
        print("nothing was accepted; logs left untouched", file=sys.stderr)
        return

    stamp = played[-1]["timestamp"] if played else "0"
    for path in logs:
        path.rename(path.with_name(f"{path.name}.{stamp}.submitted"))

    if pending:
        live = logs[0].with_name("playback.log")
        live.write_text("\n".join(pending) + "\n", encoding="utf-8")
        print(
            f"{len(pending)} plays not submitted were written back to"
            f" {live.name} for the next run",
            file=sys.stderr,
        )
    print(f"{len(logs)} log(s) set aside as .submitted", file=sys.stderr)


if __name__ == "__main__":
    sys.exit(main())