Emma Thorpe
f324b1b720
Build and publish container / build (pull_request) Successful in 5m6s
Scrobbling previously needed the on-device Last.fm plugin run by hand before each sync, to turn Rockbox's playback log into AUDIOSCROBBLER format. Forgetting that step means the sync submits nothing and quietly appears not to work. Core Rockbox writes ROCKBOX_DIR/playback.log whenever "play log" is enabled, with no plugin running at all. Each line is timestamp:elapsed_ms:length_ms:path. The only thing missing is tags, and that is exactly why the plugin exists: reading them back off the player is slow. Off the mirror it is free, because the same files are already there -- so the conversion belongs on the laptop, and the plugin can be skipped entirely. A play counts as listened at half the track's length, matching the plugin's savepct default, so the two cannot disagree about what a play was. A short play is a skip. An entry with no usable timestamp is refused rather than invented, which is the clockless case again. A path that maps to nothing in the mirror is counted and reported instead of guessed at. Rotated logs are picked up too; Rockbox starts a new one past half a megabyte. All of them are renamed aside together once Last.fm has accepted the batch. A .scrobbler.log is still read when the plugin has been run and left one.
410 lines
14 KiB
Python
Executable File
410 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Submit a Rockbox scrobbler log to Last.fm, then set it aside.
|
|
|
|
Rockbox writes /.scrobbler.log on the device in AUDIOSCROBBLER 1.1 format: one
|
|
tab-separated line per track, rated `L` for listened or `S` for skipped. Only
|
|
the listened ones are submitted; a skip is not a play.
|
|
|
|
Scrobbling is a write method, so unlike everything else here it needs the API
|
|
secret and a session key, obtained once through the browser. Read-only calls
|
|
elsewhere in these projects need neither.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
API_ROOT = "https://ws.audioscrobbler.com/2.0/"
|
|
|
|
# Last.fm's documented ceiling for one track.scrobble call.
|
|
BATCH = 50
|
|
|
|
# Rockbox names the log for whether the target has a real-time clock. Without
|
|
# one every timestamp it writes is zero, which is not a time anything can
|
|
# scrobble.
|
|
LOG_NAMES = (".scrobbler.log", ".scrobbler-timeless.log")
|
|
|
|
# Rockbox core writes this whenever "play log" is on, with no plugin running:
|
|
# timestamp:elapsed_ms:length_ms:/Music/Artist/Album/Track.mp3
|
|
# It is rotated once it grows past half a megabyte. Converting it needs tags,
|
|
# which is why the on-device plugin exists -- reading them back off the player
|
|
# is slow. Off the mirror it is free, so the plugin can be skipped entirely.
|
|
PLAYBACK_LOG_NAMES = ("playback.log", "playback_*.log")
|
|
|
|
# The plugin counts a track as listened at savepct of its length, defaulting to
|
|
# fifty. Same rule here, or the two disagree about what a play is.
|
|
LISTENED_FRACTION = 0.5
|
|
|
|
# Below this a timestamp is not a wall-clock time. Without a real-time clock
|
|
# Rockbox logs ticks in milliseconds instead, which is not a date.
|
|
EARLIEST_PLAUSIBLE = 1_000_000_000
|
|
|
|
SESSION_FILE = Path(
|
|
os.getenv("XDG_CONFIG_HOME", Path.home() / ".config")
|
|
) / "music-mirror" / "lastfm.json"
|
|
|
|
|
|
class LastfmError(Exception):
|
|
"""A Last.fm request that failed."""
|
|
|
|
|
|
def parse_log(text):
|
|
"""Return the listened tracks in an AUDIOSCROBBLER log, oldest first.
|
|
|
|
Fields are artist, album, title, track number, length, rating, timestamp
|
|
and MusicBrainz id. Rockbox converts any tab inside a field to a space
|
|
before writing, so splitting on tabs is safe.
|
|
"""
|
|
played, skipped, timeless = [], 0, 0
|
|
for line in text.splitlines():
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
fields = line.split("\t")
|
|
if len(fields) < 7:
|
|
continue
|
|
artist, album, title, number, length, rating, timestamp = fields[:7]
|
|
mbid = fields[7] if len(fields) > 7 else ""
|
|
if rating.strip().upper() != "L":
|
|
skipped += 1
|
|
continue
|
|
try:
|
|
when = int(timestamp)
|
|
except ValueError:
|
|
continue
|
|
if when <= 0:
|
|
timeless += 1
|
|
continue
|
|
if not artist or not title:
|
|
continue
|
|
played.append(
|
|
{
|
|
"artist": artist,
|
|
"album": album,
|
|
"track": title,
|
|
"trackNumber": number if number not in ("", "-1") else "",
|
|
"duration": length if length.isdigit() and int(length) > 0 else "",
|
|
"timestamp": str(when),
|
|
"mbid": mbid,
|
|
}
|
|
)
|
|
played.sort(key=lambda entry: int(entry["timestamp"]))
|
|
return played, skipped, timeless
|
|
|
|
|
|
def parse_playback_log(text):
|
|
"""Return (timestamp, elapsed_ms, length_ms, path) for each logged play."""
|
|
plays = []
|
|
for line in text.splitlines():
|
|
fields = line.strip().split(":", 3)
|
|
if len(fields) != 4:
|
|
continue
|
|
stamp, elapsed, length, path = fields
|
|
try:
|
|
plays.append((int(stamp), int(elapsed), int(length), path))
|
|
except ValueError:
|
|
continue
|
|
return plays
|
|
|
|
|
|
def device_to_local(path, device_prefix, mirror):
|
|
"""Map a path as the player sees it onto the mirror it was copied from."""
|
|
prefix = "/" + device_prefix.strip("/")
|
|
if prefix != "/":
|
|
if not path.startswith(prefix + "/"):
|
|
return None
|
|
path = path[len(prefix) :]
|
|
return Path(mirror) / path.lstrip("/")
|
|
|
|
|
|
def read_tags(path, runner=None):
|
|
"""Return the tags of a local file, via ffprobe."""
|
|
runner = runner or _ffprobe
|
|
try:
|
|
payload = json.loads(runner(path))
|
|
except (OSError, ValueError):
|
|
return {}
|
|
return {
|
|
key.lower(): value
|
|
for key, value in (payload.get("format", {}).get("tags") or {}).items()
|
|
}
|
|
|
|
|
|
def _ffprobe(path):
|
|
import subprocess
|
|
|
|
return subprocess.run(
|
|
["ffprobe", "-v", "error", "-show_entries", "format_tags",
|
|
"-of", "json", str(path)],
|
|
capture_output=True, text=True, check=True,
|
|
).stdout
|
|
|
|
|
|
def plays_from_playback_log(text, device_prefix, mirror, runner=None):
|
|
"""Return submittable entries, plus counts of what was left out.
|
|
|
|
Skips are decided by the same fraction the on-device plugin uses, so the
|
|
two never disagree about what counted as a play.
|
|
"""
|
|
played, skipped, unresolved, timeless = [], 0, 0, 0
|
|
for stamp, elapsed, length, device_path in parse_playback_log(text):
|
|
if stamp < EARLIEST_PLAUSIBLE:
|
|
timeless += 1
|
|
continue
|
|
if length > 0 and elapsed < length * LISTENED_FRACTION:
|
|
skipped += 1
|
|
continue
|
|
local = device_to_local(device_path, device_prefix, mirror)
|
|
tags = read_tags(local, runner) if local and local.is_file() else {}
|
|
artist = tags.get("artist") or tags.get("album_artist") or ""
|
|
title = tags.get("title") or ""
|
|
if not artist or not title:
|
|
unresolved += 1
|
|
continue
|
|
played.append(
|
|
{
|
|
"artist": artist,
|
|
"track": title,
|
|
"album": tags.get("album", ""),
|
|
"trackNumber": (tags.get("track") or "").split("/")[0],
|
|
"duration": str(length // 1000) if length > 0 else "",
|
|
"timestamp": str(stamp),
|
|
"mbid": tags.get("musicbrainz_trackid", ""),
|
|
}
|
|
)
|
|
played.sort(key=lambda entry: int(entry["timestamp"]))
|
|
return played, skipped, unresolved, timeless
|
|
|
|
|
|
def find_playback_logs(device):
|
|
"""Return every playback log on a device, oldest first."""
|
|
found = []
|
|
for pattern in PLAYBACK_LOG_NAMES:
|
|
found.extend(sorted(Path(device).glob(f".rockbox/{pattern}")))
|
|
return [path for path in found if path.is_file() and path.stat().st_size]
|
|
|
|
|
|
def sign(params, secret):
|
|
"""Return Last.fm's method signature for a set of parameters.
|
|
|
|
Names are sorted by the ASCII table rather than numerically, which is why
|
|
`artist[10]` comes before `artist[1]`. Getting that wrong produces an
|
|
invalid signature and nothing else.
|
|
"""
|
|
joined = "".join(f"{name}{params[name]}" for name in sorted(params))
|
|
return hashlib.md5((joined + secret).encode("utf-8")).hexdigest() # noqa: S324
|
|
|
|
|
|
def post(params, transport):
|
|
"""Sign, post, and return the decoded response."""
|
|
body = urllib.parse.urlencode(params).encode("utf-8")
|
|
request = urllib.request.Request(API_ROOT, data=body)
|
|
try:
|
|
payload = json.loads(transport(request))
|
|
except urllib.error.HTTPError as error:
|
|
detail = error.read().decode("utf-8", "replace")[:300]
|
|
raise LastfmError(f"HTTP {error.code}: {detail}") from error
|
|
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error:
|
|
raise LastfmError(str(error)) from error
|
|
if payload.get("error"):
|
|
raise LastfmError(f"error {payload['error']}: {payload.get('message', '')}")
|
|
return payload
|
|
|
|
|
|
def call(method, params, key, secret, session, transport):
|
|
"""Make one signed, authenticated call."""
|
|
full = {**params, "method": method, "api_key": key}
|
|
if session:
|
|
full["sk"] = session
|
|
full["api_sig"] = sign(full, secret)
|
|
full["format"] = "json"
|
|
return post(full, transport)
|
|
|
|
|
|
def authorise(key, secret, transport, opener=print):
|
|
"""Walk the one-time browser authorisation and return a session key."""
|
|
token = call("auth.getToken", {}, key, secret, None, transport)["token"]
|
|
url = f"https://www.last.fm/api/auth/?api_key={key}&token={token}"
|
|
opener(f"Open this, approve the application, then press Enter:\n\n {url}\n")
|
|
input()
|
|
session = call("auth.getSession", {"token": token}, key, secret, None, transport)
|
|
return session["session"]["key"]
|
|
|
|
|
|
def load_session():
|
|
if SESSION_FILE.is_file():
|
|
return json.loads(SESSION_FILE.read_text(encoding="utf-8")).get("session")
|
|
return None
|
|
|
|
|
|
def save_session(session):
|
|
SESSION_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
SESSION_FILE.write_text(json.dumps({"session": session}), encoding="utf-8")
|
|
SESSION_FILE.chmod(0o600)
|
|
|
|
|
|
def batch_params(entries):
|
|
"""Return the indexed parameters for one track.scrobble call."""
|
|
params = {}
|
|
for index, entry in enumerate(entries):
|
|
for name in ("artist", "track", "timestamp", "album", "trackNumber", "duration", "mbid"):
|
|
if entry.get(name):
|
|
params[f"{name}[{index}]"] = entry[name]
|
|
return params
|
|
|
|
|
|
def submit(entries, key, secret, session, transport, delay=1.0):
|
|
"""Submit every entry. Returns how many the service accepted.
|
|
|
|
Batches are counted as they succeed rather than at the end, so a failure
|
|
partway through leaves an honest number and the caller can keep the rest of
|
|
the log instead of losing it.
|
|
"""
|
|
accepted = 0
|
|
for start in range(0, len(entries), BATCH):
|
|
chunk = entries[start : start + BATCH]
|
|
payload = call(
|
|
"track.scrobble", batch_params(chunk), key, secret, session, transport
|
|
)
|
|
block = payload.get("scrobbles", {})
|
|
summary = block.get("@attr", block)
|
|
accepted += int(summary.get("accepted", len(chunk)))
|
|
ignored = int(summary.get("ignored", 0))
|
|
if ignored:
|
|
print(f" {ignored} of {len(chunk)} ignored by Last.fm", file=sys.stderr)
|
|
if start + BATCH < len(entries):
|
|
time.sleep(delay)
|
|
return accepted
|
|
|
|
|
|
def http_post(request, timeout=30):
|
|
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310
|
|
return response.read().decode("utf-8")
|
|
|
|
|
|
def find_log(device):
|
|
"""Return the scrobbler log on a mounted device, or None."""
|
|
for name in LOG_NAMES:
|
|
candidate = Path(device) / name
|
|
if candidate.is_file() and candidate.stat().st_size:
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def main(argv=None, transport=http_post):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("device", help="the mounted device, or a scrobbler log file")
|
|
parser.add_argument(
|
|
"--mirror",
|
|
help="the mirror the device was copied from. Given this, Rockbox's own"
|
|
" playback.log is converted here rather than needing the on-device"
|
|
" plugin run first",
|
|
)
|
|
parser.add_argument(
|
|
"--device-prefix",
|
|
default="/Music",
|
|
help="where the music sits on the device, stripped when mapping a logged"
|
|
" path back onto the mirror",
|
|
)
|
|
parser.add_argument("--api-key", default=os.getenv("LASTFM_API_KEY"))
|
|
parser.add_argument("--api-secret", default=os.getenv("LASTFM_API_SECRET"))
|
|
parser.add_argument("--dry-run", action="store_true", help="parse and report only")
|
|
parser.add_argument(
|
|
"--keep", action="store_true", help="do not set the log aside afterwards"
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
target = Path(args.device)
|
|
log = target if target.is_file() else find_log(target)
|
|
logs = []
|
|
|
|
if log is not None:
|
|
played, skipped, timeless = parse_log(
|
|
log.read_text(encoding="utf-8", errors="replace")
|
|
)
|
|
unresolved = 0
|
|
logs = [log]
|
|
print(f"{log}: {len(played)} listened, {skipped} skipped", file=sys.stderr)
|
|
elif args.mirror:
|
|
# No plugin has been run, but the core log is there. Tags come off the
|
|
# mirror, which is the only reason the plugin was needed at all.
|
|
logs = find_playback_logs(target)
|
|
if not logs:
|
|
print("no scrobbler log to submit", file=sys.stderr)
|
|
return 0
|
|
text = "\n".join(
|
|
path.read_text(encoding="utf-8", errors="replace") for path in logs
|
|
)
|
|
played, skipped, unresolved, timeless = plays_from_playback_log(
|
|
text, args.device_prefix, args.mirror
|
|
)
|
|
print(
|
|
f"{len(logs)} playback log(s): {len(played)} listened, {skipped} skipped",
|
|
file=sys.stderr,
|
|
)
|
|
if unresolved:
|
|
print(
|
|
f" {unresolved} could not be matched to a file in the mirror"
|
|
" and were left out",
|
|
file=sys.stderr,
|
|
)
|
|
else:
|
|
print(
|
|
"no scrobbler log to submit. Rockbox's own playback.log can be used"
|
|
" instead -- pass --mirror so tags can be read from it.",
|
|
file=sys.stderr,
|
|
)
|
|
return 0
|
|
|
|
if timeless:
|
|
print(
|
|
f" {timeless} entries have no timestamp, so this target has no clock."
|
|
" They cannot be scrobbled without inventing when they happened.",
|
|
file=sys.stderr,
|
|
)
|
|
if not played:
|
|
return 0
|
|
if args.dry_run:
|
|
for entry in played[:20]:
|
|
print(f"{entry['timestamp']}\t{entry['artist']}\t{entry['track']}")
|
|
return 0
|
|
|
|
if not args.api_key or not args.api_secret:
|
|
print(
|
|
"scrobbling is a write method: it needs LASTFM_API_KEY and"
|
|
" LASTFM_API_SECRET, not just the read-only key",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
session = load_session()
|
|
if not session:
|
|
session = authorise(args.api_key, args.api_secret, transport)
|
|
save_session(session)
|
|
|
|
try:
|
|
accepted = submit(played, args.api_key, args.api_secret, session, transport)
|
|
except LastfmError as error:
|
|
print(f"submission failed: {error}", file=sys.stderr)
|
|
return 1
|
|
|
|
print(f"{accepted} scrobbles accepted", file=sys.stderr)
|
|
if not args.keep and accepted:
|
|
# Renamed rather than deleted: if Last.fm quietly dropped something,
|
|
# the evidence is still on the device.
|
|
for path in logs:
|
|
aside = path.with_name(f"{path.name}.{played[-1]['timestamp']}.submitted")
|
|
path.rename(aside)
|
|
print(f"log moved to {aside.name}", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|