"""Build a local record of what actually gets listened to. Ingests a Last.fm scrobble history into SQLite and keeps it current, so that the later stages -- building playlists, and deciding what in the library has gone cold -- can query listening habits locally instead of spending an API call per question. The store is derived state. It can be deleted and rebuilt from Last.fm at any time. Nothing here writes to Last.fm, and nothing here touches the music library or Lidarr. Ingest is in two halves. A backfill walks the history backwards until it runs out, and a catch-up fetches everything scrobbled since the newest scrobble already held. Both take their bounds from the database rather than from a saved cursor, so an interrupted run resumes from what it actually has. """ import argparse import fcntl import json import logging import os import re import signal import sqlite3 import sys import time import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path logger = logging.getLogger("music-curator") API_ROOT = "https://ws.audioscrobbler.com/2.0/" # The documented ceiling for user.getRecentTracks. getLovedTracks does not state # one; asking for more than it allows is harmless, because paging is driven by # the totalPages the response reports rather than by arithmetic on this number. PAGE_SIZE = 200 # Error codes worth another attempt: backend failure, service offline, service # temporarily unavailable, rate limit. Everything else is a fault in the request # or the key and will fail again identically. RETRYABLE_ERRORS = {8, 11, 16, 29} RETRYABLE_STATUS = {429, 500, 502, 503, 504} # Last.fm asks for no more than five requests a second averaged over five # minutes. A full backfill is thousands of requests, so it is worth staying # well inside that rather than discovering error 29 halfway through. REQUEST_DELAY_SECONDS = 0.25 BACKOFF_SECONDS = 2.0 BACKOFF_CEILING_SECONDS = 60.0 SCHEMA_VERSION = "1" SCHEMA = """ -- One row per scrobble. The primary key collapses two plays of the same track -- in the same second into one: Last.fm gives scrobbles no identifier, so they -- are genuinely indistinguishable, and one lost play a decade is not worth a -- surrogate key that would make re-ingest non-idempotent. CREATE TABLE IF NOT EXISTS scrobble ( uts INTEGER NOT NULL, artist TEXT NOT NULL, track TEXT NOT NULL, album TEXT NOT NULL DEFAULT '', artist_mbid TEXT, album_mbid TEXT, track_mbid TEXT, PRIMARY KEY (uts, artist, track) ); CREATE INDEX IF NOT EXISTS scrobble_uts ON scrobble (uts); CREATE INDEX IF NOT EXISTS scrobble_artist_track ON scrobble (artist, track); CREATE INDEX IF NOT EXISTS scrobble_track_mbid ON scrobble (track_mbid); -- Loved tracks are current state, not history: a track can be unloved again. -- The table is replaced on every pass rather than accumulated. CREATE TABLE IF NOT EXISTS loved ( artist TEXT NOT NULL, track TEXT NOT NULL, track_mbid TEXT, loved_at INTEGER, PRIMARY KEY (artist, track) ); CREATE TABLE IF NOT EXISTS state ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); """ class LastfmError(Exception): """A Last.fm request that failed in a way retrying will not fix.""" @dataclass(frozen=True) class Scrobble: """One play, as Last.fm recorded it.""" uts: int artist: str track: str album: str = "" artist_mbid: str | None = None album_mbid: str | None = None track_mbid: str | None = None def parse_interval(interval): """Return seconds for an interval such as ``30m``, ``6h`` or ``90``.""" text = str(interval).strip().lower() match = re.fullmatch(r"([0-9]+)([smhd]?)", text) if not match: raise ValueError(f"unrecognised interval {interval!r}: expected e.g. 45m, 6h, 1d") value = int(match.group(1)) return value * {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400}[match.group(2)] def as_list(value): """Return a list for a field Last.fm renders as a bare object when singular.""" if value is None: return [] if isinstance(value, list): return value return [value] def name_of(value): """Return the name out of a Last.fm sub-object. The same conceptual field is ``#text`` in some responses (recent tracks) and ``name`` in others (loved tracks, and anything fetched with extended=1). """ if isinstance(value, dict): return (value.get("name") or value.get("#text") or "").strip() return (value or "").strip() def mbid_of(value): """Return an MBID, or None where the field is present but empty. Last.fm sends an empty string rather than omitting the key, and an empty string would otherwise look like a usable join key further down the line. """ if isinstance(value, dict): value = value.get("mbid") value = (value or "").strip() return value or None def parse_scrobble(entry): """Return a Scrobble for one recent-tracks entry, or None to skip it. The currently-playing track comes back with no ``date`` at all. Storing it would mean a scrobble with no timestamp, and it would arrive again on every pass until the song ended, so it is dropped until it has a real one. """ date = entry.get("date") or {} if "uts" not in date: return None artist = name_of(entry.get("artist")) track = (entry.get("name") or "").strip() if not artist or not track: return None return Scrobble( uts=int(date["uts"]), artist=artist, track=track, album=name_of(entry.get("album")), artist_mbid=mbid_of(entry.get("artist")), album_mbid=mbid_of(entry.get("album")), track_mbid=mbid_of(entry.get("mbid")), ) def http_get(url, timeout=30): """Fetch a URL and return its body. Replaced in tests.""" with urllib.request.urlopen(url, timeout=timeout) as response: # noqa: S310 - fixed https root return response.read().decode("utf-8") class Lastfm: """Minimal read-only Last.fm client. Only an API key is needed: none of the methods used here authenticate a user, so there is no session key, no signing and no secret to hold. """ def __init__( self, api_key, delay=REQUEST_DELAY_SECONDS, attempts=5, backoff=BACKOFF_SECONDS, transport=None, ): self.api_key = api_key self.delay = delay self.attempts = attempts self.backoff = backoff # Resolved here rather than as a default argument so the tests can # replace the module-level fetcher. self.transport = transport or http_get self._previous_request = None def call(self, method, params): """Return the decoded response for one API method.""" query = {key: value for key, value in params.items() if value is not None} query.update(method=method, api_key=self.api_key, format="json") url = f"{API_ROOT}?{urllib.parse.urlencode(query)}" for attempt in range(1, self.attempts + 1): self._throttle() try: payload = json.loads(self.transport(url)) except urllib.error.HTTPError as error: if error.code not in RETRYABLE_STATUS: raise LastfmError(f"{method}: HTTP {error.code}") from error self._retry_or_raise(method, attempt, f"HTTP {error.code}", error) continue except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: self._retry_or_raise(method, attempt, str(error), error) continue code = payload.get("error") if code is None: return payload detail = f"error {code}: {payload.get('message', '')}".strip() if code not in RETRYABLE_ERRORS: raise LastfmError(f"{method}: {detail}") self._retry_or_raise(method, attempt, detail, None) raise LastfmError(f"{method}: gave up after {self.attempts} attempts") def _retry_or_raise(self, method, attempt, detail, cause): """Sleep before the next attempt, or give up if this was the last one.""" if attempt >= self.attempts: raise LastfmError(f"{method}: {detail}") from cause pause = min(self.backoff * 2 ** (attempt - 1), BACKOFF_CEILING_SECONDS) logger.warning("%s: %s; retrying in %.0fs", method, detail, pause) time.sleep(pause) def _throttle(self): """Keep consecutive requests at least `delay` apart.""" if self.delay <= 0: return if self._previous_request is not None: waited = time.monotonic() - self._previous_request if waited < self.delay: time.sleep(self.delay - waited) self._previous_request = time.monotonic() class Store: """The local scrobble database.""" def __init__(self, path): self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) self.connection = sqlite3.connect(self.path) self.connection.row_factory = sqlite3.Row # WAL so a long backfill does not lock a reader out, and a relaxed sync # because the whole store can be rebuilt from Last.fm if a crash eats # the tail of it. self.connection.execute("PRAGMA journal_mode = WAL") self.connection.execute("PRAGMA synchronous = NORMAL") self.connection.executescript(SCHEMA) self._check_version() def _check_version(self): held = self.get_state("schema_version") if held is None: self.set_state("schema_version", SCHEMA_VERSION) elif held != SCHEMA_VERSION: raise RuntimeError( f"{self.path} is schema version {held}, this build expects {SCHEMA_VERSION}; " "delete it and let it rebuild" ) def close(self): self.connection.close() def get_state(self, key): row = self.connection.execute("SELECT value FROM state WHERE key = ?", (key,)).fetchone() return row["value"] if row else None def set_state(self, key, value): with self.connection: self.connection.execute( "INSERT INTO state (key, value) VALUES (?, ?)" " ON CONFLICT (key) DO UPDATE SET value = excluded.value", (key, value), ) def add_scrobbles(self, scrobbles): """Insert scrobbles, ignoring any already held. Returns how many were new.""" if not scrobbles: return 0 with self.connection: before = self.connection.total_changes self.connection.executemany( "INSERT OR IGNORE INTO scrobble" " (uts, artist, track, album, artist_mbid, album_mbid, track_mbid)" " VALUES (?, ?, ?, ?, ?, ?, ?)", [ ( item.uts, item.artist, item.track, item.album, item.artist_mbid, item.album_mbid, item.track_mbid, ) for item in scrobbles ], ) return self.connection.total_changes - before def replace_loved(self, rows): """Replace the loved-track list wholesale, in one transaction.""" with self.connection: self.connection.execute("DELETE FROM loved") self.connection.executemany( "INSERT OR IGNORE INTO loved (artist, track, track_mbid, loved_at)" " VALUES (?, ?, ?, ?)", rows, ) def scalar(self, sql): return self.connection.execute(sql).fetchone()[0] def newest_uts(self): return self.scalar("SELECT MAX(uts) FROM scrobble") def oldest_uts(self): return self.scalar("SELECT MIN(uts) FROM scrobble") def count(self): return self.scalar("SELECT COUNT(*) FROM scrobble") def fetch_page(client, user, to=None, since=None, page=1): """Return the scrobbles on one page of the recent-tracks history.""" payload = client.call( "user.getRecentTracks", {"user": user, "limit": PAGE_SIZE, "page": page, "from": since, "to": to}, ) block = payload.get("recenttracks") or {} entries = as_list(block.get("track")) parsed = [scrobble for scrobble in map(parse_scrobble, entries) if scrobble is not None] attributes = block.get("@attr") or {} return parsed, int(attributes.get("totalPages") or 1) def backfill(client, store, user, now, limit=0): """Walk the history backwards until it runs out. Returns scrobbles added. Each request asks for the newest page of everything at or before a cursor, and the cursor is then moved to the oldest scrobble that came back. There is no page number to keep across runs, so an interrupted backfill resumes from whatever the database holds. """ if store.get_state("backfill_complete") == "yes": return 0 cursor = store.oldest_uts() if cursor is None: cursor = now page = 1 added = 0 requests = 0 while limit <= 0 or requests < limit: scrobbles, _ = fetch_page(client, user, to=cursor, page=page) requests += 1 if not scrobbles: store.set_state("backfill_complete", "yes") logger.info("backfill complete: %d scrobbles held", store.count()) return added added += store.add_scrobbles(scrobbles) page_oldest = min(scrobble.uts for scrobble in scrobbles) if page_oldest < cursor: cursor, page = page_oldest, 1 else: # Every scrobble on this page shares the cursor's second. Moving the # window would step over the rest of them, so take the next page of # the same window instead. page += 1 logger.info("backfill: %d added, reached %s", added, format_time(page_oldest)) logger.info("backfill paused after %d requests; resumes next pass", requests) return added def catch_up(client, store, user, now): """Fetch everything scrobbled since the newest scrobble held. Bounded at both ends, so the window cannot shift under the paging while new scrobbles arrive mid-run; anything that lands during the pass is picked up by the next one. """ newest = store.newest_uts() if newest is None: return 0 added = 0 page = 1 while True: scrobbles, total_pages = fetch_page(client, user, to=now, since=newest, page=page) added += store.add_scrobbles(scrobbles) if page >= total_pages: return added page += 1 def sync_loved(client, store, user): """Rebuild the loved-track table. Returns how many are loved.""" rows = [] page = 1 while True: payload = client.call( "user.getLovedTracks", {"user": user, "limit": PAGE_SIZE, "page": page} ) block = payload.get("lovedtracks") or {} for entry in as_list(block.get("track")): artist = name_of(entry.get("artist")) track = (entry.get("name") or "").strip() if not artist or not track: continue loved_at = (entry.get("date") or {}).get("uts") rows.append((artist, track, mbid_of(entry.get("mbid")), int(loved_at or 0) or None)) total_pages = int((block.get("@attr") or {}).get("totalPages") or 1) if page >= total_pages: break page += 1 # Written only once the whole list has been fetched: a failure halfway # through must not leave a truncated set of protected tracks behind. store.replace_loved(rows) return len(rows) def format_time(uts): """Return a UTC timestamp as a readable date.""" if uts is None: return "never" return datetime.fromtimestamp(uts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") def report(store, now): """Log what the store holds. The MBID coverage line is the one to watch: scrobbles carrying a MusicBrainz recording id can be joined to the library exactly, and everything else has to go through name matching. That percentage predicts how well the next stage will work, and whether it can be trusted to decide what is unplayed. """ total = store.count() logger.info("scrobbles: %d", total) if not total: return with_mbid = store.scalar("SELECT COUNT(*) FROM scrobble WHERE track_mbid IS NOT NULL") logger.info( "range: %s to %s", format_time(store.oldest_uts()), format_time(store.newest_uts()) ) logger.info("distinct artists: %d", store.scalar("SELECT COUNT(DISTINCT artist) FROM scrobble")) logger.info( "distinct tracks: %d", store.scalar("SELECT COUNT(DISTINCT artist || ' - ' || track) FROM scrobble"), ) logger.info("recording MBID present: %d (%.1f%%)", with_mbid, 100 * with_mbid / total) logger.info("loved tracks: %d", store.scalar("SELECT COUNT(*) FROM loved")) logger.info("backfill complete: %s", store.get_state("backfill_complete") == "yes") top = store.connection.execute( "SELECT artist, COUNT(*) AS plays FROM scrobble" " GROUP BY artist ORDER BY plays DESC, artist LIMIT 10" ).fetchall() for position, row in enumerate(top, start=1): logger.info(" top artist %2d: %-40s %d", position, row["artist"][:40], row["plays"]) recent = store.connection.execute( "SELECT artist, track, COUNT(*) AS plays FROM scrobble WHERE uts >= ?" " GROUP BY artist, track ORDER BY plays DESC, artist LIMIT 10", (now - 90 * 86400,), ).fetchall() for position, row in enumerate(recent, start=1): label = f"{row['artist']} - {row['track']}" logger.info(" top track 90d %2d: %-50s %d", position, label[:50], row["plays"]) def run_once(client, store, user, now, backfill_limit): """Run a single ingest pass. Returns the number of scrobbles added.""" started = time.monotonic() added = catch_up(client, store, user, now) added += backfill(client, store, user, now, limit=backfill_limit) loved = sync_loved(client, store, user) logger.info( "pass complete in %.1fs: %d scrobbles added, %d loved", time.monotonic() - started, added, loved, ) return added def acquire_lock(database): """Take an exclusive lock so two passes cannot ingest into one store.""" database.parent.mkdir(parents=True, exist_ok=True) handle = open(database.with_suffix(".lock"), "w") # noqa: SIM115 - held for the process try: fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: handle.close() return None return handle def build_parser(): """Return the argument parser. Every option also reads an environment variable, so the container can be configured without a command line.""" parser = argparse.ArgumentParser( prog="music-curator", description="Ingest a Last.fm scrobble history into a local store.", ) parser.add_argument( "--user", default=os.getenv("MUSIC_CURATOR_LASTFM_USER"), help="Last.fm username to read (env MUSIC_CURATOR_LASTFM_USER)", ) parser.add_argument( "--api-key", default=os.getenv("MUSIC_CURATOR_LASTFM_API_KEY"), help="Last.fm API key (env MUSIC_CURATOR_LASTFM_API_KEY)", ) parser.add_argument( "--db", default=os.getenv("MUSIC_CURATOR_DB", "/data/curator.db"), help="path to the SQLite store (env MUSIC_CURATOR_DB)", ) parser.add_argument( "--interval", default=os.getenv("MUSIC_CURATOR_INTERVAL"), help="repeat forever, waiting this long between passes, e.g. 6h (env MUSIC_CURATOR_INTERVAL)", ) parser.add_argument( "--request-delay", type=float, default=float(os.getenv("MUSIC_CURATOR_REQUEST_DELAY", REQUEST_DELAY_SECONDS)), help="seconds between API requests (env MUSIC_CURATOR_REQUEST_DELAY)", ) parser.add_argument( "--backfill-limit", type=int, default=int(os.getenv("MUSIC_CURATOR_BACKFILL_LIMIT", "0")), help="cap the backfill at this many requests per pass; 0 for no cap" " (env MUSIC_CURATOR_BACKFILL_LIMIT)", ) parser.add_argument( "--report-only", action="store_true", help="report on the existing store without fetching anything", ) return parser def main(argv=None, clock=time.time): """Entry point. Returns a process exit code.""" logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO) args = build_parser().parse_args(argv) if not args.report_only and (not args.user or not args.api_key): logger.error("both a Last.fm user and an API key are required") return 2 try: interval = parse_interval(args.interval) if args.interval else None except ValueError as error: logger.error("%s", error) return 2 database = Path(args.db).resolve() lock = acquire_lock(database) if lock is None: logger.error("another pass is already using %s", database) return 3 store = Store(database) if args.report_only: report(store, int(clock())) store.close() lock.close() return 0 client = Lastfm(args.api_key, delay=args.request_delay) stopping = False def stop(signum, _frame): nonlocal stopping stopping = True logger.info("signal %d received; finishing the current pass", signum) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) try: while True: now = int(clock()) try: run_once(client, store, args.user, now, args.backfill_limit) except LastfmError as error: logger.error("%s", error) if interval is None: return 1 report(store, now) if interval is None or stopping: return 0 logger.info("sleeping %ds", interval) for _ in range(interval): if stopping: return 0 time.sleep(1) finally: store.close() lock.close() def run(): """Console-script entry point.""" sys.exit(main()) if __name__ == "__main__": run()