feat: write playlists into the mirror from the listening history

Six rules over the scrobble history and the library index, written as extended
M3U into <mirror>/_playlists/ and rebuilt every pass. Three look at what gets
played -- heavy rotation, all-time, and things played heavily once and silent
for a year. Three look at what does not: album tracks skipped on records
otherwise played constantly, unplayed tracks by the artists played most, and
unplayed tracks from anywhere. Four fifths of the library has never been played,
so both halves are needed.

Ninety days was the obvious window for "recent" and is the wrong one. On a real
history it holds a few hundred plays spread across a twenty-thousand track
rotation, and nothing ranks meaningfully. Twelve months does.

The two rotating playlists are shuffled by week rather than by pass. A pass runs
every few hours; a playlist that reorders itself each time is one that has to be
re-imported each time, because the Music app imports a snapshot of a file rather
than tracking it.

Lidarr knows where the lossless source is, and the playlists have to point at
the MP3s music-mirror produced from it. The library root is derived from the
common parent of the indexed artist folders, so it agrees with Lidarr by
construction instead of being kept in step by hand, and can be overridden.
Entries are relative to the playlist file, so one file works from the NAS, from
a Mac over SMB and from Linux.

A track is listed only once its mirror file has been confirmed to exist: Lidarr
holding the FLAC says nothing about whether the MP3 has been encoded. When many
are missing the run says so, because that is what a misconfigured root looks
like -- every path mapping to nothing -- and silence there would present an
empty playlist as a correct one.

music_curator.py

@@ -26,6 +26,7 @@ import re
 import signal
 import sqlite3
 import sys
+import tempfile
 import time
 import unicodedata
 import urllib.error
@@ -62,6 +63,16 @@ REQUEST_DELAY_SECONDS = 0.25
 BACKOFF_SECONDS = 2.0
 BACKOFF_CEILING_SECONDS = 60.0
 
+# What music-mirror names its output, and where playlists are written inside the
+# mirror. music-mirror's prune only deletes `*.mp3` and only removes directories
+# it finds empty, so a directory of M3Us survives it untouched.
+MIRROR_SUFFIX = ".mp3"
+PLAYLIST_DIRECTORY = "_playlists"
+
+# Group-readable, for the same reason music-mirror sets it: the mirror is read
+# back by whatever serves it, and a playlist nobody can read is not a playlist.
+GROUP_READ = 0o040
+
 SCHEMA_VERSION = "2"
 
 # Versions this build upgrades in place. Everything added since version 1 is a
@@ -1046,6 +1057,225 @@ def format_time(uts):
     return datetime.fromtimestamp(uts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M")
 
 
+# Playlists written into the mirror. Each is a rule over the listening history
+# and the library index; none of them look at the audio.
+#
+# The rotating ones are shuffled by week rather than by pass. A pass runs every
+# few hours, and a playlist that reorders itself every time is one that has to
+# be re-imported every time -- the Music app does not track a file, it imports a
+# snapshot of one.
+ROTATION_PERIOD_SECONDS = 7 * 86400
+SHUFFLE_MULTIPLIER = 2654435761
+SHUFFLE_MODULUS = 104729
+
+# Common table expressions the rules share. `played` is every library track that
+# has ever been matched to a scrobble, with its count and the last time it was
+# heard; `recent` is the same restricted to a window.
+PLAYED_CTE = """
+WITH played AS (
+    SELECT sk.track_id AS track_id, COUNT(*) AS plays, MAX(s.uts) AS last_uts
+      FROM scrobble s
+      JOIN scrobble_key sk ON sk.artist = s.artist AND sk.track = s.track
+     WHERE sk.track_id IS NOT NULL
+     GROUP BY sk.track_id
+)
+"""
+RECENT_CTE = """
+WITH recent AS (
+    SELECT sk.track_id AS track_id, COUNT(*) AS plays
+      FROM scrobble s
+      JOIN scrobble_key sk ON sk.artist = s.artist AND sk.track = s.track
+     WHERE sk.track_id IS NOT NULL AND s.uts >= :year_ago
+     GROUP BY sk.track_id
+)
+"""
+SELECT_TRACK = """
+SELECT t.id AS id, a.name AS artist, t.title AS title,
+       t.duration AS duration, t.path AS path
+  FROM lidarr_track t
+  JOIN lidarr_artist a ON a.id = t.artist_id
+"""
+PLAYABLE = " t.has_file = 1 AND t.path IS NOT NULL AND t.path <> '' "
+SHUFFLE = f" ((t.id * {SHUFFLE_MULTIPLIER}) + :week) % {SHUFFLE_MODULUS} "
+
+PLAYLISTS = (
+    (
+        "heavy-rotation",
+        "most played over the last twelve months",
+        RECENT_CTE + SELECT_TRACK + f"""
+          JOIN recent r ON r.track_id = t.id
+         WHERE {PLAYABLE}
+         ORDER BY r.plays DESC, a.name, t.title
+         LIMIT :limit
+        """,
+    ),
+    (
+        "all-time",
+        "most played ever",
+        PLAYED_CTE + SELECT_TRACK + f"""
+          JOIN played p ON p.track_id = t.id
+         WHERE {PLAYABLE}
+         ORDER BY p.plays DESC, a.name, t.title
+         LIMIT :limit
+        """,
+    ),
+    (
+        "neglected",
+        "played heavily once, silent for twelve months",
+        PLAYED_CTE + SELECT_TRACK + f"""
+          JOIN played p ON p.track_id = t.id
+         WHERE {PLAYABLE} AND p.last_uts < :year_ago
+         ORDER BY p.plays DESC, a.name, t.title
+         LIMIT :limit
+        """,
+    ),
+    (
+        "deep-cuts",
+        "never played, from albums whose other tracks you play constantly",
+        PLAYED_CTE + """,
+        album_plays AS (
+            SELECT t.album_id AS album_id, SUM(p.plays) AS plays
+              FROM played p JOIN lidarr_track t ON t.id = p.track_id
+             GROUP BY t.album_id
+        )
+        """ + SELECT_TRACK + f"""
+          JOIN album_plays ap ON ap.album_id = t.album_id
+          LEFT JOIN played p ON p.track_id = t.id
+         WHERE {PLAYABLE} AND p.track_id IS NULL
+         ORDER BY ap.plays DESC, t.id
+         LIMIT :limit
+        """,
+    ),
+    (
+        "unheard-favourites",
+        "never played, by the artists you play most; rotates weekly",
+        PLAYED_CTE + """,
+        artist_plays AS (
+            SELECT t.artist_id AS artist_id, SUM(p.plays) AS plays
+              FROM played p JOIN lidarr_track t ON t.id = p.track_id
+             GROUP BY t.artist_id
+        )
+        """ + SELECT_TRACK + f"""
+          LEFT JOIN played p ON p.track_id = t.id
+         WHERE {PLAYABLE} AND p.track_id IS NULL
+           AND t.artist_id IN (SELECT artist_id FROM artist_plays
+                                ORDER BY plays DESC LIMIT 50)
+         ORDER BY {SHUFFLE}
+         LIMIT :limit
+        """,
+    ),
+    (
+        "unheard",
+        "never played, anywhere in the library; rotates weekly",
+        PLAYED_CTE + SELECT_TRACK + f"""
+          LEFT JOIN played p ON p.track_id = t.id
+         WHERE {PLAYABLE} AND p.track_id IS NULL
+         ORDER BY {SHUFFLE}
+         LIMIT :limit
+        """,
+    ),
+)
+
+
+def library_root_of(store):
+    """Return the directory Lidarr's artist folders sit under.
+
+    Derived rather than configured, because it has to agree with what Lidarr
+    reports and no one wants to keep a second copy of that in step by hand.
+    """
+    paths = [
+        row["path"]
+        for row in store.connection.execute(
+            "SELECT path FROM lidarr_artist WHERE path IS NOT NULL AND path <> ''"
+        )
+    ]
+    if not paths:
+        return None
+    if len(paths) == 1:
+        return str(Path(paths[0]).parent)
+    try:
+        return os.path.commonpath(paths)
+    except ValueError:
+        return None
+
+
+def mirror_path_for(source, library_root, mirror_root):
+    """Return where music-mirror would have put the MP3 for a source file."""
+    try:
+        relative = Path(source).relative_to(library_root)
+    except ValueError:
+        return None
+    return (Path(mirror_root) / relative).with_suffix(MIRROR_SUFFIX)
+
+
+def write_playlist(path, entries):
+    """Write one extended M3U, atomically.
+
+    Paths are relative to the playlist file, so the same playlist works from the
+    NAS, from a Mac over SMB and from Linux without rewriting.
+    """
+    lines = ["#EXTM3U"]
+    for entry in entries:
+        seconds = round((entry["duration"] or 0) / 1000)
+        lines.append(f"#EXTINF:{seconds},{entry['artist']} - {entry['title']}")
+        lines.append(os.path.relpath(entry["mirror"], path.parent))
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    handle, temporary = tempfile.mkstemp(dir=path.parent, suffix=".m3u.part")
+    os.close(handle)
+    temporary = Path(temporary)
+    try:
+        temporary.write_text("\n".join(lines) + "\n", encoding="utf-8")
+        # The mirror is read back by something else; see music-mirror, which had
+        # to learn this the hard way.
+        mode = temporary.stat().st_mode
+        if not mode & GROUP_READ:
+            temporary.chmod(mode | GROUP_READ)
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def build_playlists(store, mirror_root, library_root, limit, now):
+    """Write every playlist into the mirror. Returns how many tracks were listed.
+
+    A track is only listed once its mirror file has been confirmed to exist. The
+    index knows what Lidarr holds, which is the lossless source; whether the MP3
+    beside it has been encoded yet is music-mirror's business and is checked
+    rather than assumed.
+    """
+    directory = Path(mirror_root) / PLAYLIST_DIRECTORY
+    parameters = {
+        "limit": limit,
+        "year_ago": now - 365 * 86400,
+        "week": now // ROTATION_PERIOD_SECONDS,
+    }
+
+    total = 0
+    missing = 0
+    for name, description, sql in PLAYLISTS:
+        entries = []
+        for row in store.connection.execute(sql, parameters):
+            mirror = mirror_path_for(row["path"], library_root, mirror_root)
+            if mirror is None or not mirror.is_file():
+                missing += 1
+                continue
+            entries.append({**dict(row), "mirror": mirror})
+        write_playlist(directory / f"{name}.m3u", entries)
+        total += len(entries)
+        logger.info("playlist %-20s %4d tracks -- %s", name, len(entries), description)
+
+    if missing:
+        logger.warning(
+            "%d selected tracks had no file in the mirror and were left out."
+            " A handful means music-mirror has not encoded them yet; a large"
+            " number means --library-root or --mirror is pointing at the wrong"
+            " place, since the paths are then being mapped to nothing.",
+            missing,
+        )
+    return total
+
+
 def coverage_report(store):
     """Log how much of the listening history could be tied to the library.
 
@@ -1262,7 +1492,10 @@ def report(store, now):
         logger.info("  top track 90d %2d: %-50s %d", position, label[:50], row["plays"])
 
 
-def run_once(client, store, user, now, backfill_limit, lidarr=None):
+def run_once(
+    client, store, user, now, backfill_limit, lidarr=None, mirror=None,
+    library_root=None, playlist_limit=100,
+):
     """Run a single pass. Returns the number of scrobbles added."""
     started = time.monotonic()
     added = catch_up(client, store, user, now)
@@ -1275,6 +1508,15 @@ def run_once(client, store, user, now, backfill_limit, lidarr=None):
     # arrived, and they need a verdict too.
     if store.scalar("SELECT COUNT(*) FROM lidarr_track"):
         match_library(store)
+        if mirror is not None:
+            root = library_root or library_root_of(store)
+            if root is None:
+                logger.warning(
+                    "cannot work out where Lidarr's music lives, so no playlists were"
+                    " written; set --library-root"
+                )
+            else:
+                build_playlists(store, mirror, root, playlist_limit, now)
 
     logger.info(
         "pass complete in %.1fs: %d scrobbles added, %d loved",
@@ -1347,6 +1589,25 @@ def build_parser():
         default=os.getenv("MUSIC_CURATOR_LIDARR_API_KEY"),
         help="Lidarr API key (env MUSIC_CURATOR_LIDARR_API_KEY)",
     )
+    parser.add_argument(
+        "--mirror",
+        default=os.getenv("MUSIC_CURATOR_MIRROR"),
+        help="root of the MP3 mirror; playlists are written into it"
+        " (env MUSIC_CURATOR_MIRROR)",
+    )
+    parser.add_argument(
+        "--library-root",
+        default=os.getenv("MUSIC_CURATOR_LIBRARY_ROOT"),
+        help="prefix to strip from Lidarr's track paths when mapping them into the"
+        " mirror; derived from the indexed artist folders when unset"
+        " (env MUSIC_CURATOR_LIBRARY_ROOT)",
+    )
+    parser.add_argument(
+        "--playlist-limit",
+        type=int,
+        default=int(os.getenv("MUSIC_CURATOR_PLAYLIST_LIMIT", "100")),
+        help="most tracks to put in any one playlist (env MUSIC_CURATOR_PLAYLIST_LIMIT)",
+    )
     parser.add_argument(
         "--skip-index",
         action="store_true",
@@ -1411,7 +1672,17 @@ def main(argv=None, clock=time.time):
         while True:
             now = int(clock())
             try:
-                run_once(client, store, args.user, now, args.backfill_limit, lidarr)
+                run_once(
+                    client,
+                    store,
+                    args.user,
+                    now,
+                    args.backfill_limit,
+                    lidarr,
+                    args.mirror,
+                    args.library_root,
+                    args.playlist_limit,
+                )
             except (LastfmError, LidarrError) as error:
                 logger.error("%s", error)
                 if interval is None: