Merge pull request 'feat: mood playlists from Last.fm crowd tags' (#10) from feat/tag-vibes into main

Reviewed-on: #10

README.md

@@ -8,7 +8,8 @@ question that mirror cannot: which of it is worth carrying, and which of it has
 not been played in years.
 
 **This is stage three.** It ingests the scrobble history, indexes the library
-from Lidarr, matches one to the other, and writes playlists into the mirror.
+from Lidarr, matches one to the other, and writes playlists into the mirror —
+by listening history and by mood.
 Nothing is written back to Lidarr — every call there is a `GET`. See "Where this
 is going" below.
 
@@ -151,6 +152,48 @@ few hours, and a playlist that reorders itself each time is one that has to be
 re-imported each time — the Music app imports a snapshot of a file, it does not
 track it.
 
+### Moods
+
+A second set of playlists selects by **Last.fm's crowd tags** rather than by
+listening history. MusicBrainz genres arrive free with the Lidarr index and are
+no use for this: they are sparse and formal, and will not tell you a record is
+screamo or synthwave. People typing tags will.
+
+Tags are fetched once per artist — one `artist.getTopTags` call each, by
+MusicBrainz id where Lidarr has one — and refreshed every ninety days. An
+artist Last.fm has never heard of is recorded as fetched with no tags, so it is
+not asked about again on every pass. `--tag-limit` spreads the first sweep over
+several passes.
+
+The built-in moods are chosen for this library rather than as a general
+taxonomy:
+
+| Mood | Selected on |
+| ------------------ | ------------------------------------------------- |
+| `80s-synths` | synthpop, new wave, synthwave — released 1975-1992 |
+| `high-energy-rock` | hard rock, punk, pop punk, alternative |
+| `screamo` | screamo, post-hardcore, metalcore, emo |
+| `drum-and-bass` | drum and bass, liquid funk, neurofunk, jungle |
+| `dance` | house, big room, hardstyle, trance, dubstep |
+| `classic-rock` | classic rock, prog, psychedelic, blues rock |
+
+An artist qualifies when their tag weights inside a mood sum to at least 30 out
+of Last.fm's 0-100 scale. One low-weight tag is not a genre, it is somebody's
+stray opinion.
+
+`years` filters on the album's release date, which is what separates eighties
+synth records from everything else a synthpop tag drags in.
+
+`--vibes` replaces the whole set with a JSON file of the same shape, so a new
+mood does not need a new release:
+
+```json
+[{ "name": "shoegaze", "tags": ["shoegaze", "dream pop"], "min_score": 40 }]
+```
+
+Names are validated when the file is read, not when the file is written. A bad
+one would otherwise surface as a playlist created somewhere unintended.
+
 ### Paths
 
 Lidarr knows where the lossless source is; the playlists have to point at the
@@ -229,6 +272,8 @@ music-curator --report-only   # report on the store, fetch nothing
 | `--mirror` | `MUSIC_CURATOR_MIRROR` | unset | Root of the MP3 mirror; playlists go here |
 | `--library-root` | `MUSIC_CURATOR_LIBRARY_ROOT` | derived | Prefix to strip from Lidarr's paths |
 | `--playlist-limit` | `MUSIC_CURATOR_PLAYLIST_LIMIT` | `100` | Most tracks in any one playlist |
+| `--vibes` | `MUSIC_CURATOR_VIBES` | built-in | JSON file of mood definitions |
+| `--tag-limit` | `MUSIC_CURATOR_TAG_LIMIT` | `0` | Cap artist tag lookups per pass |
 | `--skip-index` | — | off | Match against the index already held |
 | `--report-only` | — | off | Report without fetching |
 
@@ -278,7 +323,7 @@ nix shell nixpkgs#python3Packages.pytest -c pytest
 | Last.fm ingest and store | done |
 | Lidarr index and the scrobble-to-track matcher | done |
 | M3U playlists from the listening history | done |
-| Genre and mood playlists from Last.fm tags | next |
+| Genre and mood playlists from Last.fm tags | done |
 | Cold-music report, unmonitoring what is not played | last |
 
 The cull will unmonitor cold albums in Lidarr and tag their artists. It will

music_curator.py

@@ -69,6 +69,18 @@ BACKOFF_CEILING_SECONDS = 60.0
 MIRROR_SUFFIX = ".mp3"
 PLAYLIST_DIRECTORY = "_playlists"
 
+# Tags are re-fetched this often. They move slowly, and the first pass over a
+# library already costs one request per artist.
+TAG_REFRESH_SECONDS = 90 * 86400
+
+# How much of an artist's tag weight has to fall inside a vibe before their
+# tracks are eligible for it. Weights are Last.fm's own 0-100 popularity, summed
+# across whichever of the vibe's tags the artist carries.
+VIBE_MIN_SCORE = 30
+
+# A vibe writes a file named after itself, so the name has to be a filename.
+SAFE_VIBE_NAME = re.compile(r"^[a-z0-9][a-z0-9-]*$")
+
 # Group-readable, for the same reason music-mirror sets it: the mirror is read
 # back by whatever serves it, and a playlist nobody can read is not a playlist.
 GROUP_READ = 0o040
@@ -177,6 +189,24 @@ CREATE TABLE IF NOT EXISTS scrobble_key (
     PRIMARY KEY (artist, track)
 );
 CREATE INDEX IF NOT EXISTS scrobble_key_track ON scrobble_key (track_id);
+
+-- Last.fm's crowd tags for each library artist. MusicBrainz genres come free
+-- with the Lidarr index and are useless for this: they are sparse and formal,
+-- and will not tell you a record is screamo or synthwave. People typing tags
+-- will. Keyed on the normalised name rather than the Lidarr id so it survives
+-- an artist being removed and re-added there.
+CREATE TABLE IF NOT EXISTS artist_tag (
+    norm_artist TEXT    NOT NULL,
+    tag         TEXT    NOT NULL,
+    weight      INTEGER NOT NULL,
+    PRIMARY KEY (norm_artist, tag)
+);
+CREATE INDEX IF NOT EXISTS artist_tag_tag ON artist_tag (tag);
+
+CREATE TABLE IF NOT EXISTS artist_tag_fetched (
+    norm_artist TEXT PRIMARY KEY,
+    fetched_at  INTEGER NOT NULL
+);
 """
 
 
@@ -546,6 +576,24 @@ class Store:
                 tracks,
             )
 
+    def replace_tags(self, norm_artist, pairs, now):
+        """Replace one artist's tags, and record that they were fetched.
+
+        The fetch is recorded even when nothing came back, so an artist Last.fm
+        has never heard of is not asked about again on every pass.
+        """
+        with self.connection:
+            self.connection.execute("DELETE FROM artist_tag WHERE norm_artist = ?", (norm_artist,))
+            self.connection.executemany(
+                "INSERT OR IGNORE INTO artist_tag (norm_artist, tag, weight) VALUES (?, ?, ?)",
+                [(norm_artist, tag, weight) for tag, weight in pairs],
+            )
+            self.connection.execute(
+                "INSERT INTO artist_tag_fetched (norm_artist, fetched_at) VALUES (?, ?)"
+                " ON CONFLICT (norm_artist) DO UPDATE SET fetched_at = excluded.fetched_at",
+                (norm_artist, now),
+            )
+
     def rebuild_keys(self):
         """Collapse the scrobble history into one row per distinct track.
 
@@ -1177,6 +1225,200 @@ PLAYLISTS = (
 )
 
 
+# Moods, as sets of Last.fm tags. Chosen for this library -- drum and bass,
+# punk and its descendants, big-room dance, classic rock -- rather than as a
+# general taxonomy. `--vibes` replaces the lot with a JSON file of the same
+# shape, so a new one does not need a new release.
+#
+# `years` filters on the album's release date, which is what separates eighties
+# synth records from everything a synthpop tag would otherwise drag in.
+DEFAULT_VIBES = (
+    {
+        "name": "80s-synths",
+        "tags": [
+            "synthpop", "synth pop", "synth-pop", "new wave", "synthwave",
+            "new romantic", "electropop", "80s", "1980s",
+        ],
+        "years": [1975, 1992],
+    },
+    {
+        "name": "high-energy-rock",
+        "tags": [
+            "hard rock", "punk rock", "pop punk", "punk", "alternative rock",
+            "rock", "garage rock", "skate punk",
+        ],
+    },
+    {
+        "name": "screamo",
+        "tags": [
+            "screamo", "post-hardcore", "metalcore", "emo", "hardcore",
+            "melodic hardcore", "emocore",
+        ],
+    },
+    {
+        "name": "drum-and-bass",
+        "tags": [
+            "drum and bass", "drum n bass", "dnb", "liquid funk", "neurofunk",
+            "jungle", "breakbeat",
+        ],
+    },
+    {
+        "name": "dance",
+        "tags": [
+            "electro house", "house", "big room", "electronic dance music",
+            "edm", "hardstyle", "trance", "dubstep", "electro",
+        ],
+    },
+    {
+        "name": "classic-rock",
+        "tags": [
+            "classic rock", "progressive rock", "psychedelic rock",
+            "blues rock", "70s", "60s",
+        ],
+    },
+)
+
+
+def load_vibes(path):
+    """Return the vibe definitions, from a file when one is given.
+
+    Validated up front rather than at write time: a bad name would otherwise
+    surface as a file created somewhere unintended, which is a poor way to find
+    out about a typo.
+    """
+    if not path:
+        return DEFAULT_VIBES
+    try:
+        loaded = json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise ValueError(f"cannot read vibes from {path}: {error}") from error
+
+    if not isinstance(loaded, list):
+        raise ValueError(f"{path}: expected a list of vibes")
+    for vibe in loaded:
+        name = vibe.get("name") if isinstance(vibe, dict) else None
+        if not name or not SAFE_VIBE_NAME.fullmatch(str(name)):
+            raise ValueError(f"{path}: {name!r} is not a usable vibe name (a-z, 0-9, -)")
+        if not vibe.get("tags"):
+            raise ValueError(f"{path}: vibe {name!r} lists no tags")
+    return tuple(loaded)
+
+
+def parse_tags(payload):
+    """Return (tag, weight) pairs from an artist.getTopTags response.
+
+    The documented sample carries only a name and a URL per tag; a live response
+    also carries a 0-100 `count`. Rather than depend on which, the count is used
+    when present and the documented ordering -- by popularity -- stands in for it
+    when it is not.
+    """
+    block = payload.get("toptags") or {}
+    pairs = []
+    for position, entry in enumerate(as_list(block.get("tag"))):
+        tag = (entry.get("name") or "").strip().casefold()
+        if not tag:
+            continue
+        weight = int(entry.get("count") or 0) or max(1, 100 - position * 5)
+        pairs.append((tag, weight))
+    return pairs
+
+
+def sync_tags(client, store, now, limit=0):
+    """Fetch crowd tags for library artists that have none, or stale ones.
+
+    One request per artist, once, and then only for artists newly added. An
+    artist Last.fm has never heard of is recorded as fetched with no tags, so it
+    is not asked about again every pass.
+    """
+    stale = store.connection.execute(
+        "SELECT a.norm_name AS norm_name, a.name AS name, a.mbid AS mbid"
+        "  FROM lidarr_artist a"
+        "  LEFT JOIN artist_tag_fetched f ON f.norm_artist = a.norm_name"
+        " WHERE a.norm_name <> ''"
+        "   AND (f.fetched_at IS NULL OR f.fetched_at < :cutoff)"
+        " ORDER BY a.name",
+        {"cutoff": now - TAG_REFRESH_SECONDS},
+    ).fetchall()
+    if not stale:
+        return 0
+
+    if limit > 0 and len(stale) > limit:
+        logger.info("tagging %d of %d artists this pass; the rest follow next", limit, len(stale))
+        stale = stale[:limit]
+    else:
+        logger.info("fetching tags for %d artists", len(stale))
+
+    tagged = 0
+    for artist in stale:
+        query = {"mbid": artist["mbid"]} if artist["mbid"] else {"artist": artist["name"]}
+        try:
+            payload = client.call("artist.getTopTags", {**query, "autocorrect": 1})
+        except LastfmError as error:
+            # One artist Last.fm cannot answer for is not worth losing the pass.
+            logger.warning("no tags for %s: %s", artist["name"], error)
+            continue
+        store.replace_tags(artist["norm_name"], parse_tags(payload), now)
+        tagged += 1
+
+    logger.info("tagged %d artists", tagged)
+    return tagged
+
+
+def build_vibe_playlists(store, vibes, mirror_root, library_root, limit, now):
+    """Write one playlist per mood. Returns how many tracks were listed."""
+    if not store.scalar("SELECT COUNT(*) FROM artist_tag"):
+        return 0
+
+    directory = Path(mirror_root) / PLAYLIST_DIRECTORY
+    week = now // ROTATION_PERIOD_SECONDS
+    total = 0
+
+    for vibe in vibes:
+        tags = [str(tag).strip().casefold() for tag in vibe["tags"]]
+        placeholders = ",".join("?" * len(tags))
+        years = vibe.get("years")
+        parameters = [*tags, vibe.get("min_score", VIBE_MIN_SCORE)]
+        year_clause = ""
+        if years:
+            year_clause = (
+                " AND CAST(substr(al.release_date, 1, 4) AS INTEGER) BETWEEN ? AND ?"
+            )
+            parameters += [int(years[0]), int(years[1])]
+        parameters += [week, limit]
+
+        sql = f"""
+            WITH vibe AS (
+                SELECT norm_artist, SUM(weight) AS score
+                  FROM artist_tag
+                 WHERE tag IN ({placeholders})
+                 GROUP BY norm_artist
+                HAVING SUM(weight) >= ?
+            )
+            SELECT t.id AS id, a.name AS artist, t.title AS title,
+                   t.duration AS duration, t.path AS path
+              FROM lidarr_track t
+              JOIN lidarr_artist a ON a.id = t.artist_id
+              JOIN vibe v ON v.norm_artist = a.norm_name
+              LEFT JOIN lidarr_album al ON al.id = t.album_id
+             WHERE {PLAYABLE}{year_clause}
+             ORDER BY ((t.id * {SHUFFLE_MULTIPLIER}) + ?) % {SHUFFLE_MODULUS}
+             LIMIT ?
+        """
+
+        entries = []
+        for row in store.connection.execute(sql, parameters):
+            mirror = mirror_path_for(row["path"], library_root, mirror_root)
+            if mirror is None or not mirror.is_file():
+                continue
+            entries.append({**dict(row), "mirror": mirror})
+
+        write_playlist(directory / f"{vibe['name']}.m3u", entries)
+        total += len(entries)
+        logger.info("playlist %-20s %4d tracks -- by tag", vibe["name"], len(entries))
+
+    return total
+
+
 def library_root_of(store):
     """Return the directory Lidarr's artist folders sit under.
 
@@ -1494,7 +1736,7 @@ def report(store, now):
 
 def run_once(
     client, store, user, now, backfill_limit, lidarr=None, mirror=None,
-    library_root=None, playlist_limit=100,
+    library_root=None, playlist_limit=100, vibes=DEFAULT_VIBES, tag_limit=0,
 ):
     """Run a single pass. Returns the number of scrobbles added."""
     started = time.monotonic()
@@ -1517,6 +1759,8 @@ def run_once(
                 )
             else:
                 build_playlists(store, mirror, root, playlist_limit, now)
+                sync_tags(client, store, now, limit=tag_limit)
+                build_vibe_playlists(store, vibes, mirror, root, playlist_limit, now)
 
     logger.info(
         "pass complete in %.1fs: %d scrobbles added, %d loved",
@@ -1608,6 +1852,18 @@ def build_parser():
         default=int(os.getenv("MUSIC_CURATOR_PLAYLIST_LIMIT", "100")),
         help="most tracks to put in any one playlist (env MUSIC_CURATOR_PLAYLIST_LIMIT)",
     )
+    parser.add_argument(
+        "--vibes",
+        default=os.getenv("MUSIC_CURATOR_VIBES"),
+        help="JSON file of mood definitions, replacing the built-in set"
+        " (env MUSIC_CURATOR_VIBES)",
+    )
+    parser.add_argument(
+        "--tag-limit",
+        type=int,
+        default=int(os.getenv("MUSIC_CURATOR_TAG_LIMIT", "0")),
+        help="cap artist tag lookups per pass; 0 for no cap (env MUSIC_CURATOR_TAG_LIMIT)",
+    )
     parser.add_argument(
         "--skip-index",
         action="store_true",
@@ -1632,6 +1888,7 @@ def main(argv=None, clock=time.time):
 
     try:
         interval = parse_interval(args.interval) if args.interval else None
+        vibes = load_vibes(args.vibes)
     except ValueError as error:
         logger.error("%s", error)
         return 2
@@ -1682,6 +1939,8 @@ def main(argv=None, clock=time.time):
                     args.mirror,
                     args.library_root,
                     args.playlist_limit,
+                    vibes,
+                    args.tag_limit,
                 )
             except (LastfmError, LidarrError) as error:
                 logger.error("%s", error)

tests/conftest.py

@@ -56,7 +56,9 @@ class FakeLastfm:
     is prepended to the first page with no `date`.
     """
 
-    def __init__(self, tracks=(), loved=(), nowplaying=None, outcomes=()):
+    def __init__(self, tracks=(), loved=(), nowplaying=None, outcomes=(), tags=None):
+        # Keyed by mbid or by artist name, whichever the caller asked with.
+        self.tags = tags or {}
         self.tracks = sorted(tracks, key=lambda track: int(track["date"]["uts"]), reverse=True)
         self.loved = list(loved)
         self.nowplaying = nowplaying
@@ -82,6 +84,9 @@ class FakeLastfm:
             return json.dumps(self._recent(query))
         if method == "user.getlovedtracks":
             return json.dumps(self._loved(query))
+        if method == "artist.gettoptags":
+            key = query.get("mbid") or query.get("artist", "")
+            return json.dumps({"toptags": {"tag": self.tags.get(key, [])}})
         raise AssertionError(f"unexpected method {method}")
 
     def _recent(self, query):

tests/test_music_curator.py

@@ -983,3 +983,147 @@ def test_playlists_are_group_readable(tmp_path):
 
     for playlist in (mirror / "_playlists").glob("*.m3u"):
         assert playlist.stat().st_mode & stat.S_IRGRP, playlist
+
+
+def test_tag_weights_fall_back_to_rank_when_no_count_is_sent():
+    """The documented sample carries only a name and a URL; a live response also
+    carries a count. Neither may be relied on alone."""
+    with_count = music_curator.parse_tags(
+        {"toptags": {"tag": [{"name": "Screamo", "count": 100}, {"name": "emo", "count": 40}]}}
+    )
+    assert with_count == [("screamo", 100), ("emo", 40)]
+
+    without = music_curator.parse_tags(
+        {"toptags": {"tag": [{"name": "screamo"}, {"name": "emo"}]}}
+    )
+    assert [tag for tag, _ in without] == ["screamo", "emo"]
+    assert without[0][1] > without[1][1]
+
+
+def test_a_lone_tag_is_not_a_list():
+    assert music_curator.parse_tags({"toptags": {"tag": {"name": "dnb", "count": 90}}}) == [
+        ("dnb", 90)
+    ]
+
+
+def test_an_artist_lastfm_cannot_answer_for_is_not_asked_again(tmp_path):
+    """Recording the fetch even when it returns nothing is what stops a pass
+    spending a request per unknown artist, forever."""
+    api, source, mirror = playlist_library(tmp_path)
+    store = store_at(tmp_path)
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=api), store
+    )
+    lastfm = FakeLastfm(tags={})
+
+    assert music_curator.sync_tags(client_for(lastfm), store, NOW) == 2
+    before = len(lastfm.calls)
+
+    assert music_curator.sync_tags(client_for(lastfm), store, NOW) == 0
+    assert len(lastfm.calls) == before
+
+
+def test_tags_are_refetched_once_they_go_stale(tmp_path):
+    api, source, mirror = playlist_library(tmp_path)
+    store = store_at(tmp_path)
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=api), store
+    )
+    lastfm = FakeLastfm(tags={})
+    music_curator.sync_tags(client_for(lastfm), store, NOW)
+
+    later = NOW + music_curator.TAG_REFRESH_SECONDS + 1
+    assert music_curator.sync_tags(client_for(lastfm), store, later) == 2
+
+
+def tagged_store(tmp_path, tags):
+    api, source, mirror = playlist_library(tmp_path)
+    store = store_at(tmp_path)
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=api), store
+    )
+    music_curator.sync_tags(client_for(FakeLastfm(tags=tags)), store, NOW)
+    return store, source, mirror
+
+
+def test_a_vibe_selects_by_tag(tmp_path):
+    store, source, mirror = tagged_store(
+        tmp_path,
+        {
+            "artist-mbid-1": [{"name": "screamo", "count": 100}],
+            "artist-mbid-2": [{"name": "classic rock", "count": 100}],
+        },
+    )
+    vibes = [{"name": "screamo", "tags": ["screamo"]}]
+
+    music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW)
+
+    written = (mirror / "_playlists" / "screamo.m3u").read_text()
+    assert "Played Band" in written
+    assert "Silent Band" not in written
+
+
+def test_a_weakly_tagged_artist_is_below_the_threshold(tmp_path):
+    """A single low-weight tag is not a genre, it is somebody's stray opinion."""
+    store, source, mirror = tagged_store(
+        tmp_path, {"artist-mbid-1": [{"name": "screamo", "count": 3}]}
+    )
+    vibes = [{"name": "screamo", "tags": ["screamo"]}]
+
+    music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW)
+
+    assert (mirror / "_playlists" / "screamo.m3u").read_text() == "#EXTM3U\n"
+
+
+def test_a_vibe_can_be_restricted_by_release_year(tmp_path):
+    """What separates eighties synth records from everything else a synthpop
+    tag drags in."""
+    store, source, mirror = tagged_store(
+        tmp_path, {"artist-mbid-1": [{"name": "synthpop", "count": 100}]}
+    )
+    inside = [{"name": "eighties", "tags": ["synthpop"], "years": [1975, 1992]}]
+    outside = [{"name": "nineties", "tags": ["synthpop"], "years": [1993, 1999]}]
+
+    # FakeLidarr dates every album 2019, so neither window should catch it.
+    music_curator.build_vibe_playlists(store, inside, mirror, str(source), 100, NOW)
+    music_curator.build_vibe_playlists(store, outside, mirror, str(source), 100, NOW)
+    assert (mirror / "_playlists" / "eighties.m3u").read_text() == "#EXTM3U\n"
+    assert (mirror / "_playlists" / "nineties.m3u").read_text() == "#EXTM3U\n"
+
+    modern = [{"name": "modern", "tags": ["synthpop"], "years": [2000, 2030]}]
+    music_curator.build_vibe_playlists(store, modern, mirror, str(source), 100, NOW)
+    assert "Played Band" in (mirror / "_playlists" / "modern.m3u").read_text()
+
+
+def test_the_built_in_vibes_are_all_usable_filenames():
+    for vibe in music_curator.DEFAULT_VIBES:
+        assert music_curator.SAFE_VIBE_NAME.fullmatch(vibe["name"]), vibe["name"]
+        assert vibe["tags"]
+
+
+def test_a_vibes_file_replaces_the_built_in_set(tmp_path):
+    path = tmp_path / "vibes.json"
+    path.write_text(json.dumps([{"name": "mine", "tags": ["shoegaze"]}]))
+
+    assert music_curator.load_vibes(str(path)) == ({"name": "mine", "tags": ["shoegaze"]},)
+    assert music_curator.load_vibes(None) is music_curator.DEFAULT_VIBES
+
+
+@pytest.mark.parametrize(
+    "content",
+    [
+        '{"not": "a list"}',
+        '[{"tags": ["x"]}]',
+        '[{"name": "../escape", "tags": ["x"]}]',
+        '[{"name": "ok"}]',
+        "not json at all",
+    ],
+)
+def test_a_bad_vibes_file_is_refused_up_front(tmp_path, content):
+    """A bad name would otherwise surface as a file written somewhere
+    unintended, which is a poor way to learn about a typo."""
+    path = tmp_path / "vibes.json"
+    path.write_text(content)
+
+    with pytest.raises(ValueError):
+        music_curator.load_vibes(str(path))