From 15e5ee5aea918899719253b6c8504e4312f67383 Mon Sep 17 00:00:00 2001 From: Emma Thorpe Date: Mon, 24 Aug 2026 17:56:35 +0100 Subject: [PATCH] feat: mood playlists from Last.fm crowd tags A second set of playlists selecting by genre and mood rather than by play history: eighties synths, high energy rock, screamo, drum and bass, dance, classic rock. The tags come from Last.fm rather than MusicBrainz. MusicBrainz genres arrive free with the Lidarr index, which makes them the obvious choice and the wrong one: they are sparse and formal, and will not tell you a record is screamo or synthwave. Crowd tags will, because people typed them. One request per artist, by MusicBrainz id where Lidarr has one, refreshed every ninety days. An artist Last.fm has never heard of is recorded as fetched with no tags rather than left unmarked, so it is not asked about again on every pass forever. The tag table is keyed on the normalised artist name, not the Lidarr id, so it survives an artist being removed and re-added there. Weights are taken from the response's count where it has one. The documented sample carries only a name and a URL, a live response also carries a 0-100 count, and depending on either alone would be a guess -- so the count is used when present and the documented ordering by popularity stands in when it is not. An artist qualifies for a mood when their weights inside it sum to at least thirty. A single low-weight tag is not a genre, it is somebody's stray opinion. A mood may also restrict release years, which is what separates eighties synth records from everything else a synthpop tag drags in. The built-in set is chosen for this library rather than as a taxonomy, and --vibes replaces it wholesale with a JSON file so a new mood does not need a new release. Names are validated when that file is read: an invalid one would otherwise only surface as a playlist written somewhere unintended. --- README.md | 49 ++++++- music_curator.py | 261 +++++++++++++++++++++++++++++++++++- tests/conftest.py | 7 +- tests/test_music_curator.py | 144 ++++++++++++++++++++ 4 files changed, 457 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b0b4fb4..d70690a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ question that mirror cannot: which of it is worth carrying, and which of it has not been played in years. **This is stage three.** It ingests the scrobble history, indexes the library -from Lidarr, matches one to the other, and writes playlists into the mirror. +from Lidarr, matches one to the other, and writes playlists into the mirror — +by listening history and by mood. Nothing is written back to Lidarr — every call there is a `GET`. See "Where this is going" below. @@ -151,6 +152,48 @@ few hours, and a playlist that reorders itself each time is one that has to be re-imported each time — the Music app imports a snapshot of a file, it does not track it. +### Moods + +A second set of playlists selects by **Last.fm's crowd tags** rather than by +listening history. MusicBrainz genres arrive free with the Lidarr index and are +no use for this: they are sparse and formal, and will not tell you a record is +screamo or synthwave. People typing tags will. + +Tags are fetched once per artist — one `artist.getTopTags` call each, by +MusicBrainz id where Lidarr has one — and refreshed every ninety days. An +artist Last.fm has never heard of is recorded as fetched with no tags, so it is +not asked about again on every pass. `--tag-limit` spreads the first sweep over +several passes. + +The built-in moods are chosen for this library rather than as a general +taxonomy: + +| Mood | Selected on | +| ------------------ | ------------------------------------------------- | +| `80s-synths` | synthpop, new wave, synthwave — released 1975-1992 | +| `high-energy-rock` | hard rock, punk, pop punk, alternative | +| `screamo` | screamo, post-hardcore, metalcore, emo | +| `drum-and-bass` | drum and bass, liquid funk, neurofunk, jungle | +| `dance` | house, big room, hardstyle, trance, dubstep | +| `classic-rock` | classic rock, prog, psychedelic, blues rock | + +An artist qualifies when their tag weights inside a mood sum to at least 30 out +of Last.fm's 0-100 scale. One low-weight tag is not a genre, it is somebody's +stray opinion. + +`years` filters on the album's release date, which is what separates eighties +synth records from everything else a synthpop tag drags in. + +`--vibes` replaces the whole set with a JSON file of the same shape, so a new +mood does not need a new release: + +```json +[{ "name": "shoegaze", "tags": ["shoegaze", "dream pop"], "min_score": 40 }] +``` + +Names are validated when the file is read, not when the file is written. A bad +one would otherwise surface as a playlist created somewhere unintended. + ### Paths Lidarr knows where the lossless source is; the playlists have to point at the @@ -229,6 +272,8 @@ music-curator --report-only # report on the store, fetch nothing | `--mirror` | `MUSIC_CURATOR_MIRROR` | unset | Root of the MP3 mirror; playlists go here | | `--library-root` | `MUSIC_CURATOR_LIBRARY_ROOT` | derived | Prefix to strip from Lidarr's paths | | `--playlist-limit` | `MUSIC_CURATOR_PLAYLIST_LIMIT` | `100` | Most tracks in any one playlist | +| `--vibes` | `MUSIC_CURATOR_VIBES` | built-in | JSON file of mood definitions | +| `--tag-limit` | `MUSIC_CURATOR_TAG_LIMIT` | `0` | Cap artist tag lookups per pass | | `--skip-index` | — | off | Match against the index already held | | `--report-only` | — | off | Report without fetching | @@ -278,7 +323,7 @@ nix shell nixpkgs#python3Packages.pytest -c pytest | Last.fm ingest and store | done | | Lidarr index and the scrobble-to-track matcher | done | | M3U playlists from the listening history | done | -| Genre and mood playlists from Last.fm tags | next | +| Genre and mood playlists from Last.fm tags | done | | Cold-music report, unmonitoring what is not played | last | The cull will unmonitor cold albums in Lidarr and tag their artists. It will diff --git a/music_curator.py b/music_curator.py index 9d4d930..4b78036 100644 --- a/music_curator.py +++ b/music_curator.py @@ -69,6 +69,18 @@ BACKOFF_CEILING_SECONDS = 60.0 MIRROR_SUFFIX = ".mp3" PLAYLIST_DIRECTORY = "_playlists" +# Tags are re-fetched this often. They move slowly, and the first pass over a +# library already costs one request per artist. +TAG_REFRESH_SECONDS = 90 * 86400 + +# How much of an artist's tag weight has to fall inside a vibe before their +# tracks are eligible for it. Weights are Last.fm's own 0-100 popularity, summed +# across whichever of the vibe's tags the artist carries. +VIBE_MIN_SCORE = 30 + +# A vibe writes a file named after itself, so the name has to be a filename. +SAFE_VIBE_NAME = re.compile(r"^[a-z0-9][a-z0-9-]*$") + # Group-readable, for the same reason music-mirror sets it: the mirror is read # back by whatever serves it, and a playlist nobody can read is not a playlist. GROUP_READ = 0o040 @@ -177,6 +189,24 @@ CREATE TABLE IF NOT EXISTS scrobble_key ( PRIMARY KEY (artist, track) ); CREATE INDEX IF NOT EXISTS scrobble_key_track ON scrobble_key (track_id); + +-- Last.fm's crowd tags for each library artist. MusicBrainz genres come free +-- with the Lidarr index and are useless for this: they are sparse and formal, +-- and will not tell you a record is screamo or synthwave. People typing tags +-- will. Keyed on the normalised name rather than the Lidarr id so it survives +-- an artist being removed and re-added there. +CREATE TABLE IF NOT EXISTS artist_tag ( + norm_artist TEXT NOT NULL, + tag TEXT NOT NULL, + weight INTEGER NOT NULL, + PRIMARY KEY (norm_artist, tag) +); +CREATE INDEX IF NOT EXISTS artist_tag_tag ON artist_tag (tag); + +CREATE TABLE IF NOT EXISTS artist_tag_fetched ( + norm_artist TEXT PRIMARY KEY, + fetched_at INTEGER NOT NULL +); """ @@ -546,6 +576,24 @@ class Store: tracks, ) + def replace_tags(self, norm_artist, pairs, now): + """Replace one artist's tags, and record that they were fetched. + + The fetch is recorded even when nothing came back, so an artist Last.fm + has never heard of is not asked about again on every pass. + """ + with self.connection: + self.connection.execute("DELETE FROM artist_tag WHERE norm_artist = ?", (norm_artist,)) + self.connection.executemany( + "INSERT OR IGNORE INTO artist_tag (norm_artist, tag, weight) VALUES (?, ?, ?)", + [(norm_artist, tag, weight) for tag, weight in pairs], + ) + self.connection.execute( + "INSERT INTO artist_tag_fetched (norm_artist, fetched_at) VALUES (?, ?)" + " ON CONFLICT (norm_artist) DO UPDATE SET fetched_at = excluded.fetched_at", + (norm_artist, now), + ) + def rebuild_keys(self): """Collapse the scrobble history into one row per distinct track. @@ -1177,6 +1225,200 @@ PLAYLISTS = ( ) +# Moods, as sets of Last.fm tags. Chosen for this library -- drum and bass, +# punk and its descendants, big-room dance, classic rock -- rather than as a +# general taxonomy. `--vibes` replaces the lot with a JSON file of the same +# shape, so a new one does not need a new release. +# +# `years` filters on the album's release date, which is what separates eighties +# synth records from everything a synthpop tag would otherwise drag in. +DEFAULT_VIBES = ( + { + "name": "80s-synths", + "tags": [ + "synthpop", "synth pop", "synth-pop", "new wave", "synthwave", + "new romantic", "electropop", "80s", "1980s", + ], + "years": [1975, 1992], + }, + { + "name": "high-energy-rock", + "tags": [ + "hard rock", "punk rock", "pop punk", "punk", "alternative rock", + "rock", "garage rock", "skate punk", + ], + }, + { + "name": "screamo", + "tags": [ + "screamo", "post-hardcore", "metalcore", "emo", "hardcore", + "melodic hardcore", "emocore", + ], + }, + { + "name": "drum-and-bass", + "tags": [ + "drum and bass", "drum n bass", "dnb", "liquid funk", "neurofunk", + "jungle", "breakbeat", + ], + }, + { + "name": "dance", + "tags": [ + "electro house", "house", "big room", "electronic dance music", + "edm", "hardstyle", "trance", "dubstep", "electro", + ], + }, + { + "name": "classic-rock", + "tags": [ + "classic rock", "progressive rock", "psychedelic rock", + "blues rock", "70s", "60s", + ], + }, +) + + +def load_vibes(path): + """Return the vibe definitions, from a file when one is given. + + Validated up front rather than at write time: a bad name would otherwise + surface as a file created somewhere unintended, which is a poor way to find + out about a typo. + """ + if not path: + return DEFAULT_VIBES + try: + loaded = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read vibes from {path}: {error}") from error + + if not isinstance(loaded, list): + raise ValueError(f"{path}: expected a list of vibes") + for vibe in loaded: + name = vibe.get("name") if isinstance(vibe, dict) else None + if not name or not SAFE_VIBE_NAME.fullmatch(str(name)): + raise ValueError(f"{path}: {name!r} is not a usable vibe name (a-z, 0-9, -)") + if not vibe.get("tags"): + raise ValueError(f"{path}: vibe {name!r} lists no tags") + return tuple(loaded) + + +def parse_tags(payload): + """Return (tag, weight) pairs from an artist.getTopTags response. + + The documented sample carries only a name and a URL per tag; a live response + also carries a 0-100 `count`. Rather than depend on which, the count is used + when present and the documented ordering -- by popularity -- stands in for it + when it is not. + """ + block = payload.get("toptags") or {} + pairs = [] + for position, entry in enumerate(as_list(block.get("tag"))): + tag = (entry.get("name") or "").strip().casefold() + if not tag: + continue + weight = int(entry.get("count") or 0) or max(1, 100 - position * 5) + pairs.append((tag, weight)) + return pairs + + +def sync_tags(client, store, now, limit=0): + """Fetch crowd tags for library artists that have none, or stale ones. + + One request per artist, once, and then only for artists newly added. An + artist Last.fm has never heard of is recorded as fetched with no tags, so it + is not asked about again every pass. + """ + stale = store.connection.execute( + "SELECT a.norm_name AS norm_name, a.name AS name, a.mbid AS mbid" + " FROM lidarr_artist a" + " LEFT JOIN artist_tag_fetched f ON f.norm_artist = a.norm_name" + " WHERE a.norm_name <> ''" + " AND (f.fetched_at IS NULL OR f.fetched_at < :cutoff)" + " ORDER BY a.name", + {"cutoff": now - TAG_REFRESH_SECONDS}, + ).fetchall() + if not stale: + return 0 + + if limit > 0 and len(stale) > limit: + logger.info("tagging %d of %d artists this pass; the rest follow next", limit, len(stale)) + stale = stale[:limit] + else: + logger.info("fetching tags for %d artists", len(stale)) + + tagged = 0 + for artist in stale: + query = {"mbid": artist["mbid"]} if artist["mbid"] else {"artist": artist["name"]} + try: + payload = client.call("artist.getTopTags", {**query, "autocorrect": 1}) + except LastfmError as error: + # One artist Last.fm cannot answer for is not worth losing the pass. + logger.warning("no tags for %s: %s", artist["name"], error) + continue + store.replace_tags(artist["norm_name"], parse_tags(payload), now) + tagged += 1 + + logger.info("tagged %d artists", tagged) + return tagged + + +def build_vibe_playlists(store, vibes, mirror_root, library_root, limit, now): + """Write one playlist per mood. Returns how many tracks were listed.""" + if not store.scalar("SELECT COUNT(*) FROM artist_tag"): + return 0 + + directory = Path(mirror_root) / PLAYLIST_DIRECTORY + week = now // ROTATION_PERIOD_SECONDS + total = 0 + + for vibe in vibes: + tags = [str(tag).strip().casefold() for tag in vibe["tags"]] + placeholders = ",".join("?" * len(tags)) + years = vibe.get("years") + parameters = [*tags, vibe.get("min_score", VIBE_MIN_SCORE)] + year_clause = "" + if years: + year_clause = ( + " AND CAST(substr(al.release_date, 1, 4) AS INTEGER) BETWEEN ? AND ?" + ) + parameters += [int(years[0]), int(years[1])] + parameters += [week, limit] + + sql = f""" + WITH vibe AS ( + SELECT norm_artist, SUM(weight) AS score + FROM artist_tag + WHERE tag IN ({placeholders}) + GROUP BY norm_artist + HAVING SUM(weight) >= ? + ) + SELECT t.id AS id, a.name AS artist, t.title AS title, + t.duration AS duration, t.path AS path + FROM lidarr_track t + JOIN lidarr_artist a ON a.id = t.artist_id + JOIN vibe v ON v.norm_artist = a.norm_name + LEFT JOIN lidarr_album al ON al.id = t.album_id + WHERE {PLAYABLE}{year_clause} + ORDER BY ((t.id * {SHUFFLE_MULTIPLIER}) + ?) % {SHUFFLE_MODULUS} + LIMIT ? + """ + + entries = [] + for row in store.connection.execute(sql, parameters): + mirror = mirror_path_for(row["path"], library_root, mirror_root) + if mirror is None or not mirror.is_file(): + continue + entries.append({**dict(row), "mirror": mirror}) + + write_playlist(directory / f"{vibe['name']}.m3u", entries) + total += len(entries) + logger.info("playlist %-20s %4d tracks -- by tag", vibe["name"], len(entries)) + + return total + + def library_root_of(store): """Return the directory Lidarr's artist folders sit under. @@ -1494,7 +1736,7 @@ def report(store, now): def run_once( client, store, user, now, backfill_limit, lidarr=None, mirror=None, - library_root=None, playlist_limit=100, + library_root=None, playlist_limit=100, vibes=DEFAULT_VIBES, tag_limit=0, ): """Run a single pass. Returns the number of scrobbles added.""" started = time.monotonic() @@ -1517,6 +1759,8 @@ def run_once( ) else: build_playlists(store, mirror, root, playlist_limit, now) + sync_tags(client, store, now, limit=tag_limit) + build_vibe_playlists(store, vibes, mirror, root, playlist_limit, now) logger.info( "pass complete in %.1fs: %d scrobbles added, %d loved", @@ -1608,6 +1852,18 @@ def build_parser(): default=int(os.getenv("MUSIC_CURATOR_PLAYLIST_LIMIT", "100")), help="most tracks to put in any one playlist (env MUSIC_CURATOR_PLAYLIST_LIMIT)", ) + parser.add_argument( + "--vibes", + default=os.getenv("MUSIC_CURATOR_VIBES"), + help="JSON file of mood definitions, replacing the built-in set" + " (env MUSIC_CURATOR_VIBES)", + ) + parser.add_argument( + "--tag-limit", + type=int, + default=int(os.getenv("MUSIC_CURATOR_TAG_LIMIT", "0")), + help="cap artist tag lookups per pass; 0 for no cap (env MUSIC_CURATOR_TAG_LIMIT)", + ) parser.add_argument( "--skip-index", action="store_true", @@ -1632,6 +1888,7 @@ def main(argv=None, clock=time.time): try: interval = parse_interval(args.interval) if args.interval else None + vibes = load_vibes(args.vibes) except ValueError as error: logger.error("%s", error) return 2 @@ -1682,6 +1939,8 @@ def main(argv=None, clock=time.time): args.mirror, args.library_root, args.playlist_limit, + vibes, + args.tag_limit, ) except (LastfmError, LidarrError) as error: logger.error("%s", error) diff --git a/tests/conftest.py b/tests/conftest.py index d525641..f9ab7df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,9 @@ class FakeLastfm: is prepended to the first page with no `date`. """ - def __init__(self, tracks=(), loved=(), nowplaying=None, outcomes=()): + def __init__(self, tracks=(), loved=(), nowplaying=None, outcomes=(), tags=None): + # Keyed by mbid or by artist name, whichever the caller asked with. + self.tags = tags or {} self.tracks = sorted(tracks, key=lambda track: int(track["date"]["uts"]), reverse=True) self.loved = list(loved) self.nowplaying = nowplaying @@ -82,6 +84,9 @@ class FakeLastfm: return json.dumps(self._recent(query)) if method == "user.getlovedtracks": return json.dumps(self._loved(query)) + if method == "artist.gettoptags": + key = query.get("mbid") or query.get("artist", "") + return json.dumps({"toptags": {"tag": self.tags.get(key, [])}}) raise AssertionError(f"unexpected method {method}") def _recent(self, query): diff --git a/tests/test_music_curator.py b/tests/test_music_curator.py index f31e8a2..c6443fc 100644 --- a/tests/test_music_curator.py +++ b/tests/test_music_curator.py @@ -983,3 +983,147 @@ def test_playlists_are_group_readable(tmp_path): for playlist in (mirror / "_playlists").glob("*.m3u"): assert playlist.stat().st_mode & stat.S_IRGRP, playlist + + +def test_tag_weights_fall_back_to_rank_when_no_count_is_sent(): + """The documented sample carries only a name and a URL; a live response also + carries a count. Neither may be relied on alone.""" + with_count = music_curator.parse_tags( + {"toptags": {"tag": [{"name": "Screamo", "count": 100}, {"name": "emo", "count": 40}]}} + ) + assert with_count == [("screamo", 100), ("emo", 40)] + + without = music_curator.parse_tags( + {"toptags": {"tag": [{"name": "screamo"}, {"name": "emo"}]}} + ) + assert [tag for tag, _ in without] == ["screamo", "emo"] + assert without[0][1] > without[1][1] + + +def test_a_lone_tag_is_not_a_list(): + assert music_curator.parse_tags({"toptags": {"tag": {"name": "dnb", "count": 90}}}) == [ + ("dnb", 90) + ] + + +def test_an_artist_lastfm_cannot_answer_for_is_not_asked_again(tmp_path): + """Recording the fetch even when it returns nothing is what stops a pass + spending a request per unknown artist, forever.""" + api, source, mirror = playlist_library(tmp_path) + store = store_at(tmp_path) + music_curator.index_library( + music_curator.Lidarr("http://lidarr", "key", transport=api), store + ) + lastfm = FakeLastfm(tags={}) + + assert music_curator.sync_tags(client_for(lastfm), store, NOW) == 2 + before = len(lastfm.calls) + + assert music_curator.sync_tags(client_for(lastfm), store, NOW) == 0 + assert len(lastfm.calls) == before + + +def test_tags_are_refetched_once_they_go_stale(tmp_path): + api, source, mirror = playlist_library(tmp_path) + store = store_at(tmp_path) + music_curator.index_library( + music_curator.Lidarr("http://lidarr", "key", transport=api), store + ) + lastfm = FakeLastfm(tags={}) + music_curator.sync_tags(client_for(lastfm), store, NOW) + + later = NOW + music_curator.TAG_REFRESH_SECONDS + 1 + assert music_curator.sync_tags(client_for(lastfm), store, later) == 2 + + +def tagged_store(tmp_path, tags): + api, source, mirror = playlist_library(tmp_path) + store = store_at(tmp_path) + music_curator.index_library( + music_curator.Lidarr("http://lidarr", "key", transport=api), store + ) + music_curator.sync_tags(client_for(FakeLastfm(tags=tags)), store, NOW) + return store, source, mirror + + +def test_a_vibe_selects_by_tag(tmp_path): + store, source, mirror = tagged_store( + tmp_path, + { + "artist-mbid-1": [{"name": "screamo", "count": 100}], + "artist-mbid-2": [{"name": "classic rock", "count": 100}], + }, + ) + vibes = [{"name": "screamo", "tags": ["screamo"]}] + + music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW) + + written = (mirror / "_playlists" / "screamo.m3u").read_text() + assert "Played Band" in written + assert "Silent Band" not in written + + +def test_a_weakly_tagged_artist_is_below_the_threshold(tmp_path): + """A single low-weight tag is not a genre, it is somebody's stray opinion.""" + store, source, mirror = tagged_store( + tmp_path, {"artist-mbid-1": [{"name": "screamo", "count": 3}]} + ) + vibes = [{"name": "screamo", "tags": ["screamo"]}] + + music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW) + + assert (mirror / "_playlists" / "screamo.m3u").read_text() == "#EXTM3U\n" + + +def test_a_vibe_can_be_restricted_by_release_year(tmp_path): + """What separates eighties synth records from everything else a synthpop + tag drags in.""" + store, source, mirror = tagged_store( + tmp_path, {"artist-mbid-1": [{"name": "synthpop", "count": 100}]} + ) + inside = [{"name": "eighties", "tags": ["synthpop"], "years": [1975, 1992]}] + outside = [{"name": "nineties", "tags": ["synthpop"], "years": [1993, 1999]}] + + # FakeLidarr dates every album 2019, so neither window should catch it. + music_curator.build_vibe_playlists(store, inside, mirror, str(source), 100, NOW) + music_curator.build_vibe_playlists(store, outside, mirror, str(source), 100, NOW) + assert (mirror / "_playlists" / "eighties.m3u").read_text() == "#EXTM3U\n" + assert (mirror / "_playlists" / "nineties.m3u").read_text() == "#EXTM3U\n" + + modern = [{"name": "modern", "tags": ["synthpop"], "years": [2000, 2030]}] + music_curator.build_vibe_playlists(store, modern, mirror, str(source), 100, NOW) + assert "Played Band" in (mirror / "_playlists" / "modern.m3u").read_text() + + +def test_the_built_in_vibes_are_all_usable_filenames(): + for vibe in music_curator.DEFAULT_VIBES: + assert music_curator.SAFE_VIBE_NAME.fullmatch(vibe["name"]), vibe["name"] + assert vibe["tags"] + + +def test_a_vibes_file_replaces_the_built_in_set(tmp_path): + path = tmp_path / "vibes.json" + path.write_text(json.dumps([{"name": "mine", "tags": ["shoegaze"]}])) + + assert music_curator.load_vibes(str(path)) == ({"name": "mine", "tags": ["shoegaze"]},) + assert music_curator.load_vibes(None) is music_curator.DEFAULT_VIBES + + +@pytest.mark.parametrize( + "content", + [ + '{"not": "a list"}', + '[{"tags": ["x"]}]', + '[{"name": "../escape", "tags": ["x"]}]', + '[{"name": "ok"}]', + "not json at all", + ], +) +def test_a_bad_vibes_file_is_refused_up_front(tmp_path, content): + """A bad name would otherwise surface as a file written somewhere + unintended, which is a poor way to learn about a typo.""" + path = tmp_path / "vibes.json" + path.write_text(content) + + with pytest.raises(ValueError): + music_curator.load_vibes(str(path))