diff --git a/README.md b/README.md index b0b4fb4..d70690a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ question that mirror cannot: which of it is worth carrying, and which of it has not been played in years. **This is stage three.** It ingests the scrobble history, indexes the library -from Lidarr, matches one to the other, and writes playlists into the mirror. +from Lidarr, matches one to the other, and writes playlists into the mirror — +by listening history and by mood. Nothing is written back to Lidarr — every call there is a `GET`. See "Where this is going" below. @@ -151,6 +152,48 @@ few hours, and a playlist that reorders itself each time is one that has to be re-imported each time — the Music app imports a snapshot of a file, it does not track it. +### Moods + +A second set of playlists selects by **Last.fm's crowd tags** rather than by +listening history. MusicBrainz genres arrive free with the Lidarr index and are +no use for this: they are sparse and formal, and will not tell you a record is +screamo or synthwave. People typing tags will. + +Tags are fetched once per artist — one `artist.getTopTags` call each, by +MusicBrainz id where Lidarr has one — and refreshed every ninety days. An +artist Last.fm has never heard of is recorded as fetched with no tags, so it is +not asked about again on every pass. `--tag-limit` spreads the first sweep over +several passes. + +The built-in moods are chosen for this library rather than as a general +taxonomy: + +| Mood | Selected on | +| ------------------ | ------------------------------------------------- | +| `80s-synths` | synthpop, new wave, synthwave — released 1975-1992 | +| `high-energy-rock` | hard rock, punk, pop punk, alternative | +| `screamo` | screamo, post-hardcore, metalcore, emo | +| `drum-and-bass` | drum and bass, liquid funk, neurofunk, jungle | +| `dance` | house, big room, hardstyle, trance, dubstep | +| `classic-rock` | classic rock, prog, psychedelic, blues rock | + +An artist qualifies when their tag weights inside a mood sum to at least 30 out +of Last.fm's 0-100 scale. One low-weight tag is not a genre, it is somebody's +stray opinion. + +`years` filters on the album's release date, which is what separates eighties +synth records from everything else a synthpop tag drags in. + +`--vibes` replaces the whole set with a JSON file of the same shape, so a new +mood does not need a new release: + +```json +[{ "name": "shoegaze", "tags": ["shoegaze", "dream pop"], "min_score": 40 }] +``` + +Names are validated when the file is read, not when the file is written. A bad +one would otherwise surface as a playlist created somewhere unintended. + ### Paths Lidarr knows where the lossless source is; the playlists have to point at the @@ -229,6 +272,8 @@ music-curator --report-only # report on the store, fetch nothing | `--mirror` | `MUSIC_CURATOR_MIRROR` | unset | Root of the MP3 mirror; playlists go here | | `--library-root` | `MUSIC_CURATOR_LIBRARY_ROOT` | derived | Prefix to strip from Lidarr's paths | | `--playlist-limit` | `MUSIC_CURATOR_PLAYLIST_LIMIT` | `100` | Most tracks in any one playlist | +| `--vibes` | `MUSIC_CURATOR_VIBES` | built-in | JSON file of mood definitions | +| `--tag-limit` | `MUSIC_CURATOR_TAG_LIMIT` | `0` | Cap artist tag lookups per pass | | `--skip-index` | — | off | Match against the index already held | | `--report-only` | — | off | Report without fetching | @@ -278,7 +323,7 @@ nix shell nixpkgs#python3Packages.pytest -c pytest | Last.fm ingest and store | done | | Lidarr index and the scrobble-to-track matcher | done | | M3U playlists from the listening history | done | -| Genre and mood playlists from Last.fm tags | next | +| Genre and mood playlists from Last.fm tags | done | | Cold-music report, unmonitoring what is not played | last | The cull will unmonitor cold albums in Lidarr and tag their artists. It will diff --git a/music_curator.py b/music_curator.py index 9d4d930..4b78036 100644 --- a/music_curator.py +++ b/music_curator.py @@ -69,6 +69,18 @@ BACKOFF_CEILING_SECONDS = 60.0 MIRROR_SUFFIX = ".mp3" PLAYLIST_DIRECTORY = "_playlists" +# Tags are re-fetched this often. They move slowly, and the first pass over a +# library already costs one request per artist. +TAG_REFRESH_SECONDS = 90 * 86400 + +# How much of an artist's tag weight has to fall inside a vibe before their +# tracks are eligible for it. Weights are Last.fm's own 0-100 popularity, summed +# across whichever of the vibe's tags the artist carries. +VIBE_MIN_SCORE = 30 + +# A vibe writes a file named after itself, so the name has to be a filename. +SAFE_VIBE_NAME = re.compile(r"^[a-z0-9][a-z0-9-]*$") + # Group-readable, for the same reason music-mirror sets it: the mirror is read # back by whatever serves it, and a playlist nobody can read is not a playlist. GROUP_READ = 0o040 @@ -177,6 +189,24 @@ CREATE TABLE IF NOT EXISTS scrobble_key ( PRIMARY KEY (artist, track) ); CREATE INDEX IF NOT EXISTS scrobble_key_track ON scrobble_key (track_id); + +-- Last.fm's crowd tags for each library artist. MusicBrainz genres come free +-- with the Lidarr index and are useless for this: they are sparse and formal, +-- and will not tell you a record is screamo or synthwave. People typing tags +-- will. Keyed on the normalised name rather than the Lidarr id so it survives +-- an artist being removed and re-added there. +CREATE TABLE IF NOT EXISTS artist_tag ( + norm_artist TEXT NOT NULL, + tag TEXT NOT NULL, + weight INTEGER NOT NULL, + PRIMARY KEY (norm_artist, tag) +); +CREATE INDEX IF NOT EXISTS artist_tag_tag ON artist_tag (tag); + +CREATE TABLE IF NOT EXISTS artist_tag_fetched ( + norm_artist TEXT PRIMARY KEY, + fetched_at INTEGER NOT NULL +); """ @@ -546,6 +576,24 @@ class Store: tracks, ) + def replace_tags(self, norm_artist, pairs, now): + """Replace one artist's tags, and record that they were fetched. + + The fetch is recorded even when nothing came back, so an artist Last.fm + has never heard of is not asked about again on every pass. + """ + with self.connection: + self.connection.execute("DELETE FROM artist_tag WHERE norm_artist = ?", (norm_artist,)) + self.connection.executemany( + "INSERT OR IGNORE INTO artist_tag (norm_artist, tag, weight) VALUES (?, ?, ?)", + [(norm_artist, tag, weight) for tag, weight in pairs], + ) + self.connection.execute( + "INSERT INTO artist_tag_fetched (norm_artist, fetched_at) VALUES (?, ?)" + " ON CONFLICT (norm_artist) DO UPDATE SET fetched_at = excluded.fetched_at", + (norm_artist, now), + ) + def rebuild_keys(self): """Collapse the scrobble history into one row per distinct track. @@ -1177,6 +1225,200 @@ PLAYLISTS = ( ) +# Moods, as sets of Last.fm tags. Chosen for this library -- drum and bass, +# punk and its descendants, big-room dance, classic rock -- rather than as a +# general taxonomy. `--vibes` replaces the lot with a JSON file of the same +# shape, so a new one does not need a new release. +# +# `years` filters on the album's release date, which is what separates eighties +# synth records from everything a synthpop tag would otherwise drag in. +DEFAULT_VIBES = ( + { + "name": "80s-synths", + "tags": [ + "synthpop", "synth pop", "synth-pop", "new wave", "synthwave", + "new romantic", "electropop", "80s", "1980s", + ], + "years": [1975, 1992], + }, + { + "name": "high-energy-rock", + "tags": [ + "hard rock", "punk rock", "pop punk", "punk", "alternative rock", + "rock", "garage rock", "skate punk", + ], + }, + { + "name": "screamo", + "tags": [ + "screamo", "post-hardcore", "metalcore", "emo", "hardcore", + "melodic hardcore", "emocore", + ], + }, + { + "name": "drum-and-bass", + "tags": [ + "drum and bass", "drum n bass", "dnb", "liquid funk", "neurofunk", + "jungle", "breakbeat", + ], + }, + { + "name": "dance", + "tags": [ + "electro house", "house", "big room", "electronic dance music", + "edm", "hardstyle", "trance", "dubstep", "electro", + ], + }, + { + "name": "classic-rock", + "tags": [ + "classic rock", "progressive rock", "psychedelic rock", + "blues rock", "70s", "60s", + ], + }, +) + + +def load_vibes(path): + """Return the vibe definitions, from a file when one is given. + + Validated up front rather than at write time: a bad name would otherwise + surface as a file created somewhere unintended, which is a poor way to find + out about a typo. + """ + if not path: + return DEFAULT_VIBES + try: + loaded = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read vibes from {path}: {error}") from error + + if not isinstance(loaded, list): + raise ValueError(f"{path}: expected a list of vibes") + for vibe in loaded: + name = vibe.get("name") if isinstance(vibe, dict) else None + if not name or not SAFE_VIBE_NAME.fullmatch(str(name)): + raise ValueError(f"{path}: {name!r} is not a usable vibe name (a-z, 0-9, -)") + if not vibe.get("tags"): + raise ValueError(f"{path}: vibe {name!r} lists no tags") + return tuple(loaded) + + +def parse_tags(payload): + """Return (tag, weight) pairs from an artist.getTopTags response. + + The documented sample carries only a name and a URL per tag; a live response + also carries a 0-100 `count`. Rather than depend on which, the count is used + when present and the documented ordering -- by popularity -- stands in for it + when it is not. + """ + block = payload.get("toptags") or {} + pairs = [] + for position, entry in enumerate(as_list(block.get("tag"))): + tag = (entry.get("name") or "").strip().casefold() + if not tag: + continue + weight = int(entry.get("count") or 0) or max(1, 100 - position * 5) + pairs.append((tag, weight)) + return pairs + + +def sync_tags(client, store, now, limit=0): + """Fetch crowd tags for library artists that have none, or stale ones. + + One request per artist, once, and then only for artists newly added. An + artist Last.fm has never heard of is recorded as fetched with no tags, so it + is not asked about again every pass. + """ + stale = store.connection.execute( + "SELECT a.norm_name AS norm_name, a.name AS name, a.mbid AS mbid" + " FROM lidarr_artist a" + " LEFT JOIN artist_tag_fetched f ON f.norm_artist = a.norm_name" + " WHERE a.norm_name <> ''" + " AND (f.fetched_at IS NULL OR f.fetched_at < :cutoff)" + " ORDER BY a.name", + {"cutoff": now - TAG_REFRESH_SECONDS}, + ).fetchall() + if not stale: + return 0 + + if limit > 0 and len(stale) > limit: + logger.info("tagging %d of %d artists this pass; the rest follow next", limit, len(stale)) + stale = stale[:limit] + else: + logger.info("fetching tags for %d artists", len(stale)) + + tagged = 0 + for artist in stale: + query = {"mbid": artist["mbid"]} if artist["mbid"] else {"artist": artist["name"]} + try: + payload = client.call("artist.getTopTags", {**query, "autocorrect": 1}) + except LastfmError as error: + # One artist Last.fm cannot answer for is not worth losing the pass. + logger.warning("no tags for %s: %s", artist["name"], error) + continue + store.replace_tags(artist["norm_name"], parse_tags(payload), now) + tagged += 1 + + logger.info("tagged %d artists", tagged) + return tagged + + +def build_vibe_playlists(store, vibes, mirror_root, library_root, limit, now): + """Write one playlist per mood. Returns how many tracks were listed.""" + if not store.scalar("SELECT COUNT(*) FROM artist_tag"): + return 0 + + directory = Path(mirror_root) / PLAYLIST_DIRECTORY + week = now // ROTATION_PERIOD_SECONDS + total = 0 + + for vibe in vibes: + tags = [str(tag).strip().casefold() for tag in vibe["tags"]] + placeholders = ",".join("?" * len(tags)) + years = vibe.get("years") + parameters = [*tags, vibe.get("min_score", VIBE_MIN_SCORE)] + year_clause = "" + if years: + year_clause = ( + " AND CAST(substr(al.release_date, 1, 4) AS INTEGER) BETWEEN ? AND ?" + ) + parameters += [int(years[0]), int(years[1])] + parameters += [week, limit] + + sql = f""" + WITH vibe AS ( + SELECT norm_artist, SUM(weight) AS score + FROM artist_tag + WHERE tag IN ({placeholders}) + GROUP BY norm_artist + HAVING SUM(weight) >= ? + ) + SELECT t.id AS id, a.name AS artist, t.title AS title, + t.duration AS duration, t.path AS path + FROM lidarr_track t + JOIN lidarr_artist a ON a.id = t.artist_id + JOIN vibe v ON v.norm_artist = a.norm_name + LEFT JOIN lidarr_album al ON al.id = t.album_id + WHERE {PLAYABLE}{year_clause} + ORDER BY ((t.id * {SHUFFLE_MULTIPLIER}) + ?) % {SHUFFLE_MODULUS} + LIMIT ? + """ + + entries = [] + for row in store.connection.execute(sql, parameters): + mirror = mirror_path_for(row["path"], library_root, mirror_root) + if mirror is None or not mirror.is_file(): + continue + entries.append({**dict(row), "mirror": mirror}) + + write_playlist(directory / f"{vibe['name']}.m3u", entries) + total += len(entries) + logger.info("playlist %-20s %4d tracks -- by tag", vibe["name"], len(entries)) + + return total + + def library_root_of(store): """Return the directory Lidarr's artist folders sit under. @@ -1494,7 +1736,7 @@ def report(store, now): def run_once( client, store, user, now, backfill_limit, lidarr=None, mirror=None, - library_root=None, playlist_limit=100, + library_root=None, playlist_limit=100, vibes=DEFAULT_VIBES, tag_limit=0, ): """Run a single pass. Returns the number of scrobbles added.""" started = time.monotonic() @@ -1517,6 +1759,8 @@ def run_once( ) else: build_playlists(store, mirror, root, playlist_limit, now) + sync_tags(client, store, now, limit=tag_limit) + build_vibe_playlists(store, vibes, mirror, root, playlist_limit, now) logger.info( "pass complete in %.1fs: %d scrobbles added, %d loved", @@ -1608,6 +1852,18 @@ def build_parser(): default=int(os.getenv("MUSIC_CURATOR_PLAYLIST_LIMIT", "100")), help="most tracks to put in any one playlist (env MUSIC_CURATOR_PLAYLIST_LIMIT)", ) + parser.add_argument( + "--vibes", + default=os.getenv("MUSIC_CURATOR_VIBES"), + help="JSON file of mood definitions, replacing the built-in set" + " (env MUSIC_CURATOR_VIBES)", + ) + parser.add_argument( + "--tag-limit", + type=int, + default=int(os.getenv("MUSIC_CURATOR_TAG_LIMIT", "0")), + help="cap artist tag lookups per pass; 0 for no cap (env MUSIC_CURATOR_TAG_LIMIT)", + ) parser.add_argument( "--skip-index", action="store_true", @@ -1632,6 +1888,7 @@ def main(argv=None, clock=time.time): try: interval = parse_interval(args.interval) if args.interval else None + vibes = load_vibes(args.vibes) except ValueError as error: logger.error("%s", error) return 2 @@ -1682,6 +1939,8 @@ def main(argv=None, clock=time.time): args.mirror, args.library_root, args.playlist_limit, + vibes, + args.tag_limit, ) except (LastfmError, LidarrError) as error: logger.error("%s", error) diff --git a/tests/conftest.py b/tests/conftest.py index d525641..f9ab7df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,9 @@ class FakeLastfm: is prepended to the first page with no `date`. """ - def __init__(self, tracks=(), loved=(), nowplaying=None, outcomes=()): + def __init__(self, tracks=(), loved=(), nowplaying=None, outcomes=(), tags=None): + # Keyed by mbid or by artist name, whichever the caller asked with. + self.tags = tags or {} self.tracks = sorted(tracks, key=lambda track: int(track["date"]["uts"]), reverse=True) self.loved = list(loved) self.nowplaying = nowplaying @@ -82,6 +84,9 @@ class FakeLastfm: return json.dumps(self._recent(query)) if method == "user.getlovedtracks": return json.dumps(self._loved(query)) + if method == "artist.gettoptags": + key = query.get("mbid") or query.get("artist", "") + return json.dumps({"toptags": {"tag": self.tags.get(key, [])}}) raise AssertionError(f"unexpected method {method}") def _recent(self, query): diff --git a/tests/test_music_curator.py b/tests/test_music_curator.py index f31e8a2..c6443fc 100644 --- a/tests/test_music_curator.py +++ b/tests/test_music_curator.py @@ -983,3 +983,147 @@ def test_playlists_are_group_readable(tmp_path): for playlist in (mirror / "_playlists").glob("*.m3u"): assert playlist.stat().st_mode & stat.S_IRGRP, playlist + + +def test_tag_weights_fall_back_to_rank_when_no_count_is_sent(): + """The documented sample carries only a name and a URL; a live response also + carries a count. Neither may be relied on alone.""" + with_count = music_curator.parse_tags( + {"toptags": {"tag": [{"name": "Screamo", "count": 100}, {"name": "emo", "count": 40}]}} + ) + assert with_count == [("screamo", 100), ("emo", 40)] + + without = music_curator.parse_tags( + {"toptags": {"tag": [{"name": "screamo"}, {"name": "emo"}]}} + ) + assert [tag for tag, _ in without] == ["screamo", "emo"] + assert without[0][1] > without[1][1] + + +def test_a_lone_tag_is_not_a_list(): + assert music_curator.parse_tags({"toptags": {"tag": {"name": "dnb", "count": 90}}}) == [ + ("dnb", 90) + ] + + +def test_an_artist_lastfm_cannot_answer_for_is_not_asked_again(tmp_path): + """Recording the fetch even when it returns nothing is what stops a pass + spending a request per unknown artist, forever.""" + api, source, mirror = playlist_library(tmp_path) + store = store_at(tmp_path) + music_curator.index_library( + music_curator.Lidarr("http://lidarr", "key", transport=api), store + ) + lastfm = FakeLastfm(tags={}) + + assert music_curator.sync_tags(client_for(lastfm), store, NOW) == 2 + before = len(lastfm.calls) + + assert music_curator.sync_tags(client_for(lastfm), store, NOW) == 0 + assert len(lastfm.calls) == before + + +def test_tags_are_refetched_once_they_go_stale(tmp_path): + api, source, mirror = playlist_library(tmp_path) + store = store_at(tmp_path) + music_curator.index_library( + music_curator.Lidarr("http://lidarr", "key", transport=api), store + ) + lastfm = FakeLastfm(tags={}) + music_curator.sync_tags(client_for(lastfm), store, NOW) + + later = NOW + music_curator.TAG_REFRESH_SECONDS + 1 + assert music_curator.sync_tags(client_for(lastfm), store, later) == 2 + + +def tagged_store(tmp_path, tags): + api, source, mirror = playlist_library(tmp_path) + store = store_at(tmp_path) + music_curator.index_library( + music_curator.Lidarr("http://lidarr", "key", transport=api), store + ) + music_curator.sync_tags(client_for(FakeLastfm(tags=tags)), store, NOW) + return store, source, mirror + + +def test_a_vibe_selects_by_tag(tmp_path): + store, source, mirror = tagged_store( + tmp_path, + { + "artist-mbid-1": [{"name": "screamo", "count": 100}], + "artist-mbid-2": [{"name": "classic rock", "count": 100}], + }, + ) + vibes = [{"name": "screamo", "tags": ["screamo"]}] + + music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW) + + written = (mirror / "_playlists" / "screamo.m3u").read_text() + assert "Played Band" in written + assert "Silent Band" not in written + + +def test_a_weakly_tagged_artist_is_below_the_threshold(tmp_path): + """A single low-weight tag is not a genre, it is somebody's stray opinion.""" + store, source, mirror = tagged_store( + tmp_path, {"artist-mbid-1": [{"name": "screamo", "count": 3}]} + ) + vibes = [{"name": "screamo", "tags": ["screamo"]}] + + music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW) + + assert (mirror / "_playlists" / "screamo.m3u").read_text() == "#EXTM3U\n" + + +def test_a_vibe_can_be_restricted_by_release_year(tmp_path): + """What separates eighties synth records from everything else a synthpop + tag drags in.""" + store, source, mirror = tagged_store( + tmp_path, {"artist-mbid-1": [{"name": "synthpop", "count": 100}]} + ) + inside = [{"name": "eighties", "tags": ["synthpop"], "years": [1975, 1992]}] + outside = [{"name": "nineties", "tags": ["synthpop"], "years": [1993, 1999]}] + + # FakeLidarr dates every album 2019, so neither window should catch it. + music_curator.build_vibe_playlists(store, inside, mirror, str(source), 100, NOW) + music_curator.build_vibe_playlists(store, outside, mirror, str(source), 100, NOW) + assert (mirror / "_playlists" / "eighties.m3u").read_text() == "#EXTM3U\n" + assert (mirror / "_playlists" / "nineties.m3u").read_text() == "#EXTM3U\n" + + modern = [{"name": "modern", "tags": ["synthpop"], "years": [2000, 2030]}] + music_curator.build_vibe_playlists(store, modern, mirror, str(source), 100, NOW) + assert "Played Band" in (mirror / "_playlists" / "modern.m3u").read_text() + + +def test_the_built_in_vibes_are_all_usable_filenames(): + for vibe in music_curator.DEFAULT_VIBES: + assert music_curator.SAFE_VIBE_NAME.fullmatch(vibe["name"]), vibe["name"] + assert vibe["tags"] + + +def test_a_vibes_file_replaces_the_built_in_set(tmp_path): + path = tmp_path / "vibes.json" + path.write_text(json.dumps([{"name": "mine", "tags": ["shoegaze"]}])) + + assert music_curator.load_vibes(str(path)) == ({"name": "mine", "tags": ["shoegaze"]},) + assert music_curator.load_vibes(None) is music_curator.DEFAULT_VIBES + + +@pytest.mark.parametrize( + "content", + [ + '{"not": "a list"}', + '[{"tags": ["x"]}]', + '[{"name": "../escape", "tags": ["x"]}]', + '[{"name": "ok"}]', + "not json at all", + ], +) +def test_a_bad_vibes_file_is_refused_up_front(tmp_path, content): + """A bad name would otherwise surface as a file written somewhere + unintended, which is a poor way to learn about a typo.""" + path = tmp_path / "vibes.json" + path.write_text(content) + + with pytest.raises(ValueError): + music_curator.load_vibes(str(path))