Emma Thorpe
18f05d3d55
Build and publish container / build (push) Failing after 2m16s
First stage of a curation tool for the music library that music-mirror mirrors. Before anything can build playlists or decide what has gone cold, there has to be a local, queryable record of what is actually played; an API call per question does not scale to a library-sized analysis. Ingest is in two halves. A catch-up fetches everything scrobbled since the newest scrobble held, and a backfill walks the history backwards until it runs out. Both take their bounds from the database rather than from a saved cursor, so an interrupted run resumes from what it actually has, and both windows are bounded at each end so paging cannot shift under the fetch while new scrobbles arrive mid-run. Scrobbles carry no identifier, so the primary key is timestamp, artist and track. Two plays of one track in the same second collapse into a single row: they are indistinguishable in the data, and a surrogate key would make re-ingest non-idempotent, which is the worse trade. Three API behaviours are handled explicitly because each fails silently: the currently-playing track arrives with no timestamp and would be re-ingested on every pass; a lone result is returned as a bare object rather than a one-item list; and MBIDs are empty strings rather than absent when unknown, which would later look like a usable join key. Retries cover the rate limit and the transient backend errors with an exponential backoff. An invalid or suspended key fails immediately. The report exists to surface one number before the next stage is built: the share of scrobbles carrying a MusicBrainz recording id. Lidarr exposes the same identifier per track, so those can be joined exactly and the rest must go through name matching. That percentage bounds how far the matcher can be trusted. No runtime dependencies, and the tests run against a fake transport that reproduces the service's paging and response shapes, so they need neither network nor credentials.
244 lines
6.9 KiB
Python
244 lines
6.9 KiB
Python
import urllib.error
|
|
|
|
import pytest
|
|
from conftest import FakeLastfm, make_loved, make_tracks
|
|
|
|
import music_curator
|
|
|
|
NOW = 1_700_000_000
|
|
|
|
|
|
def store_at(tmp_path):
|
|
return music_curator.Store(tmp_path / "curator.db")
|
|
|
|
|
|
def client_for(api, **kwargs):
|
|
kwargs.setdefault("delay", 0)
|
|
kwargs.setdefault("backoff", 0)
|
|
return music_curator.Lastfm("key", transport=api, **kwargs)
|
|
|
|
|
|
def ingest(store, api, backfill_limit=0):
|
|
client = client_for(api)
|
|
return music_curator.run_once(client, store, "lyra", NOW, backfill_limit)
|
|
|
|
|
|
def test_parse_interval_units():
|
|
assert music_curator.parse_interval("90") == 90
|
|
assert music_curator.parse_interval("30m") == 1800
|
|
assert music_curator.parse_interval("6h") == 21600
|
|
with pytest.raises(ValueError):
|
|
music_curator.parse_interval("soon")
|
|
|
|
|
|
def test_artist_name_is_read_from_either_shape():
|
|
"""Recent tracks key it `#text`; loved tracks and extended=1 key it `name`."""
|
|
assert music_curator.name_of({"#text": "Autechre"}) == "Autechre"
|
|
assert music_curator.name_of({"name": "Autechre"}) == "Autechre"
|
|
assert music_curator.name_of(None) == ""
|
|
|
|
|
|
def test_empty_mbid_becomes_none():
|
|
"""An empty string would otherwise look like a usable join key later on."""
|
|
assert music_curator.mbid_of({"mbid": ""}) is None
|
|
assert music_curator.mbid_of("") is None
|
|
assert music_curator.mbid_of({"mbid": "abc"}) == "abc"
|
|
|
|
|
|
def test_backfill_ingests_the_whole_history(tmp_path):
|
|
api = FakeLastfm(make_tracks(450))
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
assert store.count() == 450
|
|
assert store.get_state("backfill_complete") == "yes"
|
|
|
|
|
|
def test_backfill_resumes_after_a_capped_pass(tmp_path):
|
|
"""An interrupted backfill picks up from what the database holds."""
|
|
api = FakeLastfm(make_tracks(450))
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api, backfill_limit=1)
|
|
partial = store.count()
|
|
assert 0 < partial < 450
|
|
assert store.get_state("backfill_complete") != "yes"
|
|
|
|
ingest(store, api)
|
|
|
|
assert store.count() == 450
|
|
assert store.get_state("backfill_complete") == "yes"
|
|
|
|
|
|
def test_now_playing_is_not_stored(tmp_path, now_playing):
|
|
"""It has no timestamp, and it would arrive again on every single pass."""
|
|
api = FakeLastfm(make_tracks(10), nowplaying=now_playing)
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
assert store.count() == 10
|
|
titles = {row["track"] for row in store.connection.execute("SELECT track FROM scrobble")}
|
|
assert "Currently Playing" not in titles
|
|
|
|
|
|
def test_a_pass_over_unchanged_history_adds_nothing(tmp_path):
|
|
api = FakeLastfm(make_tracks(300))
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
before = store.count()
|
|
|
|
assert ingest(store, api) == 0
|
|
assert store.count() == before
|
|
|
|
|
|
def test_catch_up_adds_only_what_is_new(tmp_path):
|
|
history = make_tracks(300)
|
|
api = FakeLastfm(history)
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
newest = int(history[-1]["date"]["uts"])
|
|
api.tracks = sorted(
|
|
[*history, *make_tracks(5, start=newest + 3600)],
|
|
key=lambda track: int(track["date"]["uts"]),
|
|
reverse=True,
|
|
)
|
|
|
|
assert ingest(store, api) == 5
|
|
assert store.count() == 305
|
|
|
|
|
|
def test_a_lone_result_is_not_a_list(tmp_path):
|
|
"""Last.fm returns a bare object rather than a one-item list."""
|
|
api = FakeLastfm(make_tracks(1))
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
assert store.count() == 1
|
|
|
|
|
|
def test_a_full_page_sharing_one_second_is_paged_not_skipped(tmp_path):
|
|
"""Moving the window past a same-second cluster would lose the rest of it."""
|
|
api = FakeLastfm(make_tracks(250, step=0))
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
# 250 plays at one timestamp collapse to one row per distinct track.
|
|
assert store.count() == 250
|
|
assert store.oldest_uts() == store.newest_uts()
|
|
|
|
|
|
def test_loved_tracks_are_replaced_not_accumulated(tmp_path):
|
|
api = FakeLastfm(make_tracks(5), loved=make_loved(["One", "Two", "Three"]))
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
assert store.scalar("SELECT COUNT(*) FROM loved") == 3
|
|
|
|
api.loved = make_loved(["One"])
|
|
ingest(store, api)
|
|
|
|
assert store.scalar("SELECT COUNT(*) FROM loved") == 1
|
|
|
|
|
|
def test_rate_limiting_is_retried(tmp_path):
|
|
"""Error 29 is the service asking for patience, not a broken request."""
|
|
api = FakeLastfm(make_tracks(5), outcomes=[{"error": 29, "message": "Rate Limit Exceded"}])
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
assert store.count() == 5
|
|
|
|
|
|
def test_server_errors_are_retried(tmp_path):
|
|
api = FakeLastfm(
|
|
make_tracks(5),
|
|
outcomes=[urllib.error.HTTPError("url", 503, "unavailable", {}, None)],
|
|
)
|
|
store = store_at(tmp_path)
|
|
|
|
ingest(store, api)
|
|
|
|
assert store.count() == 5
|
|
|
|
|
|
def test_an_invalid_api_key_is_not_retried():
|
|
"""Error 10 will fail identically forever; failing fast says why."""
|
|
api = FakeLastfm(outcomes=[{"error": 10, "message": "Invalid API key"}])
|
|
client = client_for(api)
|
|
|
|
with pytest.raises(music_curator.LastfmError, match="error 10"):
|
|
client.call("user.getRecentTracks", {"user": "lyra"})
|
|
|
|
assert len(api.calls) == 1
|
|
|
|
|
|
def test_giving_up_reports_the_last_failure():
|
|
api = FakeLastfm(outcomes=[{"error": 29, "message": "slow down"}] * 3)
|
|
client = client_for(api, attempts=3)
|
|
|
|
with pytest.raises(music_curator.LastfmError, match="error 29"):
|
|
client.call("user.getRecentTracks", {"user": "lyra"})
|
|
|
|
assert len(api.calls) == 3
|
|
|
|
|
|
def test_the_store_rejects_a_schema_it_does_not_understand(tmp_path):
|
|
store = store_at(tmp_path)
|
|
store.set_state("schema_version", "999")
|
|
store.close()
|
|
|
|
with pytest.raises(RuntimeError, match="schema version"):
|
|
store_at(tmp_path)
|
|
|
|
|
|
def test_missing_credentials_are_refused(tmp_path, monkeypatch):
|
|
monkeypatch.delenv("MUSIC_CURATOR_LASTFM_USER", raising=False)
|
|
monkeypatch.delenv("MUSIC_CURATOR_LASTFM_API_KEY", raising=False)
|
|
|
|
assert music_curator.main(["--db", str(tmp_path / "curator.db")]) == 2
|
|
|
|
|
|
def test_report_only_needs_no_credentials(tmp_path):
|
|
assert music_curator.main(["--db", str(tmp_path / "curator.db"), "--report-only"]) == 0
|
|
|
|
|
|
def test_a_second_pass_will_not_start_while_one_is_running(tmp_path, monkeypatch):
|
|
database = tmp_path / "curator.db"
|
|
held = music_curator.acquire_lock(database)
|
|
assert held is not None
|
|
|
|
try:
|
|
assert music_curator.main(["--db", str(database), "--report-only"]) == 3
|
|
finally:
|
|
held.close()
|
|
|
|
|
|
def test_main_ingests_through_the_module_level_fetcher(tmp_path, monkeypatch):
|
|
api = FakeLastfm(make_tracks(20))
|
|
monkeypatch.setattr(music_curator, "http_get", api)
|
|
|
|
code = music_curator.main(
|
|
[
|
|
"--db",
|
|
str(tmp_path / "curator.db"),
|
|
"--user",
|
|
"lyra",
|
|
"--api-key",
|
|
"key",
|
|
"--request-delay",
|
|
"0",
|
|
],
|
|
clock=lambda: NOW,
|
|
)
|
|
|
|
assert code == 0
|
|
assert store_at(tmp_path).count() == 20
|