feat: index the library from Lidarr and match it against the scrobbles

Stage two. The scrobble history says what was played by name; Lidarr says what
is owned, and where the files are. Neither is useful for curation until the two
are tied together, and the quality of that join is what decides whether the
later cull can be trusted at all.

The index is a wholesale rebuild of every artist, album and track Lidarr holds,
including file paths and the date each file landed -- the latter for the age
floor a cull will need. It is rebuilt rather than reconciled because Lidarr is
the authority and a deletion there has to disappear here, not linger as a
library entry with no file behind it. Every call is a GET; nothing is written
back.

Matching runs at the level of the distinct artist/track pair rather than the
individual play, because a verdict is a property of the name pair and there are
three plays for every one of them. Two tiers: a MusicBrainz recording id, which
Last.fm supplies per scrobble and Lidarr exposes as ForeignRecordingId, gives an
exact join; everything else falls to a normalised name comparison. There is
deliberately no third tier. A near-miss guess is worse than an admitted one,
since the entire purpose of the resulting number is to state how far the
matching can be relied on.

Normalisation folds the ways the two sides habitually disagree: case, accents,
guest credits that Last.fm puts in the artist field, trailing version suffixes,
ampersands, and a leading article. Punctuation needs two opposing rules and both
are load-bearing -- apostrophes are deleted so "Don't" meets "Dont", while every
other mark becomes a space so "AC/DC", "AC-DC" and "AC DC" meet as well. It errs
towards collapsing too much: a false match makes a track look played, a missed
match makes it look abandoned, and only the second one loses music.

The coverage report deliberately does not lead with matched versus unmatched.
Most unmatched listening is music that was never in the library and says nothing
about the matcher. The figure that matters is unmatched listening by an artist
the library does hold: a track that was played, sitting next to a file it should
have matched. The worst fifteen are listed by play count.

The schema gains its tables additively and migrates a version 1 store in place,
because rebuilding a nine-year history costs several thousand API requests.

tests/test_music_curator.py

@@ -1,7 +1,7 @@
 import urllib.error
 
 import pytest
-from conftest import FakeLastfm, make_loved, make_tracks
+from conftest import FakeLastfm, FakeLidarr, make_loved, make_tracks
 
 import music_curator
 
@@ -23,6 +23,62 @@ def ingest(store, api, backfill_limit=0):
     return music_curator.run_once(client, store, "lyra", NOW, backfill_limit)
 
 
+def scrobble_of(artist, track, uts=NOW - 3600, mbid=""):
+    """One recent-tracks entry, in the shape Last.fm actually sends."""
+    return {
+        "artist": {"#text": artist, "mbid": ""},
+        "album": {"#text": "Some Album", "mbid": ""},
+        "name": track,
+        "mbid": mbid,
+        "date": {"uts": str(uts)},
+    }
+
+
+LIBRARY = [
+    {
+        "name": "Yellowcard",
+        "albums": [
+            {
+                "title": "Lift a Sail",
+                "tracks": [
+                    {"title": "Here I Am Alive", "recording_mbid": "rec-alive"},
+                    {"title": "Transmission Home"},
+                ],
+            }
+        ],
+    },
+    {
+        "name": "AC/DC",
+        "albums": [{"title": "Back in Black", "tracks": [{"title": "Hells Bells"}]}],
+    },
+    {
+        "name": "The Prodigy",
+        "albums": [
+            {"title": "The Fat of the Land", "tracks": [{"title": "Breathe (Remastered)"}]}
+        ],
+    },
+]
+
+
+def indexed(tmp_path, scrobbles):
+    """Ingest scrobbles, index the canned library, and match the two."""
+    store = store_at(tmp_path)
+    ingest(store, FakeLastfm(scrobbles))
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=FakeLidarr(LIBRARY)), store
+    )
+    music_curator.match_library(store)
+    return store
+
+
+def verdict(store, artist, track):
+    row = store.connection.execute(
+        "SELECT method, track_id FROM scrobble_key WHERE artist = ? AND track = ?",
+        (artist, track),
+    ).fetchone()
+    return (row["method"], row["track_id"]) if row else (None, None)
+
+
 def test_parse_interval_units():
     assert music_curator.parse_interval("90") == 90
     assert music_curator.parse_interval("30m") == 1800
@@ -241,3 +297,206 @@ def test_main_ingests_through_the_module_level_fetcher(tmp_path, monkeypatch):
 
     assert code == 0
     assert store_at(tmp_path).count() == 20
+
+
+def test_normalise_flattens_the_ways_tags_and_scrobbles_disagree():
+    assert music_curator.normalise("Yellowcard feat. Tay Jardine") == "yellowcard"
+    assert music_curator.normalise("BABYMETAL Feat. F.Hero") == "babymetal"
+    assert music_curator.normalise("AC/DC") == "ac dc"
+    assert music_curator.normalise("The Prodigy") == "prodigy"
+    assert music_curator.normalise("Beyoncé") == "beyonce"
+    assert music_curator.normalise("Simon & Garfunkel") == "simon and garfunkel"
+    assert music_curator.normalise("Breathe (Remastered 2011)") == "breathe"
+    assert music_curator.normalise("Breathe (Live) (Remastered)") == "breathe"
+    assert music_curator.normalise("Vice Grip - Live") == "vice grip"
+
+
+def test_normalise_keeps_a_leading_parenthetical():
+    """Stripping every bracket would destroy real titles."""
+    assert music_curator.normalise("(Don't Fear) The Reaper") == "dont fear the reaper"
+
+
+def test_normalise_treats_apostrophes_and_other_punctuation_differently():
+    """The two rules pull opposite ways: one deletes, the other separates."""
+    assert music_curator.normalise("Don't") == music_curator.normalise("Dont")
+    assert music_curator.normalise("Don’t") == music_curator.normalise("Dont")
+    assert music_curator.normalise("AC/DC") == music_curator.normalise("AC-DC")
+    assert music_curator.normalise("AC/DC") == music_curator.normalise("AC DC")
+    # ...and they must not collide with each other.
+    assert music_curator.normalise("AC/DC") != music_curator.normalise("ACDC")
+
+
+def test_normalise_leaves_with_alone():
+    """`with` appears in too many genuine titles to cut on sight."""
+    assert music_curator.normalise("Dancing with Myself") == "dancing with myself"
+
+
+def test_a_recording_mbid_matches_exactly(tmp_path):
+    store = indexed(tmp_path, [scrobble_of("Yellowcard", "Here I Am Alive", mbid="rec-alive")])
+
+    assert verdict(store, "Yellowcard", "Here I Am Alive")[0] == "mbid"
+
+
+def test_a_guest_credit_still_matches_by_name(tmp_path):
+    """Last.fm puts the guest in the artist field; the file tag does not."""
+    store = indexed(
+        tmp_path, [scrobble_of("Yellowcard feat. Tay Jardine", "Here I Am Alive")]
+    )
+
+    method, track_id = verdict(store, "Yellowcard feat. Tay Jardine", "Here I Am Alive")
+    assert method == "name"
+    assert track_id is not None
+
+
+def test_a_remaster_suffix_on_the_library_side_still_matches(tmp_path):
+    store = indexed(tmp_path, [scrobble_of("The Prodigy", "Breathe")])
+
+    assert verdict(store, "The Prodigy", "Breathe")[0] == "name"
+
+
+def test_music_that_is_not_in_the_library_stays_unmatched(tmp_path):
+    store = indexed(tmp_path, [scrobble_of("Some Band", "Some Song")])
+
+    assert verdict(store, "Some Band", "Some Song") == ("none", None)
+
+
+def test_the_mbid_tier_is_preferred_over_the_name_tier(tmp_path):
+    store = indexed(
+        tmp_path,
+        [scrobble_of("Yellowcard", "Here I Am Alive", mbid="rec-alive")],
+    )
+    method, track_id = verdict(store, "Yellowcard", "Here I Am Alive")
+
+    expected = store.connection.execute(
+        "SELECT id FROM lidarr_track WHERE recording_mbid = 'rec-alive'"
+    ).fetchone()["id"]
+    assert (method, track_id) == ("mbid", expected)
+
+
+def test_keys_carry_the_play_count_and_the_span(tmp_path):
+    store = indexed(
+        tmp_path,
+        [
+            scrobble_of("AC/DC", "Hells Bells", uts=NOW - 7200),
+            scrobble_of("AC/DC", "Hells Bells", uts=NOW - 3600),
+        ],
+    )
+
+    row = store.connection.execute(
+        "SELECT plays, first_uts, last_uts FROM scrobble_key WHERE artist = 'AC/DC'"
+    ).fetchone()
+    assert row["plays"] == 2
+    assert row["last_uts"] - row["first_uts"] == 3600
+
+
+def test_re_indexing_drops_what_lidarr_no_longer_has(tmp_path):
+    """The index is Lidarr's mirror, not an accumulation of everything ever seen."""
+    store = indexed(tmp_path, [scrobble_of("AC/DC", "Hells Bells")])
+    assert store.scalar("SELECT COUNT(*) FROM lidarr_artist") == 3
+
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=FakeLidarr(LIBRARY[:1])), store
+    )
+
+    assert store.scalar("SELECT COUNT(*) FROM lidarr_artist") == 1
+    assert store.scalar("SELECT COUNT(*) FROM lidarr_track WHERE artist_id NOT IN"
+                        " (SELECT id FROM lidarr_artist)") == 0
+
+
+def test_the_index_records_paths_and_when_a_file_landed(tmp_path):
+    store = indexed(tmp_path, [])
+
+    row = store.connection.execute(
+        "SELECT path, added, has_file FROM lidarr_track WHERE title = 'Hells Bells'"
+    ).fetchone()
+    assert row["path"].endswith("Hells Bells.flac")
+    assert row["has_file"] == 1
+    assert row["added"] == music_curator.parse_added("2020-05-01T12:00:00Z")
+
+
+def test_a_track_without_a_file_has_no_path(tmp_path):
+    library = [
+        {
+            "name": "Ghost Artist",
+            "albums": [{"title": "Unowned", "tracks": [{"title": "Missing", "has_file": False}]}],
+        }
+    ]
+    store = store_at(tmp_path)
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=FakeLidarr(library)), store
+    )
+
+    row = store.connection.execute("SELECT path, has_file FROM lidarr_track").fetchone()
+    assert row["has_file"] == 0
+    assert row["path"] is None
+
+
+def test_lidarr_requires_the_api_key_header(tmp_path):
+    """The fake asserts on it, which is the point: a missing header is a 401."""
+    client = music_curator.Lidarr("http://lidarr", "key", transport=FakeLidarr(LIBRARY))
+
+    assert client.get("artist")
+
+
+def test_a_lidarr_failure_is_reported_not_swallowed():
+    def broken(url, timeout=None, headers=None):
+        raise urllib.error.HTTPError(url, 401, "unauthorised", {}, None)
+
+    client = music_curator.Lidarr("http://lidarr", "wrong", transport=broken)
+
+    with pytest.raises(music_curator.LidarrError, match="HTTP 401"):
+        client.get("artist")
+
+
+def test_an_existing_version_one_store_is_migrated_not_discarded(tmp_path):
+    """A rebuilt history is thousands of API requests; migration is additive."""
+    store = store_at(tmp_path)
+    store.add_scrobbles([music_curator.Scrobble(uts=NOW, artist="A", track="B")])
+    store.set_state("schema_version", "1")
+    store.close()
+
+    reopened = store_at(tmp_path)
+
+    assert reopened.count() == 1
+    assert reopened.get_state("schema_version") == music_curator.SCHEMA_VERSION
+
+
+def test_an_unmatched_track_by_a_known_artist_is_a_matcher_miss(tmp_path):
+    """The split that matters: music you own and played, that failed to match."""
+    store = indexed(
+        tmp_path,
+        [
+            scrobble_of("AC/DC", "A Track Lidarr Has Never Heard Of"),
+            scrobble_of("Some Band", "Some Song"),
+        ],
+    )
+
+    misses = store.connection.execute(
+        "SELECT k.artist FROM scrobble_key k WHERE k.track_id IS NULL"
+        "   AND EXISTS (SELECT 1 FROM lidarr_artist a WHERE a.norm_name = k.norm_artist)"
+    ).fetchall()
+
+    # Only the AC/DC one: "Some Band" is not in the library, so it says nothing
+    # about the matcher.
+    assert [row["artist"] for row in misses] == ["AC/DC"]
+
+
+def test_the_report_runs_over_a_matched_store(tmp_path):
+    store = indexed(tmp_path, [scrobble_of("Yellowcard", "Here I Am Alive", mbid="rec-alive")])
+
+    music_curator.report(store, NOW)
+
+
+def test_matching_is_redone_when_new_scrobbles_arrive(tmp_path):
+    """A pass with no re-index still has to give the new plays a verdict."""
+    store = indexed(tmp_path, [scrobble_of("AC/DC", "Hells Bells")])
+    api = FakeLastfm(
+        [
+            scrobble_of("AC/DC", "Hells Bells"),
+            scrobble_of("Yellowcard", "Transmission Home", uts=NOW - 60),
+        ]
+    )
+
+    music_curator.run_once(client_for(api), store, "lyra", NOW, 0)
+
+    assert verdict(store, "Yellowcard", "Transmission Home")[0] == "name"