fix: hold one connection to Lidarr open, and retry what deserves retrying

Indexing makes two requests per artist, and more when the album fallback fires.
urllib opens a new TCP connection and performs a new DNS lookup for every one of
them, so a large library becomes thousands of lookups inside a few minutes. That
is enough to exhaust a container's resolver, and the result is
"[Errno -3] Try again" on every artist at once -- a failure caused entirely by
how the requests were made rather than by anything wrong with Lidarr.

Add a transport that keeps one connection open per host, so the name is resolved
once and the socket is reused. It retries once on a connection the server has
already closed, since a stale keep-alive announces itself only on use.

Retries were previously declined on the grounds that Lidarr is on the same LAN.
That is not a safe assumption -- it may sit behind a public hostname and a
reverse proxy -- and a transient failure currently costs an artist their entire
entry for that pass. Transient failures are now retried with a backoff. HTTP 500
is deliberately excluded: it is an exception inside Lidarr's serialisation, not
a busy server, and three attempts only delay finding that out.

The same distinction gates the album probe added alongside this. Naming the
offending album costs one request per album of that artist, which is worth it
for a deterministic fault and actively harmful during a network-wide one, where
every artist fails and probing each of them multiplies the load responsible.

The keep-alive transport is tested against a real local HTTP server rather than
a fake, because connection reuse and status mapping are exactly the properties a
fake would assume rather than demonstrate.

tests/conftest.py

@@ -1,7 +1,9 @@
+import http.server
 import io
 import json
 import os
 import sys
+import threading
 import urllib.error
 import urllib.parse
 
@@ -201,6 +203,19 @@ class FakeLidarr:
                 url, 500, "Internal Server Error", {}, io.BytesIO(b'{"message": "boom"}')
             )
 
+        album_ids = query.get("albumIds")
+        if path == "album" and album_ids:
+            album_id = int(album_ids)
+            if ("albumid", album_id) in self.fail:
+                raise urllib.error.HTTPError(
+                    url,
+                    500,
+                    "Internal Server Error",
+                    {},
+                    io.BytesIO(b'{"message": "Sequence contains more than one element"}'),
+                )
+            return json.dumps([row for row in self.albums if row["id"] == album_id])
+
         if path == "artist":
             return json.dumps(self.artists)
         source = {"album": self.albums, "track": self.tracks, "trackfile": self.files}[path]
@@ -228,3 +243,53 @@ def now_playing():
         "mbid": "",
         "@attr": {"nowplaying": "true"},
     }
+
+
+class _CountingServer(http.server.ThreadingHTTPServer):
+    """Counts accepted connections, which is what connection reuse is about."""
+
+    daemon_threads = True
+
+    def __init__(self, *args, **kwargs):
+        self.connections = 0
+        super().__init__(*args, **kwargs)
+
+    def process_request(self, request, client_address):
+        self.connections += 1
+        super().process_request(request, client_address)
+
+
+class _Handler(http.server.BaseHTTPRequestHandler):
+    # Without HTTP/1.1 the server closes after every response and no client
+    # could reuse anything, which would make the test prove nothing.
+    protocol_version = "HTTP/1.1"
+
+    def do_GET(self):
+        if self.path.startswith("/boom"):
+            body, status = b'{"message": "boom"}', 500
+        else:
+            body = json.dumps(
+                {"path": self.path, "key": self.headers.get("X-Api-Key")}
+            ).encode()
+            status = 200
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):
+        pass
+
+
+@pytest.fixture
+def http_server():
+    """A real local HTTP server, for the one component that talks sockets."""
+    server = _CountingServer(("127.0.0.1", 0), _Handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield server, f"http://127.0.0.1:{server.server_port}"
+    finally:
+        server.shutdown()
+        server.server_close()

tests/test_music_curator.py

@@ -1,3 +1,4 @@
+import json
 import urllib.error
 
 import pytest
@@ -584,3 +585,105 @@ def test_a_lidarr_error_carries_the_url_and_what_the_server_said():
     assert "http://lidarr:8686/api/v1/album" in message
     assert "HTTP 500" in message
     assert "boom" in message
+
+
+def test_the_offending_album_is_named_not_just_its_artist(tmp_path, caplog):
+    """An artist's whole discography is too much to click through by hand."""
+    # AC/DC is artist 2; its only album is id 201, holding "Hells Bells".
+    api = FakeLidarr(LIBRARY, fail=[("album", 0), ("album", 2), ("albumid", 201)])
+    store = store_at(tmp_path)
+
+    with caplog.at_level("WARNING"):
+        music_curator.index_library(
+            music_curator.Lidarr("http://lidarr", "key", transport=api), store
+        )
+
+    assert "album id 201" in caplog.text
+    assert "Hells Bells" in caplog.text
+
+
+def test_a_transient_failure_is_not_probed_album_by_album(tmp_path, caplog):
+    """When the resolver is the problem every artist fails, and probing each of
+    them multiplies the load that caused it."""
+
+    def unresolvable(url, timeout=None, headers=None):
+        if "artistId" in url:
+            raise urllib.error.URLError("[Errno -3] Try again")
+        return json.dumps(FakeLidarr(LIBRARY).artists) if url.endswith("artist") else "[]"
+
+    store = store_at(tmp_path)
+    client = music_curator.Lidarr("http://lidarr", "key", transport=unresolvable, backoff=0)
+
+    with caplog.at_level("WARNING"):
+        music_curator.index_library(client, store)
+
+    assert "Try again" in caplog.text
+    assert "cannot serialise" not in caplog.text
+
+
+def test_a_transient_failure_is_retried():
+    attempts = []
+
+    def flaky(url, timeout=None, headers=None):
+        attempts.append(url)
+        if len(attempts) < 3:
+            raise urllib.error.URLError("[Errno -3] Try again")
+        return "[]"
+
+    client = music_curator.Lidarr("http://lidarr", "key", transport=flaky, backoff=0)
+
+    assert client.get("artist") == []
+    assert len(attempts) == 3
+
+
+def test_a_lidarr_500_is_not_retried():
+    """It is an exception inside Lidarr's serialisation, not a busy server."""
+    api = FakeLidarr(LIBRARY, fail=[("album", 0)])
+    client = music_curator.Lidarr("http://lidarr", "key", transport=api, backoff=0)
+
+    with pytest.raises(music_curator.LidarrError) as raised:
+        client.get("album")
+
+    assert raised.value.transient is False
+    assert len(api.calls) == 1
+
+
+def test_keep_alive_uses_one_connection_for_many_requests(http_server):
+    """The point of the whole class: one DNS lookup and one socket, not N."""
+    server, base = http_server
+    transport = music_curator.KeepAlive()
+
+    try:
+        for index in range(5):
+            body = transport(f"{base}/api/v1/artist?n={index}", headers={"X-Api-Key": "key"})
+            assert json.loads(body)["key"] == "key"
+    finally:
+        transport.close()
+
+    assert server.connections == 1
+
+
+def test_keep_alive_maps_an_error_status_onto_httperror(http_server):
+    _, base = http_server
+    transport = music_curator.KeepAlive()
+
+    try:
+        with pytest.raises(urllib.error.HTTPError) as raised:
+            transport(f"{base}/boom", headers={"X-Api-Key": "key"})
+        assert raised.value.code == 500
+        assert music_curator.error_detail(raised.value) == "boom"
+    finally:
+        transport.close()
+
+
+def test_lidarr_talks_to_a_real_server_through_keep_alive(http_server):
+    server, base = http_server
+    client = music_curator.Lidarr(base, "secret")
+
+    try:
+        assert client.get("artist", {"x": 1})["key"] == "secret"
+        assert client.get("album")["path"] == "/api/v1/album"
+    finally:
+        client.transport.close()
+
+    assert server.connections == 1