fix: write playlists as .m3u8 so Rockbox reads them as UTF-8

Rockbox's is_m3u8_name() treats every playlist extension as UTF-8 except an
explicit ".m3u", which it instead decodes through the user's configured
codepage:

    /* Default to M3U8 unless explicitly told otherwise. */
    return (!dot || strcasecmp(dot, ".m3u") != 0);

The one extension being used was therefore the only one that mangles accented
filenames, and this library holds Motley Crue, Beyonce and Sigur Ros. Renaming
the output is the whole fix.

No byte order mark is written. One would promote a .m3u file to UTF-8 as well,
but it is unnecessary at this extension and upsets players that do not expect
to find one.

Kept with the pruning change rather than raised separately, because the rename
depends on it: without pruning, seventeen dead .m3u files would sit on the
device for ever, and every one of them full of paths that still resolve.

README.md

@@ -132,7 +132,13 @@ overstates the problem and would over-block the cull.
 
 ## Playlists
 
-Written into `<mirror>/_playlists/` as extended M3U, rebuilt every pass. Six
+Written into `<mirror>/_playlists/` as extended M3U, rebuilt every pass.
+
+The extension is **`.m3u8`**, not `.m3u`. Rockbox's `is_m3u8_name()` treats
+every extension as UTF-8 *except* an explicit `.m3u`, which it decodes through
+the user's configured codepage instead — so a plain `.m3u` mangles every
+accented filename. No byte order mark is written: Rockbox does not need one at
+this extension, and a BOM upsets players that do not expect it. Six
 rules, capped at `--playlist-limit` tracks each:
 
 | Playlist | Rule |

music_curator.py

@@ -69,6 +69,12 @@ BACKOFF_CEILING_SECONDS = 60.0
 MIRROR_SUFFIX = ".mp3"
 PLAYLIST_DIRECTORY = "_playlists"
 
+# .m3u8, not .m3u. Rockbox's is_m3u8_name() treats every extension as UTF-8
+# except an explicit ".m3u", which it decodes through the user's configured
+# codepage instead -- so a plain .m3u mangles every accented filename, and this
+# library holds Mötley Crüe, Beyoncé and Sigur Rós.
+PLAYLIST_SUFFIX = ".m3u8"
+
 # Tags are re-fetched this often. They move slowly, and the first pass over a
 # library already costs one request per artist.
 TAG_REFRESH_SECONDS = 90 * 86400
@@ -1552,8 +1558,10 @@ def build_vibe_playlists(store, vibes, mirror_root, library_root, limit, now):
                 continue
             entries.append({**dict(row), "mirror": mirror})
 
-        write_playlist(directory / f"{vibe['name']}.m3u", entries, Path(mirror_root))
+        write_playlist(
-        produced.append(f"{vibe['name']}.m3u")
+            directory / f"{vibe['name']}{PLAYLIST_SUFFIX}", entries, Path(mirror_root)
+        )
+        produced.append(f"{vibe['name']}{PLAYLIST_SUFFIX}")
         total += len(entries)
         logger.info("playlist %-20s %4d tracks -- by tag", vibe["name"], len(entries))
 
@@ -1641,7 +1649,7 @@ def write_playlist(path, entries, reference=None):
     if fresh and reference is not None:
         match_ownership(path.parent, reference)
 
-    handle, temporary = tempfile.mkstemp(dir=path.parent, suffix=".m3u.part")
+    handle, temporary = tempfile.mkstemp(dir=path.parent, suffix=".m3u8.part")
     os.close(handle)
     temporary = Path(temporary)
     try:
@@ -1702,8 +1710,8 @@ def build_playlists(store, mirror_root, library_root, limit, now):
                 missing += 1
                 continue
             entries.append({**dict(row), "mirror": mirror})
-        write_playlist(directory / f"{name}.m3u", entries, Path(mirror_root))
+        write_playlist(directory / f"{name}{PLAYLIST_SUFFIX}", entries, Path(mirror_root))
-        produced.append(f"{name}.m3u")
+        produced.append(f"{name}{PLAYLIST_SUFFIX}")
         total += len(entries)
         logger.info("playlist %-20s %4d tracks -- %s", name, len(entries), description)
 

tests/test_music_curator.py

@@ -875,23 +875,23 @@ def test_playlists_are_written_into_the_mirror(tmp_path):
 
     music_curator.build_playlists(store, mirror, str(source), 100, NOW)
 
-    written = sorted(p.name for p in (mirror / "_playlists").glob("*.m3u"))
+    written = sorted(p.name for p in (mirror / "_playlists").glob("*.m3u8"))
     assert written == [
-        "all-time.m3u",
+        "all-time.m3u8",
-        "deep-cuts.m3u",
+        "deep-cuts.m3u8",
-        "heavy-rotation.m3u",
+        "heavy-rotation.m3u8",
-        "neglected.m3u",
+        "neglected.m3u8",
-        "unheard-favourites.m3u",
+        "unheard-favourites.m3u8",
-        "unheard.m3u",
+        "unheard.m3u8",
     ]
 
-    played = (mirror / "_playlists" / "all-time.m3u").read_text().splitlines()
+    played = (mirror / "_playlists" / "all-time.m3u8").read_text().splitlines()
     assert played[0] == "#EXTM3U"
     assert played[1].startswith("#EXTINF:")
     assert "Played Band - Hit" in played[1]
     # Relative to the playlist file, so the same file works from any mount.
     assert played[2] == "../Played Band/Known/Hit.mp3"
-    assert (mirror / "_playlists" / "all-time.m3u").parent.joinpath(played[2]).resolve().is_file()
+    assert (mirror / "_playlists" / "all-time.m3u8").parent.joinpath(played[2]).resolve().is_file()
 
 
 def test_an_unplayed_track_lands_in_the_unheard_playlist(tmp_path):
@@ -905,7 +905,7 @@ def test_an_unplayed_track_lands_in_the_unheard_playlist(tmp_path):
 
     music_curator.build_playlists(store, mirror, str(source), 100, NOW)
 
-    unheard = (mirror / "_playlists" / "unheard.m3u").read_text()
+    unheard = (mirror / "_playlists" / "unheard.m3u8").read_text()
     assert "Never Heard" in unheard
     assert "Album Track" in unheard
     # The one thing that was played must not be in it.
@@ -929,7 +929,7 @@ def test_a_track_with_no_mirror_file_is_left_out(tmp_path):
 
     assert total == 0
     assert produced, "the playlists are still written, they are simply empty"
-    assert (mirror / "_playlists" / "all-time.m3u").read_text() == "#EXTM3U\n"
+    assert (mirror / "_playlists" / "all-time.m3u8").read_text() == "#EXTM3U\n"
 
 
 def test_the_library_root_is_derived_from_the_artist_folders(tmp_path):
@@ -952,7 +952,7 @@ def test_the_playlist_limit_is_honoured(tmp_path):
 
     music_curator.build_playlists(store, mirror, str(source), 1, NOW)
 
-    unheard = (mirror / "_playlists" / "unheard.m3u").read_text().splitlines()
+    unheard = (mirror / "_playlists" / "unheard.m3u8").read_text().splitlines()
     assert len([line for line in unheard if line.startswith("#EXTINF")]) == 1
 
 
@@ -968,7 +968,7 @@ def test_the_rotation_moves_weekly_not_every_pass(tmp_path):
 
     def unheard_at(when):
         music_curator.build_playlists(store, mirror, str(source), 100, when)
-        return (mirror / "_playlists" / "unheard.m3u").read_text()
+        return (mirror / "_playlists" / "unheard.m3u8").read_text()
 
     same_week = unheard_at(NOW), unheard_at(NOW + 3600)
     assert same_week[0] == same_week[1]
@@ -984,7 +984,7 @@ def test_playlists_are_group_readable(tmp_path):
 
     music_curator.build_playlists(store, mirror, str(source), 100, NOW)
 
-    for playlist in (mirror / "_playlists").glob("*.m3u"):
+    for playlist in (mirror / "_playlists").glob("*.m3u8"):
         assert playlist.stat().st_mode & stat.S_IRGRP, playlist
 
 
@@ -1061,7 +1061,7 @@ def test_a_vibe_selects_by_tag(tmp_path):
 
     music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW)
 
-    written = (mirror / "_playlists" / "screamo.m3u").read_text()
+    written = (mirror / "_playlists" / "screamo.m3u8").read_text()
     assert "Played Band" in written
     assert "Silent Band" not in written
 
@@ -1075,7 +1075,7 @@ def test_a_weakly_tagged_artist_is_below_the_threshold(tmp_path):
 
     music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW)
 
-    assert (mirror / "_playlists" / "screamo.m3u").read_text() == "#EXTM3U\n"
+    assert (mirror / "_playlists" / "screamo.m3u8").read_text() == "#EXTM3U\n"
 
 
 def test_a_vibe_can_be_restricted_by_release_year(tmp_path):
@@ -1090,12 +1090,12 @@ def test_a_vibe_can_be_restricted_by_release_year(tmp_path):
     # FakeLidarr dates every album 2019, so neither window should catch it.
     music_curator.build_vibe_playlists(store, inside, mirror, str(source), 100, NOW)
     music_curator.build_vibe_playlists(store, outside, mirror, str(source), 100, NOW)
-    assert (mirror / "_playlists" / "eighties.m3u").read_text() == "#EXTM3U\n"
+    assert (mirror / "_playlists" / "eighties.m3u8").read_text() == "#EXTM3U\n"
-    assert (mirror / "_playlists" / "nineties.m3u").read_text() == "#EXTM3U\n"
+    assert (mirror / "_playlists" / "nineties.m3u8").read_text() == "#EXTM3U\n"
 
     modern = [{"name": "modern", "tags": ["synthpop"], "years": [2000, 2030]}]
     music_curator.build_vibe_playlists(store, modern, mirror, str(source), 100, NOW)
-    assert "Played Band" in (mirror / "_playlists" / "modern.m3u").read_text()
+    assert "Played Band" in (mirror / "_playlists" / "modern.m3u8").read_text()
 
 
 def test_the_built_in_vibes_are_all_usable_filenames():
@@ -1281,7 +1281,7 @@ def test_an_excluded_tag_drops_the_artist(tmp_path):
 
     music_curator.build_vibe_playlists(store, vibes, mirror, str(source), 100, NOW)
 
-    written = (mirror / "_playlists" / "eighties.m3u").read_text()
+    written = (mirror / "_playlists" / "eighties.m3u8").read_text()
     assert "Silent Band" in written
     assert "Played Band" not in written
 
@@ -1353,15 +1353,15 @@ def test_a_renamed_mood_does_not_leave_its_old_playlist_behind(tmp_path):
     directory = mirror / "_playlists"
     directory.mkdir(parents=True, exist_ok=True)
     (directory / "high-energy-rock.m3u").write_text("#EXTM3U\n")
-    (directory / "screamo.m3u").write_text("#EXTM3U\n")
+    (directory / "screamo.m3u8").write_text("#EXTM3U\n")
     store.set_state(
-        "playlist_files", json.dumps(["high-energy-rock.m3u", "screamo.m3u"])
+        "playlist_files", json.dumps(["high-energy-rock.m3u", "screamo.m3u8"])
     )
 
-    music_curator.prune_playlists(directory, ["screamo.m3u"], store)
+    music_curator.prune_playlists(directory, ["screamo.m3u8"], store)
 
     assert not (directory / "high-energy-rock.m3u").exists()
-    assert (directory / "screamo.m3u").exists()
+    assert (directory / "screamo.m3u8").exists()
 
 
 def test_pruning_leaves_a_playlist_it_never_wrote(tmp_path):
@@ -1371,7 +1371,7 @@ def test_pruning_leaves_a_playlist_it_never_wrote(tmp_path):
     directory = mirror / "_playlists"
     directory.mkdir(parents=True, exist_ok=True)
     (directory / "lyras-own-mix.m3u").write_text("#EXTM3U\n")
-    store.set_state("playlist_files", json.dumps(["screamo.m3u"]))
+    store.set_state("playlist_files", json.dumps(["screamo.m3u8"]))
 
     music_curator.prune_playlists(directory, [], store)
 
@@ -1458,3 +1458,49 @@ def test_human_bytes_reads_at_a_glance():
     assert music_curator.human_bytes(0) == "0.0 B"
     assert music_curator.human_bytes(1536) == "1.5 KiB"
     assert music_curator.human_bytes(3 * 1024**3) == "3.0 GiB"
+
+
+def test_playlists_are_written_as_m3u8(tmp_path):
+    """Rockbox's is_m3u8_name() treats every extension as UTF-8 except an
+    explicit ".m3u", which it decodes through the configured codepage instead.
+    A library with accented names needs the other extension."""
+    api, source, mirror = playlist_library(tmp_path)
+    store = store_at(tmp_path)
+    music_curator.index_library(
+        music_curator.Lidarr("http://lidarr", "key", transport=api), store
+    )
+    music_curator.match_library(store)
+
+    music_curator.build_playlists(store, mirror, str(source), 100, NOW)
+
+    written = sorted(p.suffix for p in (mirror / "_playlists").iterdir())
+    assert set(written) == {".m3u8"}
+
+
+def test_playlists_are_written_as_utf8(tmp_path):
+    playlist = tmp_path / "_playlists" / "accents.m3u8"
+    music_curator.write_playlist(
+        playlist,
+        [{"artist": "Mötley Crüe", "title": "Kickstart My Heart",
+          "duration": 283000, "mirror": tmp_path / "x.mp3"}],
+    )
+
+    # Decodes as UTF-8, and carries no BOM: Rockbox does not need one at this
+    # extension, and a BOM confuses players that do not expect it.
+    raw = playlist.read_bytes()
+    assert not raw.startswith(b"\xef\xbb\xbf")
+    assert "Mötley Crüe" in raw.decode("utf-8")
+
+
+def test_the_old_m3u_playlists_are_pruned_after_the_rename(tmp_path):
+    """Without this the seventeen dead .m3u files stay on the device for ever."""
+    api, source, mirror = playlist_library(tmp_path)
+    store = store_at(tmp_path)
+    directory = mirror / "_playlists"
+    directory.mkdir(parents=True, exist_ok=True)
+    (directory / "screamo.m3u").write_text("#EXTM3U\n")
+    store.set_state("playlist_files", json.dumps(["screamo.m3u"]))
+
+    music_curator.prune_playlists(directory, ["screamo.m3u8"], store)
+
+    assert not (directory / "screamo.m3u").exists()