Merge pull request 'ci: build the image once instead of twice' (#5) from ci/one-build-not-two into main

Reviewed-on: #5

.gitea/workflows/build-and-publish.yaml

@@ -45,7 +45,8 @@ jobs:
 
       # The suite runs inside the image, against the ffmpeg that ships, rather
       # than against whatever the runner happens to provide. A failing test
-      # fails the build. Layers are shared with the push build below.
+      # fails the build. The runtime stage below is built from the same daemon
+      # afterwards, so its layers are already in cache.
       - name: Run the test suite inside the image
         run: docker build --target test -t music-mirror:test .
 
@@ -124,9 +125,6 @@ jobs:
           echo "release=${release}" >> "$GITHUB_OUTPUT"
           echo "Computed bump=${bump}, release=${release}, base=${base}"
 
-      - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4
-
       - name: Log in to the Gitea container registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4
@@ -135,21 +133,34 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.PACKAGES_TOKEN }}
 
-      - name: Build and push
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7
-        with:
-          context: .
-          # Without this the last stage in the Dockerfile -- the test stage --
-          # would be what gets published.
-          target: runtime
-          # The NAS is the only host this runs on. Building arm64 as well would
-          # mean emulating it under QEMU for no consumer.
-          platforms: linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.version.outputs.tags }}
-          labels: |
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
+      # Plain `docker build` rather than buildx. buildx boots its own buildkit
+      # in a container with a cache of its own, so it shared nothing with the
+      # test build above and rebuilt the image from the base up -- installing
+      # ffmpeg and the package a second time, for nothing. It earns that cost
+      # when building for several platforms; this only ever targets the amd64
+      # NAS, so it does not.
+      #
+      # `--target runtime` is a strict prefix of the test stage, so every layer
+      # is already in the daemon's cache and this resolves in seconds.
+      - name: Build the runtime image
+        run: |
+          set -euo pipefail
+          tags=()
+          while IFS= read -r tag; do
+            [ -n "$tag" ] && tags+=(-t "$tag")
+          done <<< "${{ steps.version.outputs.tags }}"
+          docker build --target runtime \
+            --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \
+            --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+            "${tags[@]}" .
+
+      - name: Push
+        if: github.event_name != 'pull_request'
+        run: |
+          set -euo pipefail
+          while IFS= read -r tag; do
+            [ -n "$tag" ] && docker push "$tag"
+          done <<< "${{ steps.version.outputs.tags }}"
 
       # Record the release: write the computed version into pyproject.toml, then
       # commit and tag it, so the packaging metadata always matches the release

README.md

@@ -51,9 +51,30 @@ The mtime is read _before_ encoding rather than after. A file still being
 written when the pass reaches it would otherwise be stamped with its final
 mtime while holding truncated audio, and never be revisited.
 
-Encodes are written to a temporary file and renamed into place, so an
-interrupted run cannot leave a truncated MP3 that the next run mistakes for
-finished work. A lock file in the mirror root stops two passes overlapping.
+Both encodes and copies are written to a temporary file and renamed into place,
+so an interrupted run cannot leave a truncated MP3 that the next run mistakes
+for finished work. Copies need it as much as encodes do: the mtime comes across
+with the bytes, so a half-written copy would look current for ever. A lock file
+in the mirror root stops two passes overlapping.
+
+### Permissions
+
+Everything written into the mirror is made group-readable, and its directories
+group-traversable, so the mirror can be read back by whatever serves it. Neither
+writer does that unaided: the temporary file an encode renames into place is
+created `0600` regardless of the umask, and a straight copy of an existing MP3
+inherits the mode of a source file in a library this tool does not own.
+
+Directories are handled by clearing the owner and group read/execute bits from
+the process umask, once, at startup. Owner as well as group, because a umask
+carrying `0400` produces directories of mode `0300` — writable and enterable,
+unreadable to the very run that created them. The `other` bits are left where
+the umask puts them: whether the mirror is world-readable is a genuine policy
+question, and so is its ownership.
+
+Mirror files written before this existed are topped up on the next pass. Their
+mtimes are correct, so nothing else would revisit them — and they are not
+re-encoded, only chmod'ed.
 
 ## Usage
 
@@ -78,6 +99,20 @@ music-mirror --source /music --mirror /music-mp3 --subdir "Artist/Album"
 `--subdir` never prunes: a partial pass cannot tell an orphan from a file
 outside its own scope.
 
+### Concurrency
+
+LAME is single-threaded — ffmpeg reports `Threading capabilities: none` for
+`libmp3lame` — so throughput comes entirely from running several encoders at
+once, one process per file. `--jobs` defaults to the CPUs the process may
+actually use, which inside a container means the `cpus:` allowance rather than
+the host's core count. Each pass logs the number it settled on.
+
+As a rough guide, a Zen 3 core encodes about 40–60× realtime at V0 depending on
+clock, so six cores clear roughly 250 hours of audio per hour of wall clock.
+The first full pass is the expensive one; after that only new and changed files
+are touched. Lower `MUSIC_MIRROR_JOBS` if you would rather the NAS stayed
+responsive than finished sooner.
+
 Requires `ffmpeg` and `ffprobe` on `PATH`. The container image provides both.
 
 ## Running it on TrueNAS Scale

music_mirror.py

@@ -16,6 +16,7 @@ makes runs idempotent without a database to keep in step.
 import argparse
 import concurrent.futures
 import fcntl
+import functools
 import logging
 import os
 import re
@@ -76,6 +77,20 @@ MIRROR_SUFFIX = ".mp3"
 # Filesystems disagree about mtime precision; SMB in particular rounds.
 MTIME_TOLERANCE_SECONDS = 2
 
+# The mirror exists to be read back by something else -- an SMB share, another
+# account on the box -- so everything written into it has to be group-readable.
+# Neither writer manages that unaided: tempfile.mkstemp forces 0600 whatever the
+# umask, and shutil.copy2 carries the source file's mode across from a library
+# that may be tighter still. Directories need the execute bit too, or the group
+# cannot enter them to reach the readable files inside.
+GROUP_READ = 0o040
+
+# Cleared from the umask so directories this run creates can be listed and
+# entered. Owner as well as group: a umask carrying 0400 -- which is unusual but
+# not ours to assume away -- otherwise produces a mirror tree that not even the
+# process that built it can read back.
+DIRECTORY_ACCESS = 0o550
+
 
 @dataclass
 class Result:
@@ -123,8 +138,20 @@ def is_current(source, mirror):
     return abs(source.stat().st_mtime - mirror.stat().st_mtime) <= MTIME_TOLERANCE_SECONDS
 
 
+def make_group_readable(path):
+    """Add the group-read bit to a mirror file, leaving the rest of the mode alone."""
+    mode = path.stat().st_mode
+    if not mode & GROUP_READ:
+        path.chmod(mode | GROUP_READ)
+
+
+@functools.lru_cache(maxsize=4096)
 def find_cover(directory):
-    """Return an external cover image for a directory, if one is present."""
+    """Return an external cover image for a directory, if one is present.
+
+    Cached because an album's tracks all ask the same question, and the answer
+    costs one stat per candidate name.
+    """
     for name in COVER_NAMES:
         candidate = directory / name
         if candidate.is_file():
@@ -201,7 +228,12 @@ def encode(source, mirror, quality_args, dry_run):
         return Result("encoded", mirror)
 
     mirror.parent.mkdir(parents=True, exist_ok=True)
-    cover = None if has_embedded_picture(source) else find_cover(source.parent)
+    # Probing costs an ffprobe process per file, so only ask when the answer
+    # can change the command. With no cover file beside the track, `-map
+    # 0:v:0?` carries embedded art if there is any and shrugs if there is not.
+    cover = find_cover(source.parent)
+    if cover is not None and has_embedded_picture(source):
+        cover = None
 
     # Read the source's mtime before encoding, not after. If the file is still
     # being written -- a Lidarr import landing mid-pass -- stamping the mirror
@@ -223,6 +255,9 @@ def encode(source, mirror, quality_args, dry_run):
             lines = completed.stderr.strip().splitlines()
             return Result("failed", source, lines[-1] if lines else "ffmpeg failed")
         os.utime(temporary, (stat.st_atime, stat.st_mtime))
+        # Before the rename, so the file is never visible in the mirror without
+        # the bit.
+        make_group_readable(temporary)
         os.replace(temporary, mirror)
     except Exception as error:  # noqa: BLE001 - reported per file, run continues
         return Result("failed", source, str(error))
@@ -234,16 +269,31 @@ def encode(source, mirror, quality_args, dry_run):
 
 
 def copy(source, mirror, dry_run):
-    """Copy an already-MP3 source into the mirror."""
+    """Copy an already-MP3 source into the mirror, atomically."""
     if dry_run:
         logger.info("would copy %s", source)
         return Result("copied", mirror)
 
     mirror.parent.mkdir(parents=True, exist_ok=True)
+
+    # Through a temporary file and a rename, for the same reason encodes go
+    # that way, and a sharper one: copy2 reproduces the source's mtime as well
+    # as its bytes, so a copy cut short by a full disk or a killed container
+    # would leave a truncated MP3 that every later pass reads as current.
+    handle, temporary = tempfile.mkstemp(dir=mirror.parent, suffix=".mp3.part")
+    os.close(handle)
+    temporary = Path(temporary)
+
     try:
-        shutil.copy2(source, mirror)
+        shutil.copy2(source, temporary)
+        # copy2 brings the source's mode with it, and the source library is not
+        # ours to have permissions opinions about.
+        make_group_readable(temporary)
+        os.replace(temporary, mirror)
     except OSError as error:
         return Result("failed", source, str(error))
+    finally:
+        temporary.unlink(missing_ok=True)
 
     logger.info("copied %s", source)
     return Result("copied", mirror)
@@ -252,6 +302,14 @@ def copy(source, mirror, dry_run):
 def process(source, mirror, quality_args, dry_run):
     """Bring one source file's mirror entry up to date."""
     if is_current(source, mirror):
+        # A mirror written before this bit was set has a correct mtime, so
+        # nothing else in the pass would ever revisit it. Top it up here
+        # instead: one stat per file, and no chmod at all once it is right.
+        if not dry_run:
+            try:
+                make_group_readable(mirror)
+            except OSError as error:
+                return Result("failed", mirror, str(error))
         return Result("skipped", mirror)
     if source.suffix.lower() in COPY_EXTENSIONS:
         return copy(source, mirror, dry_run)
@@ -331,6 +389,7 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d
     computed against; they differ only for a partial pass over one directory.
     """
     started = time.monotonic()
+    logger.info("pass starting with %d concurrent encoders", jobs)
     counts = {"encoded": 0, "copied": 0, "skipped": 0, "failed": 0}
     failures = []
 
@@ -376,6 +435,25 @@ def acquire_lock(mirror_root):
     return handle
 
 
+def default_jobs():
+    """Return the number of CPUs this process may actually use.
+
+    os.cpu_count() reports the host's total, which in a container with a `cpus:`
+    limit means starting several times more encoders than there is CPU to run
+    them. libmp3lame is single-threaded, so one process per available CPU is the
+    whole of the concurrency story.
+    """
+    try:
+        quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split()
+        if quota != "max":
+            return max(1, round(int(quota) / int(period)))
+    except (OSError, ValueError):
+        pass
+    if hasattr(os, "process_cpu_count"):  # 3.13+, respects CPU affinity
+        return os.process_cpu_count() or 4
+    return os.cpu_count() or 4
+
+
 def build_parser():
     """Return the argument parser. Every option also reads an env var, so the
     container can be configured without a command line."""
@@ -401,8 +479,8 @@ def build_parser():
     parser.add_argument(
         "--jobs",
         type=int,
-        default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or (os.cpu_count() or 4),
-        help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: CPU count)",
+        default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or default_jobs(),
+        help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: available CPUs)",
     )
     parser.add_argument(
         "--interval",
@@ -432,6 +510,15 @@ def main(argv=None):
     logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO)
     args = build_parser().parse_args(argv)
 
+    # Directories are created with 0o777 masked by the umask, so clear the bits
+    # that matter from it once here rather than chmod'ing every directory the
+    # walk creates. The `other` bits are left alone, since whether the mirror is
+    # world-readable is a real policy question; owner and group access is not.
+    # Files cannot be handled this way -- mkstemp and copy2 both set a mode
+    # outright, ignoring the umask -- so they get an explicit chmod instead.
+    inherited = os.umask(0o077)
+    os.umask(inherited & ~DIRECTORY_ACCESS)
+
     if not args.source or not args.mirror:
         logger.error("both --source and --mirror are required")
         return 2

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "music-mirror"
-version = "0.1.0"
+version = "0.1.2"
 description = "Maintain a lossy MP3 mirror of a lossless music library"
 readme = "README.md"
 requires-python = ">=3.11"

tests/conftest.py

@@ -19,6 +19,29 @@ def require_ffmpeg():
             pytest.skip(f"{tool} is not on PATH", allow_module_level=True)
 
 
+@pytest.fixture
+def tight_umask():
+    """Run a test under a umask that would otherwise make the mirror private."""
+    previous = os.umask(0o077)
+    yield
+    os.umask(previous)
+
+
+@pytest.fixture
+def owner_hostile_umask():
+    """Return a callable applying a umask that masks off the owner's read bit.
+
+    Unusual, but it is what produces a mirror tree of mode 0300 -- writable and
+    enterable, unreadable to the very process that built it. Applied on demand
+    rather than for the whole test, because the source library is built by
+    something else entirely and the same umask would make the test's own
+    fixtures unreadable before the run under test even started.
+    """
+    previous = os.umask(0o022)
+    yield lambda: os.umask(0o477)
+    os.umask(previous)
+
+
 @pytest.fixture
 def make_flac():
     """Return a factory writing a short tagged FLAC file."""

tests/test_music_mirror.py

@@ -1,7 +1,9 @@
 import os
 import shutil
+import stat
 import subprocess
 import time
+from pathlib import Path
 
 import pytest
 
@@ -153,6 +155,126 @@ def test_existing_mp3_is_copied_not_re_encoded(tmp_path, make_flac):
     assert (mirror / "b.mp3").read_bytes() == (source / "b.mp3").read_bytes()
 
 
+def test_interrupted_copy_leaves_nothing_behind(tmp_path, make_flac, monkeypatch):
+    """copy2 reproduces the source mtime, so a truncated copy left in the mirror
+    would be read as current by every later pass."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    flac = make_flac(source / "a.flac")
+    subprocess.run(
+        ["ffmpeg", "-loglevel", "error", "-y", "-i", str(flac), str(source / "b.mp3")],
+        check=True,
+        capture_output=True,
+    )
+    flac.unlink()
+
+    def truncated(src, destination, **kwargs):
+        Path(destination).write_bytes(Path(src).read_bytes()[:64])
+        raise OSError("no space left on device")
+
+    monkeypatch.setattr(music_mirror.shutil, "copy2", truncated)
+
+    assert run(source, mirror) == 1
+    assert not (mirror / "b.mp3").exists()
+    assert list(mirror.rglob("*.part")) == []
+
+
+def test_encoded_file_is_group_readable(tmp_path, make_flac, tight_umask):
+    """mkstemp creates 0600 whatever the umask, so the bit has to be added."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    run(source, mirror)
+
+    assert (mirror / "a.mp3").stat().st_mode & stat.S_IRGRP
+
+
+def test_copied_file_is_group_readable(tmp_path, make_flac, tight_umask):
+    """copy2 carries the source's mode across, and the source may be private."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    flac = make_flac(source / "a.flac")
+    subprocess.run(
+        ["ffmpeg", "-loglevel", "error", "-y", "-i", str(flac), str(source / "b.mp3")],
+        check=True,
+        capture_output=True,
+    )
+    flac.unlink()
+    (source / "b.mp3").chmod(0o600)
+
+    run(source, mirror)
+
+    assert (mirror / "b.mp3").stat().st_mode & stat.S_IRGRP
+
+
+def test_mirror_directories_are_group_traversable(tmp_path, make_flac, tight_umask):
+    """A readable file is unreachable if the group cannot enter its directory."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Artist" / "Album" / "a.flac")
+
+    run(source, mirror)
+
+    for directory in (mirror, mirror / "Artist", mirror / "Artist" / "Album"):
+        mode = directory.stat().st_mode
+        assert mode & stat.S_IRGRP, directory
+        assert mode & stat.S_IXGRP, directory
+
+
+def test_mirror_directories_survive_an_owner_hostile_umask(
+    tmp_path, make_flac, owner_hostile_umask
+):
+    """A umask carrying 0400 otherwise builds a tree the run cannot read back."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "Artist" / "Album" / "a.flac")
+
+    # Applied only now: the library already exists, and the umask under test is
+    # the one the container starts this run with.
+    owner_hostile_umask()
+    run(source, mirror)
+
+    for directory in (mirror, mirror / "Artist", mirror / "Artist" / "Album"):
+        mode = directory.stat().st_mode
+        assert mode & stat.S_IRUSR, directory
+        assert mode & stat.S_IXUSR, directory
+        assert mode & stat.S_IRGRP, directory
+        assert mode & stat.S_IXGRP, directory
+
+
+def test_private_mirror_file_is_repaired_without_re_encoding(tmp_path, make_flac):
+    """A mirror written by an older version has a correct mtime, so nothing
+    else in the pass would revisit it."""
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    run(source, mirror)
+    output = mirror / "a.mp3"
+    output.chmod(output.stat().st_mode & ~stat.S_IRGRP)
+    before = output.stat().st_mtime_ns
+
+    run(source, mirror)
+
+    assert output.stat().st_mode & stat.S_IRGRP
+    assert output.stat().st_mtime_ns == before
+
+
+def test_dry_run_does_not_change_permissions(tmp_path, make_flac):
+    source = tmp_path / "src"
+    mirror = tmp_path / "dst"
+    make_flac(source / "a.flac")
+
+    run(source, mirror)
+    output = mirror / "a.mp3"
+    output.chmod(output.stat().st_mode & ~stat.S_IRGRP)
+
+    run(source, mirror, "--dry-run")
+
+    assert not output.stat().st_mode & stat.S_IRGRP
+
+
 def test_format_upgrade_replaces_rather_than_duplicating(tmp_path, make_flac):
     """Lidarr replacing an MP3 with a FLAC must not leave two mirror files."""
     source = tmp_path / "src"