From 2b831daa1a754d80de7e7fe49dece9ecae68a103 Mon Sep 17 00:00:00 2001 From: Emma Thorpe Date: Fri, 21 Aug 2026 16:50:53 +0100 Subject: [PATCH] perf: size the encoder pool to the CPUs the container may use libmp3lame is single-threaded, so concurrency is one ffmpeg process per file and the pool width is the whole of it. The width came from os.cpu_count(), which reports the host's cores and ignores a container's cpus: allowance -- on a 12-thread host limited to 4 CPUs that is threefold oversubscription, which costs context switches and NAS responsiveness for no throughput. The default now reads the cgroup v2 quota, falling back to process CPU affinity and then to the host count. Each pass logs the number it chose. Also stops probing every file for embedded cover art. The probe only changes the command when a cover file sits beside the track, so ask only then; with no cover file, -map 0:v:0? already carries embedded art if there is any. Worth roughly 30 ms per track against about 4 s of encoding, so this is tidiness rather than a speed-up. The per-directory cover lookup is cached alongside. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 14 ++++++++++++++ music_mirror.py | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7c5fffe..f74904a 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,20 @@ music-mirror --source /music --mirror /music-mp3 --subdir "Artist/Album" `--subdir` never prunes: a partial pass cannot tell an orphan from a file outside its own scope. +### Concurrency + +LAME is single-threaded — ffmpeg reports `Threading capabilities: none` for +`libmp3lame` — so throughput comes entirely from running several encoders at +once, one process per file. `--jobs` defaults to the CPUs the process may +actually use, which inside a container means the `cpus:` allowance rather than +the host's core count. Each pass logs the number it settled on. + +As a rough guide, a Zen 3 core encodes about 40–60× realtime at V0 depending on +clock, so six cores clear roughly 250 hours of audio per hour of wall clock. +The first full pass is the expensive one; after that only new and changed files +are touched. Lower `MUSIC_MIRROR_JOBS` if you would rather the NAS stayed +responsive than finished sooner. + Requires `ffmpeg` and `ffprobe` on `PATH`. The container image provides both. ## Running it on TrueNAS Scale diff --git a/music_mirror.py b/music_mirror.py index 97a57dd..4756609 100644 --- a/music_mirror.py +++ b/music_mirror.py @@ -16,6 +16,7 @@ makes runs idempotent without a database to keep in step. import argparse import concurrent.futures import fcntl +import functools import logging import os import re @@ -123,8 +124,13 @@ def is_current(source, mirror): return abs(source.stat().st_mtime - mirror.stat().st_mtime) <= MTIME_TOLERANCE_SECONDS +@functools.lru_cache(maxsize=4096) def find_cover(directory): - """Return an external cover image for a directory, if one is present.""" + """Return an external cover image for a directory, if one is present. + + Cached because an album's tracks all ask the same question, and the answer + costs one stat per candidate name. + """ for name in COVER_NAMES: candidate = directory / name if candidate.is_file(): @@ -201,7 +207,12 @@ def encode(source, mirror, quality_args, dry_run): return Result("encoded", mirror) mirror.parent.mkdir(parents=True, exist_ok=True) - cover = None if has_embedded_picture(source) else find_cover(source.parent) + # Probing costs an ffprobe process per file, so only ask when the answer + # can change the command. With no cover file beside the track, `-map + # 0:v:0?` carries embedded art if there is any and shrugs if there is not. + cover = find_cover(source.parent) + if cover is not None and has_embedded_picture(source): + cover = None # Read the source's mtime before encoding, not after. If the file is still # being written -- a Lidarr import landing mid-pass -- stamping the mirror @@ -331,6 +342,7 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d computed against; they differ only for a partial pass over one directory. """ started = time.monotonic() + logger.info("pass starting with %d concurrent encoders", jobs) counts = {"encoded": 0, "copied": 0, "skipped": 0, "failed": 0} failures = [] @@ -376,6 +388,25 @@ def acquire_lock(mirror_root): return handle +def default_jobs(): + """Return the number of CPUs this process may actually use. + + os.cpu_count() reports the host's total, which in a container with a `cpus:` + limit means starting several times more encoders than there is CPU to run + them. libmp3lame is single-threaded, so one process per available CPU is the + whole of the concurrency story. + """ + try: + quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split() + if quota != "max": + return max(1, round(int(quota) / int(period))) + except (OSError, ValueError): + pass + if hasattr(os, "process_cpu_count"): # 3.13+, respects CPU affinity + return os.process_cpu_count() or 4 + return os.cpu_count() or 4 + + def build_parser(): """Return the argument parser. Every option also reads an env var, so the container can be configured without a command line.""" @@ -401,8 +432,8 @@ def build_parser(): parser.add_argument( "--jobs", type=int, - default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or (os.cpu_count() or 4), - help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: CPU count)", + default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or default_jobs(), + help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: available CPUs)", ) parser.add_argument( "--interval", -- 2.54.0