diff --git a/README.md b/README.md index 7c5fffe..f74904a 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,20 @@ music-mirror --source /music --mirror /music-mp3 --subdir "Artist/Album" `--subdir` never prunes: a partial pass cannot tell an orphan from a file outside its own scope. +### Concurrency + +LAME is single-threaded — ffmpeg reports `Threading capabilities: none` for +`libmp3lame` — so throughput comes entirely from running several encoders at +once, one process per file. `--jobs` defaults to the CPUs the process may +actually use, which inside a container means the `cpus:` allowance rather than +the host's core count. Each pass logs the number it settled on. + +As a rough guide, a Zen 3 core encodes about 40–60× realtime at V0 depending on +clock, so six cores clear roughly 250 hours of audio per hour of wall clock. +The first full pass is the expensive one; after that only new and changed files +are touched. Lower `MUSIC_MIRROR_JOBS` if you would rather the NAS stayed +responsive than finished sooner. + Requires `ffmpeg` and `ffprobe` on `PATH`. The container image provides both. ## Running it on TrueNAS Scale diff --git a/music_mirror.py b/music_mirror.py index 97a57dd..4756609 100644 --- a/music_mirror.py +++ b/music_mirror.py @@ -16,6 +16,7 @@ makes runs idempotent without a database to keep in step. import argparse import concurrent.futures import fcntl +import functools import logging import os import re @@ -123,8 +124,13 @@ def is_current(source, mirror): return abs(source.stat().st_mtime - mirror.stat().st_mtime) <= MTIME_TOLERANCE_SECONDS +@functools.lru_cache(maxsize=4096) def find_cover(directory): - """Return an external cover image for a directory, if one is present.""" + """Return an external cover image for a directory, if one is present. + + Cached because an album's tracks all ask the same question, and the answer + costs one stat per candidate name. + """ for name in COVER_NAMES: candidate = directory / name if candidate.is_file(): @@ -201,7 +207,12 @@ def encode(source, mirror, quality_args, dry_run): return Result("encoded", mirror) mirror.parent.mkdir(parents=True, exist_ok=True) - cover = None if has_embedded_picture(source) else find_cover(source.parent) + # Probing costs an ffprobe process per file, so only ask when the answer + # can change the command. With no cover file beside the track, `-map + # 0:v:0?` carries embedded art if there is any and shrugs if there is not. + cover = find_cover(source.parent) + if cover is not None and has_embedded_picture(source): + cover = None # Read the source's mtime before encoding, not after. If the file is still # being written -- a Lidarr import landing mid-pass -- stamping the mirror @@ -331,6 +342,7 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d computed against; they differ only for a partial pass over one directory. """ started = time.monotonic() + logger.info("pass starting with %d concurrent encoders", jobs) counts = {"encoded": 0, "copied": 0, "skipped": 0, "failed": 0} failures = [] @@ -376,6 +388,25 @@ def acquire_lock(mirror_root): return handle +def default_jobs(): + """Return the number of CPUs this process may actually use. + + os.cpu_count() reports the host's total, which in a container with a `cpus:` + limit means starting several times more encoders than there is CPU to run + them. libmp3lame is single-threaded, so one process per available CPU is the + whole of the concurrency story. + """ + try: + quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split() + if quota != "max": + return max(1, round(int(quota) / int(period))) + except (OSError, ValueError): + pass + if hasattr(os, "process_cpu_count"): # 3.13+, respects CPU affinity + return os.process_cpu_count() or 4 + return os.cpu_count() or 4 + + def build_parser(): """Return the argument parser. Every option also reads an env var, so the container can be configured without a command line.""" @@ -401,8 +432,8 @@ def build_parser(): parser.add_argument( "--jobs", type=int, - default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or (os.cpu_count() or 4), - help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: CPU count)", + default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or default_jobs(), + help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: available CPUs)", ) parser.add_argument( "--interval",