3 Commits
Author SHA1 Message Date
lyrathorpe 147c4c77c8 chore(release): v0.1.1 2026-08-24 10:41:58 +00:00
lyrathorpe 9984bedd02 Merge pull request 'perf: size the encoder pool to the CPUs the container may use' (#3) from perf/encode-concurrency into main
Build and publish container / build (push) Successful in 7m6s
Reviewed-on: #3
2026-08-24 11:35:03 +01:00
Emma ThorpeandClaude Opus 5 2b831daa1a perf: size the encoder pool to the CPUs the container may use
Build and publish container / build (pull_request) Successful in 11m13s
libmp3lame is single-threaded, so concurrency is one ffmpeg process per file
and the pool width is the whole of it. The width came from os.cpu_count(),
which reports the host's cores and ignores a container's cpus: allowance -- on
a 12-thread host limited to 4 CPUs that is threefold oversubscription, which
costs context switches and NAS responsiveness for no throughput.

The default now reads the cgroup v2 quota, falling back to process CPU
affinity and then to the host count. Each pass logs the number it chose.

Also stops probing every file for embedded cover art. The probe only changes
the command when a cover file sits beside the track, so ask only then; with no
cover file, -map 0:v:0? already carries embedded art if there is any. Worth
roughly 30 ms per track against about 4 s of encoding, so this is tidiness
rather than a speed-up. The per-directory cover lookup is cached alongside.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-21 16:50:53 +01:00
3 changed files with 50 additions and 5 deletions
+14
View File
@@ -78,6 +78,20 @@ music-mirror --source /music --mirror /music-mp3 --subdir "Artist/Album"
`--subdir` never prunes: a partial pass cannot tell an orphan from a file
outside its own scope.
### Concurrency
LAME is single-threaded — ffmpeg reports `Threading capabilities: none` for
`libmp3lame` — so throughput comes entirely from running several encoders at
once, one process per file. `--jobs` defaults to the CPUs the process may
actually use, which inside a container means the `cpus:` allowance rather than
the host's core count. Each pass logs the number it settled on.
As a rough guide, a Zen 3 core encodes about 4060× realtime at V0 depending on
clock, so six cores clear roughly 250 hours of audio per hour of wall clock.
The first full pass is the expensive one; after that only new and changed files
are touched. Lower `MUSIC_MIRROR_JOBS` if you would rather the NAS stayed
responsive than finished sooner.
Requires `ffmpeg` and `ffprobe` on `PATH`. The container image provides both.
## Running it on TrueNAS Scale
+35 -4
View File
@@ -16,6 +16,7 @@ makes runs idempotent without a database to keep in step.
import argparse
import concurrent.futures
import fcntl
import functools
import logging
import os
import re
@@ -123,8 +124,13 @@ def is_current(source, mirror):
return abs(source.stat().st_mtime - mirror.stat().st_mtime) <= MTIME_TOLERANCE_SECONDS
@functools.lru_cache(maxsize=4096)
def find_cover(directory):
"""Return an external cover image for a directory, if one is present."""
"""Return an external cover image for a directory, if one is present.
Cached because an album's tracks all ask the same question, and the answer
costs one stat per candidate name.
"""
for name in COVER_NAMES:
candidate = directory / name
if candidate.is_file():
@@ -201,7 +207,12 @@ def encode(source, mirror, quality_args, dry_run):
return Result("encoded", mirror)
mirror.parent.mkdir(parents=True, exist_ok=True)
cover = None if has_embedded_picture(source) else find_cover(source.parent)
# Probing costs an ffprobe process per file, so only ask when the answer
# can change the command. With no cover file beside the track, `-map
# 0:v:0?` carries embedded art if there is any and shrugs if there is not.
cover = find_cover(source.parent)
if cover is not None and has_embedded_picture(source):
cover = None
# Read the source's mtime before encoding, not after. If the file is still
# being written -- a Lidarr import landing mid-pass -- stamping the mirror
@@ -331,6 +342,7 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d
computed against; they differ only for a partial pass over one directory.
"""
started = time.monotonic()
logger.info("pass starting with %d concurrent encoders", jobs)
counts = {"encoded": 0, "copied": 0, "skipped": 0, "failed": 0}
failures = []
@@ -376,6 +388,25 @@ def acquire_lock(mirror_root):
return handle
def default_jobs():
"""Return the number of CPUs this process may actually use.
os.cpu_count() reports the host's total, which in a container with a `cpus:`
limit means starting several times more encoders than there is CPU to run
them. libmp3lame is single-threaded, so one process per available CPU is the
whole of the concurrency story.
"""
try:
quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split()
if quota != "max":
return max(1, round(int(quota) / int(period)))
except (OSError, ValueError):
pass
if hasattr(os, "process_cpu_count"): # 3.13+, respects CPU affinity
return os.process_cpu_count() or 4
return os.cpu_count() or 4
def build_parser():
"""Return the argument parser. Every option also reads an env var, so the
container can be configured without a command line."""
@@ -401,8 +432,8 @@ def build_parser():
parser.add_argument(
"--jobs",
type=int,
default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or (os.cpu_count() or 4),
help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: CPU count)",
default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or default_jobs(),
help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: available CPUs)",
)
parser.add_argument(
"--interval",
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "music-mirror"
version = "0.1.0"
version = "0.1.1"
description = "Maintain a lossy MP3 mirror of a lossless music library"
readme = "README.md"
requires-python = ">=3.11"