perf: size the encoder pool to the CPUs the container may use
Build and publish container / build (pull_request) Successful in 11m13s
Build and publish container / build (pull_request) Successful in 11m13s
libmp3lame is single-threaded, so concurrency is one ffmpeg process per file and the pool width is the whole of it. The width came from os.cpu_count(), which reports the host's cores and ignores a container's cpus: allowance -- on a 12-thread host limited to 4 CPUs that is threefold oversubscription, which costs context switches and NAS responsiveness for no throughput. The default now reads the cgroup v2 quota, falling back to process CPU affinity and then to the host count. Each pass logs the number it chose. Also stops probing every file for embedded cover art. The probe only changes the command when a cover file sits beside the track, so ask only then; with no cover file, -map 0:v:0? already carries embedded art if there is any. Worth roughly 30 ms per track against about 4 s of encoding, so this is tidiness rather than a speed-up. The per-directory cover lookup is cached alongside. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
d4ccff3b75
commit
2b831daa1a
+35
-4
@@ -16,6 +16,7 @@ makes runs idempotent without a database to keep in step.
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import fcntl
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -123,8 +124,13 @@ def is_current(source, mirror):
|
||||
return abs(source.stat().st_mtime - mirror.stat().st_mtime) <= MTIME_TOLERANCE_SECONDS
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=4096)
|
||||
def find_cover(directory):
|
||||
"""Return an external cover image for a directory, if one is present."""
|
||||
"""Return an external cover image for a directory, if one is present.
|
||||
|
||||
Cached because an album's tracks all ask the same question, and the answer
|
||||
costs one stat per candidate name.
|
||||
"""
|
||||
for name in COVER_NAMES:
|
||||
candidate = directory / name
|
||||
if candidate.is_file():
|
||||
@@ -201,7 +207,12 @@ def encode(source, mirror, quality_args, dry_run):
|
||||
return Result("encoded", mirror)
|
||||
|
||||
mirror.parent.mkdir(parents=True, exist_ok=True)
|
||||
cover = None if has_embedded_picture(source) else find_cover(source.parent)
|
||||
# Probing costs an ffprobe process per file, so only ask when the answer
|
||||
# can change the command. With no cover file beside the track, `-map
|
||||
# 0:v:0?` carries embedded art if there is any and shrugs if there is not.
|
||||
cover = find_cover(source.parent)
|
||||
if cover is not None and has_embedded_picture(source):
|
||||
cover = None
|
||||
|
||||
# Read the source's mtime before encoding, not after. If the file is still
|
||||
# being written -- a Lidarr import landing mid-pass -- stamping the mirror
|
||||
@@ -331,6 +342,7 @@ def run_once(scan_root, source_root, mirror_root, quality_args, jobs, dry_run, d
|
||||
computed against; they differ only for a partial pass over one directory.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
logger.info("pass starting with %d concurrent encoders", jobs)
|
||||
counts = {"encoded": 0, "copied": 0, "skipped": 0, "failed": 0}
|
||||
failures = []
|
||||
|
||||
@@ -376,6 +388,25 @@ def acquire_lock(mirror_root):
|
||||
return handle
|
||||
|
||||
|
||||
def default_jobs():
|
||||
"""Return the number of CPUs this process may actually use.
|
||||
|
||||
os.cpu_count() reports the host's total, which in a container with a `cpus:`
|
||||
limit means starting several times more encoders than there is CPU to run
|
||||
them. libmp3lame is single-threaded, so one process per available CPU is the
|
||||
whole of the concurrency story.
|
||||
"""
|
||||
try:
|
||||
quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split()
|
||||
if quota != "max":
|
||||
return max(1, round(int(quota) / int(period)))
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
if hasattr(os, "process_cpu_count"): # 3.13+, respects CPU affinity
|
||||
return os.process_cpu_count() or 4
|
||||
return os.cpu_count() or 4
|
||||
|
||||
|
||||
def build_parser():
|
||||
"""Return the argument parser. Every option also reads an env var, so the
|
||||
container can be configured without a command line."""
|
||||
@@ -401,8 +432,8 @@ def build_parser():
|
||||
parser.add_argument(
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or (os.cpu_count() or 4),
|
||||
help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: CPU count)",
|
||||
default=int(os.getenv("MUSIC_MIRROR_JOBS", "0")) or default_jobs(),
|
||||
help="concurrent encodes (env MUSIC_MIRROR_JOBS; default: available CPUs)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--interval",
|
||||
|
||||
Reference in New Issue
Block a user