backpack-backcrack 0.2.0__py3-none-any.whl

backcrack/disc.py

@@ -0,0 +1,178 @@
+"""Find discs sitting in optical drives, of any kind, and tell them
+apart. All macOS-specific (diskutil, drutil, mount) - the disc hardware layer
+is the one part of this pipeline that doesn't travel to another OS for free.
+"""
+import hashlib
+import re
+import subprocess
+import time
+from pathlib import Path
+
+from backcrack import config as cfg
+
+
+# Optical drive speeds are quoted in multiples of DVD 1x.
+DVD_1X_BPS = 1_385_000
+
+
+def raw(dev: str) -> str:
+    return "/dev/r" + dev.removeprefix("/dev/")
+
+
+def cooked(dev: str) -> str:
+    return "/dev/" + dev.removeprefix("/dev/").removeprefix("r")
+
+
+def optical_mounts():
+    """Yields (device, mountpoint) for every mounted UDF/ISO9660 volume:
+    DVD-Video, Blu-ray, and any other data disc. Audio CDs never mount a
+    filesystem, so they never show up here - see audio_discs below.
+    """
+    try:
+        out = subprocess.run(["mount"], capture_output=True, text=True, timeout=10).stdout
+    except subprocess.TimeoutExpired:
+        return
+    for line in out.splitlines():
+        m = re.match(r"^(\S+) on (.+) \((udf|cd9660)[,)]", line)
+        if m:
+            yield m.group(1), m.group(2)
+
+
+def audio_discs():
+    """Yields device paths for every whole-disk carrying a CD_DA (audio)
+    partition. diskutil lists these even though nothing gets mounted.
+    """
+    try:
+        out = subprocess.run(["diskutil", "list"], capture_output=True, text=True, timeout=10).stdout
+    except subprocess.TimeoutExpired:
+        return
+    dev = None
+    for line in out.splitlines():
+        m = re.match(r"^(/dev/disk\S+)", line)
+        if m:
+            dev = m.group(1)
+        elif "CD_DA" in line and dev:
+            yield dev
+
+
+def pending_discs():
+    """Yields (device, kind, label) for every disc worth ripping. kind is
+    "video" or "audio". label is the volume name for video. For audio it is
+    only the device name: reading the CD's id here, on every poll, would
+    disturb a rip already running in that drive, so rip_audio_disc names
+    the CD with audio_label() once it holds the drive's lock.
+    """
+    for dev, mnt in optical_mounts():
+        yield dev, "video", Path(mnt).name
+    for dev in audio_discs():
+        yield dev, "audio", Path(dev).name
+
+
+def cd_discid(rdev: str) -> list:
+    """cd-discid's fields for the CD in `rdev` (freedb id, track count,
+    offsets..., seconds), or [] if it isn't installed or can't read it."""
+    if not cfg.CD_DISCID:
+        return []
+    try:
+        return subprocess.run([cfg.CD_DISCID, rdev], capture_output=True, text=True, timeout=15).stdout.split()
+    except (OSError, subprocess.TimeoutExpired):
+        return []
+
+
+def cd_toc(rdev: str):
+    """The CD's table of contents as MusicBrainz wants it: (first track, last
+    track, lead-out, [track offsets]) in sectors, or None. Uses cd-discid's
+    exact --musicbrainz output; an older cd-discid only gives the length in
+    whole seconds, so the lead-out is then approximate (MusicBrainz's toc
+    lookup still finds near matches)."""
+    if not cfg.CD_DISCID:
+        return None
+    try:
+        out = subprocess.run([cfg.CD_DISCID, "--musicbrainz", rdev], capture_output=True, text=True, timeout=15).stdout.split()
+    except (OSError, subprocess.TimeoutExpired):
+        out = []
+    nums = [int(x) for x in out if x.isdigit()]
+    if len(nums) >= 3 and nums[0] == len(nums) - 2:          # count, offsets..., lead-out
+        return 1, nums[0], nums[-1], nums[1:-1]
+    fields = cd_discid(rdev)                                 # freedb: id, count, offsets..., seconds
+    if len(fields) < 3 or not fields[1].isdigit():
+        return None
+    n = int(fields[1])
+    try:
+        offsets = [int(x) for x in fields[2:2 + n]]
+        return 1, n, int(fields[2 + n]) * 75, offsets
+    except (ValueError, IndexError):
+        return None
+
+
+def musicbrainz_discid(toc) -> str:
+    """MusicBrainz's disc id for a (first, last, lead-out, offsets) toc: the
+    SHA-1 of the toc in fixed-width hex, base64 with MusicBrainz's alphabet."""
+    import base64
+    first, last, leadout, offsets = toc
+    frames = [leadout] + list(offsets) + [0] * (99 - len(offsets))
+    text = f"{first:02X}{last:02X}" + "".join(f"{f:08X}" for f in frames)
+    digest = base64.b64encode(hashlib.sha1(text.encode()).digest()).decode()
+    return digest.translate(str.maketrans("+/=", "._-"))
+
+
+def audio_label(rdev: str):
+    """A name for the CD in `rdev` that stays the same whichever drive it is
+    in: "AudioCD-<n>tracks-<id>". The id is cd-discid's, or a hash of
+    cdparanoia's track table when cd-discid isn't installed. None if
+    neither can read the disc.
+    """
+    fields = cd_discid(rdev)
+    if len(fields) >= 2 and fields[1].isdigit():
+        return f"AudioCD-{fields[1]}tracks-{fields[0]}"
+    if not cfg.CDPARANOIA:
+        return None
+    try:
+        toc = subprocess.run([cfg.CDPARANOIA, "-Q", "-d", rdev], capture_output=True, text=True, timeout=30).stderr
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    tracks = [line.split() for line in toc.splitlines() if re.match(r"^\s*\d+\.\s+\d+", line)]
+    if not tracks:
+        return None
+    digest = hashlib.sha1(" ".join(t[1] for t in tracks).encode()).hexdigest()[:8]
+    return f"AudioCD-{len(tracks)}tracks-{digest}"
+
+
+_DRV_RE = re.compile(r'^DRV:(\d+),2,\d+,\d+,"([^"]*)","([^"]*)","([^"]*)"')
+
+
+def mkv_drives(timeout: int = 60) -> list:
+    """(index, drive name, disc label, raw device) for every drive MakeMKV
+    sees with a disc loaded."""
+    try:
+        out = subprocess.run([cfg.MKVCON, "-r", "--cache=1", "info", "disc:9999"],
+                             capture_output=True, text=True, timeout=timeout).stdout
+    except (OSError, subprocess.TimeoutExpired):
+        return []
+    return [m.groups() for m in map(_DRV_RE.match, out.splitlines()) if m]
+
+
+# MakeMKV drive indices move between insertions, so never cache one.
+# Enumeration touches every drive, so it's serialised behind a lock to avoid
+# disturbing a rip already running in the other drive. video_ts mode only.
+def mkv_index_for(dev: str):
+    want = raw(dev)
+    lockdir = cfg.STATE / "enum.lock"
+    tries = 0
+    while True:
+        try:
+            lockdir.mkdir()
+            break
+        except FileExistsError:
+            tries += 1
+            if tries > 90:
+                break
+            time.sleep(2)
+    try:
+        drives = mkv_drives()
+    finally:
+        try:
+            lockdir.rmdir()
+        except OSError:
+            pass
+    return next((idx for idx, _, _, device in drives if device == want), None)

backcrack/diskspeed.py

@@ -0,0 +1,129 @@
+"""Measure sustained read throughput of every loaded optical
+drive. It's a raw block read, so any disc works. Speeds are shown in MB/s and
+in DVD "x" units (1x = 1,385,000 bytes/s), so a CD or Blu-ray reads as a
+multiple of DVD 1x, not of its own format's 1x. Reports live progress and
+detects a stalled read instead of hanging silently.
+
+    diskspeed
+
+Put the SAME disc in each drive in turn so the comparison is fair.
+Results accumulate in RESULTS (default ~/diskspeed.txt). READ_MB, SKIP_MB,
+STALL_S and DISC_BYTES (the size behind the minutes-per-disc estimate,
+default a 7 GB DVD) are settings like any other.
+"""
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from datetime import datetime
+from pathlib import Path
+
+from backcrack import config as cfg
+from backcrack.disc import DVD_1X_BPS, cooked, mkv_drives
+
+READ_MB, SKIP_MB, STALL_S, DISC_BYTES, RESULTS = cfg.READ_MB, cfg.SKIP_MB, cfg.STALL_S, cfg.DISC_BYTES, cfg.RESULTS
+
+
+def hr() -> None:
+    print("-" * 70)
+
+
+def main() -> None:
+    if not Path(cfg.MKVCON).exists():
+        print(f"makemkvcon not found at {cfg.MKVCON}. See `backcrack doctor`.", file=sys.stderr)
+        sys.exit(1)
+
+    disc_drives = mkv_drives(timeout=30)
+    if not disc_drives:
+        print("No discs loaded. Insert a disc in each drive and re-run.", file=sys.stderr)
+        sys.exit(1)
+
+    sudo = []
+    first_raw = disc_drives[0][3]
+    if subprocess.run(["dd", f"if={first_raw}", "of=/dev/null", "bs=2048", "count=1"],
+                      capture_output=True).returncode != 0:
+        print("Raw device reads need admin rights - password prompt incoming.")
+        if subprocess.run(["sudo", "-v"]).returncode != 0:
+            sys.exit(1)
+        sudo = ["sudo"]
+
+    print(f"\nReading {READ_MB}MB from each drive starting at {SKIP_MB}MB.")
+    print("Progress prints every 5s. Ctrl-C is safe at any point.")
+
+    for idx, name, label, raw in disc_drives:
+        short = name[:40]
+        hr()
+        print(short)
+        print(f"  disc: {label or '<unlabelled>'}   device: {raw}")
+
+        subprocess.run(["diskutil", "unmountDisk", cooked(raw)], capture_output=True)
+
+        tmp = tempfile.NamedTemporaryFile(prefix="diskspeed", delete=False)
+        tmp.close()
+        with open(tmp.name, "wb") as errf:
+            proc = subprocess.Popen(
+                [*sudo, "dd", f"if={raw}", "of=/dev/null", "bs=1m", f"count={READ_MB}", f"skip={SKIP_MB}"],
+                stderr=errf,
+            )
+
+        last_bytes = stalled = quiet = 0
+        while proc.poll() is None:
+            time.sleep(5)
+            try:
+                proc.send_signal(signal.SIGINFO)
+            except (AttributeError, ProcessLookupError):
+                pass
+            time.sleep(1)
+            text = Path(tmp.name).read_text(errors="ignore")
+            lines = [l for l in text.splitlines() if "bytes transferred" in l]
+            b = 0
+            if lines:
+                m = re.match(r"^(\d+) bytes", lines[-1])
+                if m:
+                    b = int(m.group(1))
+
+            if b > last_bytes:
+                last_bytes, quiet = b, 0
+                m = re.search(r"in ([\d.]+) secs", lines[-1])
+                t = float(m.group(1)) if m else 0
+                if t > 0:
+                    print(f"   {b/1e6:6.0f} MB   {b/1e6/t:6.2f} MB/s   {b/DVD_1X_BPS/t:4.1f}x")
+            else:
+                quiet += 6
+                if quiet >= STALL_S:
+                    print(f"   STALLED at {last_bytes // 1_000_000}MB - no progress for {quiet}s, aborting")
+                    proc.terminate()   # SIGTERM, which sudo passes on to dd
+                    stalled = 1
+                    break
+                print(f"   ...no progress for {quiet}s")
+        proc.wait()
+
+        text = Path(tmp.name).read_text(errors="ignore")
+        os.unlink(tmp.name)
+        final = [l for l in text.splitlines() if "bytes transferred" in l]
+        bps = None
+        if final:
+            m = re.search(r"\((\d+) bytes/sec\)", final[-1])
+            if m:
+                bps = int(m.group(1))
+
+        if stalled or bps is None:
+            print("  RESULT: no clean measurement (likely bad/protected sectors - try SKIP_MB=4000)")
+            continue
+
+        out_line = (f"{short:<42} {label or '?':<20} {bps/1e6:6.2f} MB/s  {bps/DVD_1X_BPS:4.1f}x  "
+                    f"~{(DISC_BYTES/bps)/60:.0f} min/disc")
+        print(f"  RESULT: {out_line}")
+        with open(RESULTS, "a") as f:
+            f.write(f"{datetime.now():%Y-%m-%d %H:%M:%S}  {out_line}\n")
+
+    hr()
+    print(f"Saved to {RESULTS}")
+    print("Now swap that disc into the other drive and re-run to compare like with like.")
+
+
+if __name__ == "__main__":
+    main()

backcrack/encd.py

@@ -0,0 +1,42 @@
+"""Encode worker. Video jobs go through HandBrake, sorting the
+main feature and special features into separate folders; audio jobs get
+compressed losslessly. Start it once and leave it; idle is normal. Safe to
+stop and restart - finished files are never redone.
+
+    encd
+"""
+import time
+from pathlib import Path
+
+from backcrack import config as cfg
+from backcrack.encode import audio_ext, process_job
+from backcrack.common import log
+
+
+def main() -> None:
+    cfg.make_library_dirs()
+    if not Path(cfg.HBCLI).exists():
+        print(f"WARNING: HandBrakeCLI not found at {cfg.HBCLI} - video jobs will fail. See `backcrack doctor`.")
+    if audio_ext() != cfg.AUDIO_FORMAT:
+        need = "flac or ffmpeg" if cfg.AUDIO_FORMAT == "flac" else "ffmpeg"
+        print(f"WARNING: {need} not found - audio tracks will be copied as .wav. See `backcrack doctor`.")
+
+    classes = ", ".join(f"{c.name} RF={c.quality}->{c.folder}/" for c in cfg.DURATION_CLASSES)
+    print(f"Video: {cfg.VIDEO_ENCODER} preset={cfg.ENCODER_PRESET}  {classes}  ({cfg.ENCODE_JOBS} at a time)")
+    print(f"Audio: {audio_ext()}  -> encoded/")
+    print("Idle until ripd finishes a disc - that's normal.")
+    log(cfg.ENCLOG, "encd started")
+
+    try:
+        while True:
+            jobfiles = [p for p in cfg.QUEUE.iterdir() if p.is_file()]
+            for jobfile in jobfiles:
+                process_job(jobfile)
+            if not jobfiles:
+                time.sleep(15)
+    except KeyboardInterrupt:
+        print("\nencoder stopped - queue kept in", cfg.QUEUE)
+
+
+if __name__ == "__main__":
+    main()

backcrack/encode.py

@@ -0,0 +1,198 @@
+"""Turn a ripped disc's source/ into encoded/ (and extras/ for
+video). Shared by encd.py.
+"""
+import subprocess
+from pathlib import Path
+
+from backcrack import config as cfg
+from backcrack.common import log, notify, classify_class, hb_titles, file_duration
+
+
+PART = ".part"   # an encode in progress is written as <stem>.part<suffix>
+
+
+def _encode_to(out: Path, cmd_for, log_path: Path) -> int:
+    """Runs cmd_for(tmp) to write `out` under a temporary name, renaming it to
+    `out` only once the command succeeds, so an interrupted encode never
+    leaves a file that looks finished."""
+    out.parent.mkdir(parents=True, exist_ok=True)
+    tmp = out.with_name(out.stem + PART + out.suffix)
+    with open(log_path, "ab") as logf:
+        rc = subprocess.run(cmd_for(tmp), stdout=logf, stderr=subprocess.STDOUT, timeout=None).returncode
+    if rc == 0 and tmp.exists():
+        tmp.replace(out)
+    else:
+        tmp.unlink(missing_ok=True)
+        log(cfg.ENCLOG, f"FAIL  {out.name} (rc={rc}) - see {log_path}")
+    return rc
+
+
+def encode_video_one(src: str, title, out: Path, quality: str) -> int:
+    def cmd_for(tmp: Path) -> list:
+        return [
+            cfg.HBCLI, "-i", src,
+            *(["-t", str(title)] if title is not None else []),
+            "-o", str(tmp),
+            "--format", "av_mkv",
+            "--encoder", cfg.VIDEO_ENCODER, "--encoder-preset", cfg.ENCODER_PRESET, "--quality", quality,
+            "-x", f"threads={cfg.ENCODE_THREADS}",
+            *cfg.DEINTERLACE_ARGS,
+            "--all-audio", "--aencoder", "copy", "--audio-fallback", "ca_aac",
+            "--all-subtitles", "--markers",
+        ]
+    return _encode_to(out, cmd_for, cfg.LOGDIR / f"{out.name}.hb.log")
+
+
+def audio_ext() -> str:
+    """The encoded tracks' extension: AUDIO_FORMAT, or "wav" when no encoder
+    for it is installed and the ripped WAV is copied as it is."""
+    if (cfg.FLAC and cfg.AUDIO_FORMAT == "flac") or cfg.FFMPEG:
+        return cfg.AUDIO_FORMAT
+    return "wav"
+
+
+def encode_audio_one(src: Path, out: Path) -> None:
+    def cmd_for(tmp: Path) -> list:
+        if cfg.FLAC and cfg.AUDIO_FORMAT == "flac":
+            return [cfg.FLAC, "-s", "-f", "-o", str(tmp), str(src)]
+        if cfg.FFMPEG:
+            return [cfg.FFMPEG, "-y", "-i", str(src), "-c:a", cfg.AUDIO_FORMAT, str(tmp)]
+        return ["cp", str(src), str(tmp)]
+    _encode_to(out, cmd_for, cfg.LOGDIR / f"{out.name}.enc.log")
+
+
+def _run_batched(jobs, worker) -> None:
+    """Runs `worker(job)` for each job, ENCODE_JOBS at a time in the
+    background, waiting for each batch before starting the next - matches
+    HandBrake's own single-threaded-per-process model, so ENCODE_JOBS
+    processes run in parallel rather than one process trying to use them all.
+    """
+    import threading
+    batch = []
+    for job in jobs:
+        t = threading.Thread(target=worker, args=(job,))
+        t.start()
+        batch.append(t)
+        if len(batch) >= cfg.ENCODE_JOBS:
+            for t in batch:
+                t.join()
+            batch = []
+    for t in batch:
+        t.join()
+
+
+def process_video_job(label: str, dest: Path) -> None:
+    counts = {c.name: 0 for c in cfg.DURATION_CLASSES}
+    n_already = 0
+    seen = {c.name: set() for c in cfg.DURATION_CLASSES if c.dedup}
+
+    def route(secs, ident):
+        """The DurationClass for secs, logging + returning None if the
+        title is skipped (no matching class, or a deduped repeat)."""
+        cls = classify_class(secs)
+        if cls is None:
+            log(cfg.ENCLOG, f"  skip {ident} ({secs}s - no matching duration class)")
+            return None
+        if cls.dedup:
+            if secs in seen[cls.name]:
+                log(cfg.ENCLOG, f"  dup  {ident} ({secs}s) already taken")
+                return None
+            seen[cls.name].add(secs)
+        return cls
+
+    video_ts = dest / "VIDEO_TS"
+    if cfg.RIP_MODE == "video_ts" and video_ts.is_dir():
+        jobs = []
+        for idx, secs in hb_titles(str(video_ts)):
+            if idx is None:
+                continue
+            cls = route(secs, f"{label} t{idx}")
+            if cls is None:
+                continue
+            counts[cls.name] += 1
+            outdir = dest / cls.folder
+            out = outdir / f"title{idx:02d}.mkv"
+            if out.exists() and out.stat().st_size > 0:
+                n_already += 1
+                continue
+            log(cfg.ENCLOG, f"  ->   {label} t{idx}  {secs // 60}m  {cls.name}  -> {outdir.name}/{out.name}")
+            jobs.append((str(video_ts), idx, out, cls.quality))
+        _run_batched(jobs, lambda j: encode_video_one(*j))
+    else:
+        src = dest / "source"
+        if not src.is_dir():
+            log(cfg.ENCLOG, f"no source/ in {dest}")
+            return
+        jobs = []
+        for f in sorted(src.glob("*.mkv")):
+            secs = file_duration(str(f))
+            cls = route(secs, f.name)
+            if cls is None:
+                continue
+            counts[cls.name] += 1
+            outdir = dest / cls.folder
+            out = outdir / f.name
+            if out.exists() and out.stat().st_size > 0:
+                n_already += 1
+                continue
+            log(cfg.ENCLOG, f"  ->   {f.name}  {secs // 60}m  {cls.name}  -> {outdir.name}/")
+            jobs.append((str(f), None, out, cls.quality))
+        _run_batched(jobs, lambda j: encode_video_one(*j))
+
+    if sum(counts.values()) + n_already == 0:
+        log(cfg.ENCLOG, f"WARN  {label} - nothing encodable found. Check {cfg.LOGDIR}/")
+        notify(f"backcrack - {label} nothing found", "No titles matched a duration class.", "high", "warning")
+    else:
+        breakdown = ", ".join(f"{n} {name}" for name, n in counts.items() if n) or "nothing new"
+        log(cfg.ENCLOG, f"DONE  {label} - {breakdown}, {n_already} already present")
+        if cfg.NOTIFY_ENCODES:
+            notify(f"backcrack - {label} encoded", breakdown, "low", "film_projector")
+
+
+def process_audio_job(label: str, dest: Path) -> None:
+    src = dest / "source"
+    if not src.is_dir():
+        log(cfg.ENCLOG, f"no source/ in {dest}")
+        return
+
+    n_done = n_skip = 0
+    jobs = []
+    ext = audio_ext()
+    for f in sorted(src.glob("*.wav")):
+        # cdparanoia -B names tracks track01.cdda.wav
+        out = dest / "encoded" / f"{f.stem.removesuffix('.cdda')}.{ext}"
+        if out.exists() and out.stat().st_size > 0:
+            n_skip += 1
+            continue
+        log(cfg.ENCLOG, f"  ->   {f.name}  -> encoded/{out.name}")
+        jobs.append((f, out))
+        n_done += 1
+    _run_batched(jobs, lambda j: encode_audio_one(*j))
+
+    if n_done + n_skip == 0:
+        log(cfg.ENCLOG, f"WARN  {label} - no tracks found in source/. Check {cfg.LOGDIR}/")
+        notify(f"backcrack - {label} nothing found", "No tracks in source/.", "high", "warning")
+    else:
+        log(cfg.ENCLOG, f"DONE  {label} - {n_done} tracks, {n_skip} already present")
+        if cfg.NOTIFY_ENCODES:
+            notify(f"backcrack - {label} encoded", f"{n_done} tracks", "low", "film_projector")
+
+
+def process_job(jobfile: Path) -> None:
+    fields = dict(
+        line.split("=", 1) for line in jobfile.read_text().splitlines() if "=" in line
+    )
+    label, dest_s, kind = fields.get("label"), fields.get("dest"), fields.get("kind")
+    if not label or not dest_s or not Path(dest_s).is_dir():
+        log(cfg.ENCLOG, f"BAD JOB {jobfile}")
+        jobfile.unlink(missing_ok=True)
+        return
+
+    dest = Path(dest_s)
+    log(cfg.ENCLOG, f"ENCODE {label} ({kind})")
+    if kind == "audio":
+        process_audio_job(label, dest)
+    else:
+        process_video_job(label, dest)
+    (cfg.ENCDONE / label).touch()
+    jobfile.unlink(missing_ok=True)