scourgify 1.0.0__py3-none-any.whl

scourgify/__init__.py

@@ -0,0 +1,3 @@
+"""scourgify — normalize a FanFicFare-imported Calibre library."""
+
+__version__ = "1.0.0"

scourgify/__main__.py

@@ -0,0 +1,4 @@
+from scourgify.cli import main
+
+if __name__ == "__main__":
+    main()

scourgify/_writer.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Internal Calibre write helper — invoked UNDER calibre-debug by wrangle.py / classify.py, never by hand.
+
+The standalone (system-python) tools compute everything via read-only sqlite, then shell out to:
+    calibre-debug -e _writer.py -- <ops.json>
+to perform the writes through Calibre's API — the only fast batched write path (calibredb's set_metadata is
+one book per process). One generic op list keeps Calibre access in a single ~30-line stub.
+
+ops.json = [ {op: ...}, ... ]:
+  {"op":"create_column","label":"wrangled","name":"Wrangled","datatype":"datetime","is_multiple":false}
+  {"op":"set_field","field":"tags","values":{"<book_id>": ["a","b"] | "scalar"}}   # shape coerced to the column
+  {"op":"stamp_now","field":"#wrangled","books":[id,...] | null}                    # null = all books
+  {"op":"set_pref","key":"namespaced:...","value":{...}}
+"""
+import os, sys, json
+from calibre.library import db as DB
+
+LIB = os.path.expanduser(os.environ.get("CALIBRE_LIBRARY", ""))
+if not LIB: raise SystemExit("_writer: CALIBRE_LIBRARY not set")
+ops = json.load(open(sys.argv[-1]))
+legacy = DB(LIB); api = legacy.new_api
+
+for op in ops:
+    kind = op["op"]
+    if kind == "create_column":
+        if "#" + op["label"].lstrip("#") not in api.field_metadata.all_field_keys():
+            legacy.create_custom_column(op["label"].lstrip("#"), op["name"], op["datatype"], op.get("is_multiple", False))
+            legacy = DB(LIB); api = legacy.new_api          # reopen so the new column is usable
+            print(f"  created #{op['label'].lstrip('#')}")
+    elif kind == "set_field":
+        field = op["field"]
+        mult = bool(api.field_metadata.all_metadata().get(field, {}).get("is_multiple"))
+        vals = {}
+        for b, v in op["values"].items():
+            vals[int(b)] = (tuple(v) if isinstance(v, list) else (v,)) if mult else ((v[0] if isinstance(v, list) and v else v) or None)
+        api.set_field(field, vals)
+        print(f"  set {field}: {len(vals)} books")
+    elif kind == "stamp_now":
+        from calibre.utils.date import now as cal_now
+        books = op["books"] if op.get("books") is not None else list(api.all_book_ids())
+        ts = cal_now(); api.set_field(op["field"], {int(b): ts for b in books})
+        print(f"  stamped {op['field']}: {len(books)} books")
+    elif kind == "set_pref":
+        api.set_pref(op["key"], op["value"]); print(f"  set pref {op['key']}")
+print("WROTE.")

scourgify/afm.swift

@@ -0,0 +1,30 @@
+// Apple Foundation Models bridge for calibre-wrangler (macOS 26+, Apple Intelligence).
+// Persistent stdin loop: each input line is one prompt (literal newlines escaped as 
);
+// prints one response line per prompt. Used by classify.py --engine apple.
+//   build:  swiftc -O afm.swift -o afm     (or run directly: swift afm.swift)
+import Foundation
+import FoundationModels
+
+@available(macOS 26.0, *)
+func loop() async {
+    guard case .available = SystemLanguageModel.default.availability else {
+        FileHandle.standardError.write(Data("AFM_UNAVAILABLE\n".utf8)); exit(2)
+    }
+    while let line = readLine(strippingNewline: true) {
+        let prompt = line.replacingOccurrences(of: "\u{01}", with: "\n")
+        var out = "ERR"
+        do {
+            let resp = try await LanguageModelSession().respond(to: prompt)
+            out = resp.content.replacingOccurrences(of: "\n", with: " ")
+        } catch { out = "ERR: \(error)".replacingOccurrences(of: "\n", with: " ") }
+        print(out); fflush(stdout)
+    }
+}
+
+if #available(macOS 26.0, *) {
+    let sem = DispatchSemaphore(value: 0)
+    Task { await loop(); sem.signal() }
+    sem.wait()
+} else {
+    FileHandle.standardError.write(Data("needs macOS 26+\n".utf8)); exit(1)
+}

scourgify/classify.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""Content-based tagging from a controlled vocabulary, with TWO outputs per book:
+  1) added_tags    — tags chosen from defaults/classify_vocab.txt (the consolidated set); these get APPLIED.
+  2) proposed_new  — short reusable tags the model thinks apply but are NOT in the vocab yet; aggregated into
+                     classify_newtags_ranked.csv for review, so the vocabulary grows cleanly (promote -> vocab).
+
+  scourgify classify [--engine apple|claude|openai|gemini] [--incremental] [--workers N] [--batch N] [--fresh]
+  scourgify classify --apply                    # apply 'added_tags' + stamp #wrangled (Calibre CLOSED; writes shell to calibre-debug)
+
+Engines (--engine):  apple = on-device Apple Foundation Models via ./afm (free; macOS 26+).
+          claude = Anthropic (ANTHROPIC_API_KEY) | openai = OpenAI (OPENAI_API_KEY) | gemini = Google (GEMINI_API_KEY).
+          --model overrides the per-engine default. Only books with < --min-tags tags AND a description are processed.
+          Runs are resumable (skip books already in the proposal; --fresh restarts). Dry-run until --apply.
+--incremental = cheap maintenance after new downloads: (re)process only books whose #updated is newer than their own
+          #wrangled marker (or never wrangled), plus any still untagged. --apply auto-creates the #wrangled datetime
+          column and stamps each tagged book, so the state lives IN the library — no external file. (--since DATE
+          forces an explicit cutoff against #updated.) Avoids the ~full-library cost of a --fresh pass."""
+import argparse, os, re, csv, json, subprocess, collections, time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from scourgify.common import HERE, DATA, ro_connect, read_custom_column, custom_column_id, run_writer, library
+try:                                              # rich is optional: live dashboard/tables in system python3
+    from rich.console import Console, Group
+    from rich.live import Live
+    from rich.panel import Panel
+    from rich.progress import Progress, BarColumn, TextColumn, MofNCompleteColumn, TimeRemainingColumn
+    from rich.table import Table
+    from rich.text import Text
+    _con = Console(stderr=True); RICH = True
+except ImportError:
+    RICH = False
+
+VOCAB = [l.strip() for l in open(f"{HERE}/defaults/classify_vocab.txt") if l.strip() and not l.startswith("#")]
+VLOW = {v.lower(): v for v in VOCAB}
+PROP = f"{DATA}/classify_proposal.csv"
+RANK = f"{DATA}/classify_newtags_ranked.csv"
+FAIL = f"{DATA}/classify_failures.csv"
+SPEND_GATE = 200        # cloud runs above this many books require an explicit yes
+
+
+def prompt_for(desc, maxtags):
+    return ("You are tagging a fanfiction story. Return ONLY a JSON object with two arrays:\n"
+            f'  "tags": tags from the CONTROLLED LIST below that clearly apply (exact spelling, at most {maxtags}; '
+            "be conservative; [] if vague; do NOT echo the whole list).\n"
+            '  "new": up to 3 SHORT reusable trope/genre/theme tags (Title Case) that clearly apply but are NOT in the '
+            "list and would be worth adding to the vocabulary. No plot specifics, character names, or fandoms; [] if none.\n"
+            f"CONTROLLED LIST: {', '.join(VOCAB)}\n\nDESCRIPTION:\n{desc[:1500]}\n\nJSON:")
+
+def parse_resp(text, maxtags=6):
+    m = re.search(r"\{.*\}", text, re.S)
+    if not m: return [], []
+    try: obj = json.loads(m.group(0))
+    except Exception: return [], []
+    vt = [VLOW[str(t).strip().lower()] for t in obj.get("tags", []) if str(t).strip().lower() in VLOW]
+    if len(vt) > maxtags * 2: vt = []          # model echoed the list, not selecting
+    nt, seen = [], set()
+    for t in obj.get("new", []):
+        t = str(t).strip()
+        # first char must be alphanumeric: also blocks =/+/-/@ spreadsheet-formula injection in the review CSVs
+        if t and t[0].isalnum() and 1 < len(t) <= 40 and t.lower() not in VLOW and t.lower() not in seen:
+            seen.add(t.lower()); nt.append(t)
+    return vt[:maxtags], nt[:3]
+
+
+# ---- engines ----
+class Apple:
+    def __init__(self, model, timeout):
+        exe = f"{HERE}/afm" if os.path.exists(f"{HERE}/afm") else None
+        cmd = [exe] if exe else ["swift", f"{HERE}/afm.swift"]
+        self.p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1)
+    def ask(self, prompt):
+        self.p.stdin.write(prompt.replace("\n", "") + "\n"); self.p.stdin.flush()
+        return self.p.stdout.readline()
+
+class Claude:
+    def __init__(self, model, timeout):
+        self.key = os.environ.get("ANTHROPIC_API_KEY")
+        if not self.key: raise SystemExit("claude engine needs ANTHROPIC_API_KEY (or use --engine apple).")
+        self.model = model or "claude-haiku-4-5-20251001"; self.timeout = timeout
+    def ask(self, prompt):
+        import urllib.request
+        body = json.dumps({"model": self.model, "max_tokens": 300,
+                           "messages": [{"role": "user", "content": prompt}]}).encode()
+        req = urllib.request.Request("https://api.anthropic.com/v1/messages", data=body,
+            headers={"x-api-key": self.key, "anthropic-version": "2023-06-01", "content-type": "application/json"})
+        return json.load(urllib.request.urlopen(req, timeout=self.timeout))["content"][0]["text"]
+
+class OpenAI:
+    def __init__(self, model, timeout):
+        self.key = os.environ.get("OPENAI_API_KEY")
+        if not self.key: raise SystemExit("openai engine needs OPENAI_API_KEY.")
+        self.model = model or "gpt-4o-mini"; self.timeout = timeout
+    def ask(self, prompt):
+        import urllib.request
+        body = json.dumps({"model": self.model, "max_tokens": 300,
+                           "messages": [{"role": "user", "content": prompt}]}).encode()
+        req = urllib.request.Request("https://api.openai.com/v1/chat/completions", data=body,
+            headers={"Authorization": f"Bearer {self.key}", "content-type": "application/json"})
+        return json.load(urllib.request.urlopen(req, timeout=self.timeout))["choices"][0]["message"]["content"]
+
+class Gemini:
+    # personal fanfic library: don't let safety filters drop mature/dark stories (the tag list itself lists such terms)
+    SAFE = [{"category": c, "threshold": "BLOCK_NONE"} for c in
+            ("HARM_CATEGORY_HARASSMENT", "HARM_CATEGORY_HATE_SPEECH",
+             "HARM_CATEGORY_SEXUALLY_EXPLICIT", "HARM_CATEGORY_DANGEROUS_CONTENT")]
+    def __init__(self, model, timeout):
+        self.key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+        if not self.key: raise SystemExit("gemini engine needs GEMINI_API_KEY (or GOOGLE_API_KEY).")
+        self.model = model or "gemini-2.5-flash"; self.timeout = timeout
+    def ask(self, prompt):
+        import urllib.request
+        url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model}:generateContent"
+        body = json.dumps({"contents": [{"parts": [{"text": prompt}]}], "safetySettings": self.SAFE,
+                           "generationConfig": {"maxOutputTokens": 2048}}).encode()   # bound cost; roomy for thinking models
+        req = urllib.request.Request(url, data=body,
+            headers={"content-type": "application/json", "x-goog-api-key": self.key})  # key in header, never in the URL
+        r = json.load(urllib.request.urlopen(req, timeout=self.timeout))
+        cands = r.get("candidates")
+        if not cands: raise RuntimeError("blocked:" + str(r.get("promptFeedback", {}).get("blockReason")))
+        parts = cands[0].get("content", {}).get("parts")
+        if not parts: raise RuntimeError("nocontent:" + str(cands[0].get("finishReason")))
+        return parts[0]["text"]
+
+ENGINES = {"apple": Apple, "claude": Claude, "openai": OpenAI, "gemini": Gemini}
+
+
+# ---- live run display ----
+def sparkline(vals, width=28):
+    """Unicode sparkline of a numeric series (last `width` points), scaled to its max."""
+    vals = [v for v in vals][-width:]
+    if not vals: return ""
+    blocks = "▁▂▃▄▅▆▇█"
+    hi = max(vals)
+    if hi <= 0: return blocks[0] * len(vals)
+    return "".join(blocks[min(7, int(v * 8 / hi))] for v in vals)
+
+class _Dashboard:
+    """Live display for a classify run: progress bar, running numbers (tagged / failed /
+    no-match / rate), a throughput sparkline, and the rising new-tag candidates.
+    rich renders it live; without rich it degrades to a checkpoint line every 25 books."""
+    BUCKET = 5.0                                   # seconds per throughput bucket
+
+    def __init__(self, todo_n, done_before, targets_n):
+        self.total, self.done_before, self.targets = todo_n, done_before, targets_n
+        self.n = self.tagged = self.fails = 0
+        self.newtags = collections.Counter()
+        self.t0 = time.monotonic(); self.hist = [0]
+        self.live = self.prog = self.task = None
+
+    def __enter__(self):
+        if RICH and self.total:
+            self.prog = Progress(TextColumn("[cyan]classifying"), BarColumn(bar_width=None),
+                                 MofNCompleteColumn(), TimeRemainingColumn(), console=_con)
+            self.task = self.prog.add_task("", total=self.total)
+            self.live = Live(self._render(), console=_con, refresh_per_second=4)
+            self.live.__enter__()
+        return self
+
+    def __exit__(self, *exc):
+        if self.live: self.live.__exit__(*exc)
+        return False
+
+    def update(self, vt, nt, err):
+        self.n += 1
+        if err: self.fails += 1
+        elif vt: self.tagged += 1
+        self.newtags.update(nt)
+        b = int((time.monotonic() - self.t0) // self.BUCKET)
+        while len(self.hist) <= b: self.hist.append(0)
+        self.hist[b] += 1
+        if self.live:
+            self.prog.update(self.task, advance=1)
+            self.live.update(self._render())
+        elif self.n % 25 == 0:
+            el = time.monotonic() - self.t0
+            print(f"  +{self.n}/{self.total} … {self.tagged} tagged, {self.fails} failed, {self.n / el:.1f}/s")
+
+    def _render(self):
+        el = time.monotonic() - self.t0
+        rate = self.n / el if el > 1 else 0.0
+        g = Table.grid(padding=(0, 2))
+        g.add_row("[bold]this run[/]", f"{self.n}/{self.total}",
+                  "[green]tagged[/]", str(self.tagged),
+                  "[red]failed[/]", str(self.fails),
+                  "[dim]no match[/]", str(max(0, self.n - self.tagged - self.fails)),
+                  "[bold]rate[/]", f"{rate:.1f}/s")
+        parts = [self.prog, g]
+        spark = sparkline(self.hist)
+        if spark: parts.append(Text.assemble(("throughput  ", "bold"), (spark, "cyan")))
+        if self.newtags:
+            top = " · ".join(f"{t} ×{c}" for t, c in self.newtags.most_common(5))
+            parts.append(Text.assemble(("rising candidates  ", "bold"), (top, "magenta")))
+        return Panel(Group(*parts), border_style="cyan", padding=(0, 1),
+                     title=f"classify — {self.done_before + self.n}/{self.targets} total")
+
+
+# ---- apply: 'added_tags' + stamp #wrangled — standalone, no LLM calls ----
+def apply_proposal():
+    if not os.path.exists(PROP):
+        raise SystemExit(f"no proposal to apply ({os.path.basename(PROP)} not found — run a classify pass first).")
+    con = ro_connect()
+    cur = collections.defaultdict(list)
+    for b, t in con.execute("SELECT l.book, t.name FROM books_tags_link l JOIN tags t ON t.id=l.tag"): cur[b].append(t)
+    have_wrangled = custom_column_id(con, "wrangled") is not None
+    chg = {}
+    for r in csv.DictReader(open(PROP)):
+        b = int(r["book_id"]); tags = [t for t in r.get("added_tags", "").split("; ") if t.strip()]
+        if tags: chg[str(b)] = sorted(set(cur.get(b, [])) | set(tags))   # union with current tags
+    ops = []
+    if not have_wrangled:                                             # first run: create + backfill whole library as wrangled-now
+        ops.append({"op": "create_column", "label": "wrangled", "name": "Wrangled", "datatype": "datetime", "is_multiple": False})
+        ops.append({"op": "stamp_now", "field": "#wrangled", "books": None})
+    ops.append({"op": "set_field", "field": "tags", "values": chg})
+    ops.append({"op": "stamp_now", "field": "#wrangled", "books": [int(b) for b in chg]})
+    run_writer(ops)
+    # archive so a later --apply can't re-add tags you've since hand-removed (stale rows never re-apply)
+    arch = PROP.replace(".csv", f"_applied_{time.strftime('%Y%m%d-%H%M%S')}.csv")
+    os.rename(PROP, arch)
+    print(f"applied tags to {len(chg)} books + stamped #wrangled; proposal archived -> {os.path.basename(arch)}")
+
+
+# ---- gather books (read-only) ----
+def strip_html(s): return re.sub(r"<[^>]+>", " ", s or "").strip()
+
+def book_text(path, limit=6000):
+    if not path or not os.path.exists(path): return ""
+    if path.lower().endswith(".epub"):                  # fast path: epub is a zip of XHTML
+        import zipfile
+        try:
+            z = zipfile.ZipFile(path); out = []
+            for n in z.namelist():
+                if not n.lower().endswith((".xhtml", ".html", ".htm")): continue
+                if z.getinfo(n).file_size > 2_000_000: continue        # untrusted download: skip zip-bomb members
+                t = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", z.read(n).decode("utf-8", "ignore"))).strip()
+                if len(t) > 200: out.append(t)           # skip nav/title pages
+                if sum(len(x) for x in out) > limit: break
+            return " ".join(out)[:limit]
+        except Exception: return ""
+    import tempfile                                      # other formats (MOBI/PDF/DOCX/…): let calibre extract
+    try:
+        with tempfile.TemporaryDirectory() as td:
+            o = os.path.join(td, "o.txt")
+            subprocess.run(["ebook-convert", path, o], capture_output=True, timeout=180)
+            return re.sub(r"\s+", " ", open(o, errors="ignore").read()).strip()[:limit] if os.path.exists(o) else ""
+    except Exception: return ""
+
+def gather(a):
+    """-> (targets [(book, text)], titles, needs) for books under --min-tags or changed since last wrangle."""
+    con = ro_connect(); c = con.cursor()
+    tagn = {b: 0 for (b,) in c.execute("SELECT id FROM books")}
+    for (b,) in c.execute("SELECT book FROM books_tags_link"): tagn[b] = tagn.get(b, 0) + 1
+    desc = {b: t for b, t in c.execute("SELECT book, text FROM comments")}
+    updated = wrangled = {}
+    if a.since or a.incremental:
+        updated = {b: str(v)[:10] for b, v in (read_custom_column(con, "#updated") or {}).items()}
+        if a.incremental: wrangled = {b: str(v)[:10] for b, v in (read_custom_column(con, "#wrangled") or {}).items()}
+    def needs(b):                                 # changed since we last tagged it, or after an explicit --since date
+        if a.incremental and (b not in wrangled or updated.get(b, "") > wrangled.get(b, "")): return True
+        if a.since and updated.get(b, "") >= a.since: return True
+        return False
+    bookfile = {}
+    if a.text_fallback:                           # when the description is thin, sample the book's own text
+        bp = {b: p for b, p in c.execute("SELECT id, path FROM books")}
+        byb = {}
+        for b, fmt, name in c.execute("SELECT book, format, name FROM data"):
+            byb.setdefault(b, {})[fmt.upper()] = os.path.join(library(), bp[b], name + "." + fmt.lower())
+        for b, fm in byb.items():
+            bookfile[b] = fm.get("EPUB") or next(iter(fm.values()))   # prefer EPUB, else any available format
+    def text_for(b):
+        d = strip_html(desc.get(b, ""))
+        if len(d) >= 80 or not a.text_fallback: return d
+        et = book_text(bookfile.get(b, ""))
+        return (d + " " + et).strip() if et else d
+    targets = [(b, text_for(b)) for b in tagn if tagn[b] < a.min_tags or needs(b)]
+    if a.incremental or a.since: print(f"  incremental: {sum(1 for b in tagn if needs(b))} books changed since last wrangle")
+    targets = [(b, t) for b, t in targets if t and len(t) >= 40]
+    titles = {b: t for b, t in c.execute("SELECT id, title FROM books")}
+    if a.limit: targets = targets[:a.limit]
+    return targets, titles, needs
+
+
+def classify_run(a):
+    targets, titles, needs = gather(a)
+    print(f"engine={a.engine}  books to process (< {a.min_tags} tags, has description): {len(targets)}")
+
+    proposal, done = {}, set()                     # book -> (vocab_tags, proposed_new_tags)
+    if os.path.exists(PROP) and not a.fresh:       # resume: skip books already in proposal
+        for r in csv.DictReader(open(PROP)):
+            bid = int(r["book_id"]); at = [t for t in r.get("added_tags", "").split("; ") if t.strip()]
+            proposal[bid] = (at, [t for t in r.get("proposed_new", "").split("; ") if t.strip()])
+            if (at or not a.text_fallback) and not needs(bid): done.add(bid)   # re-process books changed since last wrangle
+        if done: print(f"  resuming: {len(done)} already in proposal (pass --fresh to restart)")
+    def dump():
+        with open(PROP, "w", newline="") as f:
+            w = csv.writer(f); w.writerow(["book_id", "title", "added_tags", "proposed_new"])
+            for b, (vt, nt) in proposal.items(): w.writerow([b, titles.get(b, ""), "; ".join(vt), "; ".join(nt)])
+
+    todo = [(b, d) for b, d in targets if b not in done]
+    if a.batch: todo = todo[:a.batch]
+    if a.engine != "apple" and len(todo) > SPEND_GATE and not a.yes:   # spend gate: cloud runs cost real money
+        import sys
+        msg = f"about to send {len(todo)} books to the {a.engine} API (costs money; --incremental/--batch shrink it)."
+        if sys.stdin.isatty() and input(f"  {msg} proceed? [y/N] ").strip().lower() not in ("y", "yes"):
+            raise SystemExit("aborted (nothing sent).")
+        elif not sys.stdin.isatty():
+            raise SystemExit(f"  {msg}\n  non-interactive: re-run with --yes to confirm.")
+
+    eng = ENGINES[a.engine](a.model, a.timeout)
+    def ask_retry(prompt, tries=4):
+        err = ""
+        for k in range(tries):
+            try: return eng.ask(prompt), ""
+            except RuntimeError as e:              # deterministic content block (no candidates) — retrying is futile
+                return "", str(e)[:140]
+            except Exception as e:                 # transient (HTTP 429/503, network) — back off and retry
+                err = f"{type(e).__name__}: {e}"[:140]
+                if k == tries - 1: return "", err
+                time.sleep(2 ** k)
+    def work(b, d):
+        out, err = ask_retry(prompt_for(d, a.max_tags)); vt, nt = parse_resp(out, a.max_tags); return b, err, vt, nt
+
+    failures = []
+    print(f"  {len(todo)} to do this run, {a.workers} concurrent")
+    ex = ThreadPoolExecutor(max_workers=a.workers)
+    interrupted = False
+    try:
+        with _Dashboard(len(todo), len(done), len(targets)) as dash:
+            futs = [ex.submit(work, b, d) for b, d in todo]
+            for fut in as_completed(futs):
+                b, err, vt, nt = fut.result()
+                if err: failures.append((b, err))
+                if vt or nt: proposal[b] = (vt, nt)
+                dash.update(vt, nt, err)
+                if dash.n % 50 == 0: dump()           # checkpoint regardless of UI
+    except KeyboardInterrupt:
+        # Ctrl+C: never start queued work, don't wait for in-flight requests (they're
+        # abandoned; runs are resumable so nothing is lost beyond the requests in the air)
+        interrupted = True
+        ex.shutdown(wait=False, cancel_futures=True)
+    else:
+        ex.shutdown()
+    dump()
+    if interrupted:
+        print(f"\n  interrupted — {len(proposal)} results saved to the proposal; re-run to resume where you left off.")
+    if failures:
+        with open(FAIL, "w", newline="") as f:
+            w = csv.writer(f); w.writerow(["book_id", "title", "reason"])
+            for b, e in failures: w.writerow([b, titles.get(b, ""), e])
+        bytype = collections.Counter(e.split(":")[0].split(" ")[0] for _, e in failures)
+        print(f"failures: {len(failures)} -> {os.path.basename(FAIL)}  by type: {dict(bytype)}")
+        print("  (recover blocked books with a no-policy engine: scourgify classify --engine apple)")
+
+    ranked = collections.Counter()
+    for vt, nt in proposal.values():
+        for t in nt: ranked[t] += 1
+    with open(RANK, "w", newline="") as f:
+        w = csv.writer(f); w.writerow(["proposed_tag", "count"])
+        for t, cnt in ranked.most_common(): w.writerow([t, cnt])
+    print(f"\nOutput 1 (apply): {sum(1 for v in proposal.values() if v[0])} books with vocab tags -> {os.path.basename(PROP)} (col 'added_tags')")
+    print(f"Output 2 (grow):  {len(ranked)} distinct new-tag candidates -> {os.path.basename(RANK)} (review -> promote into defaults/classify_vocab.txt)")
+    if RICH and ranked:
+        tbl = Table(title="top new-tag candidates (review → promote into the vocab)")
+        tbl.add_column("count", justify="right", style="cyan"); tbl.add_column("proposed tag")
+        for tag, cnt in ranked.most_common(25): tbl.add_row(str(cnt), tag)
+        _con.print(tbl)
+    else:
+        print("top new-tag candidates:")
+        for t, cnt in ranked.most_common(25): print(f"  {cnt:4}  {t}")
+    print("\nApply vocab tags with: scourgify classify --apply   (Calibre closed)")
+
+
+def build_parser():
+    p = argparse.ArgumentParser(description="Content-based tagging from a controlled vocabulary (LLM engines; dry-run until --apply).")
+    p.add_argument("--engine", default="apple", choices=sorted(ENGINES), help="apple = on-device, free (default)")
+    p.add_argument("--apply", action="store_true", help="apply 'added_tags' from the proposal + stamp #wrangled (Calibre closed)")
+    p.add_argument("--incremental", action="store_true", help="only books whose #updated is newer than their #wrangled marker")
+    p.add_argument("--since", default="", metavar="DATE", help="(re)process books with #updated >= this ISO date")
+    p.add_argument("--fresh", action="store_true", help="ignore the existing proposal and restart (a full cloud pass costs real money)")
+    p.add_argument("--batch", type=int, default=0, metavar="N", help="process only N new books this run (re-run resumes)")
+    p.add_argument("--limit", type=int, default=0, metavar="N", help="hard cap on candidate books")
+    p.add_argument("--workers", type=int, default=8, metavar="N", help="concurrent API requests (cloud engines are I/O-bound)")
+    p.add_argument("--min-tags", type=int, default=2, metavar="N", help="process books with fewer than N tags")
+    p.add_argument("--max-tags", type=int, default=6, metavar="N", help="max vocab tags per book")
+    p.add_argument("--model", default="", help="override the per-engine default model")
+    p.add_argument("--timeout", type=int, default=60, metavar="S", help="per-request HTTP timeout")
+    p.add_argument("--text-fallback", action="store_true", help="sample the book's own prose when the description is thin")
+    p.add_argument("--yes", "-y", action="store_true", help="skip the large-cloud-run confirmation")
+    return p
+
+def normalize(a):
+    """Post-parse invariants shared by the CLI and the wizard."""
+    if a.engine == "apple": a.workers = 1        # apple = one subprocess pipe, not thread-safe
+    library()                                    # fail fast with a clear message
+    os.makedirs(DATA, exist_ok=True)
+    return a
+
+def main():
+    a = normalize(build_parser().parse_args())
+    if a.apply: apply_proposal()
+    else: classify_run(a)
+
+
+if __name__ == "__main__":
+    main()

scourgify/cli.py

@@ -0,0 +1,31 @@
+"""Single `scourgify` command.
+
+Dispatch: bare -> wizard (via wrangle); setup/audit/apply -> wrangle;
+classify -> classify; staleness -> staleness. Each tool keeps its own argparse,
+so we just hand off argv. Imports are lazy so `scourgify --version` stays cheap.
+"""
+import sys
+
+from scourgify import __version__
+
+
+def main():
+    argv = sys.argv[1:]
+    if argv and argv[0] in ("-V", "--version"):
+        print(f"scourgify {__version__}")
+        return
+    if argv and argv[0] == "classify":
+        from scourgify import classify
+        sys.argv = ["scourgify classify", *argv[1:]]
+        return classify.main()
+    if argv and argv[0] == "staleness":
+        from scourgify import staleness
+        sys.argv = ["scourgify staleness", *argv[1:]]
+        return staleness.main()
+    # setup / audit / apply / (none -> wizard) all live in wrangle's main()
+    from scourgify import wrangle
+    return wrangle.main()
+
+
+if __name__ == "__main__":
+    main()

scourgify/common.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Shared core for the standalone tools (wrangle / classify / staleness).
+
+Everything here runs under plain system python3. It owns the four things the tools
+used to each carry a private copy of:
+  - library resolution (CALIBRE_LIBRARY) — checked lazily, never at import time
+  - read-only sqlite access + link-table-aware custom-column reading
+  - normalization helpers (norm, ascii_fold) and the minimal TOML config reader
+  - the single write funnel: run_writer() -> calibre-debug -e _writer.py
+    (backs up metadata.db to /tmp before every write; refuses to run while Calibre is open)
+"""
+import os, re, sys, sqlite3, collections, unicodedata
+
+HERE = os.path.dirname(os.path.abspath(__file__))   # the installed package dir (read-only)
+DEFAULTS = os.path.join(HERE, "defaults")            # bundled generic maps — ship inside the package
+# Per-run + per-user files live in the working directory, not site-packages: `data/`
+# (proposals/intermediates), config.toml, and overrides/ are all resolved against CWD.
+# When run from the repo via `uv run`, CWD == repo root, so dev layout is unchanged.
+DATA = os.path.join(os.getcwd(), "data")             # personal review maps, proposals, intermediates (gitignored)
+
+
+# ---------------- library resolution (lazy — importing this module never exits) ----------------
+def library():
+    lib = os.path.expanduser(os.environ.get("CALIBRE_LIBRARY", ""))
+    if not lib:
+        raise SystemExit("Set CALIBRE_LIBRARY to your Calibre library folder (the one containing metadata.db).")
+    return lib
+
+def db_path():
+    return os.path.join(library(), "metadata.db")
+
+def ro_connect():
+    return sqlite3.connect(f"file:{db_path()}?mode=ro", uri=True)
+
+
+# ---------------- normalization ----------------
+def norm(s):
+    s = str(s).strip().lower(); s = re.sub(r"[\[\]\(\)]", "", s); s = s.replace("&", "and")
+    s = re.sub(r"[\s_\-/]+", " ", s); s = re.sub(r"[^\w ]", "", s, flags=re.UNICODE); return s.strip()
+
+def ascii_fold(s):
+    s = s.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"').replace("…", "...").replace("–", "-").replace("—", "-")
+    return unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode()
+
+
+# ---------------- custom columns (single- and multi-value; link-table aware) ----------------
+def custom_column_id(con, label):
+    r = con.execute("SELECT id FROM custom_columns WHERE label=?", (label.lstrip("#"),)).fetchone()
+    return r[0] if r else None
+
+def read_custom_column(con, label, multi=False):
+    """{book: value} (or {book: [values]} with multi=True) for a custom column; None if it doesn't exist.
+    Handles both storage shapes: a books_custom_column_N_link table, or an inline `book` column."""
+    i = custom_column_id(con, label)
+    if i is None: return None
+    link = f"books_custom_column_{i}_link"
+    has_link = con.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (link,)).fetchone()
+    q = (f"SELECT l.book, v.value FROM {link} l JOIN custom_column_{i} v ON v.id=l.value" if has_link
+         else f"SELECT book, value FROM custom_column_{i}")
+    out = collections.defaultdict(list) if multi else {}
+    for b, v in con.execute(q):
+        if multi: out[b].append(v)
+        else: out[b] = v
+    return dict(out)
+
+
+# ---------------- config (minimal TOML reader; no tomllib dependency) ----------------
+def load_config(path=None):
+    cfg = {"columns": {"fandoms": "#fandoms", "characters": "#characters", "relationships": "#relationships",
+                       "genres": "#genres", "status": "#status", "tags": "tags"},
+           "behavior": {"fold_characters": True, "ascii_only_tags": True, "au_as": "genre", "crossover_as": "genre",
+                        "reincarnation_as": "genre", "time_travel_as": "genre", "fold_ratings": False,
+                        "keep_categories": True, "tropes_as": "tag"},
+           "overrides": {"dir": "overrides"}}
+    p = path or os.path.join(os.getcwd(), "config.toml")   # user config: CWD, not the package
+    if os.path.exists(p):
+        sec = None
+        for raw in open(p):
+            ln = raw.strip()
+            if not ln or ln.startswith("#"): continue
+            if ln.startswith("["): sec = ln[1:ln.index("]")].strip(); cfg.setdefault(sec, {}); continue
+            if "=" in ln and sec:
+                k, v = ln.split("=", 1); k = k.strip(); v = v.strip()
+                if v[:1] in ("\"", "'"):                 # quoted string -> value between the quotes (# allowed inside)
+                    v = v[1:].split(v[0], 1)[0]
+                else:                                    # bool/number -> strip any trailing inline comment
+                    v = v.split("#", 1)[0].strip()
+                    if v.lower() in ("true", "false"): v = v.lower() == "true"
+                cfg[sec][k] = v
+    return cfg
+
+
+# ---------------- the write funnel ----------------
+def calibre_open():
+    import subprocess, shutil
+    if not shutil.which("pgrep"): return False
+    out = subprocess.run(["pgrep", "-fl", "calibre"], capture_output=True, text=True).stdout
+    return any("calibre" in l.lower() and not any(x in l for x in ("calibre-debug", "pgrep", "wrangle", "_writer", "classify"))
+               for l in out.splitlines())
+
+def run_writer(ops):
+    """Apply a list of write-ops through Calibre by shelling out to `calibre-debug -e _writer.py`.
+    Automatically snapshots metadata.db to /tmp first — every write path gets a rollback point for free."""
+    import json, time, tempfile, subprocess, shutil
+    ops = [o for o in ops if o.get("op") != "set_field" or o.get("values")]
+    if not ops: print("  (nothing to write)"); return
+    if calibre_open(): raise SystemExit("Calibre is running — close it first (it locks metadata.db), then re-run.")
+    cb = shutil.which("calibre-debug") or "/Applications/calibre.app/Contents/MacOS/calibre-debug"
+    if not (shutil.which("calibre-debug") or os.path.exists(cb)): raise SystemExit("calibre-debug not found (install Calibre's CLI tools).")
+    bak = f"/tmp/ff_{int(time.time())}.db"
+    shutil.copy2(db_path(), bak)
+    print(f"  backup: {bak}")
+    f = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False); json.dump(ops, f); f.close()
+    print("  → writing via calibre-debug …")
+    rc = subprocess.run([cb, "-e", os.path.join(HERE, "_writer.py"), "--", f.name],
+                        env={**os.environ, "CALIBRE_LIBRARY": library()}).returncode
+    os.unlink(f.name)
+    if rc != 0: raise SystemExit(f"writer failed (exit {rc}) — library backup at {bak}")