pdf2wiki 0.1.0__py3-none-any.whl

pdf2wiki/__init__.py

@@ -0,0 +1,14 @@
+"""pdf2wiki — convert technical books (native-text PDFs) into clean, chapter-split,
+LLM-ready Markdown, built on a dual-pass MinerU pipeline.
+
+Pipeline stages:
+  1. convert  — MinerU pipeline pass (embedded text layer, byte-perfect code) merged with a
+                MinerU hybrid/VLM pass (correct table grids, Mermaid diagram transcription).
+  2. phase5   — post-processing chain: caption unbleed -> code-fence language retag ->
+                dash normalize -> mermaid repair -> code unescape -> chapter split with
+                YAML frontmatter.
+  3. qa       — reproducible page sampling + per-page review artifacts for manual back-checks.
+  4. batch    — manifest-driven multi-book runs, resumable, with optional SSH remote execution.
+"""
+
+__version__ = "0.1.0"

pdf2wiki/batch.py

@@ -0,0 +1,138 @@
+"""Manifest-driven batch conversion: convert -> (fetch) -> phase5 -> optional vault placement.
+
+The book list is a user-supplied TOML file — never code:
+
+    # books.toml
+    [[book]]
+    pdf = "Some_Technical_Book.pdf"   # path (local mode) or filename under remote books_dir
+    slug = "some-technical-book"      # output name, used everywhere downstream
+    domain = "distributed-systems"    # optional: subfolder under the output/vault root
+
+Resumable: a JSON status manifest records per-book status next to the book list; a book whose
+status is exactly "done" is skipped on re-run. One book's failure is logged and does NOT abort
+the batch. Sequential only — a single GPU cannot run concurrent VLM passes.
+
+A STOP file next to the status manifest halts cleanly between books (it is consumed on halt).
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import time
+import tomllib
+
+from .executor import ExecutionError, LocalExecutor, SSHExecutor
+from .phase5 import run_chain
+
+
+def load_books(books_toml: str) -> list[dict]:
+    with open(books_toml, "rb") as f:
+        data = tomllib.load(f)
+    books = data.get("book", [])
+    for b in books:
+        if "pdf" not in b or "slug" not in b:
+            raise ValueError(f"each [[book]] needs `pdf` and `slug`: {b}")
+    return books
+
+
+def run_batch(books_toml: str, cfg, stage_dir: str, remote: str | None = None,
+              max_books: int | None = None, only: str | None = None,
+              vault: str | None = None) -> dict:
+    """Run the batch. Returns the final status manifest."""
+    stage = os.path.expanduser(stage_dir)
+    os.makedirs(stage, exist_ok=True)
+    manifest_path = os.path.join(stage, "manifest.json")
+    stop_file = os.path.join(stage, "STOP")
+
+    manifest: dict = {}
+    if os.path.exists(manifest_path):
+        try:
+            with open(manifest_path) as f:
+                manifest = json.load(f)
+        except json.JSONDecodeError as e:
+            raise SystemExit(
+                f"status manifest {manifest_path} is corrupt ({e}). "
+                f"Repair or delete it (deleting restarts every book) and re-run."
+            )
+
+    def save():
+        # atomic: a kill mid-dump must never truncate the manifest (it's the resume backbone)
+        tmp = manifest_path + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump(manifest, f, indent=1)
+        os.replace(tmp, manifest_path)
+
+    if remote:
+        ex = SSHExecutor(remote, cfg.remote.books_dir, cfg.remote.workdir,
+                         cfg.remote.connect_timeout, cfg.remote.convert_timeout)
+    else:
+        ex = LocalExecutor()
+    ex.check()  # fail fast before touching any book
+
+    books = load_books(books_toml)
+    attempted = 0
+    for b in books:
+        slug, domain = b["slug"], b.get("domain", "")
+        if only is not None and slug != only:
+            continue
+        if max_books is not None and attempted >= max_books:
+            print(f"--max-books {max_books} reached — stopping cleanly (done books skip on re-run).")
+            break
+        if os.path.exists(stop_file):
+            print(f"STOP file present ({stop_file}) — halting cleanly between books.")
+            os.remove(stop_file)
+            break
+        if manifest.get(slug, {}).get("status") == "done":
+            print(f"[skip] {slug} already done")
+            continue
+        attempted += 1
+        print(f"=== {slug} ({domain or 'no domain'}) ===", flush=True)
+        t0 = time.time()
+
+        timeout = cfg.remote.convert_timeout if remote else cfg.convert.timeout
+        ok, log = ex.convert(b["pdf"], slug, cfg.convert.out_root, timeout)
+        if not ok:
+            print(f"  CONVERT FAILED: {slug} — log tail:\n{log[-2000:]}")
+            manifest[slug] = {"status": "convert_failed", "domain": domain}
+            save()
+            continue
+
+        work = os.path.join(stage, slug)
+        if not ex.fetch(slug, cfg.convert.out_root, work):
+            manifest[slug] = {"status": "fetch_failed", "domain": domain}
+            save()
+            continue
+        if isinstance(ex, LocalExecutor):
+            work = ex.artifacts_dir(slug, cfg.convert.out_root)
+
+        md = os.path.join(work, f"{slug}.md")
+        try:
+            run_chain(md, slug, out_dir=os.path.join(work, "chapters"),
+                      source_name=os.path.basename(b["pdf"]), apply=True)
+        except Exception as e:
+            print(f"  PHASE5 FAILED: {slug}: {e}")
+            manifest[slug] = {"status": "phase5_failed", "domain": domain}
+            save()
+            continue
+        # chapters need the shared images/ dir next to them for relative refs to resolve
+        img_src = os.path.join(work, "images")
+        img_dst = os.path.join(work, "chapters", "images")
+        if os.path.isdir(img_src):
+            os.makedirs(img_dst, exist_ok=True)
+            for f in os.listdir(img_src):
+                shutil.copy(os.path.join(img_src, f), os.path.join(img_dst, f))
+
+        entry = {"status": "done", "domain": domain, "minutes": round((time.time() - t0) / 60, 1)}
+        if vault:
+            dest = os.path.join(os.path.expanduser(vault), domain, slug) if domain \
+                else os.path.join(os.path.expanduser(vault), slug)
+            shutil.copytree(os.path.join(work, "chapters"), dest, dirs_exist_ok=True)
+            entry["vault_path"] = dest
+            print(f"  DONE {slug} -> {dest} ({entry['minutes']} min)")
+        else:
+            print(f"  DONE {slug} -> {os.path.join(work, 'chapters')} ({entry['minutes']} min)")
+        manifest[slug] = entry
+        save()
+    print("BATCH COMPLETE")
+    return manifest

pdf2wiki/cli.py

@@ -0,0 +1,144 @@
+"""pdf2wiki command-line interface.
+
+Convention: every mutating command is DRY-RUN by default and requires --apply to write
+(exceptions: convert and qa, whose whole purpose is producing new artifacts in their own
+output directories — they never modify existing files in place).
+"""
+import argparse
+import json
+import sys
+
+from .config import load_config
+
+
+def _cmd_convert(a, cfg):
+    from .executor import LocalExecutor, SSHExecutor
+    if a.remote or cfg.remote.host:
+        host = a.remote or cfg.remote.host
+        ex = SSHExecutor(host, cfg.remote.books_dir, cfg.remote.workdir,
+                         cfg.remote.connect_timeout, cfg.remote.convert_timeout)
+        ex.check()
+        ok, log = ex.convert(a.pdf, a.name, a.out or cfg.convert.out_root)
+        print(log)               # remote log was captured on the remote host — show it
+    else:
+        ex = LocalExecutor()
+        ok, log = ex.convert(a.pdf, a.name, a.out or cfg.convert.out_root,
+                             cfg.convert.timeout)
+        # local progress already streamed live by convert_book — don't print the log twice
+    return 0 if ok else 1
+
+
+def _cmd_phase5(a, cfg):
+    from .phase5 import run_chain
+    report = run_chain(a.md, a.book, out_dir=a.out, source_name=a.source_name, apply=a.apply)
+    print(f"caption_unbleed: {report['caption_unbleed']['unwrapped']} unwrapped")
+    lr = report["lang_retag"]
+    print(f"lang_retag: {lr['changes']} changes {lr['stats']}")
+    print(f"dash_normalize: {report['dash_normalize']['fixes']} fixes")
+    mr = report["mermaid_repair"]
+    print(f"mermaid_repair: {mr['blocks_changed']} blocks, "
+          f"parse-breaker score {mr['score_before']} -> {mr['score_after']}")
+    cs = report["chapter_split"]
+    print(f"chapter_split: {cs['boundaries']} boundaries")
+    for i, t in enumerate(cs["titles"], 1):
+        print(f"  {i:2d}. {t}")
+    if a.apply:
+        print(f"\nAPPLIED — wrote {len(cs['files'])} chapter files")
+    else:
+        print(f"\n(dry-run — would write {len(cs['files'])} chapter files; pass --apply)")
+    return 0
+
+
+def _cmd_qa_sample(a, cfg):
+    from .qa.sample import sample_pages
+    r = sample_pages(a.pdf, a.name, a.qa_root or cfg.qa.root,
+                     n=a.pages or cfg.qa.pages, seed=a.seed if a.seed is not None else cfg.qa.seed,
+                     dpi=a.dpi or cfg.qa.dpi)
+    print(f"{a.name}: sampled {len(r['pages'])} pages (seed {r['seed']}) "
+          f"from range {r['range'][0]}-{r['range'][1]} of {r['page_count']}")
+    print("pages:", r["pages"])
+    print(f"sample pdf: {r['sample_pdf']} ; PNGs in {r['qa_dir']}/pages/")
+    return 0
+
+
+def _cmd_qa_review(a, cfg):
+    from .qa.review import build_review
+    r = build_review(a.qa_dir, a.name, blocks_path=a.blocks)
+    print(f"wrote {r['review']} ; pages with content: {r['pages_with_content']}/{r['sampled']}")
+    return 0
+
+
+def _cmd_scan(a, cfg):
+    from .scan import scan_dir
+    print(json.dumps(scan_dir(a.directory), indent=1))
+    return 0
+
+
+def _cmd_batch(a, cfg):
+    from .batch import run_batch
+    run_batch(a.books, cfg, a.stage, remote=a.remote or (cfg.remote.host or None),
+              max_books=a.max_books, only=a.only, vault=a.vault or (cfg.output.vault or None))
+    return 0
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="pdf2wiki",
+                                 description="Convert technical books (native-text PDFs) into "
+                                             "clean, chapter-split, LLM-ready Markdown.")
+    ap.add_argument("--config", default=None, help="explicit config TOML path")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p = sub.add_parser("convert", help="convert one PDF (dual-pass MinerU merge)")
+    p.add_argument("pdf")
+    p.add_argument("--name", required=True, help="output slug")
+    p.add_argument("--out", default=None, help="output root (default from config)")
+    p.add_argument("--remote", default=None, help="ssh host to run the conversion on")
+    p.set_defaults(fn=_cmd_convert)
+
+    p = sub.add_parser("phase5", help="post-process a converted .md (dry-run by default)")
+    p.add_argument("md")
+    p.add_argument("--book", required=True, help="book slug for frontmatter")
+    p.add_argument("--out", default=None, help="chapter output dir (default: <md dir>/chapters)")
+    p.add_argument("--source-name", default=None,
+                   help="original PDF filename for frontmatter `source:` "
+                        "(avoids leaking a staging path)")
+    p.add_argument("--apply", action="store_true", help="write changes (default: dry-run)")
+    p.set_defaults(fn=_cmd_phase5)
+
+    pqa = sub.add_parser("qa", help="conversion QA tools")
+    qsub = pqa.add_subparsers(dest="qa_cmd", required=True)
+    p = qsub.add_parser("sample", help="sample N random pages -> sample PDF + PNGs")
+    p.add_argument("pdf")
+    p.add_argument("name")
+    p.add_argument("-n", "--pages", type=int, default=None)
+    p.add_argument("--seed", type=int, default=None)
+    p.add_argument("--dpi", type=int, default=None)
+    p.add_argument("--qa-root", default=None)
+    p.set_defaults(fn=_cmd_qa_sample)
+    p = qsub.add_parser("review", help="build per-page review.txt from blocks.json")
+    p.add_argument("qa_dir")
+    p.add_argument("name")
+    p.add_argument("--blocks", default=None, help="explicit blocks.json path")
+    p.set_defaults(fn=_cmd_qa_review)
+
+    p = sub.add_parser("scan", help="scan a directory of PDFs -> title/year guesses (JSON)")
+    p.add_argument("directory")
+    p.set_defaults(fn=_cmd_scan)
+
+    p = sub.add_parser("batch", help="manifest-driven multi-book run (resumable)")
+    p.add_argument("books", help="books TOML file ([[book]] entries with pdf/slug/domain)")
+    p.add_argument("--stage", default="~/pdf2wiki/stage", help="staging + status-manifest dir")
+    p.add_argument("--remote", default=None, help="ssh host to convert on")
+    p.add_argument("--max-books", type=int, default=None,
+                   help="stop after this many books attempted this run")
+    p.add_argument("--only", default=None, help="run only this slug")
+    p.add_argument("--vault", default=None, help="final placement root (e.g. an Obsidian vault)")
+    p.set_defaults(fn=_cmd_batch)
+
+    a = ap.parse_args(argv)
+    cfg = load_config(a.config)
+    return a.fn(a, cfg)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pdf2wiki/config.py

@@ -0,0 +1,105 @@
+"""Configuration for pdf2wiki.
+
+Resolution order (first hit wins, per key):
+  1. CLI flags
+  2. ./pdf2wiki.toml (project-local)
+  3. ~/.config/pdf2wiki/config.toml (user)
+  4. built-in defaults below
+
+All values that were once hardcoded for a specific machine live here instead.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import tomllib
+from dataclasses import dataclass, field, fields
+
+
+@dataclass
+class MineruConfig:
+    binary: str = ""                    # empty -> discover `mineru` on PATH
+    model_source: str = "huggingface"   # MINERU_MODEL_SOURCE
+    effort: str = "high"                # hybrid-engine effort for the VLM pass
+
+    def resolve_binary(self) -> str:
+        if self.binary:
+            return os.path.expanduser(self.binary)
+        found = shutil.which("mineru")
+        if not found:
+            raise FileNotFoundError(
+                "mineru CLI not found on PATH. Install MinerU (https://github.com/opendatalab/MinerU) "
+                "or set [mineru].binary in pdf2wiki.toml"
+            )
+        return found
+
+
+@dataclass
+class ConvertConfig:
+    out_root: str = "~/pdf2wiki/out"
+    workdir: str = "~/.pdf2wiki/run"    # clean cwd for MinerU subprocesses (see README: stdlib-shadow gotcha)
+    timeout: int = 7200                 # seconds per MinerU pass (local conversion)
+    gap: int = 3                        # merge nearby rich pages into one hybrid run if gap <= this
+    seg: int = 40                       # pipeline segment size (pages) for chunked passes
+    maxrun: int = 25                    # cap on a single hybrid run length (pages)
+    tiny_px2: int = 2500                # ignore images smaller than this (px^2) when profiling pages
+
+
+@dataclass
+class QaConfig:
+    root: str = "~/pdf2wiki/qa"
+    dpi: int = 140
+    seed: int = 42
+    pages: int = 20
+
+
+@dataclass
+class RemoteConfig:
+    host: str = ""                      # ssh destination (alias or user@host); empty -> local execution
+    books_dir: str = ""                 # directory on the remote host holding the PDFs
+    workdir: str = "~/pdf2wiki-remote"  # remote working directory (converter output lives under it)
+    connect_timeout: int = 8
+    convert_timeout: int = 7200
+
+
+@dataclass
+class OutputConfig:
+    vault: str = ""                     # optional: final placement root (e.g. an Obsidian vault)
+
+
+@dataclass
+class Config:
+    mineru: MineruConfig = field(default_factory=MineruConfig)
+    convert: ConvertConfig = field(default_factory=ConvertConfig)
+    qa: QaConfig = field(default_factory=QaConfig)
+    remote: RemoteConfig = field(default_factory=RemoteConfig)
+    output: OutputConfig = field(default_factory=OutputConfig)
+
+
+def _apply_section(obj, data: dict):
+    valid = {f.name for f in fields(obj)}
+    for k, v in data.items():
+        if k in valid:
+            setattr(obj, k, v)
+
+
+def load_config(explicit_path: str | None = None) -> Config:
+    """Load config, merging user config then project-local config over defaults."""
+    cfg = Config()
+    candidates = []
+    if explicit_path:
+        candidates = [explicit_path]
+    else:
+        candidates = [
+            os.path.expanduser("~/.config/pdf2wiki/config.toml"),
+            "pdf2wiki.toml",
+        ]
+    for path in candidates:
+        if not os.path.exists(path):
+            continue
+        with open(path, "rb") as f:
+            data = tomllib.load(f)
+        for section_name in ("mineru", "convert", "qa", "remote", "output"):
+            if section_name in data:
+                _apply_section(getattr(cfg, section_name), data[section_name])
+    return cfg

pdf2wiki/convert/__init__.py

@@ -0,0 +1,18 @@
+"""Dual-pass converter (MinerU pipeline + hybrid merge).
+
+Base-driven merge: the pipeline `-m txt` pass gives a byte-perfect skeleton from the PDF's
+embedded text layer; the hybrid/VLM pass is grafted in for table grids, Mermaid diagram
+transcriptions, LaTeX equations and chart data, matched by page + bbox-IoU. Code blocks are
+token-verified between the passes and flagged on divergence. A coverage gate hard-stops if any
+text-bearing page produced no blocks (zero-fail scrape); all MinerU passes are cached with
+`.done` sentinels for safe resume.
+
+Output layout (consumed by phase5, qa.review, batch):
+    <out_root>/<slug>/<slug>.md     merged markdown
+    <out_root>/<slug>/images/       extracted figure images (relative refs from the md)
+    <out_root>/<slug>/blocks.json   per-block records with abs_page (for QA review)
+    <out_root>/<slug>/*.log         per-pass MinerU logs (never suppressed)
+"""
+from .merge import convert_book
+
+__all__ = ["convert_book"]