pdf2wiki 0.2.2__tar.gz → 0.2.4__tar.gz

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/CHANGELOG.md

@@ -4,6 +4,32 @@ All notable changes to this project are documented here. The format is based on
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.4] - 2026-07-23
+
+### Fixed
+- **Remote convert produced no output** (`no content_list.json`). `run_mineru` handed MinerU a
+  *relative* `-o`/`-p`, but MinerU runs with a different working directory (the stdlib-shadow-safe
+  `clean_cwd`), so its output landed where pdf2wiki couldn't find it. Only surfaced in `--remote` mode,
+  where `--out` is passed home-relative. Now the paths handed to MinerU are absolutized
+  (`os.path.abspath`) — idempotent for the already-absolute local case. Found by the first real
+  end-to-end remote run; +1 regression test reproducing the cwd divergence.
+
+## [0.2.3] - 2026-07-23
+
+QA + diagnostics from an external review of the repo. No breaking changes; +4 tests (103 → 107).
+
+### Added
+- **`pdf2wiki qa flags PATHS...`** — per-book report of the code blocks where the VLM diverged from the
+  byte-clean text layer (`_code_flag`), or where hybrid indentation failed a Python ast check
+  (`_indent_flag`), read from `blocks.json`. Ranks multiple books by flagged count (which books to
+  trust least) and lists each flagged block (page / language / snippet) for a single book — the
+  highest-signal spots to spot-check.
+
+### Changed
+- **Batch summary rolls up `error_class`** — a partial batch now prints, e.g.,
+  `5 book(s) not done — by class: permanent×3, timeout×1, fetch×1` (plus the slug list), so a cluster
+  of same-kind failures reads as one diagnosis instead of N separate slugs. Exit code unchanged.
+
 ## [0.2.2] - 2026-07-23
 
 Resilience + security hardening from a book-grounded review (Tech-Books vault: Backoff-Retries,

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pdf2wiki
-Version: 0.2.2
+Version: 0.2.4
 Summary: Convert technical books (native-text PDFs) into clean, chapter-split, LLM-ready Markdown using a dual-pass MinerU pipeline plus fidelity-preserving post-processing.
 Project-URL: Homepage, https://github.com/Sevthered/pdf2wiki
 Project-URL: Repository, https://github.com/Sevthered/pdf2wiki

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pdf2wiki"
-version = "0.2.2"
+version = "0.2.4"
 description = "Convert technical books (native-text PDFs) into clean, chapter-split, LLM-ready Markdown using a dual-pass MinerU pipeline plus fidelity-preserving post-processing."
 authors = [{ name = "Sevthered" }]
 readme = "README.md"

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/src/pdf2wiki/__init__.py

@@ -11,4 +11,4 @@ Pipeline stages:
   4. batch    — manifest-driven multi-book runs, resumable, with optional SSH remote execution.
 """
 
-__version__ = "0.2.2"
+__version__ = "0.2.4"

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/src/pdf2wiki/cli.py

@@ -99,6 +99,19 @@ def _cmd_qa_review(a, cfg):
     return 0
 
 
+def _cmd_qa_flagged(a, cfg):
+    from .qa.flagged import flagged_report
+    reports = sorted((flagged_report(p) for p in a.blocks), key=lambda r: r["flagged"], reverse=True)
+    print(f"{'flagged':>7} {'diverged':>8} {'indent':>6} {'code':>5}  book")
+    for r in reports:
+        print(f"{r['flagged']:>7} {r['diverged']:>8} {r['indent_suspect']:>6} {r['code_blocks']:>5}  {r['name']}")
+    if len(reports) == 1:                    # single book -> also list the blocks to eyeball
+        print()
+        for e in reports[0]["blocks"]:
+            print(f"  p{e['page']:<4} [{e['lang'] or '?'}] {e['flag']}: {e['snippet']}")
+    return 0
+
+
 def _cmd_scan(a, cfg):
     from .scan import scan_dir
     print(json.dumps(scan_dir(a.directory), indent=1))
@@ -109,9 +122,13 @@ def _cmd_batch(a, cfg):
     from .batch import run_batch
     manifest = run_batch(a.books, cfg, a.stage, remote=a.remote or (cfg.remote.host or None),
                          max_books=a.max_books, only=a.only, vault=a.vault or (cfg.output.vault or None))
-    failed = [s for s, e in manifest.items() if e.get("status") != "done"]
+    failed = [(s, e) for s, e in manifest.items() if e.get("status") != "done"]
     if failed:                       # non-zero exit so CI/automation can detect a partial batch
-        print(f"batch: {len(failed)} book(s) not done: {failed}", file=sys.stderr)
+        from collections import Counter
+        by_class = Counter(e.get("error_class", "unknown") for _, e in failed)
+        rollup = ", ".join(f"{k}×{v}" for k, v in by_class.most_common())
+        print(f"batch: {len(failed)} book(s) not done — by class: {rollup}", file=sys.stderr)
+        print(f"  not done: {[s for s, _ in failed]}", file=sys.stderr)
         return 1
     return 0
 
@@ -166,6 +183,10 @@ def main(argv=None):
     p.add_argument("name")
     p.add_argument("--blocks", default=None, help="explicit blocks.json path")
     p.set_defaults(fn=_cmd_qa_review)
+    p = qsub.add_parser("flags", help="report code blocks the VLM diverged on (from blocks.json), "
+                                      "ranked across books — the highest-signal QA sample")
+    p.add_argument("blocks", nargs="+", help="one or more converted-book blocks.json paths")
+    p.set_defaults(fn=_cmd_qa_flagged)
 
     p = sub.add_parser("scan", help="scan a directory of PDFs -> title/year guesses (JSON)")
     p.add_argument("directory")

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/src/pdf2wiki/convert/merge.py

@@ -179,7 +179,12 @@ def run_mineru(mineru_bin, pdf, a, b, backend, extra, outdir, clean_cwd, env,
     cl = glob.glob(f"{outdir}/*/*/*content_list.json")
     if not (os.path.exists(done) and cl):
         os.makedirs(outdir, exist_ok=True)
-        cmd = [mineru_bin, "-p", pdf, "-o", outdir, "-b", backend, "-s", str(a), "-e", str(b)] + extra
+        # MinerU runs with cwd=clean_cwd (stdlib-shadow-safe), NOT pdf2wiki's cwd — so any relative path
+        # handed to it resolves against the wrong directory and its output lands where our glob can't see
+        # it (remote mode passes a home-relative --out). Absolutize the paths MinerU receives; abspath is
+        # idempotent for already-absolute (local) paths. See bug-pdf2wiki-remote-relative-outdir.
+        cmd = ([mineru_bin, "-p", os.path.abspath(pdf), "-o", os.path.abspath(outdir),
+                "-b", backend, "-s", str(a), "-e", str(b)] + extra)
         print("  run:", " ".join(cmd[3:]))
         with open(f"{outdir}.log", "w", encoding="utf-8") as log:
             # start_new_session=True puts MinerU in its own process group so a timeout can SIGKILL the

pdf2wiki-0.2.4/src/pdf2wiki/qa/flagged.py

@@ -0,0 +1,42 @@
+"""Per-book flagged-block report: which code blocks the VLM diverged on — the highest-signal QA
+sample. Pure read over a converted book's blocks.json (no converter or schema change). The flags are
+set during merge in convert/merge.py: `_code_flag` = the hybrid/VLM code diverged from the byte-clean
+text layer (output shows the pipeline tokens); `_indent_flag` = tokens agreed but the hybrid
+indentation failed a Python ast sanity check. These are exactly the places worth eyeballing."""
+import json
+import os
+
+
+def flagged_report(blocks_path: str, name: str | None = None) -> dict:
+    """Summarize the flagged code blocks in one book's blocks.json.
+
+    Returns {name, code_blocks, flagged, diverged, indent_suspect, blocks} where `blocks` is the
+    page-sorted list of flagged entries {page, lang, flag, snippet}; `page` is 1-based for display.
+    """
+    path = os.path.expanduser(blocks_path)
+    with open(path, encoding="utf-8") as f:
+        blocks = json.load(f)
+    name = name or os.path.basename(os.path.dirname(os.path.abspath(path)))
+
+    code_blocks = 0
+    flagged = []
+    for b in blocks:
+        if b.get("type") != "code":
+            continue
+        code_blocks += 1
+        if b.get("_code_flag"):
+            flag = "diverged"
+        elif b.get("_indent_flag"):
+            flag = "indent"
+        else:
+            continue
+        body = str(b.get("code_body") or "")
+        snippet = next((ln.strip() for ln in body.splitlines() if ln.strip()), "")[:80]
+        flagged.append({"page": int(b.get("abs_page", 0)) + 1, "lang": b.get("sub_type") or "",
+                        "flag": flag, "snippet": snippet})
+
+    flagged.sort(key=lambda e: e["page"])
+    return {"name": name, "code_blocks": code_blocks, "flagged": len(flagged),
+            "diverged": sum(1 for e in flagged if e["flag"] == "diverged"),
+            "indent_suspect": sum(1 for e in flagged if e["flag"] == "indent"),
+            "blocks": flagged}

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/tests/test_batch.py

@@ -125,3 +125,17 @@ def test_no_breaker_when_executor_stays_healthy(tmp_path, monkeypatch):
     failed = [s for s, e in manifest.items() if e["status"] == "convert_failed"]
     assert len(failed) == 5                            # all attempted, no premature abort
     assert manifest["book-0"]["error_class"] == "permanent"
+
+
+def test_cmd_batch_rolls_up_error_class(monkeypatch, capsys):
+    # a partial batch must aggregate error_class so a cluster of same-kind failures reads as one line.
+    monkeypatch.setattr(batch, "run_batch",
+                        lambda *a, **k: {"book-a": {"status": "done"},
+                                         "book-b": {"status": "convert_failed", "error_class": "permanent"},
+                                         "book-c": {"status": "convert_failed", "error_class": "permanent"},
+                                         "book-d": {"status": "convert_failed", "error_class": "TimeoutExpired"}})
+    assert cli.main(["batch", "books.toml"]) == 1
+    err = capsys.readouterr().err
+    assert "3 book(s) not done" in err
+    assert "permanent×2" in err and "TimeoutExpired×1" in err
+    assert "not done:" in err                        # slug detail retained

{pdf2wiki-0.2.2 → pdf2wiki-0.2.4}/tests/test_convert.py

@@ -1,4 +1,5 @@
 """Unit tests for the converter's pure functions (no MinerU / GPU needed)."""
+import json
 import os
 import subprocess
 import sys
@@ -346,3 +347,35 @@ def test_render_emits_code_verify_flag():
     out = render(flagged)
     assert out.startswith("<!-- code-verify:")
     assert "private_key" in out
+
+
+def test_run_mineru_passes_absolute_paths_to_mineru(tmp_path, monkeypatch):
+    # MinerU runs in clean_cwd, NOT pdf2wiki's cwd. If run_mineru hands it a relative -o/-p, MinerU's
+    # output lands under clean_cwd and pdf2wiki's glob (against its own cwd) misses it -> the remote-mode
+    # "no content_list.json" bug. run_mineru must absolutize the paths it hands MinerU.
+    proj = tmp_path / "proj"; proj.mkdir()
+    clean = tmp_path / "clean"; clean.mkdir()          # a DIFFERENT cwd, as clean_cwd would be on the box
+    monkeypatch.chdir(proj)                            # pdf2wiki's cwd
+    (proj / "in.pdf").write_bytes(b"%PDF-1.4")
+    captured = {}
+
+    class FakeProc:
+        pid = 4321
+        def __init__(self, cmd, cwd):
+            captured["cmd"] = cmd
+            odir = cmd[cmd.index("-o") + 1]
+            # simulate MinerU resolving its -o against ITS cwd (clean_cwd) when relative
+            base = odir if os.path.isabs(odir) else os.path.join(cwd, odir)
+            dst = os.path.join(base, "in", "txt"); os.makedirs(dst, exist_ok=True)
+            with open(os.path.join(dst, "in_content_list.json"), "w") as f:
+                json.dump([{"type": "text", "page_idx": 0, "bbox": [0, 0, 1, 1], "text": "x"}], f)
+        def wait(self, timeout=None): return 0
+
+    monkeypatch.setattr(merge_mod.subprocess, "Popen",
+                        lambda cmd, **k: FakeProc(cmd, k.get("cwd")))
+    blocks, _ = run_mineru("mineru", "in.pdf", 0, 1, "pipeline", ["-m", "txt"],
+                           "reldir", str(clean), {}, label="pipeline 0-1", timeout=60)
+    oi, pi = captured["cmd"].index("-o"), captured["cmd"].index("-p")
+    assert os.path.isabs(captured["cmd"][oi + 1]), "MinerU -o must be absolute"
+    assert os.path.isabs(captured["cmd"][pi + 1]), "MinerU -p must be absolute"
+    assert blocks and blocks[0]["text"] == "x"          # found the content_list MinerU 'wrote'

pdf2wiki-0.2.4/tests/test_qa.py

@@ -0,0 +1,94 @@
+"""Tests for qa.sample page-window math on short books (no GPU/MinerU needed).
+
+Regression: `random.sample(range(lo, hi), n)` raised `ValueError: Sample larger than
+population` on any book whose 5..95% window was smaller than the requested sample size.
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+import pymupdf  # runtime dependency
+
+from pdf2wiki.qa.sample import sample_pages
+
+
+def _make_pdf(path, pages):
+    d = pymupdf.open()
+    for _ in range(pages):
+        d.new_page(width=200, height=200)
+    d.save(path)
+    d.close()
+
+
+def test_sample_short_book_clamps_instead_of_crashing(tmp_path):
+    pdf = str(tmp_path / "short.pdf")
+    _make_pdf(pdf, 5)                                   # window 0..4 < default n=20
+    r = sample_pages(pdf, "short", str(tmp_path / "qa"), n=20)
+    assert 0 < len(r["pages"]) <= 4                     # clamped to the window, no ValueError
+    assert len(set(r["pages"])) == len(r["pages"])      # still distinct
+
+
+def test_sample_single_page_book(tmp_path):
+    pdf = str(tmp_path / "one.pdf")
+    _make_pdf(pdf, 1)                                   # 5..95% window empty -> falls back to whole book
+    r = sample_pages(pdf, "one", str(tmp_path / "qa"), n=20)
+    assert r["pages"] == [0]
+
+
+def test_sample_normal_book_unchanged(tmp_path):
+    pdf = str(tmp_path / "big.pdf")
+    _make_pdf(pdf, 100)                                 # window 5..95 holds 90 pages
+    r = sample_pages(pdf, "big", str(tmp_path / "qa"), n=20)
+    assert len(r["pages"]) == 20
+    assert all(5 <= p < 95 for p in r["pages"])
+
+
+# ---------- qa.flagged: per-book flagged-block report (T3) ----------
+import json
+
+import pdf2wiki.cli as cli
+from pdf2wiki.qa.flagged import flagged_report
+
+
+def _write_blocks(dirpath, specs):
+    """specs: list of (type, flag) where flag in {None, '_code_flag', '_indent_flag'}."""
+    os.makedirs(dirpath, exist_ok=True)
+    blocks = []
+    for i, (t, flag) in enumerate(specs):
+        b = {"type": t, "abs_page": i, "sub_type": "python", "code_body": f"line_{i} = {i}\nmore"}
+        if flag:
+            b[flag] = True
+        blocks.append(b)
+    p = os.path.join(dirpath, "blocks.json")
+    with open(p, "w", encoding="utf-8") as f:
+        json.dump(blocks, f)
+    return p
+
+
+def test_flagged_report_counts(tmp_path):
+    p = _write_blocks(str(tmp_path / "mybook"), [
+        ("code", None), ("code", "_code_flag"), ("code", "_indent_flag"),
+        ("code", "_code_flag"), ("text", None)])
+    r = flagged_report(p)
+    assert r["name"] == "mybook"
+    assert r["code_blocks"] == 4                       # 4 code blocks, the text block ignored
+    assert r["diverged"] == 2 and r["indent_suspect"] == 1 and r["flagged"] == 3
+    assert [e["flag"] for e in r["blocks"]] == ["diverged", "indent", "diverged"]  # page-sorted
+    assert r["blocks"][0]["page"] == 2 and r["blocks"][0]["lang"] == "python"       # abs_page 1 -> pg 2
+    assert r["blocks"][0]["snippet"].startswith("line_1")
+
+
+def test_flagged_report_empty(tmp_path):
+    p = _write_blocks(str(tmp_path / "clean"), [("code", None), ("code", None), ("text", None)])
+    r = flagged_report(p)
+    assert r["flagged"] == 0 and r["blocks"] == [] and r["code_blocks"] == 2
+
+
+def test_cmd_qa_flags_ranks_books(tmp_path, capsys):
+    a = _write_blocks(str(tmp_path / "alpha"), [("code", "_code_flag"), ("code", None)])          # 1 flag
+    b = _write_blocks(str(tmp_path / "beta"),
+                      [("code", "_code_flag"), ("code", "_code_flag"), ("code", "_indent_flag")])  # 3 flags
+    assert cli.main(["qa", "flags", a, b]) == 0
+    out = capsys.readouterr().out
+    assert out.index("beta") < out.index("alpha")      # most-flagged book ranked first

pdf2wiki-0.2.2/tests/test_qa.py

@@ -1,44 +0,0 @@
-"""Tests for qa.sample page-window math on short books (no GPU/MinerU needed).
-
-Regression: `random.sample(range(lo, hi), n)` raised `ValueError: Sample larger than
-population` on any book whose 5..95% window was smaller than the requested sample size.
-"""
-import os
-import sys
-
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
-
-import pymupdf  # runtime dependency
-
-from pdf2wiki.qa.sample import sample_pages
-
-
-def _make_pdf(path, pages):
-    d = pymupdf.open()
-    for _ in range(pages):
-        d.new_page(width=200, height=200)
-    d.save(path)
-    d.close()
-
-
-def test_sample_short_book_clamps_instead_of_crashing(tmp_path):
-    pdf = str(tmp_path / "short.pdf")
-    _make_pdf(pdf, 5)                                   # window 0..4 < default n=20
-    r = sample_pages(pdf, "short", str(tmp_path / "qa"), n=20)
-    assert 0 < len(r["pages"]) <= 4                     # clamped to the window, no ValueError
-    assert len(set(r["pages"])) == len(r["pages"])      # still distinct
-
-
-def test_sample_single_page_book(tmp_path):
-    pdf = str(tmp_path / "one.pdf")
-    _make_pdf(pdf, 1)                                   # 5..95% window empty -> falls back to whole book
-    r = sample_pages(pdf, "one", str(tmp_path / "qa"), n=20)
-    assert r["pages"] == [0]
-
-
-def test_sample_normal_book_unchanged(tmp_path):
-    pdf = str(tmp_path / "big.pdf")
-    _make_pdf(pdf, 100)                                 # window 5..95 holds 90 pages
-    r = sample_pages(pdf, "big", str(tmp_path / "qa"), n=20)
-    assert len(r["pages"]) == 20
-    assert all(5 <= p < 95 for p in r["pages"])