pdf2wiki 0.2.2__tar.gz → 0.2.3__tar.gz

{pdf2wiki-0.2.2 → pdf2wiki-0.2.3}/CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project are documented here. The format is based on
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.3] - 2026-07-23
+
+QA + diagnostics from an external review of the repo. No breaking changes; +4 tests (103 → 107).
+
+### Added
+- **`pdf2wiki qa flags PATHS...`** — per-book report of the code blocks where the VLM diverged from the
+  byte-clean text layer (`_code_flag`), or where hybrid indentation failed a Python ast check
+  (`_indent_flag`), read from `blocks.json`. Ranks multiple books by flagged count (which books to
+  trust least) and lists each flagged block (page / language / snippet) for a single book — the
+  highest-signal spots to spot-check.
+
+### Changed
+- **Batch summary rolls up `error_class`** — a partial batch now prints, e.g.,
+  `5 book(s) not done — by class: permanent×3, timeout×1, fetch×1` (plus the slug list), so a cluster
+  of same-kind failures reads as one diagnosis instead of N separate slugs. Exit code unchanged.
+
 ## [0.2.2] - 2026-07-23
 
 Resilience + security hardening from a book-grounded review (Tech-Books vault: Backoff-Retries,

{pdf2wiki-0.2.2 → pdf2wiki-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pdf2wiki
-Version: 0.2.2
+Version: 0.2.3
 Summary: Convert technical books (native-text PDFs) into clean, chapter-split, LLM-ready Markdown using a dual-pass MinerU pipeline plus fidelity-preserving post-processing.
 Project-URL: Homepage, https://github.com/Sevthered/pdf2wiki
 Project-URL: Repository, https://github.com/Sevthered/pdf2wiki

{pdf2wiki-0.2.2 → pdf2wiki-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pdf2wiki"
-version = "0.2.2"
+version = "0.2.3"
 description = "Convert technical books (native-text PDFs) into clean, chapter-split, LLM-ready Markdown using a dual-pass MinerU pipeline plus fidelity-preserving post-processing."
 authors = [{ name = "Sevthered" }]
 readme = "README.md"

{pdf2wiki-0.2.2 → pdf2wiki-0.2.3}/src/pdf2wiki/__init__.py

@@ -11,4 +11,4 @@ Pipeline stages:
   4. batch    — manifest-driven multi-book runs, resumable, with optional SSH remote execution.
 """
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"

{pdf2wiki-0.2.2 → pdf2wiki-0.2.3}/src/pdf2wiki/cli.py

@@ -99,6 +99,19 @@ def _cmd_qa_review(a, cfg):
     return 0
 
 
+def _cmd_qa_flagged(a, cfg):
+    from .qa.flagged import flagged_report
+    reports = sorted((flagged_report(p) for p in a.blocks), key=lambda r: r["flagged"], reverse=True)
+    print(f"{'flagged':>7} {'diverged':>8} {'indent':>6} {'code':>5}  book")
+    for r in reports:
+        print(f"{r['flagged']:>7} {r['diverged']:>8} {r['indent_suspect']:>6} {r['code_blocks']:>5}  {r['name']}")
+    if len(reports) == 1:                    # single book -> also list the blocks to eyeball
+        print()
+        for e in reports[0]["blocks"]:
+            print(f"  p{e['page']:<4} [{e['lang'] or '?'}] {e['flag']}: {e['snippet']}")
+    return 0
+
+
 def _cmd_scan(a, cfg):
     from .scan import scan_dir
     print(json.dumps(scan_dir(a.directory), indent=1))
@@ -109,9 +122,13 @@ def _cmd_batch(a, cfg):
     from .batch import run_batch
     manifest = run_batch(a.books, cfg, a.stage, remote=a.remote or (cfg.remote.host or None),
                          max_books=a.max_books, only=a.only, vault=a.vault or (cfg.output.vault or None))
-    failed = [s for s, e in manifest.items() if e.get("status") != "done"]
+    failed = [(s, e) for s, e in manifest.items() if e.get("status") != "done"]
     if failed:                       # non-zero exit so CI/automation can detect a partial batch
-        print(f"batch: {len(failed)} book(s) not done: {failed}", file=sys.stderr)
+        from collections import Counter
+        by_class = Counter(e.get("error_class", "unknown") for _, e in failed)
+        rollup = ", ".join(f"{k}×{v}" for k, v in by_class.most_common())
+        print(f"batch: {len(failed)} book(s) not done — by class: {rollup}", file=sys.stderr)
+        print(f"  not done: {[s for s, _ in failed]}", file=sys.stderr)
         return 1
     return 0
 
@@ -166,6 +183,10 @@ def main(argv=None):
     p.add_argument("name")
     p.add_argument("--blocks", default=None, help="explicit blocks.json path")
     p.set_defaults(fn=_cmd_qa_review)
+    p = qsub.add_parser("flags", help="report code blocks the VLM diverged on (from blocks.json), "
+                                      "ranked across books — the highest-signal QA sample")
+    p.add_argument("blocks", nargs="+", help="one or more converted-book blocks.json paths")
+    p.set_defaults(fn=_cmd_qa_flagged)
 
     p = sub.add_parser("scan", help="scan a directory of PDFs -> title/year guesses (JSON)")
     p.add_argument("directory")

pdf2wiki-0.2.3/src/pdf2wiki/qa/flagged.py

@@ -0,0 +1,42 @@
+"""Per-book flagged-block report: which code blocks the VLM diverged on — the highest-signal QA
+sample. Pure read over a converted book's blocks.json (no converter or schema change). The flags are
+set during merge in convert/merge.py: `_code_flag` = the hybrid/VLM code diverged from the byte-clean
+text layer (output shows the pipeline tokens); `_indent_flag` = tokens agreed but the hybrid
+indentation failed a Python ast sanity check. These are exactly the places worth eyeballing."""
+import json
+import os
+
+
+def flagged_report(blocks_path: str, name: str | None = None) -> dict:
+    """Summarize the flagged code blocks in one book's blocks.json.
+
+    Returns {name, code_blocks, flagged, diverged, indent_suspect, blocks} where `blocks` is the
+    page-sorted list of flagged entries {page, lang, flag, snippet}; `page` is 1-based for display.
+    """
+    path = os.path.expanduser(blocks_path)
+    with open(path, encoding="utf-8") as f:
+        blocks = json.load(f)
+    name = name or os.path.basename(os.path.dirname(os.path.abspath(path)))
+
+    code_blocks = 0
+    flagged = []
+    for b in blocks:
+        if b.get("type") != "code":
+            continue
+        code_blocks += 1
+        if b.get("_code_flag"):
+            flag = "diverged"
+        elif b.get("_indent_flag"):
+            flag = "indent"
+        else:
+            continue
+        body = str(b.get("code_body") or "")
+        snippet = next((ln.strip() for ln in body.splitlines() if ln.strip()), "")[:80]
+        flagged.append({"page": int(b.get("abs_page", 0)) + 1, "lang": b.get("sub_type") or "",
+                        "flag": flag, "snippet": snippet})
+
+    flagged.sort(key=lambda e: e["page"])
+    return {"name": name, "code_blocks": code_blocks, "flagged": len(flagged),
+            "diverged": sum(1 for e in flagged if e["flag"] == "diverged"),
+            "indent_suspect": sum(1 for e in flagged if e["flag"] == "indent"),
+            "blocks": flagged}

{pdf2wiki-0.2.2 → pdf2wiki-0.2.3}/tests/test_batch.py

@@ -125,3 +125,17 @@ def test_no_breaker_when_executor_stays_healthy(tmp_path, monkeypatch):
     failed = [s for s, e in manifest.items() if e["status"] == "convert_failed"]
     assert len(failed) == 5                            # all attempted, no premature abort
     assert manifest["book-0"]["error_class"] == "permanent"
+
+
+def test_cmd_batch_rolls_up_error_class(monkeypatch, capsys):
+    # a partial batch must aggregate error_class so a cluster of same-kind failures reads as one line.
+    monkeypatch.setattr(batch, "run_batch",
+                        lambda *a, **k: {"book-a": {"status": "done"},
+                                         "book-b": {"status": "convert_failed", "error_class": "permanent"},
+                                         "book-c": {"status": "convert_failed", "error_class": "permanent"},
+                                         "book-d": {"status": "convert_failed", "error_class": "TimeoutExpired"}})
+    assert cli.main(["batch", "books.toml"]) == 1
+    err = capsys.readouterr().err
+    assert "3 book(s) not done" in err
+    assert "permanent×2" in err and "TimeoutExpired×1" in err
+    assert "not done:" in err                        # slug detail retained

pdf2wiki-0.2.3/tests/test_qa.py

@@ -0,0 +1,94 @@
+"""Tests for qa.sample page-window math on short books (no GPU/MinerU needed).
+
+Regression: `random.sample(range(lo, hi), n)` raised `ValueError: Sample larger than
+population` on any book whose 5..95% window was smaller than the requested sample size.
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+import pymupdf  # runtime dependency
+
+from pdf2wiki.qa.sample import sample_pages
+
+
+def _make_pdf(path, pages):
+    d = pymupdf.open()
+    for _ in range(pages):
+        d.new_page(width=200, height=200)
+    d.save(path)
+    d.close()
+
+
+def test_sample_short_book_clamps_instead_of_crashing(tmp_path):
+    pdf = str(tmp_path / "short.pdf")
+    _make_pdf(pdf, 5)                                   # window 0..4 < default n=20
+    r = sample_pages(pdf, "short", str(tmp_path / "qa"), n=20)
+    assert 0 < len(r["pages"]) <= 4                     # clamped to the window, no ValueError
+    assert len(set(r["pages"])) == len(r["pages"])      # still distinct
+
+
+def test_sample_single_page_book(tmp_path):
+    pdf = str(tmp_path / "one.pdf")
+    _make_pdf(pdf, 1)                                   # 5..95% window empty -> falls back to whole book
+    r = sample_pages(pdf, "one", str(tmp_path / "qa"), n=20)
+    assert r["pages"] == [0]
+
+
+def test_sample_normal_book_unchanged(tmp_path):
+    pdf = str(tmp_path / "big.pdf")
+    _make_pdf(pdf, 100)                                 # window 5..95 holds 90 pages
+    r = sample_pages(pdf, "big", str(tmp_path / "qa"), n=20)
+    assert len(r["pages"]) == 20
+    assert all(5 <= p < 95 for p in r["pages"])
+
+
+# ---------- qa.flagged: per-book flagged-block report (T3) ----------
+import json
+
+import pdf2wiki.cli as cli
+from pdf2wiki.qa.flagged import flagged_report
+
+
+def _write_blocks(dirpath, specs):
+    """specs: list of (type, flag) where flag in {None, '_code_flag', '_indent_flag'}."""
+    os.makedirs(dirpath, exist_ok=True)
+    blocks = []
+    for i, (t, flag) in enumerate(specs):
+        b = {"type": t, "abs_page": i, "sub_type": "python", "code_body": f"line_{i} = {i}\nmore"}
+        if flag:
+            b[flag] = True
+        blocks.append(b)
+    p = os.path.join(dirpath, "blocks.json")
+    with open(p, "w", encoding="utf-8") as f:
+        json.dump(blocks, f)
+    return p
+
+
+def test_flagged_report_counts(tmp_path):
+    p = _write_blocks(str(tmp_path / "mybook"), [
+        ("code", None), ("code", "_code_flag"), ("code", "_indent_flag"),
+        ("code", "_code_flag"), ("text", None)])
+    r = flagged_report(p)
+    assert r["name"] == "mybook"
+    assert r["code_blocks"] == 4                       # 4 code blocks, the text block ignored
+    assert r["diverged"] == 2 and r["indent_suspect"] == 1 and r["flagged"] == 3
+    assert [e["flag"] for e in r["blocks"]] == ["diverged", "indent", "diverged"]  # page-sorted
+    assert r["blocks"][0]["page"] == 2 and r["blocks"][0]["lang"] == "python"       # abs_page 1 -> pg 2
+    assert r["blocks"][0]["snippet"].startswith("line_1")
+
+
+def test_flagged_report_empty(tmp_path):
+    p = _write_blocks(str(tmp_path / "clean"), [("code", None), ("code", None), ("text", None)])
+    r = flagged_report(p)
+    assert r["flagged"] == 0 and r["blocks"] == [] and r["code_blocks"] == 2
+
+
+def test_cmd_qa_flags_ranks_books(tmp_path, capsys):
+    a = _write_blocks(str(tmp_path / "alpha"), [("code", "_code_flag"), ("code", None)])          # 1 flag
+    b = _write_blocks(str(tmp_path / "beta"),
+                      [("code", "_code_flag"), ("code", "_code_flag"), ("code", "_indent_flag")])  # 3 flags
+    assert cli.main(["qa", "flags", a, b]) == 0
+    out = capsys.readouterr().out
+    assert out.index("beta") < out.index("alpha")      # most-flagged book ranked first

pdf2wiki-0.2.2/tests/test_qa.py

@@ -1,44 +0,0 @@
-"""Tests for qa.sample page-window math on short books (no GPU/MinerU needed).
-
-Regression: `random.sample(range(lo, hi), n)` raised `ValueError: Sample larger than
-population` on any book whose 5..95% window was smaller than the requested sample size.
-"""
-import os
-import sys
-
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
-
-import pymupdf  # runtime dependency
-
-from pdf2wiki.qa.sample import sample_pages
-
-
-def _make_pdf(path, pages):
-    d = pymupdf.open()
-    for _ in range(pages):
-        d.new_page(width=200, height=200)
-    d.save(path)
-    d.close()
-
-
-def test_sample_short_book_clamps_instead_of_crashing(tmp_path):
-    pdf = str(tmp_path / "short.pdf")
-    _make_pdf(pdf, 5)                                   # window 0..4 < default n=20
-    r = sample_pages(pdf, "short", str(tmp_path / "qa"), n=20)
-    assert 0 < len(r["pages"]) <= 4                     # clamped to the window, no ValueError
-    assert len(set(r["pages"])) == len(r["pages"])      # still distinct
-
-
-def test_sample_single_page_book(tmp_path):
-    pdf = str(tmp_path / "one.pdf")
-    _make_pdf(pdf, 1)                                   # 5..95% window empty -> falls back to whole book
-    r = sample_pages(pdf, "one", str(tmp_path / "qa"), n=20)
-    assert r["pages"] == [0]
-
-
-def test_sample_normal_book_unchanged(tmp_path):
-    pdf = str(tmp_path / "big.pdf")
-    _make_pdf(pdf, 100)                                 # window 5..95 holds 90 pages
-    r = sample_pages(pdf, "big", str(tmp_path / "qa"), n=20)
-    assert len(r["pages"]) == 20
-    assert all(5 <= p < 95 for p in r["pages"])