regen.mde 0.2.2 → 0.8.0

package/tools/compare_pptx_inputs_outputs.py

@@ -1,87 +1,87 @@
-from __future__ import annotations
-
-import argparse
-import json
-import re
-from pathlib import Path
-from zipfile import ZipFile
-
-
-def count_input_features(path: Path) -> dict:
-    with ZipFile(path) as zf:
-        names = zf.namelist()
-        slide_count = len([n for n in names if n.startswith("ppt/slides/slide") and n.endswith(".xml")])
-        media_count = len([n for n in names if n.startswith("ppt/media/")])
-        table_hint = 0
-        slide_referenced_images = 0
-        for n in names:
-            if not (n.startswith("ppt/slides/slide") and n.endswith(".xml")):
-                continue
-            xml = zf.read(n)
-            table_hint += len(re.findall(rb"<a:tbl\b", xml))
-            rels_path = f"{Path(n).parent.as_posix()}/_rels/{Path(n).name}.rels"
-            if rels_path in names:
-                rels = zf.read(rels_path)
-                slide_referenced_images += len(re.findall(rb"/media/[^\"']+", rels))
-    return {
-        "slides_in": slide_count,
-        "images_in_package": media_count,
-        "images_in_slides": slide_referenced_images,
-        "tables_in": table_hint,
-    }
-
-
-def count_output_features(md_path: Path) -> dict:
-    text = md_path.read_text(encoding="utf-8") if md_path.exists() else ""
-    slides_out = len(re.findall(r"^## Slide \d+:", text, flags=re.MULTILINE))
-    images_out = len(re.findall(r"!\[[^\]]*\]\([^)]+\)|<img\s+[^>]*src=", text, flags=re.IGNORECASE))
-    tables_out = len(re.findall(r"^\|\s.*\|\s*$", text, flags=re.MULTILINE))
-    return {
-        "slides_out": slides_out,
-        "images_out": images_out,
-        "table_lines_out": tables_out,
-    }
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--manifest", type=Path, required=True)
-    parser.add_argument("--out", type=Path, required=True, help="conversion output root")
-    parser.add_argument("--report", type=Path, required=True)
-    args = parser.parse_args()
-
-    payload = json.loads(args.manifest.read_text(encoding="utf-8"))
-    items = payload.get("items", [])
-    rows = []
-    for item in items:
-        source = Path(item["file"])
-        in_stats = count_input_features(source)
-        md = args.out / source.stem / f"{source.stem}.md"
-        out_stats = count_output_features(md)
-        rows.append({
-            "file": source.name,
-            **in_stats,
-            **out_stats,
-            "slide_delta": out_stats["slides_out"] - in_stats["slides_in"],
-            "image_delta": out_stats["images_out"] - in_stats["images_in_slides"],
-        })
-
-    summary = {
-        "count": len(rows),
-        "slides_in_total": sum(r["slides_in"] for r in rows),
-        "slides_out_total": sum(r["slides_out"] for r in rows),
-        "images_in_package_total": sum(r["images_in_package"] for r in rows),
-        "images_in_slides_total": sum(r["images_in_slides"] for r in rows),
-        "images_out_total": sum(r["images_out"] for r in rows),
-        "files_with_slide_mismatch": len([r for r in rows if r["slide_delta"] != 0]),
-        "files_with_image_gap": len([r for r in rows if r["images_out"] < r["images_in_slides"]]),
-    }
-    report = {"summary": summary, "rows": rows}
-    args.report.parent.mkdir(parents=True, exist_ok=True)
-    args.report.write_text(json.dumps(report, indent=2), encoding="utf-8")
-    print(json.dumps(summary, indent=2))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from zipfile import ZipFile
+
+
+def count_input_features(path: Path) -> dict:
+    with ZipFile(path) as zf:
+        names = zf.namelist()
+        slide_count = len([n for n in names if n.startswith("ppt/slides/slide") and n.endswith(".xml")])
+        media_count = len([n for n in names if n.startswith("ppt/media/")])
+        table_hint = 0
+        slide_referenced_images = 0
+        for n in names:
+            if not (n.startswith("ppt/slides/slide") and n.endswith(".xml")):
+                continue
+            xml = zf.read(n)
+            table_hint += len(re.findall(rb"<a:tbl\b", xml))
+            rels_path = f"{Path(n).parent.as_posix()}/_rels/{Path(n).name}.rels"
+            if rels_path in names:
+                rels = zf.read(rels_path)
+                slide_referenced_images += len(re.findall(rb"/media/[^\"']+", rels))
+    return {
+        "slides_in": slide_count,
+        "images_in_package": media_count,
+        "images_in_slides": slide_referenced_images,
+        "tables_in": table_hint,
+    }
+
+
+def count_output_features(md_path: Path) -> dict:
+    text = md_path.read_text(encoding="utf-8") if md_path.exists() else ""
+    slides_out = len(re.findall(r"^## Slide \d+:", text, flags=re.MULTILINE))
+    images_out = len(re.findall(r"!\[[^\]]*\]\([^)]+\)|<img\s+[^>]*src=", text, flags=re.IGNORECASE))
+    tables_out = len(re.findall(r"^\|\s.*\|\s*$", text, flags=re.MULTILINE))
+    return {
+        "slides_out": slides_out,
+        "images_out": images_out,
+        "table_lines_out": tables_out,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True, help="conversion output root")
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+
+    payload = json.loads(args.manifest.read_text(encoding="utf-8"))
+    items = payload.get("items", [])
+    rows = []
+    for item in items:
+        source = Path(item["file"])
+        in_stats = count_input_features(source)
+        md = args.out / source.stem / f"{source.stem}.md"
+        out_stats = count_output_features(md)
+        rows.append({
+            "file": source.name,
+            **in_stats,
+            **out_stats,
+            "slide_delta": out_stats["slides_out"] - in_stats["slides_in"],
+            "image_delta": out_stats["images_out"] - in_stats["images_in_slides"],
+        })
+
+    summary = {
+        "count": len(rows),
+        "slides_in_total": sum(r["slides_in"] for r in rows),
+        "slides_out_total": sum(r["slides_out"] for r in rows),
+        "images_in_package_total": sum(r["images_in_package"] for r in rows),
+        "images_in_slides_total": sum(r["images_in_slides"] for r in rows),
+        "images_out_total": sum(r["images_out"] for r in rows),
+        "files_with_slide_mismatch": len([r for r in rows if r["slide_delta"] != 0]),
+        "files_with_image_gap": len([r for r in rows if r["images_out"] < r["images_in_slides"]]),
+    }
+    report = {"summary": summary, "rows": rows}
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    args.report.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/roundtrip_docx_corpus.py

@@ -1,171 +1,171 @@
-from __future__ import annotations
-
-import argparse
-import difflib
-import json
-import re
-import shutil
-import sys
-from collections import Counter
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parents[1]
-SRC = ROOT / "src"
-if str(SRC) not in sys.path:
-    sys.path.insert(0, str(SRC))
-
-from build_corpus.docx_exporter import export_markdown_to_docx
-from build_corpus.exporter import BuildCorpusExporter
-
-
-def markdown_stats(text: str) -> dict[str, int]:
-    return {
-        "headings": len(re.findall(r"^#+\s", text, re.M)),
-        "tables": len(re.findall(r"^\| .* \|$", text, re.M)),
-        "images": len(re.findall(r"!\[[^\]]*\]\([^)]+\)", text)),
-        "links": len(re.findall(r"(?<!!)\[[^\]]+\]\([^)]+\)", text)),
-        "math_inline": len(re.findall(r"\$[^$\n]+\$", text)),
-        "math_block": len(re.findall(r"^\$\$$", text, re.M)),
-        "code_fences": len(re.findall(r"^```", text, re.M)) // 2,
-        "lists": len(re.findall(r"^(?:\s*)(?:[-*+]|\d+\.)\s+", text, re.M)),
-        "chars": len(text),
-        "lines": len(text.splitlines()),
-    }
-
-
-def compare_markdown(source: str, roundtrip: str) -> dict:
-    source_lines = source.splitlines()
-    roundtrip_lines = roundtrip.splitlines()
-    diff = list(difflib.unified_diff(source_lines, roundtrip_lines, fromfile="pass1.md", tofile="pass2.md", lineterm=""))
-    return {
-        "diff_lines": sum(
-            1
-            for line in diff
-            if line.startswith(("+", "-")) and not line.startswith(("+++", "---"))
-        ),
-        "diff_preview": diff[:40],
-        "source_stats": markdown_stats(source),
-        "roundtrip_stats": markdown_stats(roundtrip),
-    }
-
-
-def collect_inputs(path: Path) -> list[Path]:
-    if path.is_file():
-        return [path]
-    return sorted(doc for doc in path.rglob("*.docx") if not doc.name.startswith("~$"))
-
-
-def run_one(docx_path: Path, out_root: Path) -> dict:
-    slug = docx_path.stem
-    file_root = out_root / slug
-    pass1_root = file_root / "pass1"
-    pass2_root = file_root / "pass2"
-    if file_root.exists():
-        shutil.rmtree(file_root)
-    pass1_root.mkdir(parents=True, exist_ok=True)
-    pass2_root.mkdir(parents=True, exist_ok=True)
-
-    first = BuildCorpusExporter(docx_path, pass1_root).export()
-    md1 = Path(first["output"])
-    second = export_markdown_to_docx(md1, pass1_root, out_same_dir=False)
-    regenerated_docx = Path(second["output"])
-    third = BuildCorpusExporter(regenerated_docx, pass2_root).export()
-    md2 = Path(third["output"])
-
-    source_text = md1.read_text(encoding="utf-8")
-    roundtrip_text = md2.read_text(encoding="utf-8")
-    compare = compare_markdown(source_text, roundtrip_text)
-    compare["input"] = str(docx_path)
-    compare["pass1_markdown"] = str(md1)
-    compare["pass2_markdown"] = str(md2)
-    compare["roundtrip_docx"] = str(regenerated_docx)
-    compare["pass1_report"] = first
-    compare["pass2_report"] = third
-    return compare
-
-
-def summarize(results: list[dict]) -> dict:
-    counter: Counter[str] = Counter()
-    total_diff = 0
-    for result in results:
-        total_diff += result.get("diff_lines", 0)
-        source_stats = result["source_stats"]
-        roundtrip_stats = result["roundtrip_stats"]
-        for key in ("headings", "tables", "images", "links", "math_inline", "math_block", "code_fences", "lists"):
-            if source_stats.get(key) != roundtrip_stats.get(key):
-                counter[f"{key}_mismatch"] += 1
-        if result.get("diff_lines", 0):
-            counter["files_with_diff"] += 1
-        if result.get("pass1_report", {}).get("stats", {}).get("warnings"):
-            counter["pass1_warnings"] += 1
-        if result.get("pass2_report", {}).get("stats", {}).get("warnings"):
-            counter["pass2_warnings"] += 1
-    return {
-        "files": len(results),
-        "files_with_diff": counter.get("files_with_diff", 0),
-        "avg_diff_lines": (total_diff / len(results)) if results else 0,
-        "max_diff_lines": max((result.get("diff_lines", 0) for result in results), default=0),
-        "mismatch_counts": dict(counter),
-        "worst_examples": sorted(
-            [
-                {
-                    "input": result["input"],
-                    "diff_lines": result["diff_lines"],
-                    "source_stats": result["source_stats"],
-                    "roundtrip_stats": result["roundtrip_stats"],
-                    "diff_preview": result["diff_preview"][:12],
-                }
-                for result in results
-            ],
-            key=lambda item: item["diff_lines"],
-            reverse=True,
-        )[:25],
-    }
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Run DOCX -> Markdown -> DOCX -> Markdown round-trip audits across a corpus.")
-    parser.add_argument("--source", type=Path, required=True, help="Single DOCX file or directory of DOCX files")
-    parser.add_argument("--out", type=Path, required=True, help="Output directory for round-trip artifacts and report")
-    parser.add_argument("--limit", type=int, default=0, help="Optional max number of files to process")
-    args = parser.parse_args()
-
-    inputs = collect_inputs(args.source)
-    if args.limit > 0:
-        inputs = inputs[: args.limit]
-
-    args.out.mkdir(parents=True, exist_ok=True)
-    results: list[dict] = []
-    for index, docx_path in enumerate(inputs, 1):
-        try:
-            result = run_one(docx_path, args.out)
-            results.append(result)
-            print(f"[{index}/{len(inputs)}] {docx_path.name} diff={result['diff_lines']}", flush=True)
-        except Exception as exc:
-            failure = {
-                "input": str(docx_path),
-                "error": repr(exc),
-                "diff_lines": -1,
-                "source_stats": {},
-                "roundtrip_stats": {},
-                "diff_preview": [],
-            }
-            results.append(failure)
-            print(f"[{index}/{len(inputs)}] {docx_path.name} FAILED {exc!r}", flush=True)
-
-    summary = summarize([result for result in results if result.get("diff_lines", 0) >= 0])
-    report = {
-        "source": str(args.source),
-        "out": str(args.out),
-        "summary": summary,
-        "results": results,
-    }
-    report_path = args.out / "roundtrip-report.json"
-    report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
-    print(json.dumps(summary, indent=2))
-    print(f"WROTE {report_path}")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
+from __future__ import annotations
+
+import argparse
+import difflib
+import json
+import re
+import shutil
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+from build_corpus.docx_exporter import export_markdown_to_docx
+from build_corpus.exporter import BuildCorpusExporter
+
+
+def markdown_stats(text: str) -> dict[str, int]:
+    return {
+        "headings": len(re.findall(r"^#+\s", text, re.M)),
+        "tables": len(re.findall(r"^\| .* \|$", text, re.M)),
+        "images": len(re.findall(r"!\[[^\]]*\]\([^)]+\)", text)),
+        "links": len(re.findall(r"(?<!!)\[[^\]]+\]\([^)]+\)", text)),
+        "math_inline": len(re.findall(r"\$[^$\n]+\$", text)),
+        "math_block": len(re.findall(r"^\$\$$", text, re.M)),
+        "code_fences": len(re.findall(r"^```", text, re.M)) // 2,
+        "lists": len(re.findall(r"^(?:\s*)(?:[-*+]|\d+\.)\s+", text, re.M)),
+        "chars": len(text),
+        "lines": len(text.splitlines()),
+    }
+
+
+def compare_markdown(source: str, roundtrip: str) -> dict:
+    source_lines = source.splitlines()
+    roundtrip_lines = roundtrip.splitlines()
+    diff = list(difflib.unified_diff(source_lines, roundtrip_lines, fromfile="pass1.md", tofile="pass2.md", lineterm=""))
+    return {
+        "diff_lines": sum(
+            1
+            for line in diff
+            if line.startswith(("+", "-")) and not line.startswith(("+++", "---"))
+        ),
+        "diff_preview": diff[:40],
+        "source_stats": markdown_stats(source),
+        "roundtrip_stats": markdown_stats(roundtrip),
+    }
+
+
+def collect_inputs(path: Path) -> list[Path]:
+    if path.is_file():
+        return [path]
+    return sorted(doc for doc in path.rglob("*.docx") if not doc.name.startswith("~$"))
+
+
+def run_one(docx_path: Path, out_root: Path) -> dict:
+    slug = docx_path.stem
+    file_root = out_root / slug
+    pass1_root = file_root / "pass1"
+    pass2_root = file_root / "pass2"
+    if file_root.exists():
+        shutil.rmtree(file_root)
+    pass1_root.mkdir(parents=True, exist_ok=True)
+    pass2_root.mkdir(parents=True, exist_ok=True)
+
+    first = BuildCorpusExporter(docx_path, pass1_root).export()
+    md1 = Path(first["output"])
+    second = export_markdown_to_docx(md1, pass1_root, out_same_dir=False)
+    regenerated_docx = Path(second["output"])
+    third = BuildCorpusExporter(regenerated_docx, pass2_root).export()
+    md2 = Path(third["output"])
+
+    source_text = md1.read_text(encoding="utf-8")
+    roundtrip_text = md2.read_text(encoding="utf-8")
+    compare = compare_markdown(source_text, roundtrip_text)
+    compare["input"] = str(docx_path)
+    compare["pass1_markdown"] = str(md1)
+    compare["pass2_markdown"] = str(md2)
+    compare["roundtrip_docx"] = str(regenerated_docx)
+    compare["pass1_report"] = first
+    compare["pass2_report"] = third
+    return compare
+
+
+def summarize(results: list[dict]) -> dict:
+    counter: Counter[str] = Counter()
+    total_diff = 0
+    for result in results:
+        total_diff += result.get("diff_lines", 0)
+        source_stats = result["source_stats"]
+        roundtrip_stats = result["roundtrip_stats"]
+        for key in ("headings", "tables", "images", "links", "math_inline", "math_block", "code_fences", "lists"):
+            if source_stats.get(key) != roundtrip_stats.get(key):
+                counter[f"{key}_mismatch"] += 1
+        if result.get("diff_lines", 0):
+            counter["files_with_diff"] += 1
+        if result.get("pass1_report", {}).get("stats", {}).get("warnings"):
+            counter["pass1_warnings"] += 1
+        if result.get("pass2_report", {}).get("stats", {}).get("warnings"):
+            counter["pass2_warnings"] += 1
+    return {
+        "files": len(results),
+        "files_with_diff": counter.get("files_with_diff", 0),
+        "avg_diff_lines": (total_diff / len(results)) if results else 0,
+        "max_diff_lines": max((result.get("diff_lines", 0) for result in results), default=0),
+        "mismatch_counts": dict(counter),
+        "worst_examples": sorted(
+            [
+                {
+                    "input": result["input"],
+                    "diff_lines": result["diff_lines"],
+                    "source_stats": result["source_stats"],
+                    "roundtrip_stats": result["roundtrip_stats"],
+                    "diff_preview": result["diff_preview"][:12],
+                }
+                for result in results
+            ],
+            key=lambda item: item["diff_lines"],
+            reverse=True,
+        )[:25],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run DOCX -> Markdown -> DOCX -> Markdown round-trip audits across a corpus.")
+    parser.add_argument("--source", type=Path, required=True, help="Single DOCX file or directory of DOCX files")
+    parser.add_argument("--out", type=Path, required=True, help="Output directory for round-trip artifacts and report")
+    parser.add_argument("--limit", type=int, default=0, help="Optional max number of files to process")
+    args = parser.parse_args()
+
+    inputs = collect_inputs(args.source)
+    if args.limit > 0:
+        inputs = inputs[: args.limit]
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    results: list[dict] = []
+    for index, docx_path in enumerate(inputs, 1):
+        try:
+            result = run_one(docx_path, args.out)
+            results.append(result)
+            print(f"[{index}/{len(inputs)}] {docx_path.name} diff={result['diff_lines']}", flush=True)
+        except Exception as exc:
+            failure = {
+                "input": str(docx_path),
+                "error": repr(exc),
+                "diff_lines": -1,
+                "source_stats": {},
+                "roundtrip_stats": {},
+                "diff_preview": [],
+            }
+            results.append(failure)
+            print(f"[{index}/{len(inputs)}] {docx_path.name} FAILED {exc!r}", flush=True)
+
+    summary = summarize([result for result in results if result.get("diff_lines", 0) >= 0])
+    report = {
+        "source": str(args.source),
+        "out": str(args.out),
+        "summary": summary,
+        "results": results,
+    }
+    report_path = args.out / "roundtrip-report.json"
+    report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+    print(f"WROTE {report_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())