regen.mde 0.2.2

package/tools/collect_online_pptx_corpus.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import time
+from pathlib import Path
+from typing import Iterable
+from urllib.parse import parse_qs, quote_plus, unquote, urlparse
+from zipfile import BadZipFile, ZipFile
+
+import requests
+
+
+DEFAULT_QUERIES = [
+    'filetype:pptx "roadmap"',
+    'filetype:pptx "quarterly business review"',
+    'filetype:pptx "deck"',
+    'filetype:pptx "strategy presentation"',
+    'filetype:pptx "project update"',
+    'filetype:pptx "financial presentation"',
+    'filetype:pptx "education presentation"',
+]
+
+GITHUB_REPO_QUERIES = [
+    "pptx presentation template",
+    "pptx deck examples",
+    "powerpoint pptx samples",
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+    )
+}
+
+PPTX_CT_HINTS = (
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/octet-stream",
+    "application/zip",
+    "binary/octet-stream",
+)
+
+
+def extract_urls(html: str) -> list[str]:
+    raw = re.findall(r'https?://[^"\'<>\s)]+', html)
+    urls: list[str] = []
+    for url in raw:
+        url = unquote(url).replace("&amp;", "&")
+        parsed = urlparse(url)
+        if parsed.netloc.endswith("bing.com") and parsed.path == "/ck/a":
+            qs = parse_qs(parsed.query)
+            for key in ("u", "r"):
+                if key in qs:
+                    url = unquote(qs[key][0])
+                    break
+        if "duckduckgo.com/l/?" in url:
+            qs = parse_qs(urlparse(url).query)
+            if "uddg" in qs:
+                url = unquote(qs["uddg"][0])
+        urls.append(url.rstrip(".,;"))
+    return urls
+
+
+def search(query: str, pages: int = 2) -> Iterable[str]:
+    endpoints = [
+        "https://www.bing.com/search?q={query}&first={offset}",
+        "https://html.duckduckgo.com/html/?q={query}&s={offset}",
+    ]
+    for endpoint in endpoints:
+        for page in range(pages):
+            offset = page * 10 + 1
+            url = endpoint.format(query=quote_plus(query), offset=offset)
+            try:
+                response = requests.get(url, headers=HEADERS, timeout=20)
+                if response.status_code >= 400:
+                    continue
+                yield from extract_urls(response.text)
+                time.sleep(0.4)
+            except requests.RequestException:
+                continue
+
+
+def github_json(url: str) -> dict | None:
+    try:
+        response = requests.get(url, headers={**HEADERS, "Accept": "application/vnd.github+json"}, timeout=25)
+        if response.status_code >= 400:
+            return None
+        return response.json()
+    except requests.RequestException:
+        return None
+
+
+def github_pptx_urls(max_repos_per_query: int = 10) -> Iterable[str]:
+    seen_repos: set[str] = set()
+    for query in GITHUB_REPO_QUERIES:
+        search_url = (
+            "https://api.github.com/search/repositories"
+            f"?q={quote_plus(query)}&per_page={max_repos_per_query}"
+        )
+        payload = github_json(search_url)
+        if not payload:
+            continue
+        for repo in payload.get("items", []):
+            full_name = repo.get("full_name")
+            branch = repo.get("default_branch") or "main"
+            if not full_name or full_name in seen_repos:
+                continue
+            seen_repos.add(full_name)
+            tree_url = f"https://api.github.com/repos/{full_name}/git/trees/{branch}?recursive=1"
+            tree = github_json(tree_url)
+            if not tree:
+                continue
+            for item in tree.get("tree", []):
+                path = item.get("path", "")
+                if item.get("type") == "blob" and path.lower().endswith(".pptx"):
+                    yield f"https://raw.githubusercontent.com/{full_name}/{branch}/{quote_path(path)}"
+            time.sleep(0.25)
+
+
+def quote_path(path: str) -> str:
+    return "/".join(quote_plus(part).replace("+", "%20") for part in path.split("/"))
+
+
+def looks_like_pptx_url(url: str) -> bool:
+    low = url.lower()
+    return ".pptx" in low and not any(bad in low for bad in ("?format=pdf", "/view?", "webcache"))
+
+
+def safe_name(index: int, url: str, content: bytes) -> str:
+    parsed = urlparse(url)
+    stem = Path(unquote(parsed.path)).name or f"online-{index:03d}.pptx"
+    stem = re.sub(r"[^A-Za-z0-9._ -]+", "_", stem)
+    if not stem.lower().endswith(".pptx"):
+        stem += ".pptx"
+    digest = hashlib.sha256(content).hexdigest()[:10]
+    return f"{index:03d}-{digest}-{stem}"
+
+
+def validate_pptx(path: Path) -> dict:
+    with ZipFile(path) as zf:
+        names = set(zf.namelist())
+        if "[Content_Types].xml" not in names or "ppt/presentation.xml" not in names:
+            raise BadZipFile("not a PowerPoint PPTX package")
+        slides = len([n for n in names if n.startswith("ppt/slides/slide") and n.endswith(".xml")])
+        media = len([n for n in names if n.startswith("ppt/media/")])
+        return {
+            "slides": slides,
+            "media_parts": media,
+        }
+
+
+def download(url: str, out_dir: Path, index: int, max_mb: int) -> dict | None:
+    try:
+        with requests.get(url, headers=HEADERS, timeout=30, stream=True, allow_redirects=True) as response:
+            if response.status_code >= 400:
+                return None
+            content_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
+            if content_type and content_type not in PPTX_CT_HINTS and "presentationml" not in content_type:
+                if ".pptx" not in response.url.lower():
+                    return None
+            chunks: list[bytes] = []
+            total = 0
+            limit = max_mb * 1024 * 1024
+            for chunk in response.iter_content(chunk_size=1024 * 256):
+                if not chunk:
+                    continue
+                total += len(chunk)
+                if total > limit:
+                    return None
+                chunks.append(chunk)
+        content = b"".join(chunks)
+    except requests.RequestException:
+        return None
+
+    if not content.startswith(b"PK"):
+        return None
+
+    out_path = out_dir / safe_name(index, response.url, content)
+    out_path.write_bytes(content)
+    try:
+        stats = validate_pptx(out_path)
+    except (BadZipFile, KeyError):
+        out_path.unlink(missing_ok=True)
+        return None
+
+    return {
+        "file": str(out_path),
+        "source_url": response.url,
+        "bytes": len(content),
+        "content_type": content_type,
+        **stats,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--target", type=int, default=20)
+    parser.add_argument("--max-mb", type=int, default=50)
+    args = parser.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    seen: set[str] = set()
+    manifest: list[dict] = []
+    index = len(list(args.out.glob("*.pptx"))) + 1
+
+    print("source: github repositories", flush=True)
+    for url in github_pptx_urls():
+        if len(manifest) >= args.target:
+            break
+        normalized = url.split("#", 1)[0]
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        item = download(normalized, args.out, index, args.max_mb)
+        if not item:
+            continue
+        manifest.append(item)
+        index += 1
+        print(f"kept {len(manifest):02d}: {Path(item['file']).name} slides={item['slides']}", flush=True)
+
+    for query in DEFAULT_QUERIES:
+        print(f"search: {query}", flush=True)
+        for url in search(query):
+            if len(manifest) >= args.target:
+                break
+            if not looks_like_pptx_url(url):
+                continue
+            normalized = url.split("#", 1)[0]
+            if normalized in seen:
+                continue
+            seen.add(normalized)
+            item = download(normalized, args.out, index, args.max_mb)
+            if not item:
+                continue
+            manifest.append(item)
+            index += 1
+            print(f"kept {len(manifest):02d}: {Path(item['file']).name} slides={item['slides']}", flush=True)
+        if len(manifest) >= args.target:
+            break
+
+    manifest_path = args.out / "online-pptx-manifest.json"
+    manifest_path.write_text(json.dumps({"count": len(manifest), "items": manifest}, indent=2), encoding="utf-8")
+    print(f"saved manifest: {manifest_path}", flush=True)
+    return 0 if len(manifest) >= args.target else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/compare_pptx_inputs_outputs.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from zipfile import ZipFile
+
+
+def count_input_features(path: Path) -> dict:
+    with ZipFile(path) as zf:
+        names = zf.namelist()
+        slide_count = len([n for n in names if n.startswith("ppt/slides/slide") and n.endswith(".xml")])
+        media_count = len([n for n in names if n.startswith("ppt/media/")])
+        table_hint = 0
+        slide_referenced_images = 0
+        for n in names:
+            if not (n.startswith("ppt/slides/slide") and n.endswith(".xml")):
+                continue
+            xml = zf.read(n)
+            table_hint += len(re.findall(rb"<a:tbl\b", xml))
+            rels_path = f"{Path(n).parent.as_posix()}/_rels/{Path(n).name}.rels"
+            if rels_path in names:
+                rels = zf.read(rels_path)
+                slide_referenced_images += len(re.findall(rb"/media/[^\"']+", rels))
+    return {
+        "slides_in": slide_count,
+        "images_in_package": media_count,
+        "images_in_slides": slide_referenced_images,
+        "tables_in": table_hint,
+    }
+
+
+def count_output_features(md_path: Path) -> dict:
+    text = md_path.read_text(encoding="utf-8") if md_path.exists() else ""
+    slides_out = len(re.findall(r"^## Slide \d+:", text, flags=re.MULTILINE))
+    images_out = len(re.findall(r"!\[[^\]]*\]\([^)]+\)|<img\s+[^>]*src=", text, flags=re.IGNORECASE))
+    tables_out = len(re.findall(r"^\|\s.*\|\s*$", text, flags=re.MULTILINE))
+    return {
+        "slides_out": slides_out,
+        "images_out": images_out,
+        "table_lines_out": tables_out,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True, help="conversion output root")
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+
+    payload = json.loads(args.manifest.read_text(encoding="utf-8"))
+    items = payload.get("items", [])
+    rows = []
+    for item in items:
+        source = Path(item["file"])
+        in_stats = count_input_features(source)
+        md = args.out / source.stem / f"{source.stem}.md"
+        out_stats = count_output_features(md)
+        rows.append({
+            "file": source.name,
+            **in_stats,
+            **out_stats,
+            "slide_delta": out_stats["slides_out"] - in_stats["slides_in"],
+            "image_delta": out_stats["images_out"] - in_stats["images_in_slides"],
+        })
+
+    summary = {
+        "count": len(rows),
+        "slides_in_total": sum(r["slides_in"] for r in rows),
+        "slides_out_total": sum(r["slides_out"] for r in rows),
+        "images_in_package_total": sum(r["images_in_package"] for r in rows),
+        "images_in_slides_total": sum(r["images_in_slides"] for r in rows),
+        "images_out_total": sum(r["images_out"] for r in rows),
+        "files_with_slide_mismatch": len([r for r in rows if r["slide_delta"] != 0]),
+        "files_with_image_gap": len([r for r in rows if r["images_out"] < r["images_in_slides"]]),
+    }
+    report = {"summary": summary, "rows": rows}
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    args.report.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/roundtrip_docx_corpus.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import argparse
+import difflib
+import json
+import re
+import shutil
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+from build_corpus.docx_exporter import export_markdown_to_docx
+from build_corpus.exporter import BuildCorpusExporter
+
+
+def markdown_stats(text: str) -> dict[str, int]:
+    return {
+        "headings": len(re.findall(r"^#+\s", text, re.M)),
+        "tables": len(re.findall(r"^\| .* \|$", text, re.M)),
+        "images": len(re.findall(r"!\[[^\]]*\]\([^)]+\)", text)),
+        "links": len(re.findall(r"(?<!!)\[[^\]]+\]\([^)]+\)", text)),
+        "math_inline": len(re.findall(r"\$[^$\n]+\$", text)),
+        "math_block": len(re.findall(r"^\$\$$", text, re.M)),
+        "code_fences": len(re.findall(r"^```", text, re.M)) // 2,
+        "lists": len(re.findall(r"^(?:\s*)(?:[-*+]|\d+\.)\s+", text, re.M)),
+        "chars": len(text),
+        "lines": len(text.splitlines()),
+    }
+
+
+def compare_markdown(source: str, roundtrip: str) -> dict:
+    source_lines = source.splitlines()
+    roundtrip_lines = roundtrip.splitlines()
+    diff = list(difflib.unified_diff(source_lines, roundtrip_lines, fromfile="pass1.md", tofile="pass2.md", lineterm=""))
+    return {
+        "diff_lines": sum(
+            1
+            for line in diff
+            if line.startswith(("+", "-")) and not line.startswith(("+++", "---"))
+        ),
+        "diff_preview": diff[:40],
+        "source_stats": markdown_stats(source),
+        "roundtrip_stats": markdown_stats(roundtrip),
+    }
+
+
+def collect_inputs(path: Path) -> list[Path]:
+    if path.is_file():
+        return [path]
+    return sorted(doc for doc in path.rglob("*.docx") if not doc.name.startswith("~$"))
+
+
+def run_one(docx_path: Path, out_root: Path) -> dict:
+    slug = docx_path.stem
+    file_root = out_root / slug
+    pass1_root = file_root / "pass1"
+    pass2_root = file_root / "pass2"
+    if file_root.exists():
+        shutil.rmtree(file_root)
+    pass1_root.mkdir(parents=True, exist_ok=True)
+    pass2_root.mkdir(parents=True, exist_ok=True)
+
+    first = BuildCorpusExporter(docx_path, pass1_root).export()
+    md1 = Path(first["output"])
+    second = export_markdown_to_docx(md1, pass1_root, out_same_dir=False)
+    regenerated_docx = Path(second["output"])
+    third = BuildCorpusExporter(regenerated_docx, pass2_root).export()
+    md2 = Path(third["output"])
+
+    source_text = md1.read_text(encoding="utf-8")
+    roundtrip_text = md2.read_text(encoding="utf-8")
+    compare = compare_markdown(source_text, roundtrip_text)
+    compare["input"] = str(docx_path)
+    compare["pass1_markdown"] = str(md1)
+    compare["pass2_markdown"] = str(md2)
+    compare["roundtrip_docx"] = str(regenerated_docx)
+    compare["pass1_report"] = first
+    compare["pass2_report"] = third
+    return compare
+
+
+def summarize(results: list[dict]) -> dict:
+    counter: Counter[str] = Counter()
+    total_diff = 0
+    for result in results:
+        total_diff += result.get("diff_lines", 0)
+        source_stats = result["source_stats"]
+        roundtrip_stats = result["roundtrip_stats"]
+        for key in ("headings", "tables", "images", "links", "math_inline", "math_block", "code_fences", "lists"):
+            if source_stats.get(key) != roundtrip_stats.get(key):
+                counter[f"{key}_mismatch"] += 1
+        if result.get("diff_lines", 0):
+            counter["files_with_diff"] += 1
+        if result.get("pass1_report", {}).get("stats", {}).get("warnings"):
+            counter["pass1_warnings"] += 1
+        if result.get("pass2_report", {}).get("stats", {}).get("warnings"):
+            counter["pass2_warnings"] += 1
+    return {
+        "files": len(results),
+        "files_with_diff": counter.get("files_with_diff", 0),
+        "avg_diff_lines": (total_diff / len(results)) if results else 0,
+        "max_diff_lines": max((result.get("diff_lines", 0) for result in results), default=0),
+        "mismatch_counts": dict(counter),
+        "worst_examples": sorted(
+            [
+                {
+                    "input": result["input"],
+                    "diff_lines": result["diff_lines"],
+                    "source_stats": result["source_stats"],
+                    "roundtrip_stats": result["roundtrip_stats"],
+                    "diff_preview": result["diff_preview"][:12],
+                }
+                for result in results
+            ],
+            key=lambda item: item["diff_lines"],
+            reverse=True,
+        )[:25],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run DOCX -> Markdown -> DOCX -> Markdown round-trip audits across a corpus.")
+    parser.add_argument("--source", type=Path, required=True, help="Single DOCX file or directory of DOCX files")
+    parser.add_argument("--out", type=Path, required=True, help="Output directory for round-trip artifacts and report")
+    parser.add_argument("--limit", type=int, default=0, help="Optional max number of files to process")
+    args = parser.parse_args()
+
+    inputs = collect_inputs(args.source)
+    if args.limit > 0:
+        inputs = inputs[: args.limit]
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    results: list[dict] = []
+    for index, docx_path in enumerate(inputs, 1):
+        try:
+            result = run_one(docx_path, args.out)
+            results.append(result)
+            print(f"[{index}/{len(inputs)}] {docx_path.name} diff={result['diff_lines']}", flush=True)
+        except Exception as exc:
+            failure = {
+                "input": str(docx_path),
+                "error": repr(exc),
+                "diff_lines": -1,
+                "source_stats": {},
+                "roundtrip_stats": {},
+                "diff_preview": [],
+            }
+            results.append(failure)
+            print(f"[{index}/{len(inputs)}] {docx_path.name} FAILED {exc!r}", flush=True)
+
+    summary = summarize([result for result in results if result.get("diff_lines", 0) >= 0])
+    report = {
+        "source": str(args.source),
+        "out": str(args.out),
+        "summary": summary,
+        "results": results,
+    }
+    report_path = args.out / "roundtrip-report.json"
+    report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+    print(f"WROTE {report_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())