regen.mde 0.2.2

package/tools/audit_corpus.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from zipfile import ZipFile
+from xml.etree import ElementTree as ET
+
+
+W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+W = f"{{{W_NS}}}"
+
+TEXT_TAGS = {
+    f"{W}t",
+    f"{W}delText",
+    f"{W}instrText",
+}
+
+CONTENT_TAGS = TEXT_TAGS | {
+    f"{W}drawing",
+    f"{W}object",
+    f"{W}pict",
+    f"{W}oMath",
+    f"{W}oMathPara",
+    f"{W}noBreakHyphen",
+    f"{W}softHyphen",
+    f"{W}tab",
+    f"{W}br",
+    f"{W}cr",
+}
+
+IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]+\)")
+IMAGE_GLUE_RE = re.compile(
+    r"!\[[^\]]*\]\([^)]+\)(?=[^\s<>)\].,;:!?])|(?<=[^\s<(\[.,;:!?])!\[[^\]]*\]\([^)]+\)"
+)
+FOUR_PLUS_STARS_RE = re.compile(r"(?<!^)\*{4,}(?!$)", re.M)
+ODD_BACKTICK_LINE_RE = re.compile(r"`")
+
+
+@dataclass
+class SourceStats:
+    total_paragraphs: int = 0
+    nonempty_paragraphs: int = 0
+
+
+def local_name(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1] if "}" in tag else tag
+
+
+def paragraph_has_content(p: ET.Element) -> bool:
+    for node in p.iter():
+        if node.tag in TEXT_TAGS and (node.text or "").strip():
+            return True
+        if node.tag in CONTENT_TAGS - TEXT_TAGS:
+            return True
+    return False
+
+
+def source_stats(docx_path: Path) -> SourceStats:
+    stats = SourceStats()
+    with ZipFile(docx_path) as zf:
+        root = ET.fromstring(zf.read("word/document.xml"))
+    for p in root.iter(f"{W}p"):
+        stats.total_paragraphs += 1
+        if paragraph_has_content(p):
+            stats.nonempty_paragraphs += 1
+    return stats
+
+
+def load_json(path: Path) -> dict | list:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def count_odd_backtick_lines(text: str) -> int:
+    count = 0
+    for line in text.splitlines():
+        if line.count("`") % 2:
+            count += 1
+    return count
+
+
+def scan_markdown(md_path: Path) -> dict[str, int]:
+    text = md_path.read_text(encoding="utf-8", errors="replace")
+    four_plus = 0
+    for match in FOUR_PLUS_STARS_RE.finditer(text):
+        if match.group(0).strip("*"):
+            four_plus += 1
+    return {
+        "image_count": len(IMAGE_RE.findall(text)),
+        "image_glue_count": len(IMAGE_GLUE_RE.findall(text)),
+        "four_plus_stars_count": four_plus,
+        "odd_backtick_line_count": count_odd_backtick_lines(text),
+    }
+
+
+def audit_entry(entry: dict) -> dict:
+    input_path = Path(entry["input"])
+    output_path = Path(entry["output"])
+    export_report_path = output_path.parent / "export-report.json"
+
+    problems: list[str] = []
+    source = source_stats(input_path)
+
+    if not output_path.exists():
+        problems.append("missing_markdown_output")
+        return {
+            "input": str(input_path),
+            "output": str(output_path),
+            "problems": problems,
+        }
+
+    if not export_report_path.exists():
+        problems.append("missing_export_report")
+        return {
+            "input": str(input_path),
+            "output": str(output_path),
+            "problems": problems,
+        }
+
+    export_report = load_json(export_report_path)
+    report_stats = export_report.get("stats", {})
+    batch_stats = entry.get("stats", {})
+    markdown = scan_markdown(output_path)
+
+    rendered_block_count = (
+        int(report_stats.get("paragraphs", 0))
+        + int(report_stats.get("headings", 0))
+        + int(report_stats.get("lists", 0))
+        + int(report_stats.get("code_blocks", 0))
+    )
+
+    if report_stats != batch_stats:
+        problems.append("batch_report_mismatch")
+    if rendered_block_count != source.nonempty_paragraphs:
+        problems.append("paragraph_count_mismatch")
+    if markdown["image_count"] != int(report_stats.get("images", 0)):
+        problems.append("image_count_mismatch")
+    if markdown["image_glue_count"]:
+        problems.append("image_glue")
+    if markdown["four_plus_stars_count"]:
+        problems.append("four_plus_stars")
+    if markdown["odd_backtick_line_count"]:
+        problems.append("odd_backtick_lines")
+
+    return {
+        "input": str(input_path),
+        "output": str(output_path),
+        "source_total_paragraphs": source.total_paragraphs,
+        "source_nonempty_paragraphs": source.nonempty_paragraphs,
+        "report_rendered_blocks": rendered_block_count,
+        "report_images": int(report_stats.get("images", 0)),
+        "markdown_images": markdown["image_count"],
+        "image_glue_count": markdown["image_glue_count"],
+        "four_plus_stars_count": markdown["four_plus_stars_count"],
+        "odd_backtick_line_count": markdown["odd_backtick_line_count"],
+        "warnings": list(report_stats.get("warnings", [])),
+        "problems": problems,
+    }
+
+
+def summarize(results: list[dict]) -> dict:
+    problem_counts: dict[str, int] = {}
+    for result in results:
+        for problem in result.get("problems", []):
+            problem_counts[problem] = problem_counts.get(problem, 0) + 1
+    return {
+        "files_audited": len(results),
+        "files_with_problems": sum(1 for result in results if result.get("problems")),
+        "problem_counts": problem_counts,
+        "problem_examples": [result for result in results if result.get("problems")][:25],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Audit build-corpus batch output against source .docx files.")
+    parser.add_argument("--batch-report", required=True, help="Path to build-corpus-batch-report.json")
+    parser.add_argument("--out", default="", help="Optional path for the audit JSON report")
+    args = parser.parse_args()
+
+    batch_report_path = Path(args.batch_report).resolve()
+    entries = load_json(batch_report_path)
+    if not isinstance(entries, list):
+        raise SystemExit("Batch report must be a JSON array.")
+
+    results = [audit_entry(entry) for entry in entries]
+    summary = summarize(results)
+    payload = {
+        "batch_report": str(batch_report_path),
+        "summary": summary,
+        "results": results,
+    }
+
+    out_path = Path(args.out).resolve() if args.out else batch_report_path.with_name("corpus-audit-report.json")
+    out_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    print(json.dumps(summary, indent=2))
+    print(f"WROTE {out_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/collect_microsoft_word_templates.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import time
+from collections import deque
+from pathlib import Path
+from typing import Iterable
+from urllib.parse import urljoin, urlparse
+from zipfile import BadZipFile, ZipFile
+
+import requests
+
+
+BASE_URL = "https://word.cloud.microsoft"
+START_URLS = [
+    "https://word.cloud.microsoft/create/en/templates/",
+    "https://word.cloud.microsoft/create/en/resume-templates/",
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+    )
+}
+
+PAGE_LINK_RE = re.compile(r'(?:"|href=")(/create/en/[^"#? ]+/)')
+FILE_URL_RE = re.compile(r'"fileUrl":"(https://cdn\.create\.microsoft\.com/[^"]+\.docx)"')
+TITLE_RE = re.compile(
+    r'"fileUrl":"(?P<url>https://cdn\.create\.microsoft\.com/[^"]+\.docx)","title":"(?P<title>[^"]+)"'
+)
+
+
+def normalize_page(url: str) -> str:
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+
+
+def should_visit(url: str) -> bool:
+    parsed = urlparse(url)
+    if parsed.netloc != "word.cloud.microsoft":
+        return False
+    if not parsed.path.startswith("/create/en/"):
+        return False
+    path = parsed.path.lower()
+    if "/blog/" in path:
+        return False
+    if any(token in path for token in ("/copilot-", "/document-editor/", "/grammar-checker/", "/ai-", "/new/")):
+        return False
+    return True
+
+
+def fetch(session: requests.Session, url: str) -> str | None:
+    try:
+        response = session.get(url, headers=HEADERS, timeout=30)
+        if response.status_code >= 400:
+            return None
+        return response.text
+    except requests.RequestException:
+        return None
+
+
+def extract_links(html: str) -> Iterable[str]:
+    for match in PAGE_LINK_RE.finditer(html):
+        yield urljoin(BASE_URL, match.group(1))
+
+
+def extract_templates(html: str) -> list[dict[str, str]]:
+    titles_by_url = {match.group("url"): match.group("title") for match in TITLE_RE.finditer(html)}
+    found = []
+    for match in FILE_URL_RE.finditer(html):
+        url = match.group(1).replace("\\u0026", "&")
+        found.append({
+            "file_url": url,
+            "title": titles_by_url.get(url, ""),
+        })
+    return found
+
+
+def validate_docx(path: Path) -> dict:
+    with ZipFile(path) as zf:
+        names = set(zf.namelist())
+        if "[Content_Types].xml" not in names or "word/document.xml" not in names:
+            raise BadZipFile("not a Word DOCX package")
+        document_xml = zf.read("word/document.xml")
+        return {
+            "omml_nodes": len(re.findall(rb"<m:oMath\b|<m:oMathPara\b", document_xml)),
+            "media_parts": len([name for name in names if name.startswith("word/media/")]),
+            "document_xml_bytes": len(document_xml),
+        }
+
+
+def safe_name(index: int, title: str, content: bytes, file_url: str) -> str:
+    digest = hashlib.sha256(content).hexdigest()[:10]
+    stem = title.strip() or Path(urlparse(file_url).path).name
+    stem = re.sub(r"[^A-Za-z0-9._ -]+", "_", stem).strip(" ._")
+    if not stem.lower().endswith(".docx"):
+        stem += ".docx"
+    return f"{index:03d}-{digest}-{stem}"
+
+
+def download_template(
+    session: requests.Session,
+    item: dict[str, str],
+    out_dir: Path,
+    index: int,
+    max_mb: int,
+) -> dict | None:
+    try:
+        with session.get(item["file_url"], headers=HEADERS, timeout=60, stream=True) as response:
+            if response.status_code >= 400:
+                return None
+            chunks: list[bytes] = []
+            total = 0
+            limit = max_mb * 1024 * 1024
+            for chunk in response.iter_content(chunk_size=1024 * 256):
+                if not chunk:
+                    continue
+                total += len(chunk)
+                if total > limit:
+                    return None
+                chunks.append(chunk)
+        content = b"".join(chunks)
+    except requests.RequestException:
+        return None
+
+    if not content.startswith(b"PK"):
+        return None
+
+    filename = safe_name(index, item.get("title", ""), content, item["file_url"])
+    out_path = out_dir / filename
+    out_path.write_bytes(content)
+    try:
+        stats = validate_docx(out_path)
+    except (BadZipFile, KeyError):
+        out_path.unlink(missing_ok=True)
+        return None
+
+    return {
+        "file": str(out_path),
+        "title": item.get("title", ""),
+        "source_url": item["source_url"],
+        "file_url": item["file_url"],
+        "bytes": len(content),
+        **stats,
+    }
+
+
+def crawl_template_pages(session: requests.Session, max_pages: int) -> list[dict[str, str]]:
+    queue = deque(START_URLS)
+    seen_pages: set[str] = set()
+    seen_file_urls: set[str] = set()
+    template_items: list[dict[str, str]] = []
+
+    while queue and len(seen_pages) < max_pages:
+        current = normalize_page(queue.popleft())
+        if current in seen_pages or not should_visit(current):
+            continue
+        seen_pages.add(current)
+        html = fetch(session, current)
+        if not html:
+            continue
+
+        for template in extract_templates(html):
+            file_url = template["file_url"]
+            if file_url in seen_file_urls:
+                continue
+            seen_file_urls.add(file_url)
+            template_items.append({
+                **template,
+                "source_url": current,
+            })
+
+        for link in extract_links(html):
+            normalized = normalize_page(link)
+            if normalized not in seen_pages and should_visit(normalized):
+                queue.append(normalized)
+
+        time.sleep(0.15)
+
+    return template_items
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--max-pages", type=int, default=80)
+    parser.add_argument("--max-mb", type=int, default=40)
+    parser.add_argument("--limit", type=int, default=0, help="0 means no explicit template limit")
+    args = parser.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    session = requests.Session()
+
+    template_items = crawl_template_pages(session, max_pages=args.max_pages)
+    if args.limit > 0:
+        template_items = template_items[: args.limit]
+
+    manifest: list[dict] = []
+    for index, item in enumerate(template_items, 1):
+        downloaded = download_template(session, item, args.out, index, args.max_mb)
+        if not downloaded:
+            continue
+        manifest.append(downloaded)
+        print(f"kept {len(manifest):03d}: {Path(downloaded['file']).name}", flush=True)
+        time.sleep(0.1)
+
+    manifest_path = args.out / "microsoft-word-templates-manifest.json"
+    manifest_path.write_text(
+        json.dumps(
+            {
+                "count": len(manifest),
+                "items": manifest,
+            },
+            indent=2,
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    print(f"saved manifest: {manifest_path}", flush=True)
+    return 0 if manifest else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/tools/collect_online_docx_corpus.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Iterable
+from urllib.parse import parse_qs, quote_plus, unquote, urlparse
+from zipfile import BadZipFile, ZipFile
+
+import requests
+
+
+DEFAULT_QUERIES = [
+    'filetype:docx "equation"',
+    'filetype:docx "equations"',
+    'filetype:docx "Cambria Math"',
+    'filetype:docx "math equations"',
+    'filetype:docx "Microsoft Equation"',
+    'filetype:docx "OMML"',
+    'filetype:docx "quadratic equation"',
+    'filetype:docx "integral"',
+    'filetype:docx "matrix"',
+    'filetype:docx "calculus"',
+    'filetype:docx "physics" "equation"',
+    'filetype:docx "engineering" "equation"',
+    'filetype:docx "statistics" "equation"',
+    'filetype:docx "algebra" "equation"',
+    'filetype:docx "geometry" "equation"',
+]
+
+GITHUB_REPO_QUERIES = [
+    "docx equation math",
+    "docx equations latex",
+    "docx omml latex",
+    "word equation docx",
+    "markdown docx equation",
+    "pandoc docx equation",
+    "math docx converter",
+    "equation-heavy docx",
+    "ooxml math docx",
+    "Cambria Math docx",
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+    )
+}
+
+DOCX_CT_HINTS = (
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/octet-stream",
+    "application/zip",
+    "binary/octet-stream",
+)
+
+
+def extract_urls(html: str) -> list[str]:
+    raw = re.findall(r'https?://[^"\'<>\s)]+', html)
+    urls: list[str] = []
+    for url in raw:
+        url = unquote(url).replace("&amp;", "&")
+        parsed = urlparse(url)
+        if parsed.netloc.endswith("bing.com") and parsed.path == "/ck/a":
+            qs = parse_qs(parsed.query)
+            for key in ("u", "r"):
+                if key in qs:
+                    url = unquote(qs[key][0])
+                    break
+        if "duckduckgo.com/l/?" in url:
+            qs = parse_qs(urlparse(url).query)
+            if "uddg" in qs:
+                url = unquote(qs["uddg"][0])
+        urls.append(url.rstrip(".,;"))
+    return urls
+
+
+def search(query: str, pages: int = 2) -> Iterable[str]:
+    endpoints = [
+        "https://www.bing.com/search?q={query}&first={offset}",
+        "https://html.duckduckgo.com/html/?q={query}&s={offset}",
+    ]
+    for endpoint in endpoints:
+        for page in range(pages):
+            offset = page * 10 + 1
+            url = endpoint.format(query=quote_plus(query), offset=offset)
+            try:
+                response = requests.get(url, headers=HEADERS, timeout=20)
+                if response.status_code >= 400:
+                    continue
+                yield from extract_urls(response.text)
+                time.sleep(0.4)
+            except requests.RequestException:
+                continue
+
+
+def github_json(url: str) -> dict | None:
+    try:
+        response = requests.get(url, headers={**HEADERS, "Accept": "application/vnd.github+json"}, timeout=25)
+        if response.status_code >= 400:
+            return None
+        return response.json()
+    except requests.RequestException:
+        return None
+
+
+def github_docx_urls(max_repos_per_query: int = 8) -> Iterable[str]:
+    seen_repos: set[str] = set()
+    for query in GITHUB_REPO_QUERIES:
+        search_url = (
+            "https://api.github.com/search/repositories"
+            f"?q={quote_plus(query)}&per_page={max_repos_per_query}"
+        )
+        payload = github_json(search_url)
+        if not payload:
+            continue
+        for repo in payload.get("items", []):
+            full_name = repo.get("full_name")
+            branch = repo.get("default_branch") or "main"
+            if not full_name or full_name in seen_repos:
+                continue
+            seen_repos.add(full_name)
+            tree_url = f"https://api.github.com/repos/{full_name}/git/trees/{branch}?recursive=1"
+            tree = github_json(tree_url)
+            if not tree:
+                continue
+            for item in tree.get("tree", []):
+                path = item.get("path", "")
+                if item.get("type") == "blob" and path.lower().endswith(".docx"):
+                    yield f"https://raw.githubusercontent.com/{full_name}/{branch}/{quote_path(path)}"
+            time.sleep(0.25)
+
+
+def quote_path(path: str) -> str:
+    return "/".join(quote_plus(part).replace("+", "%20") for part in path.split("/"))
+
+
+def looks_like_docx_url(url: str) -> bool:
+    low = url.lower()
+    return ".docx" in low and not any(bad in low for bad in ("?format=pdf", "/view?", "webcache"))
+
+
+def safe_name(index: int, url: str, content: bytes) -> str:
+    parsed = urlparse(url)
+    stem = Path(unquote(parsed.path)).name or f"online-{index:03d}.docx"
+    stem = re.sub(r"[^A-Za-z0-9._ -]+", "_", stem)
+    if not stem.lower().endswith(".docx"):
+        stem += ".docx"
+    digest = hashlib.sha256(content).hexdigest()[:10]
+    return f"{index:03d}-{digest}-{stem}"
+
+
+def validate_docx(path: Path) -> dict:
+    with ZipFile(path) as zf:
+        names = set(zf.namelist())
+        if "[Content_Types].xml" not in names or "word/document.xml" not in names:
+            raise BadZipFile("not a Word DOCX package")
+        document_xml = zf.read("word/document.xml")
+        return {
+            "omml_nodes": len(re.findall(rb"<m:oMath\b|<m:oMathPara\b", document_xml)),
+            "media_parts": len([name for name in names if name.startswith("word/media/")]),
+            "document_xml_bytes": len(document_xml),
+        }
+
+
+def download(url: str, out_dir: Path, index: int, max_mb: int) -> dict | None:
+    try:
+        with requests.get(url, headers=HEADERS, timeout=30, stream=True, allow_redirects=True) as response:
+            if response.status_code >= 400:
+                return None
+            content_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
+            if content_type and content_type not in DOCX_CT_HINTS and "wordprocessingml" not in content_type:
+                if ".docx" not in response.url.lower():
+                    return None
+            chunks: list[bytes] = []
+            total = 0
+            limit = max_mb * 1024 * 1024
+            for chunk in response.iter_content(chunk_size=1024 * 256):
+                if not chunk:
+                    continue
+                total += len(chunk)
+                if total > limit:
+                    return None
+                chunks.append(chunk)
+        content = b"".join(chunks)
+    except requests.RequestException:
+        return None
+
+    if not content.startswith(b"PK"):
+        return None
+
+    out_path = out_dir / safe_name(index, response.url, content)
+    out_path.write_bytes(content)
+    try:
+        stats = validate_docx(out_path)
+    except (BadZipFile, KeyError):
+        out_path.unlink(missing_ok=True)
+        return None
+
+    return {
+        "file": str(out_path),
+        "source_url": response.url,
+        "bytes": len(content),
+        "content_type": content_type,
+        **stats,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--target", type=int, default=50)
+    parser.add_argument("--max-mb", type=int, default=40)
+    parser.add_argument("--prefer-equations", action="store_true")
+    args = parser.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    seen: set[str] = set()
+    manifest: list[dict] = []
+    index = len(list(args.out.glob("*.docx"))) + 1
+
+    print("source: github repositories", flush=True)
+    for url in github_docx_urls():
+        if len(manifest) >= args.target:
+            break
+        normalized = url.split("#", 1)[0]
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        item = download(normalized, args.out, index, args.max_mb)
+        if not item:
+            continue
+        manifest.append(item)
+        index += 1
+        print(f"kept {len(manifest):02d}: {Path(item['file']).name} omml={item['omml_nodes']}", flush=True)
+
+    for query in DEFAULT_QUERIES:
+        print(f"search: {query}", flush=True)
+        for url in search(query):
+            if len(manifest) >= args.target:
+                break
+            if not looks_like_docx_url(url):
+                continue
+            normalized = url.split("#", 1)[0]
+            if normalized in seen:
+                continue
+            seen.add(normalized)
+            item = download(normalized, args.out, index, args.max_mb)
+            if not item:
+                continue
+            if args.prefer_equations and item["omml_nodes"] == 0 and len(manifest) < args.target // 2:
+                Path(item["file"]).unlink(missing_ok=True)
+                continue
+            manifest.append(item)
+            index += 1
+            print(f"kept {len(manifest):02d}: {Path(item['file']).name} omml={item['omml_nodes']}", flush=True)
+        if len(manifest) >= args.target:
+            break
+
+    manifest_path = args.out / "online-docx-manifest.json"
+    manifest_path.write_text(json.dumps({"count": len(manifest), "items": manifest}, indent=2), encoding="utf-8")
+    print(f"saved manifest: {manifest_path}", flush=True)
+    return 0 if len(manifest) >= args.target else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())