regen.mde 0.8.0 → 0.8.2

package/tools/collect_microsoft_word_templates.py

@@ -1,228 +1,228 @@
-from __future__ import annotations
-
-import argparse
-import hashlib
-import json
-import re
-import time
-from collections import deque
-from pathlib import Path
-from typing import Iterable
-from urllib.parse import urljoin, urlparse
-from zipfile import BadZipFile, ZipFile
-
-import requests
-
-
-BASE_URL = "https://word.cloud.microsoft"
-START_URLS = [
-    "https://word.cloud.microsoft/create/en/templates/",
-    "https://word.cloud.microsoft/create/en/resume-templates/",
-]
-
-HEADERS = {
-    "User-Agent": (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
-    )
-}
-
-PAGE_LINK_RE = re.compile(r'(?:"|href=")(/create/en/[^"#? ]+/)')
-FILE_URL_RE = re.compile(r'"fileUrl":"(https://cdn\.create\.microsoft\.com/[^"]+\.docx)"')
-TITLE_RE = re.compile(
-    r'"fileUrl":"(?P<url>https://cdn\.create\.microsoft\.com/[^"]+\.docx)","title":"(?P<title>[^"]+)"'
-)
-
-
-def normalize_page(url: str) -> str:
-    parsed = urlparse(url)
-    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
-
-
-def should_visit(url: str) -> bool:
-    parsed = urlparse(url)
-    if parsed.netloc != "word.cloud.microsoft":
-        return False
-    if not parsed.path.startswith("/create/en/"):
-        return False
-    path = parsed.path.lower()
-    if "/blog/" in path:
-        return False
-    if any(token in path for token in ("/copilot-", "/document-editor/", "/grammar-checker/", "/ai-", "/new/")):
-        return False
-    return True
-
-
-def fetch(session: requests.Session, url: str) -> str | None:
-    try:
-        response = session.get(url, headers=HEADERS, timeout=30)
-        if response.status_code >= 400:
-            return None
-        return response.text
-    except requests.RequestException:
-        return None
-
-
-def extract_links(html: str) -> Iterable[str]:
-    for match in PAGE_LINK_RE.finditer(html):
-        yield urljoin(BASE_URL, match.group(1))
-
-
-def extract_templates(html: str) -> list[dict[str, str]]:
-    titles_by_url = {match.group("url"): match.group("title") for match in TITLE_RE.finditer(html)}
-    found = []
-    for match in FILE_URL_RE.finditer(html):
-        url = match.group(1).replace("\\u0026", "&")
-        found.append({
-            "file_url": url,
-            "title": titles_by_url.get(url, ""),
-        })
-    return found
-
-
-def validate_docx(path: Path) -> dict:
-    with ZipFile(path) as zf:
-        names = set(zf.namelist())
-        if "[Content_Types].xml" not in names or "word/document.xml" not in names:
-            raise BadZipFile("not a Word DOCX package")
-        document_xml = zf.read("word/document.xml")
-        return {
-            "omml_nodes": len(re.findall(rb"<m:oMath\b|<m:oMathPara\b", document_xml)),
-            "media_parts": len([name for name in names if name.startswith("word/media/")]),
-            "document_xml_bytes": len(document_xml),
-        }
-
-
-def safe_name(index: int, title: str, content: bytes, file_url: str) -> str:
-    digest = hashlib.sha256(content).hexdigest()[:10]
-    stem = title.strip() or Path(urlparse(file_url).path).name
-    stem = re.sub(r"[^A-Za-z0-9._ -]+", "_", stem).strip(" ._")
-    if not stem.lower().endswith(".docx"):
-        stem += ".docx"
-    return f"{index:03d}-{digest}-{stem}"
-
-
-def download_template(
-    session: requests.Session,
-    item: dict[str, str],
-    out_dir: Path,
-    index: int,
-    max_mb: int,
-) -> dict | None:
-    try:
-        with session.get(item["file_url"], headers=HEADERS, timeout=60, stream=True) as response:
-            if response.status_code >= 400:
-                return None
-            chunks: list[bytes] = []
-            total = 0
-            limit = max_mb * 1024 * 1024
-            for chunk in response.iter_content(chunk_size=1024 * 256):
-                if not chunk:
-                    continue
-                total += len(chunk)
-                if total > limit:
-                    return None
-                chunks.append(chunk)
-        content = b"".join(chunks)
-    except requests.RequestException:
-        return None
-
-    if not content.startswith(b"PK"):
-        return None
-
-    filename = safe_name(index, item.get("title", ""), content, item["file_url"])
-    out_path = out_dir / filename
-    out_path.write_bytes(content)
-    try:
-        stats = validate_docx(out_path)
-    except (BadZipFile, KeyError):
-        out_path.unlink(missing_ok=True)
-        return None
-
-    return {
-        "file": str(out_path),
-        "title": item.get("title", ""),
-        "source_url": item["source_url"],
-        "file_url": item["file_url"],
-        "bytes": len(content),
-        **stats,
-    }
-
-
-def crawl_template_pages(session: requests.Session, max_pages: int) -> list[dict[str, str]]:
-    queue = deque(START_URLS)
-    seen_pages: set[str] = set()
-    seen_file_urls: set[str] = set()
-    template_items: list[dict[str, str]] = []
-
-    while queue and len(seen_pages) < max_pages:
-        current = normalize_page(queue.popleft())
-        if current in seen_pages or not should_visit(current):
-            continue
-        seen_pages.add(current)
-        html = fetch(session, current)
-        if not html:
-            continue
-
-        for template in extract_templates(html):
-            file_url = template["file_url"]
-            if file_url in seen_file_urls:
-                continue
-            seen_file_urls.add(file_url)
-            template_items.append({
-                **template,
-                "source_url": current,
-            })
-
-        for link in extract_links(html):
-            normalized = normalize_page(link)
-            if normalized not in seen_pages and should_visit(normalized):
-                queue.append(normalized)
-
-        time.sleep(0.15)
-
-    return template_items
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--out", type=Path, required=True)
-    parser.add_argument("--max-pages", type=int, default=80)
-    parser.add_argument("--max-mb", type=int, default=40)
-    parser.add_argument("--limit", type=int, default=0, help="0 means no explicit template limit")
-    args = parser.parse_args()
-
-    args.out.mkdir(parents=True, exist_ok=True)
-    session = requests.Session()
-
-    template_items = crawl_template_pages(session, max_pages=args.max_pages)
-    if args.limit > 0:
-        template_items = template_items[: args.limit]
-
-    manifest: list[dict] = []
-    for index, item in enumerate(template_items, 1):
-        downloaded = download_template(session, item, args.out, index, args.max_mb)
-        if not downloaded:
-            continue
-        manifest.append(downloaded)
-        print(f"kept {len(manifest):03d}: {Path(downloaded['file']).name}", flush=True)
-        time.sleep(0.1)
-
-    manifest_path = args.out / "microsoft-word-templates-manifest.json"
-    manifest_path.write_text(
-        json.dumps(
-            {
-                "count": len(manifest),
-                "items": manifest,
-            },
-            indent=2,
-            ensure_ascii=False,
-        ),
-        encoding="utf-8",
-    )
-    print(f"saved manifest: {manifest_path}", flush=True)
-    return 0 if manifest else 1
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import time
+from collections import deque
+from pathlib import Path
+from typing import Iterable
+from urllib.parse import urljoin, urlparse
+from zipfile import BadZipFile, ZipFile
+
+import requests
+
+
+BASE_URL = "https://word.cloud.microsoft"
+START_URLS = [
+    "https://word.cloud.microsoft/create/en/templates/",
+    "https://word.cloud.microsoft/create/en/resume-templates/",
+]
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+    )
+}
+
+PAGE_LINK_RE = re.compile(r'(?:"|href=")(/create/en/[^"#? ]+/)')
+FILE_URL_RE = re.compile(r'"fileUrl":"(https://cdn\.create\.microsoft\.com/[^"]+\.docx)"')
+TITLE_RE = re.compile(
+    r'"fileUrl":"(?P<url>https://cdn\.create\.microsoft\.com/[^"]+\.docx)","title":"(?P<title>[^"]+)"'
+)
+
+
+def normalize_page(url: str) -> str:
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+
+
+def should_visit(url: str) -> bool:
+    parsed = urlparse(url)
+    if parsed.netloc != "word.cloud.microsoft":
+        return False
+    if not parsed.path.startswith("/create/en/"):
+        return False
+    path = parsed.path.lower()
+    if "/blog/" in path:
+        return False
+    if any(token in path for token in ("/copilot-", "/document-editor/", "/grammar-checker/", "/ai-", "/new/")):
+        return False
+    return True
+
+
+def fetch(session: requests.Session, url: str) -> str | None:
+    try:
+        response = session.get(url, headers=HEADERS, timeout=30)
+        if response.status_code >= 400:
+            return None
+        return response.text
+    except requests.RequestException:
+        return None
+
+
+def extract_links(html: str) -> Iterable[str]:
+    for match in PAGE_LINK_RE.finditer(html):
+        yield urljoin(BASE_URL, match.group(1))
+
+
+def extract_templates(html: str) -> list[dict[str, str]]:
+    titles_by_url = {match.group("url"): match.group("title") for match in TITLE_RE.finditer(html)}
+    found = []
+    for match in FILE_URL_RE.finditer(html):
+        url = match.group(1).replace("\\u0026", "&")
+        found.append({
+            "file_url": url,
+            "title": titles_by_url.get(url, ""),
+        })
+    return found
+
+
+def validate_docx(path: Path) -> dict:
+    with ZipFile(path) as zf:
+        names = set(zf.namelist())
+        if "[Content_Types].xml" not in names or "word/document.xml" not in names:
+            raise BadZipFile("not a Word DOCX package")
+        document_xml = zf.read("word/document.xml")
+        return {
+            "omml_nodes": len(re.findall(rb"<m:oMath\b|<m:oMathPara\b", document_xml)),
+            "media_parts": len([name for name in names if name.startswith("word/media/")]),
+            "document_xml_bytes": len(document_xml),
+        }
+
+
+def safe_name(index: int, title: str, content: bytes, file_url: str) -> str:
+    digest = hashlib.sha256(content).hexdigest()[:10]
+    stem = title.strip() or Path(urlparse(file_url).path).name
+    stem = re.sub(r"[^A-Za-z0-9._ -]+", "_", stem).strip(" ._")
+    if not stem.lower().endswith(".docx"):
+        stem += ".docx"
+    return f"{index:03d}-{digest}-{stem}"
+
+
+def download_template(
+    session: requests.Session,
+    item: dict[str, str],
+    out_dir: Path,
+    index: int,
+    max_mb: int,
+) -> dict | None:
+    try:
+        with session.get(item["file_url"], headers=HEADERS, timeout=60, stream=True) as response:
+            if response.status_code >= 400:
+                return None
+            chunks: list[bytes] = []
+            total = 0
+            limit = max_mb * 1024 * 1024
+            for chunk in response.iter_content(chunk_size=1024 * 256):
+                if not chunk:
+                    continue
+                total += len(chunk)
+                if total > limit:
+                    return None
+                chunks.append(chunk)
+        content = b"".join(chunks)
+    except requests.RequestException:
+        return None
+
+    if not content.startswith(b"PK"):
+        return None
+
+    filename = safe_name(index, item.get("title", ""), content, item["file_url"])
+    out_path = out_dir / filename
+    out_path.write_bytes(content)
+    try:
+        stats = validate_docx(out_path)
+    except (BadZipFile, KeyError):
+        out_path.unlink(missing_ok=True)
+        return None
+
+    return {
+        "file": str(out_path),
+        "title": item.get("title", ""),
+        "source_url": item["source_url"],
+        "file_url": item["file_url"],
+        "bytes": len(content),
+        **stats,
+    }
+
+
+def crawl_template_pages(session: requests.Session, max_pages: int) -> list[dict[str, str]]:
+    queue = deque(START_URLS)
+    seen_pages: set[str] = set()
+    seen_file_urls: set[str] = set()
+    template_items: list[dict[str, str]] = []
+
+    while queue and len(seen_pages) < max_pages:
+        current = normalize_page(queue.popleft())
+        if current in seen_pages or not should_visit(current):
+            continue
+        seen_pages.add(current)
+        html = fetch(session, current)
+        if not html:
+            continue
+
+        for template in extract_templates(html):
+            file_url = template["file_url"]
+            if file_url in seen_file_urls:
+                continue
+            seen_file_urls.add(file_url)
+            template_items.append({
+                **template,
+                "source_url": current,
+            })
+
+        for link in extract_links(html):
+            normalized = normalize_page(link)
+            if normalized not in seen_pages and should_visit(normalized):
+                queue.append(normalized)
+
+        time.sleep(0.15)
+
+    return template_items
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--max-pages", type=int, default=80)
+    parser.add_argument("--max-mb", type=int, default=40)
+    parser.add_argument("--limit", type=int, default=0, help="0 means no explicit template limit")
+    args = parser.parse_args()
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    session = requests.Session()
+
+    template_items = crawl_template_pages(session, max_pages=args.max_pages)
+    if args.limit > 0:
+        template_items = template_items[: args.limit]
+
+    manifest: list[dict] = []
+    for index, item in enumerate(template_items, 1):
+        downloaded = download_template(session, item, args.out, index, args.max_mb)
+        if not downloaded:
+            continue
+        manifest.append(downloaded)
+        print(f"kept {len(manifest):03d}: {Path(downloaded['file']).name}", flush=True)
+        time.sleep(0.1)
+
+    manifest_path = args.out / "microsoft-word-templates-manifest.json"
+    manifest_path.write_text(
+        json.dumps(
+            {
+                "count": len(manifest),
+                "items": manifest,
+            },
+            indent=2,
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    print(f"saved manifest: {manifest_path}", flush=True)
+    return 0 if manifest else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())