regen.mde 0.2.2

package/src/build_corpus/ppt_exporter.py

@@ -0,0 +1,532 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import json
+import math
+import mimetypes
+import re
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+from zipfile import ZipFile
+from xml.etree import ElementTree as ET
+
+from PIL import Image
+
+NS = {
+    "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
+    "p": "http://schemas.openxmlformats.org/presentationml/2006/main",
+    "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+}
+
+
+@dataclass
+class ShapeBlock:
+    kind: str
+    x: int
+    y: int
+    cx: int
+    cy: int
+    markdown: str
+    text_norm: str = ""
+    image_sig: str = ""
+    is_title: bool = False
+
+
+@dataclass
+class SlideContent:
+    index: int
+    title: str
+    blocks: list[ShapeBlock] = field(default_factory=list)
+
+
+@dataclass
+class PresentationStats:
+    slides: int = 0
+    removed_repetitive_blocks: int = 0
+    tables: int = 0
+    text_blocks: int = 0
+    images: int = 0
+    removed_logo_images: int = 0
+    low_dpi_images: int = 0
+    warnings: list[str] = field(default_factory=list)
+
+
+def clean_text(value: str) -> str:
+    return re.sub(r"\s+", " ", (value or "").replace("\u00a0", " ")).strip()
+
+
+def md_escape_cell(value: str) -> str:
+    return clean_text(value).replace("|", r"\|").replace("\n", "<br>")
+
+
+def normalize_for_repeat(value: str) -> str:
+    return re.sub(r"\s+", " ", value).strip().lower()
+
+
+def local_name(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1] if "}" in tag else tag
+
+
+def emu_box(node: ET.Element) -> tuple[int, int, int, int]:
+    xfrm = node.find("./p:spPr/a:xfrm", NS)
+    if xfrm is None:
+        return 0, 0, 0, 0
+    off = xfrm.find("./a:off", NS)
+    ext = xfrm.find("./a:ext", NS)
+    try:
+        x = int((off.attrib.get("x", "0") if off is not None else "0"))
+        y = int((off.attrib.get("y", "0") if off is not None else "0"))
+        cx = int((ext.attrib.get("cx", "0") if ext is not None else "0"))
+        cy = int((ext.attrib.get("cy", "0") if ext is not None else "0"))
+        return x, y, cx, cy
+    except ValueError:
+        return 0, 0, 0, 0
+
+
+class PptxMarkdownExporter:
+    def __init__(self, input_path: Path, output_dir: Path, image_mode: str = "assets"):
+        self.input_path = input_path
+        self.output_dir = output_dir
+        self.output_md = output_dir / f"{input_path.stem}.md"
+        self.report_path = output_dir / "export-report.json"
+        self.assets_dir = output_dir / "assets"
+        self.asset_ref_prefix = self.assets_dir.name
+        self.image_mode = image_mode
+        self.media_map: dict[str, str] = {}
+        self.media_bytes: dict[str, bytes] = {}
+        self.variant_map: dict[str, str] = {}
+        self.stats = PresentationStats()
+
+    def export(self) -> dict:
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        if self.image_mode == "assets":
+            self.assets_dir.mkdir(parents=True, exist_ok=True)
+
+        with ZipFile(self.input_path) as zf:
+            self._copy_media(zf)
+            slides = self._parse_all_slides(zf)
+
+        self.stats.slides = len(slides)
+        self._remove_repetitive_text_blocks(slides)
+        self._remove_repeated_logo_images(slides)
+        markdown = self._render_markdown(slides)
+        self.output_md.write_text(markdown, encoding="utf-8")
+
+        report = {
+            "input": str(self.input_path),
+            "output": str(self.output_md),
+            "stats": self.stats.__dict__,
+        }
+        self.report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+        return report
+
+    def _parse_all_slides(self, zf: ZipFile) -> list[SlideContent]:
+        slide_paths = self._ordered_slide_paths(zf)
+        slides: list[SlideContent] = []
+        for i, slide_path in enumerate(slide_paths, 1):
+            root = ET.fromstring(zf.read(slide_path))
+            rels = self._relationships(zf, slide_path)
+            content = SlideContent(index=i, title="")
+            self._walk_shape_tree(root.find("./p:cSld/p:spTree", NS), content, rels, slide_path)
+            self._sort_blocks(content)
+            if not content.title:
+                image_only = bool(content.blocks) and all(b.kind == "image" for b in content.blocks)
+                content.title = "Image" if image_only else f"Slide {i}"
+            slides.append(content)
+        return slides
+
+    def _walk_shape_tree(
+        self,
+        node: Optional[ET.Element],
+        content: SlideContent,
+        rels: dict[str, str],
+        slide_path: str,
+    ) -> None:
+        if node is None:
+            return
+        for child in list(node):
+            tag = local_name(child.tag)
+            if tag == "sp":
+                block = self._parse_text_shape(child)
+                if block is not None:
+                    if block.is_title and not content.title:
+                        content.title = clean_text(block.markdown.replace("- ", " "))
+                    else:
+                        self.stats.text_blocks += 1
+                        content.blocks.append(block)
+            elif tag == "pic":
+                block = self._parse_picture_shape(child, rels, slide_path)
+                if block is not None:
+                    self.stats.images += 1
+                    content.blocks.append(block)
+            elif tag == "graphicFrame":
+                block = self._parse_table_shape(child)
+                if block is not None:
+                    self.stats.tables += 1
+                    content.blocks.append(block)
+            elif tag == "grpSp":
+                self._walk_shape_tree(child, content, rels, slide_path)
+
+    def _parse_text_shape(self, shape: ET.Element) -> Optional[ShapeBlock]:
+        ph = shape.find("./p:nvSpPr/p:nvPr/p:ph", NS)
+        ph_type = (ph.attrib.get("type", "") if ph is not None else "").lower()
+        if ph_type in {"ftr", "dt", "sldnum"}:
+            return None
+
+        tx_body = shape.find("./p:txBody", NS)
+        if tx_body is None:
+            return None
+
+        lines: list[str] = []
+        for p in tx_body.findall("./a:p", NS):
+            txt = self._paragraph_text(p)
+            if not txt:
+                continue
+            lvl = self._paragraph_level(p)
+            if lvl is None:
+                lines.append(txt)
+            else:
+                lines.append(f"{'  ' * max(lvl, 0)}- {txt}")
+        if not lines:
+            return None
+        x, y, cx, cy = emu_box(shape)
+        md = "\n".join(lines)
+        return ShapeBlock(
+            kind="text",
+            x=x,
+            y=y,
+            cx=cx,
+            cy=cy,
+            markdown=md,
+            text_norm=normalize_for_repeat(md),
+            is_title=ph_type in {"title", "ctrtitle"},
+        )
+
+    def _parse_table_shape(self, frame: ET.Element) -> Optional[ShapeBlock]:
+        tbl = frame.find(".//a:tbl", NS)
+        if tbl is None:
+            return None
+        rows: list[list[str]] = []
+        for tr in tbl.findall("./a:tr", NS):
+            row: list[str] = []
+            for tc in tr.findall("./a:tc", NS):
+                cell_lines = []
+                for p in tc.findall(".//a:p", NS):
+                    txt = self._paragraph_text(p)
+                    if txt:
+                        cell_lines.append(txt)
+                row.append(md_escape_cell("\n".join(cell_lines)))
+            rows.append(row)
+        if not rows:
+            return None
+        width = max(len(r) for r in rows)
+        padded = [r + [""] * (width - len(r)) for r in rows]
+        md_lines = []
+        md_lines.append("| " + " | ".join(padded[0]) + " |")
+        md_lines.append("| " + " | ".join("---" for _ in range(width)) + " |")
+        for row in padded[1:]:
+            md_lines.append("| " + " | ".join(row) + " |")
+        x, y, cx, cy = emu_box(frame)
+        md = "\n".join(md_lines)
+        return ShapeBlock(kind="table", x=x, y=y, cx=cx, cy=cy, markdown=md, text_norm=normalize_for_repeat(md))
+
+    def _parse_picture_shape(self, pic: ET.Element, rels: dict[str, str], slide_path: str) -> Optional[ShapeBlock]:
+        blip = pic.find(".//a:blip", NS)
+        if blip is None:
+            return None
+        rid = blip.attrib.get(f"{{{NS['r']}}}embed") or blip.attrib.get(f"{{{NS['r']}}}link")
+        if not rid:
+            return None
+        target = rels.get(rid, "")
+        if not target:
+            return None
+        part = self._resolve_part_path(slide_path, target)
+        ref = self.media_map.get(part, "")
+        if not ref:
+            return None
+
+        src_rect = self._source_crop(pic)
+        image_bytes = self.media_bytes.get(part, b"")
+        if src_rect is not None:
+            cropped = self._render_cropped_variant(part, src_rect)
+            if cropped:
+                ref = cropped
+                image_bytes = self._variant_bytes(part, src_rect) or image_bytes
+
+        x, y, cx, cy = emu_box(pic)
+        self._track_dpi(cx, cy, image_bytes, Path(part).suffix.lower())
+        sig = hashlib.sha256(image_bytes).hexdigest()[:16]
+        return ShapeBlock(
+            kind="image",
+            x=x,
+            y=y,
+            cx=cx,
+            cy=cy,
+            markdown=f"![image]({ref})",
+            image_sig=sig,
+        )
+
+    def _track_dpi(self, cx: int, cy: int, data: bytes, suffix: str) -> None:
+        dims = self._image_dimensions(data, suffix)
+        if not dims or cx <= 0 or cy <= 0:
+            return
+        w, h = dims
+        dpi_x = w / (cx / 914400)
+        dpi_y = h / (cy / 914400)
+        if min(dpi_x, dpi_y) < 150:
+            self.stats.low_dpi_images += 1
+
+    def _sort_blocks(self, slide: SlideContent) -> None:
+        row_height = 0.35 * 914400
+        slide.blocks.sort(key=lambda b: (round(b.y / row_height), b.y, b.x))
+
+    def _remove_repetitive_text_blocks(self, slides: list[SlideContent]) -> None:
+        counts: dict[str, int] = {}
+        for slide in slides:
+            seen = set()
+            for b in slide.blocks:
+                if b.kind != "text" or not b.text_norm:
+                    continue
+                if b.text_norm in seen:
+                    continue
+                seen.add(b.text_norm)
+                counts[b.text_norm] = counts.get(b.text_norm, 0) + 1
+        threshold = max(4, math.ceil(len(slides) * 0.6)) if slides else 99999
+        repetitive = {k for k, v in counts.items() if v >= threshold}
+        for slide in slides:
+            kept: list[ShapeBlock] = []
+            for b in slide.blocks:
+                if b.kind == "text" and b.text_norm and b.text_norm in repetitive:
+                    self.stats.removed_repetitive_blocks += 1
+                    continue
+                kept.append(b)
+            slide.blocks = kept
+
+    def _remove_repeated_logo_images(self, slides: list[SlideContent]) -> None:
+        counts: dict[str, int] = {}
+        geom_counts: dict[str, int] = {}
+        for slide in slides:
+            for b in slide.blocks:
+                if b.kind == "image" and b.image_sig:
+                    counts[b.image_sig] = counts.get(b.image_sig, 0) + 1
+                    if self._looks_like_corner_logo(b):
+                        g = self._logo_geom_bucket(b)
+                        geom_counts[g] = geom_counts.get(g, 0) + 1
+        threshold = max(3, math.ceil(len(slides) * 0.15)) if slides else 99999
+        repetitive = {k for k, v in counts.items() if v >= threshold}
+        repetitive_geom = {k for k, v in geom_counts.items() if v >= threshold}
+        for slide in slides:
+            kept: list[ShapeBlock] = []
+            for b in slide.blocks:
+                if b.kind == "image" and self._looks_like_corner_logo(b):
+                    geom_match = self._logo_geom_bucket(b) in repetitive_geom
+                    hash_match = b.image_sig in repetitive
+                    if geom_match or hash_match:
+                        self.stats.removed_logo_images += 1
+                        continue
+                kept.append(b)
+            slide.blocks = kept
+
+    @staticmethod
+    def _looks_like_corner_logo(b: ShapeBlock) -> bool:
+        # Small and near top/bottom edge is a typical logo/footer marker.
+        w_in = b.cx / 914400 if b.cx else 0
+        h_in = b.cy / 914400 if b.cy else 0
+        y_in = b.y / 914400 if b.y else 0
+        return w_in <= 2.2 and h_in <= 1.2 and (y_in <= 1.4 or y_in >= 5.5)
+
+    @staticmethod
+    def _logo_geom_bucket(b: ShapeBlock) -> str:
+        return f"{round(b.x/300000)}:{round(b.y/300000)}:{round(b.cx/300000)}:{round(b.cy/300000)}"
+
+    def _render_markdown(self, slides: list[SlideContent]) -> str:
+        out: list[str] = []
+        for slide in slides:
+            out.append(f"## Slide {slide.index}: {slide.title}")
+            if slide.blocks:
+                out.append("")
+                for b in slide.blocks:
+                    out.append(b.markdown)
+                    out.append("")
+            else:
+                out.append("")
+            if out and out[-1] != "":
+                out.append("")
+        return "\n".join(out).strip() + "\n"
+
+    def _copy_media(self, zf: ZipFile) -> None:
+        for name in zf.namelist():
+            if not name.startswith("ppt/media/"):
+                continue
+            data = zf.read(name)
+            self.media_bytes[name] = data
+            mime = mimetypes.guess_type(Path(name).name)[0] or "application/octet-stream"
+            if self.image_mode == "base64":
+                self.media_map[name] = f"data:{mime};base64,{base64.b64encode(data).decode('ascii')}"
+            elif self.image_mode == "assets":
+                target = self.assets_dir / Path(name).name
+                target.write_bytes(data)
+                self.media_map[name] = f"{self.asset_ref_prefix}/{target.name}"
+            else:
+                self.media_map[name] = ""
+
+    def _source_crop(self, pic: ET.Element) -> tuple[int, int, int, int] | None:
+        src = pic.find("./p:blipFill/a:srcRect", NS)
+        if src is None:
+            return None
+        try:
+            return (
+                int(src.attrib.get("l", "0")),
+                int(src.attrib.get("t", "0")),
+                int(src.attrib.get("r", "0")),
+                int(src.attrib.get("b", "0")),
+            )
+        except ValueError:
+            return None
+
+    def _variant_bytes(self, part: str, rect: tuple[int, int, int, int]) -> bytes | None:
+        key = f"{part}:{rect[0]}:{rect[1]}:{rect[2]}:{rect[3]}:bytes"
+        ref = self.variant_map.get(key, "")
+        if not ref:
+            return None
+        return base64.b64decode(ref)
+
+    def _render_cropped_variant(self, part: str, src_rect: tuple[int, int, int, int]) -> str:
+        cache_key_ref = f"{part}:{src_rect[0]}:{src_rect[1]}:{src_rect[2]}:{src_rect[3]}:{self.image_mode}"
+        cache_key_bytes = f"{part}:{src_rect[0]}:{src_rect[1]}:{src_rect[2]}:{src_rect[3]}:bytes"
+        if cache_key_ref in self.variant_map:
+            return self.variant_map[cache_key_ref]
+        data = self.media_bytes.get(part, b"")
+        if not data:
+            return ""
+        ext = Path(part).suffix.lower()
+        if ext not in {".png", ".jpg", ".jpeg"}:
+            return ""
+        with Image.open(io.BytesIO(data)) as img:
+            width, height = img.size
+            l, t, r, b = src_rect
+            left = max(0, min(width, round(width * (l / 100000.0))))
+            top = max(0, min(height, round(height * (t / 100000.0))))
+            right = max(0, min(width, round(width * ((100000 - r) / 100000.0))))
+            bottom = max(0, min(height, round(height * ((100000 - b) / 100000.0))))
+            if right <= left or bottom <= top:
+                return ""
+            cropped = img.crop((left, top, right, bottom))
+            out = io.BytesIO()
+            fmt = "PNG" if ext == ".png" else "JPEG"
+            kwargs = {"quality": 95} if fmt == "JPEG" else {}
+            cropped.save(out, format=fmt, **kwargs)
+            out_bytes = out.getvalue()
+        self.variant_map[cache_key_bytes] = base64.b64encode(out_bytes).decode("ascii")
+        digest = hashlib.sha256(out_bytes).hexdigest()[:12]
+        out_ext = ".jpg" if ext == ".jpeg" else ext
+        name = f"{Path(part).stem}.crop.{digest}{out_ext}"
+        if self.image_mode == "base64":
+            mime = mimetypes.guess_type(name)[0] or "application/octet-stream"
+            ref = f"data:{mime};base64,{base64.b64encode(out_bytes).decode('ascii')}"
+        else:
+            target = self.assets_dir / name
+            target.write_bytes(out_bytes)
+            ref = f"{self.asset_ref_prefix}/{target.name}"
+        self.variant_map[cache_key_ref] = ref
+        return ref
+
+    @staticmethod
+    def _image_dimensions(data: bytes, suffix: str) -> tuple[int, int] | None:
+        if suffix == ".png" and data.startswith(b"\x89PNG\r\n\x1a\n") and len(data) >= 24:
+            return int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big")
+        if suffix in {".jpg", ".jpeg"} and data.startswith(b"\xff\xd8"):
+            i = 2
+            while i + 9 < len(data):
+                if data[i] != 0xFF:
+                    i += 1
+                    continue
+                marker = data[i + 1]
+                if marker in {0xC0, 0xC1, 0xC2, 0xC3, 0xC9, 0xCA, 0xCB}:
+                    if i + 9 <= len(data):
+                        return int.from_bytes(data[i + 7:i + 9], "big"), int.from_bytes(data[i + 5:i + 7], "big")
+                if i + 4 > len(data):
+                    break
+                seg_len = int.from_bytes(data[i + 2:i + 4], "big")
+                i += 2 + seg_len
+        return None
+
+    def _paragraph_text(self, p: ET.Element) -> str:
+        chunks = []
+        for n in p.iter():
+            if local_name(n.tag) == "t" and n.text:
+                chunks.append(n.text)
+        return clean_text("".join(chunks))
+
+    def _paragraph_level(self, p: ET.Element) -> Optional[int]:
+        ppr = p.find("./a:pPr", NS)
+        if ppr is None:
+            return None
+        lvl = ppr.attrib.get("lvl")
+        if lvl is None:
+            return None
+        try:
+            return int(lvl)
+        except ValueError:
+            return None
+
+    def _ordered_slide_paths(self, zf: ZipFile) -> list[str]:
+        presentation = ET.fromstring(zf.read("ppt/presentation.xml"))
+        rels = self._relationships(zf, "ppt/presentation.xml")
+        paths: list[str] = []
+        for sld_id in presentation.findall("./p:sldIdLst/p:sldId", NS):
+            rid = sld_id.attrib.get(f"{{{NS['r']}}}id")
+            target = rels.get(rid or "")
+            if not target:
+                continue
+            full = self._resolve_part_path("ppt/presentation.xml", target)
+            if full in zf.namelist():
+                paths.append(full)
+        return paths
+
+    def _relationships(self, zf: ZipFile, part: str) -> dict[str, str]:
+        rels_path = str(Path(part).parent / "_rels" / f"{Path(part).name}.rels").replace("\\", "/")
+        if rels_path not in zf.namelist():
+            return {}
+        root = ET.fromstring(zf.read(rels_path))
+        return {rel.attrib["Id"]: rel.attrib.get("Target", "") for rel in root if "Id" in rel.attrib}
+
+    @staticmethod
+    def _resolve_part_path(part: str, target: str) -> str:
+        joined = (Path(part).parent / target).as_posix()
+        norm: list[str] = []
+        for piece in joined.split("/"):
+            if piece in {"", "."}:
+                continue
+            if piece == "..":
+                if norm:
+                    norm.pop()
+                continue
+            norm.append(piece)
+        return "/".join(norm)
+
+
+def export_presentation(input_path: Path, output_root: Path, out_same_dir: bool, image_mode: str = "assets") -> dict:
+    if input_path.suffix.lower() == ".ppt":
+        input_path = convert_legacy_ppt(input_path)
+    output_dir = input_path.parent if out_same_dir else output_root / input_path.stem
+    return PptxMarkdownExporter(input_path, output_dir, image_mode=image_mode).export()
+
+
+def convert_legacy_ppt(input_path: Path) -> Path:
+    with tempfile.TemporaryDirectory(prefix="build-corpus-ppt-") as tmp:
+        tmp_dir = Path(tmp)
+        cmd = ["soffice", "--headless", "--convert-to", "pptx", "--outdir", str(tmp_dir), str(input_path)]
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        converted = tmp_dir / f"{input_path.stem}.pptx"
+        if result.returncode != 0 or not converted.exists():
+            message = (result.stderr or result.stdout or "conversion failed").strip()
+            raise RuntimeError(f"Legacy .ppt conversion requires LibreOffice; failed for {input_path}: {message}")
+        persistent = input_path.with_suffix(".converted.pptx")
+        persistent.write_bytes(converted.read_bytes())
+        return persistent

package/src/build_corpus/templates/__init__.py

@@ -0,0 +1 @@
+

package/src/build_corpus/validate_assets.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)]+)\)")
+
+
+def validate_file(path: Path) -> dict:
+    text = path.read_text(encoding="utf-8", errors="replace")
+    refs = IMAGE_RE.findall(text)
+    missing = []
+    for ref in refs:
+        if ref.startswith(("http://", "https://", "data:")):
+            continue
+        if not (path.parent / ref).exists() and not Path(ref).exists():
+            missing.append(ref)
+    return {
+        "file": str(path),
+        "image_refs": len(refs),
+        "missing_refs": len(missing),
+        "missing_samples": missing[:50],
+    }
+
+
+def collect_markdown(path: Path) -> list[Path]:
+    if path.is_file():
+        return [path]
+    return sorted(path.rglob("*.md"))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("target", type=Path)
+    args = parser.parse_args()
+
+    results = [validate_file(path) for path in collect_markdown(args.target)]
+    print(json.dumps({"files": len(results), "results": results}, indent=2))
+    raise SystemExit(1 if any(result["missing_refs"] for result in results) else 0)
+
+
+if __name__ == "__main__":
+    main()