regen.mde 0.2.2 → 0.7.0

package/src/build_corpus/frontmatter.py

@@ -0,0 +1,302 @@
+"""MDK YAML frontmatter — emit on forward (docx/pptx -> md), strip on reverse
+(md -> docx) and persist verbatim into the Office package's ``docProps/custom.xml``.
+
+The design contract (see ``HANDOFF-build-corpus-mdk-frontmatter``):
+
+* Every generated ``.md`` carries an inline MDK frontmatter block by default, so
+  ``@regen/codeflow`` (MDK) can compile it with zero changes on the MDK side.
+* The frontmatter is **stripped** on the ``md -> docx`` reverse so the visible
+  Word body never contains YAML.
+* The stripped YAML is stored **verbatim** in one custom document property,
+  ``mdk_frontmatter`` (``vt:lpwstr``), inside ``docProps/custom.xml``. That is the
+  durable home that travels inside the Word/PowerPoint file, so the round-trip is
+  lossless: forward re-reads it verbatim if present, else generates defaults.
+
+No external YAML dependency is used; the frontmatter we emit is a small, fixed
+shape, so we build/parse it with plain string handling and keep the body verbatim.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+from pathlib import Path
+from xml.sax.saxutils import escape as _xml_escape
+from zipfile import ZIP_DEFLATED, ZipFile
+
+FRONTMATTER_PROPERTY_NAME = "mdk_frontmatter"
+CUSTOM_XML_PART = "docProps/custom.xml"
+CONTENT_TYPES_PART = "[Content_Types].xml"
+ROOT_RELS_PART = "_rels/.rels"
+
+CUSTOM_PROPS_NS = "http://schemas.openxmlformats.org/officeDocument/2006/custom-properties"
+VT_NS = "http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes"
+CUSTOM_PROPS_CONTENT_TYPE = (
+    "application/vnd.openxmlformats-officedocument.custom-properties+xml"
+)
+CUSTOM_PROPS_REL_TYPE = (
+    "http://schemas.openxmlformats.org/officeDocument/2006/relationships/custom-properties"
+)
+CUSTOM_PROPS_FMTID = "{D5CDD505-2E9C-101B-9397-08002B2CF9AE}"
+
+# An evidence-handle HTML comment immediately follows the frontmatter so MDK can
+# anchor the converted Markdown to the original binary it was derived from.
+_EVIDENCE_OPEN = "<!-- mdk:evidence-handle"
+_EVIDENCE_CLOSE = "-->"
+
+
+# ---------------------------------------------------------------------------
+# Emit (forward: docx/pptx -> md)
+# ---------------------------------------------------------------------------
+
+def _yaml_block_default(source_path: Path) -> str:
+    """Render the default MDK frontmatter YAML block for a converted document."""
+    try:
+        mtime = _dt.date.fromtimestamp(source_path.stat().st_mtime).isoformat()
+        created = f'"{mtime}"'
+    except OSError:
+        created = "null"
+    return (
+        "---\n"
+        'mdk_schema_version: "1.0"\n'
+        "doc_type: corpus_note\n"
+        "system: other\n"
+        "status: active\n"
+        "owner: human\n"
+        "source_of_truth: false\n"
+        f"created: {created}\n"
+        "last_reviewed: null\n"
+        "supersedes: []\n"
+        "depends_on: []\n"
+        "tags: []\n"
+        "---\n"
+    )
+
+
+def _evidence_handle(source_path: Path) -> str:
+    """Render the ``mdk:evidence-handle`` comment for the original binary."""
+    ext = source_path.suffix.lstrip(".") or "binary"
+    try:
+        digest = hashlib.sha256(source_path.read_bytes()).hexdigest()
+    except OSError:
+        digest = ""
+    abs_path = str(source_path.resolve())
+    return (
+        f"{_EVIDENCE_OPEN}\n"
+        "kind: local\n"
+        f"source_path: {abs_path}\n"
+        f"hash: {digest}\n"
+        f"description: Original {ext} the corpus Markdown was converted from "
+        f"(build-corpus).\n"
+        f"{_EVIDENCE_CLOSE}\n"
+    )
+
+
+def build_frontmatter(source_path: Path, prior_frontmatter: str | None = None) -> str:
+    """Return the YAML frontmatter block (``---`` … ``---\\n``) for a document.
+
+    If ``prior_frontmatter`` (recovered verbatim from a source ``custom.xml``)
+    is supplied, it is used as-is so round-trips stay lossless. Otherwise a
+    default block is generated from the source file's metadata.
+    """
+    if prior_frontmatter:
+        block = prior_frontmatter.strip("\n") + "\n"
+        if not block.startswith("---"):
+            block = "---\n" + block
+        if not block.rstrip("\n").endswith("---"):
+            block = block.rstrip("\n") + "\n---\n"
+        return block
+    return _yaml_block_default(source_path)
+
+
+def add_mdk_frontmatter(
+    markdown: str,
+    source_path: Path,
+    prior_frontmatter: str | None = None,
+) -> str:
+    """Prepend MDK frontmatter (+ an evidence-handle when freshly generated).
+
+    When ``prior_frontmatter`` is supplied (a verbatim restore from a source
+    ``custom.xml``), only that block is emitted — no new evidence-handle is added,
+    because the verbatim block already carries whatever the source held.
+    """
+    block = build_frontmatter(source_path, prior_frontmatter)
+    if prior_frontmatter:
+        return f"{block}\n{markdown.lstrip(chr(10))}"
+    evidence = _evidence_handle(source_path)
+    return f"{block}{evidence}\n{markdown.lstrip(chr(10))}"
+
+
+# ---------------------------------------------------------------------------
+# Strip (reverse: md -> docx)
+# ---------------------------------------------------------------------------
+
+def strip_mdk_frontmatter(markdown: str) -> tuple[str | None, str]:
+    """Split a leading ``---`` … ``---`` YAML frontmatter block off the body.
+
+    Returns ``(frontmatter_block_or_None, body)``. The returned frontmatter
+    block, when present, includes its ``---`` fences and a trailing newline so it
+    can be stored verbatim. A following ``mdk:evidence-handle`` comment is also
+    stripped from the body (it is conversion provenance, not document content).
+    """
+    if not markdown.startswith("---"):
+        return None, markdown
+    lines = markdown.split("\n")
+    if not lines or lines[0].strip() != "---":
+        return None, markdown
+    close_idx = None
+    for idx in range(1, len(lines)):
+        if lines[idx].strip() == "---":
+            close_idx = idx
+            break
+    if close_idx is None:
+        return None, markdown
+    block = "\n".join(lines[: close_idx + 1]) + "\n"
+    rest_lines = lines[close_idx + 1 :]
+    body = "\n".join(rest_lines)
+    body = _strip_leading_evidence_handle(body)
+    return block, body.lstrip("\n")
+
+
+def _strip_leading_evidence_handle(body: str) -> str:
+    """Remove a leading ``mdk:evidence-handle`` comment block from the body."""
+    stripped = body.lstrip("\n")
+    if not stripped.startswith(_EVIDENCE_OPEN):
+        return body
+    end = stripped.find(_EVIDENCE_CLOSE)
+    if end == -1:
+        return body
+    return stripped[end + len(_EVIDENCE_CLOSE) :]
+
+
+# ---------------------------------------------------------------------------
+# Source custom.xml read (forward)
+# ---------------------------------------------------------------------------
+
+def read_frontmatter_from_zip(zf: ZipFile) -> str | None:
+    """Return the verbatim ``mdk_frontmatter`` property from an OPC package, if any."""
+    if CUSTOM_XML_PART not in zf.namelist():
+        return None
+    try:
+        raw = zf.read(CUSTOM_XML_PART).decode("utf-8")
+    except (KeyError, UnicodeDecodeError):
+        return None
+    return _extract_property_value(raw, FRONTMATTER_PROPERTY_NAME)
+
+
+def _extract_property_value(custom_xml: str, name: str) -> str | None:
+    """Pull a custom property's ``<vt:lpwstr>`` text out of ``custom.xml``."""
+    from xml.etree import ElementTree as ET
+
+    try:
+        root = ET.fromstring(custom_xml)
+    except ET.ParseError:
+        return None
+    for prop in root:
+        if prop.attrib.get("name") != name:
+            continue
+        for child in prop:
+            if child.text is not None:
+                return _xml_unescape(child.text)
+            return ""
+    return None
+
+
+def _xml_unescape(value: str) -> str:
+    return (
+        value.replace("&lt;", "<")
+        .replace("&gt;", ">")
+        .replace("&quot;", '"')
+        .replace("&apos;", "'")
+        .replace("&amp;", "&")
+    )
+
+
+# ---------------------------------------------------------------------------
+# Inject custom.xml (reverse) — write frontmatter into the saved Office package
+# ---------------------------------------------------------------------------
+
+def _build_custom_xml(frontmatter: str) -> str:
+    body = _xml_escape(frontmatter)
+    return (
+        '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
+        f'<Properties xmlns="{CUSTOM_PROPS_NS}" xmlns:vt="{VT_NS}">'
+        f'<property fmtid="{CUSTOM_PROPS_FMTID}" pid="2" '
+        f'name="{FRONTMATTER_PROPERTY_NAME}">'
+        f"<vt:lpwstr>{body}</vt:lpwstr>"
+        "</property></Properties>"
+    )
+
+
+def _add_content_type_override(content_types_xml: str) -> str:
+    override = (
+        f'<Override PartName="/{CUSTOM_XML_PART}" '
+        f'ContentType="{CUSTOM_PROPS_CONTENT_TYPE}"/>'
+    )
+    if f'PartName="/{CUSTOM_XML_PART}"' in content_types_xml:
+        return content_types_xml
+    return content_types_xml.replace("</Types>", f"{override}</Types>")
+
+
+def _add_root_relationship(rels_xml: str) -> str:
+    if f'Target="{CUSTOM_XML_PART}"' in rels_xml or 'Id="rIdMDK"' in rels_xml:
+        return rels_xml
+    relationship = (
+        f'<Relationship Id="rIdMDK" Type="{CUSTOM_PROPS_REL_TYPE}" '
+        f'Target="{CUSTOM_XML_PART}"/>'
+    )
+    return rels_xml.replace("</Relationships>", f"{relationship}</Relationships>")
+
+
+def inject_frontmatter_into_package(package_path: Path, frontmatter: str) -> None:
+    """Add/replace the ``mdk_frontmatter`` custom property in an OPC package.
+
+    Rewrites ``docProps/custom.xml`` with the verbatim frontmatter, registers the
+    Content-Type Override, and adds the ``_rels/.rels`` Relationship. Works for
+    any OPC package (``.docx``/``.pptx``/``.xlsx``) since they share OPC packaging.
+    """
+    package_path = Path(package_path)
+    custom_xml = _build_custom_xml(frontmatter)
+    import tempfile
+
+    with tempfile.TemporaryDirectory(prefix="build-corpus-fm-") as tmp:
+        patched = Path(tmp) / package_path.name
+        with ZipFile(package_path) as src, ZipFile(patched, "w", ZIP_DEFLATED) as dst:
+            names = set(src.namelist())
+            for name in src.namelist():
+                if name == CUSTOM_XML_PART:
+                    continue  # replaced below
+                data = src.read(name)
+                if name == CONTENT_TYPES_PART:
+                    data = _add_content_type_override(data.decode("utf-8")).encode("utf-8")
+                elif name == ROOT_RELS_PART:
+                    data = _add_root_relationship(data.decode("utf-8")).encode("utf-8")
+                dst.writestr(name, data)
+            dst.writestr(CUSTOM_XML_PART, custom_xml)
+            if CONTENT_TYPES_PART not in names:
+                dst.writestr(CONTENT_TYPES_PART, _default_content_types())
+            if ROOT_RELS_PART not in names:
+                dst.writestr(ROOT_RELS_PART, _default_root_rels())
+        import shutil
+
+        shutil.move(str(patched), str(package_path))
+
+
+def _default_content_types() -> str:
+    return (
+        '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
+        '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+        f'<Override PartName="/{CUSTOM_XML_PART}" '
+        f'ContentType="{CUSTOM_PROPS_CONTENT_TYPE}"/>'
+        "</Types>"
+    )
+
+
+def _default_root_rels() -> str:
+    return (
+        '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
+        '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+        f'<Relationship Id="rIdMDK" Type="{CUSTOM_PROPS_REL_TYPE}" '
+        f'Target="{CUSTOM_XML_PART}"/>'
+        "</Relationships>"
+    )