metaobjects 0.9.0__py3-none-any.whl

metaobjects/render/extract/types.py

@@ -0,0 +1,332 @@
+"""FR-010 extract model + report.
+
+Frozen cross-port vocabularies (``FieldExtraction``, ``FieldKind``, ``Tolerance``,
+``Format``) plus the immutable schema/option/outcome dataclasses and the mutable
+``ExtractionReport`` accumulator.
+
+The corpus serializes ``FieldExtraction`` with SCREAMING_SNAKE values
+(``EXTRACTED`` / ``DEFAULTED`` / ``LOST_OPTIONAL`` / ``LOST_REQUIRED`` /
+``MALFORMED``) and ``Format`` / ``FieldKind`` as UPPER tokens; the conformance
+runner maps the schema-json tokens onto these enums.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Callable, Generic, TypeVar
+
+from metaobjects.render.extract.normalize import DEFAULT as _NORMALIZE_DEFAULT
+
+T = TypeVar("T")
+
+# A bespoke per-field coercion hook: (field_path, raw_value, spec) -> coerced | None.
+# Returning ``None`` falls through to the default coercion. Forward-referenced as a
+# string in ExtractOptions to avoid a forward-declaration cycle with FieldSpec.
+OnField = Callable[[str, str, "FieldSpec"], object | None]
+Normalizer = Callable[[str], object | None]
+
+
+class Format(Enum):
+    """Document format the extract pipeline targets."""
+
+    JSON = "JSON"
+    XML = "XML"
+
+
+class FieldKind(Enum):
+    """The coercion target kinds the engine understands. ``OBJECT`` = nested schema."""
+
+    STRING = "STRING"
+    INT = "INT"
+    LONG = "LONG"
+    DOUBLE = "DOUBLE"
+    BOOLEAN = "BOOLEAN"
+    ENUM = "ENUM"
+    OBJECT = "OBJECT"
+
+
+class FieldExtraction(Enum):
+    """FROZEN cross-port per-field extraction classification.
+
+    Do not reorder or add without an ADR. Values match the corpus serialization.
+    """
+
+    EXTRACTED = "EXTRACTED"
+    # FR-011: a value reached via @coerceDefault (present-but-uncoercible fallback)
+    # or @default (absent-fill) — distinct from a cleanly EXTRACTED value.
+    DEFAULTED = "DEFAULTED"
+    LOST_OPTIONAL = "LOST_OPTIONAL"
+    LOST_REQUIRED = "LOST_REQUIRED"
+    MALFORMED = "MALFORMED"
+
+
+class Tolerance(Enum):
+    """STRICT: case-sensitive, minimal repair. NORMAL: case-insensitive (default). LOOSE: maximal repair.
+
+    NOTE: LOOSE currently behaves identically to NORMAL (case-insensitive). Reserved
+    for future maximal-repair behavior.
+    """
+
+    STRICT = "STRICT"
+    NORMAL = "NORMAL"
+    LOOSE = "LOOSE"
+
+
+@dataclass(frozen=True, slots=True)
+class Coercion:
+    """A recorded normalization/coercion.
+
+    ``kind`` e.g. ``"normalize"``, ``"alias"``, ``"runtime-alias-override"``,
+    ``"clamp"``, ``"coerceDefault"``, ``"default"``.
+    """
+
+    field_path: str
+    from_: str
+    to: str
+    kind: str
+
+
+@dataclass(frozen=True, slots=True)
+class FieldSpec:
+    """One field's extract descriptor.
+
+    ``enum_values``/``enum_alias`` non-None only for ENUM; ``min``/``max`` non-None
+    only for numeric range constraints; ``nested`` non-None only for OBJECT.
+    """
+
+    name: str
+    kind: FieldKind
+    required: bool = False
+    array: bool = False
+    enum_values: list[str] | None = None
+    enum_alias: dict[str, str] | None = None
+    min: float | None = None
+    max: float | None = None
+    nested: "ExtractSchema | None" = None
+    # FR-011: present-but-uncoercible fallback member (from ``@coerceDefault``).
+    # ENUM-only; None = none.
+    coerce_default: str | None = None
+    # FR-011: absent-fill member (from ``@default``). ENUM-only; None = none.
+    default_value: str | None = None
+    # FR-011: resolved enum normalization mode (from ``@normalize``; default ``"strip"``).
+    normalize: str = _NORMALIZE_DEFAULT
+    # ``@xmlText``: this field receives its element's TEXT CONTENT (analogous to JAXB
+    # ``@XmlValue`` / Jackson ``@JacksonXmlText`` / .NET ``[XmlText]``). The extract engine
+    # reads it from the ``#text`` sentinel the lenient XML reader carries when an element has
+    # both attributes and a text body, instead of a same-named child. False for normal/JSON.
+    text_content: bool = False
+
+    @staticmethod
+    def scalar(
+        name: str,
+        kind: FieldKind,
+        required: bool,
+        default_value: str | None = None,
+    ) -> "FieldSpec":
+        """Phase B (generalized ``@default``): a scalar field optionally carrying an
+        absent-fill ``@default``. When the field is ABSENT, tolerant extract coerces
+        this string to ``kind`` (via the pure ``scalar_coerce``) and classifies the
+        field DEFAULTED (which satisfies ``required``). ``default_value is None`` is
+        the no-default case (back-compat)."""
+        return FieldSpec(
+            name=name, kind=kind, required=required, default_value=default_value
+        )
+
+    @staticmethod
+    def text_content_field(name: str, kind: FieldKind, required: bool) -> "FieldSpec":
+        """A field that receives its element's TEXT CONTENT — the ``@xmlText`` marker
+        (see ``text_content``). A scalar with the flag set; coerced to ``kind``."""
+        return FieldSpec(name=name, kind=kind, required=required, text_content=True)
+
+    @staticmethod
+    def scalar_array(name: str, kind: FieldKind, required: bool) -> "FieldSpec":
+        """A scalar-array FieldSpec (``array == True``); each element is coerced via
+        the scalar pipeline. No per-element default fill."""
+        return FieldSpec(name=name, kind=kind, required=required, array=True)
+
+    @staticmethod
+    def enum_array(
+        name: str,
+        required: bool,
+        values: list[str] | None,
+        aliases: dict[str, str] | None,
+        coerce_default: str | None = None,
+        normalize: str = _NORMALIZE_DEFAULT,
+        default_value: str | None = None,
+    ) -> "FieldSpec":
+        """Phase B (array-of-enum): an enum field that is a ``list[enum]``
+        (``array == True``). Each element flows through the SAME enum pipeline a
+        scalar enum uses (exact → normalize → ``@enumAlias`` → ``@coerceDefault`` →
+        MALFORMED) and is classified independently by indexed path (``tags[0]``,
+        ``tags[1]``, …). Mirrors :meth:`enum_field` but with ``array = True``."""
+        return FieldSpec(
+            name=name,
+            kind=FieldKind.ENUM,
+            required=required,
+            array=True,
+            enum_values=None if values is None else list(values),
+            enum_alias={} if aliases is None else dict(aliases),
+            coerce_default=coerce_default,
+            default_value=default_value,
+            normalize=normalize,
+        )
+
+    @staticmethod
+    def enum_field(
+        name: str,
+        required: bool,
+        values: list[str] | None,
+        aliases: dict[str, str] | None,
+        coerce_default: str | None = None,
+        normalize: str = _NORMALIZE_DEFAULT,
+        default_value: str | None = None,
+    ) -> "FieldSpec":
+        return FieldSpec(
+            name=name,
+            kind=FieldKind.ENUM,
+            required=required,
+            enum_values=None if values is None else list(values),
+            enum_alias={} if aliases is None else dict(aliases),
+            coerce_default=coerce_default,
+            default_value=default_value,
+            normalize=normalize,
+        )
+
+    @staticmethod
+    def range_(
+        name: str,
+        kind: FieldKind,
+        required: bool,
+        min: float | None,
+        max: float | None,
+    ) -> "FieldSpec":
+        return FieldSpec(name=name, kind=kind, required=required, min=min, max=max)
+
+    @staticmethod
+    def object_(
+        name: str,
+        required: bool,
+        array: bool,
+        nested: "ExtractSchema | None",
+    ) -> "FieldSpec":
+        return FieldSpec(
+            name=name,
+            kind=FieldKind.OBJECT,
+            required=required,
+            array=array,
+            nested=nested,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractSchema:
+    """Top-level extract descriptor.
+
+    ``root_name`` = the XML root tag / logical JSON root name.
+    """
+
+    format: Format
+    root_name: str
+    fields: list[FieldSpec] = field(default_factory=list)
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractOptions:
+    """Bounded runtime override surface (the "20%").
+
+    ``aliases``/``normalizers`` are MERGED with the schema's, runtime winning on key
+    conflict. ``on_field`` is the single bespoke-coercion hook.
+
+    ``rootless`` (XML only): when ``True``, the input has NO enclosing root element —
+    the payload's fields ARE the top-level elements (a flat sequence like
+    ``<a>..</a><b>..</b>``). Mirrors Java ``ExtractOptions.rootless``.
+    """
+
+    tolerance: Tolerance = Tolerance.NORMAL
+    aliases: dict[str, str] = field(default_factory=dict)
+    normalizers: dict[str, Normalizer] = field(default_factory=dict)
+    on_field: OnField | None = None
+    rootless: bool = False
+
+    @staticmethod
+    def defaults() -> "ExtractOptions":
+        return ExtractOptions()
+
+    def with_tolerance(self, t: Tolerance) -> "ExtractOptions":
+        return ExtractOptions(
+            tolerance=t,
+            aliases=dict(self.aliases),
+            normalizers=dict(self.normalizers),
+            on_field=self.on_field,
+            rootless=self.rootless,
+        )
+
+    def with_rootless(self, r: bool) -> "ExtractOptions":
+        """XML only: parse a rootless flat element sequence directly (no wrapper
+        root). Returns a copy with ``rootless`` set. Mirrors Java ``withRootless``."""
+        return ExtractOptions(
+            tolerance=self.tolerance,
+            aliases=dict(self.aliases),
+            normalizers=dict(self.normalizers),
+            on_field=self.on_field,
+            rootless=r,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractionOutcome:
+    """Engine return.
+
+    ``data`` is a forgiving ``dict[str, object]``; Plan 2 wraps it into a typed
+    ``ExtractionResult``.
+    """
+
+    data: dict[str, object]
+    report: "ExtractionReport"
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractionResult(Generic[T]):
+    """Typed result of a generated ``extract(...)``: best-effort value + report."""
+
+    data: T | None
+    report: "ExtractionReport"
+
+
+class ExtractionReport:
+    """Mutable accumulator of per-field classification, the empty flag, and coercion notes."""
+
+    def __init__(self) -> None:
+        self._states: dict[str, FieldExtraction] = {}
+        self._coercions: list[Coercion] = []
+        self._empty: bool = False
+
+    def set(self, field_path: str, state: FieldExtraction) -> None:
+        self._states[field_path] = state
+
+    def add_coercion(self, c: Coercion) -> None:
+        self._coercions.append(c)
+
+    def mark_empty(self) -> None:
+        self._empty = True
+
+    def is_empty(self) -> bool:
+        return self._empty
+
+    def states(self) -> dict[str, FieldExtraction]:
+        return dict(self._states)
+
+    def coercions(self) -> list[Coercion]:
+        return list(self._coercions)
+
+    def lost_required(self) -> list[str]:
+        return self._by_state(FieldExtraction.LOST_REQUIRED)
+
+    def malformed(self) -> list[str]:
+        return self._by_state(FieldExtraction.MALFORMED)
+
+    def has_lost_required(self) -> bool:
+        return len(self.lost_required()) > 0
+
+    def _by_state(self, s: FieldExtraction) -> list[str]:
+        return [k for k, v in self._states.items() if v == s]

metaobjects/render/extract/xml_forgiving_reader.py

@@ -0,0 +1,162 @@
+"""Stage-4 tolerant XML reader for the bounded corpus malformation set. Never throws.
+
+Mirrors Java XmlForgivingReader: maps an element's child elements, text, AND attributes
+into the field map, and handles self-closing tags (``<x a="1"/>``).
+
+Representation:
+
+- text-only element, no attributes      → its trimmed text (``str``) — unchanged
+- self-closing / attributes-only element → a dict of attribute name→value ("" when none)
+- element with child elements (± attrs)  → a dict merging attributes + child entries
+  (a child element wins a name collision)
+- element with text AND attributes       → a dict of the attributes plus the body text under
+  :data:`TEXT_KEY` (a scalar consumer unwraps it)
+- repeated sibling tags                  → a list
+
+Carries the FR-010 fixed-behavior edge cases:
+
+- No-throw on a leading ``</x>``.
+- Unclosed tags extract their text up to the next sibling open tag.
+"""
+from __future__ import annotations
+
+import re
+
+#: Reserved key holding an element's own text content when the element is represented as a
+#: dict (because it also carries attributes). ``#`` is not a legal XML name char, so it never
+#: collides with a real attribute or child-element name.
+TEXT_KEY = "#text"
+
+# tag name + everything up to the closing '>' (attributes and/or a trailing '/' for a
+# self-closing tag). Non-greedy so the first '>' closes the open tag.
+_OPEN_TAG = re.compile(r"<([A-Za-z_][A-Za-z0-9_]*)([^>]*?)>")
+_OPEN_TAG_CI = re.compile(r"<([A-Za-z_][A-Za-z0-9_]*)([^>]*?)>", re.IGNORECASE)
+# one attribute: name = "double" | 'single' | bareword.
+_ATTR = re.compile(r"""([A-Za-z_:][A-Za-z0-9_:.\-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s/>]+))""")
+
+
+class XmlForgivingReader:
+    def read(self, span: str | None, case_insensitive: bool) -> dict[str, object]:
+        out: dict[str, object] = {}
+        if span is None or span.strip() == "":
+            return out
+        gt = span.find(">")
+        if gt < 0:
+            return out
+        root_end = span.rfind("</")
+        inner_end = len(span) if (root_end < 0 or root_end <= gt) else root_end
+        inner = span[gt + 1 : inner_end]
+        self._parse_children(inner, case_insensitive, out)
+        return out
+
+    def read_rootless(self, text: str | None, case_insensitive: bool) -> dict[str, object]:
+        """Rootless read: parse the WHOLE text's top-level elements directly, with no
+        enclosing root element to strip (a flat sequence like ``<a>..</a><b>..</b>``).
+        Used for :attr:`ExtractOptions.rootless` responses. Leading/trailing non-element
+        text is ignored. Never throws. Mirrors Java ``readRootless``."""
+        out: dict[str, object] = {}
+        if text is None or text.strip() == "":
+            return out
+        self._parse_children(text, case_insensitive, out)
+        return out
+
+    def _parse_children(self, inner: str, ci: bool, out: dict[str, object]) -> None:
+        open_tag = _OPEN_TAG_CI if ci else _OPEN_TAG
+        pos = 0
+        while True:
+            m = open_tag.search(inner, pos)
+            if m is None:
+                return
+            tag = m.group(1)
+            key = tag.lower() if ci else tag
+
+            raw_attrs = (m.group(2) or "").strip()
+            self_closing = raw_attrs.endswith("/")
+            if self_closing:
+                raw_attrs = raw_attrs[:-1].strip()
+            attrs = self._parse_attrs(raw_attrs, ci)
+
+            if self_closing:
+                self._accumulate(out, key, "" if not attrs else attrs)
+                pos = m.end()
+                continue
+
+            content_start = m.end()
+            close_re = re.compile(
+                "</" + re.escape(tag) + r"\s*>", re.IGNORECASE if ci else 0
+            )
+            close_m = close_re.search(inner, content_start)
+            if close_m is not None:
+                content_end = close_m.start()
+                nxt = close_m.end()
+            else:
+                # unclosed tag: extract content up to the next sibling open tag.
+                sib = open_tag.search(inner, content_start)
+                if sib is not None:
+                    # When the unclosed element's content begins IMMEDIATELY with a child
+                    # open tag (no leading text), that child was almost certainly meant to
+                    # be NESTED, not a sibling — a common LLM malformation is dropping the
+                    # parent's close tag while still emitting a real child element
+                    # (e.g. <check ...><payoff>text). Absorb the remainder of this span as
+                    # the unclosed element's content so the child nests under it. When there
+                    # IS leading text before the first child tag (e.g. <t>hi<c>..), keep the
+                    # sibling split — the leading text is the unclosed element's body and the
+                    # following tag is its sibling. Mirrors Java XmlForgivingReader.
+                    no_leading_text = inner[content_start : sib.start()].strip() == ""
+                    if no_leading_text:
+                        content_end = len(inner)
+                        nxt = len(inner)
+                    else:
+                        content_end = sib.start()
+                        nxt = content_end
+                else:
+                    content_end = len(inner)
+                    nxt = len(inner)
+            content = inner[content_start:content_end]
+            self._accumulate(out, key, self._combine(attrs, content, ci))
+            pos = nxt
+
+    def _combine(self, attrs: dict[str, object], content: str, ci: bool) -> object:
+        """Combine an element's attributes with its body (nested children or plain text)."""
+        if "<" in content:
+            nested: dict[str, object] = {}
+            self._parse_children(content, ci, nested)
+            if nested:
+                # attributes first; a child element wins a name collision
+                merged: dict[str, object] = dict(attrs)
+                merged.update(nested)
+                return merged
+        return self._text_value(attrs, content)
+
+    def _text_value(self, attrs: dict[str, object], content: str) -> object:
+        text = content.strip()
+        if not attrs:
+            return text
+        m: dict[str, object] = dict(attrs)
+        m[TEXT_KEY] = text
+        return m
+
+    def _parse_attrs(self, raw_attrs: str, ci: bool) -> dict[str, object]:
+        attrs: dict[str, object] = {}
+        if not raw_attrs:
+            return attrs
+        for a in _ATTR.finditer(raw_attrs):
+            name = a.group(1).lower() if ci else a.group(1)
+            val = a.group(2) if a.group(2) is not None else (
+                a.group(3) if a.group(3) is not None else (
+                    a.group(4) if a.group(4) is not None else ""
+                )
+            )
+            if name not in attrs:
+                attrs[name] = val
+        return attrs
+
+    def _accumulate(self, out: dict[str, object], key: str, value: object) -> None:
+        if key not in out:
+            out[key] = value
+            return
+        existing = out[key]
+        if isinstance(existing, list):
+            existing.append(value)
+        else:
+            out[key] = [existing, value]

metaobjects/render/filesystem_provider.py

@@ -0,0 +1,51 @@
+"""Filesystem-backed :class:`~metaobjects.render.verify.Provider`.
+
+Resolves a ``group/source`` reference to ``<root>/group/source.mustache``. The render
+engine + verify delegate all I/O to a provider, so this is the only file-touching piece
+of the render tier. Mirrors the C# ``MetaObjects.Render.FilesystemProvider`` and the Java
+``com.metaobjects.render.FilesystemProvider`` semantics: read the file if present, return
+``None`` when absent/unreadable, and reject refs that escape the root (via ``..``).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class FilesystemProvider:
+    """Resolves ``group/source`` references to text files under a root directory:
+    ``resolve("npc/turn")`` → ``<root>/npc/turn.mustache``. Returns ``None`` when the file
+    is absent. Refs that escape the root (via ``..``) are rejected.
+
+    Implements the :class:`~metaobjects.render.verify.Provider` protocol.
+    """
+
+    def __init__(self, root: str | Path, extension: str = ".mustache") -> None:
+        self._root = Path(root).resolve()
+        self._extension = extension
+
+    def resolve(self, ref: str | None) -> str | None:
+        # Build <root>/<seg>/<seg> from the slash-separated ref, dropping empties.
+        if not ref:
+            return None
+        segments = [s for s in ref.split("/") if s]
+        if not segments or any(s == ".." for s in segments):
+            return None
+
+        base = self._root.joinpath(*segments)
+        candidate = base.with_name(base.name + self._extension)
+        try:
+            resolved = candidate.resolve()
+        except OSError:
+            return None
+
+        # Path-traversal guard: the resolved file must stay under the root.
+        if resolved != self._root and self._root not in resolved.parents:
+            return None
+
+        if not resolved.is_file():
+            return None
+        try:
+            return resolved.read_text(encoding="utf-8")
+        except OSError:
+            return None

metaobjects/render/prompt/__init__.py

@@ -0,0 +1,32 @@
+"""FR-010 artifact 1 — output-format prompt renderer ("produce your answer like this").
+
+Renders an :class:`OutputFormatSpec` into a prompt fragment that teaches an LLM how
+to shape its answer. Three comment-free styles (guide / inline / exampleOnly) × two
+formats (json / xml). Guidance is carried in prose / inline placeholders / a filled
+skeleton — NEVER in comments (models ignore them).
+
+Cross-port INVARIANT: the rendered text is byte-identical to the Java/C#/Kotlin/TS
+reference (``com.metaobjects.render.prompt.OutputFormatRenderer``).
+"""
+from __future__ import annotations
+
+from metaobjects.render.prompt.output_format_renderer import render_output_format
+from metaobjects.render.prompt.output_format_spec import OutputFormatSpec
+from metaobjects.render.prompt.prompt_field import PromptField
+from metaobjects.render.prompt.prompt_overrides import (
+    PROMPT_OVERRIDES_NONE,
+    PromptOverrides,
+    no_overrides,
+)
+from metaobjects.render.prompt.prompt_style import PromptStyle, prompt_style_from
+
+__all__ = [
+    "PROMPT_OVERRIDES_NONE",
+    "OutputFormatSpec",
+    "PromptField",
+    "PromptOverrides",
+    "PromptStyle",
+    "no_overrides",
+    "prompt_style_from",
+    "render_output_format",
+]