xstructured 0.1.0__py3-none-any.whl

xstructured/__init__.py

@@ -0,0 +1,40 @@
+"""Schema-guided structured output parsing."""
+
+from .core import ParserConfig, ParseResult, RecoveryConfig, RepairConfig, RepairError
+from .envelope import EnvelopeScanner, EnvelopeSpec
+from .langchain import XStructuredResult, XStructuredRunnable, with_xstructured_output
+from .parser import StructuredParser
+from .schema import (
+    NamedSchemas,
+    NamedSchemaSpec,
+    SchemaInfo,
+    fingerprint_schema,
+    inspect_named_schemas,
+    inspect_schema,
+    schema_instructions,
+)
+from .streaming import StreamDecoder, StreamEvent, StreamEventKind
+
+__all__ = [
+    "EnvelopeScanner",
+    "EnvelopeSpec",
+    "NamedSchemaSpec",
+    "NamedSchemas",
+    "ParseResult",
+    "ParserConfig",
+    "RecoveryConfig",
+    "RepairConfig",
+    "RepairError",
+    "SchemaInfo",
+    "StreamDecoder",
+    "StreamEvent",
+    "StreamEventKind",
+    "StructuredParser",
+    "XStructuredResult",
+    "XStructuredRunnable",
+    "fingerprint_schema",
+    "inspect_named_schemas",
+    "inspect_schema",
+    "schema_instructions",
+    "with_xstructured_output",
+]

xstructured/core/__init__.py

@@ -0,0 +1,25 @@
+"""Core configuration, errors, and result types."""
+
+from .config import ParserConfig, RecoveryConfig, RepairConfig
+from .errors import (
+    EnvelopeError,
+    ParseError,
+    RecoveryError,
+    RepairError,
+    SchemaError,
+    XStructuredError,
+)
+from .result import ParseResult
+
+__all__ = [
+    "EnvelopeError",
+    "ParseError",
+    "ParseResult",
+    "ParserConfig",
+    "RecoveryConfig",
+    "RecoveryError",
+    "RepairConfig",
+    "RepairError",
+    "SchemaError",
+    "XStructuredError",
+]

xstructured/core/config.py

@@ -0,0 +1,58 @@
+"""Configuration for structured parsing."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class RecoveryConfig(BaseModel):
+    """Controls conservative normalization before retrying JSON parsing."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    enabled: bool = True
+    strip_markdown_fences: bool = True
+    strip_surrounding_text: bool = True
+    max_candidates: int = Field(default=8, ge=1, le=100)
+
+
+class RepairConfig(BaseModel):
+    """Controls the optional, bounded LLM-assisted repair fallback.
+
+    Repair only runs when the caller explicitly supplies a repair
+    ``Runnable`` (for example via
+    ``with_xstructured_output(..., repair=repair_runnable)``); this config
+    only tunes how that opt-in fallback behaves once wired in. It is never
+    consulted when no repair ``Runnable`` is supplied, so existing
+    single-schema callers are unaffected by its defaults.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    enabled: bool = True
+    max_attempts: int = Field(default=1, ge=1, le=5)
+
+
+class ParserConfig(BaseModel):
+    """Controls parser limits and recovery behavior."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    max_input_chars: int = Field(default=1_000_000, ge=1)
+    max_envelope_chars: int = Field(default=1_000_000, ge=1)
+    max_payload_chars: int = Field(default=1_000_000, ge=1)
+    max_nesting_depth: int = Field(default=100, ge=1)
+    recovery: RecoveryConfig = Field(default_factory=RecoveryConfig)
+    require_envelope: bool = False
+
+    @field_validator(
+        "max_input_chars",
+        "max_envelope_chars",
+        "max_payload_chars",
+        "max_nesting_depth",
+    )
+    @classmethod
+    def validate_positive_limit(cls, value: int) -> int:
+        if value < 1:
+            raise ValueError("resource limits must be positive")
+        return value

xstructured/core/errors.py

@@ -0,0 +1,51 @@
+"""Domain-specific exceptions with parse context."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+class XStructuredError(Exception):
+    """Base exception for xstructured failures."""
+
+
+class SchemaError(XStructuredError):
+    """Raised when a schema target cannot be introspected or validated."""
+
+
+class EnvelopeError(XStructuredError):
+    """Raised for invalid envelope specifications or envelope extraction."""
+
+
+@dataclass(eq=False)
+class ParseError(XStructuredError):
+    """Raised when content cannot be parsed as JSON."""
+
+    message: str
+    text: str
+    cause: Exception | None = None
+
+    def __str__(self) -> str:
+        return self.message
+
+
+@dataclass(eq=False)
+class RecoveryError(ParseError):
+    """Raised after every configured recovery candidate has failed."""
+
+    attempts: tuple[str, ...] = ()
+    validation_errors: Any | None = None
+
+
+@dataclass(eq=False)
+class RepairError(ParseError):
+    """Raised when bounded, opt-in LLM-assisted repair exhausts its attempts.
+
+    Only raised when a repair ``Runnable`` was explicitly configured (see
+    ``RepairConfig``); repair is never attempted, and this error is never
+    raised, by default.
+    """
+
+    attempt_count: int = 0
+    repair_errors: tuple[str, ...] = ()

xstructured/core/result.py

@@ -0,0 +1,32 @@
+"""Result values returned by structured parsing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True, slots=True)
+class ParseResult(Generic[T]):
+    """A validated value and the exact parsing path used to produce it."""
+
+    value: T
+    raw: str
+    json_text: str
+    recovered: bool = False
+    envelope_found: bool = False
+    schema_name: str | None = None
+    """The matched schema name, when parsing against named multiple schemas.
+
+    ``None`` when the parser was constructed with a single schema target.
+    """
+    repaired: bool = False
+    """Whether this value came from the optional, bounded LLM repair fallback.
+
+    Always ``False`` for a plain `StructuredParser.parse` call; only ever
+    ``True`` when `xstructured.langchain.repair.Repairer` produced the value
+    after conservative recovery was exhausted.
+    """
+    repair_attempt_count: int = 0

xstructured/envelope/__init__.py

@@ -0,0 +1,6 @@
+"""Delimited envelope specifications and streaming scanner."""
+
+from .scanner import EnvelopeScanner, EnvelopeState, ScanEvent
+from .spec import EnvelopeSpec
+
+__all__ = ["EnvelopeScanner", "EnvelopeSpec", "EnvelopeState", "ScanEvent"]

xstructured/envelope/scanner.py

@@ -0,0 +1,174 @@
+"""Incremental scanner for delimited structured payloads."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from xstructured.core.errors import EnvelopeError
+
+from .spec import EnvelopeSpec
+
+
+class EnvelopeState(StrEnum):
+    SEEKING_START = "seeking_start"
+    COLLECTING = "collecting"
+    COMPLETE = "complete"
+
+
+@dataclass(frozen=True, slots=True)
+class ScanEvent:
+    """The scanner's observable state after accepting input."""
+
+    state: EnvelopeState
+    payload: str | None = None
+
+
+class EnvelopeScanner:
+    """Find one envelope across arbitrary text chunks."""
+
+    def __init__(
+        self,
+        spec: EnvelopeSpec | None = None,
+        *,
+        max_envelope_chars: int = 1_000_000,
+        max_payload_chars: int = 1_000_000,
+    ) -> None:
+        if max_envelope_chars < 1 or max_payload_chars < 1:
+            raise ValueError("Envelope limits must be positive")
+        self.spec = spec or EnvelopeSpec()
+        self._max_envelope_chars = max_envelope_chars
+        self._max_payload_chars = max_payload_chars
+        self._state = EnvelopeState.SEEKING_START
+        self._buffer = ""
+        self._payload: str | None = None
+        self._payload_parts: list[str] = []
+        self._payload_chars = 0
+        self._in_string = False
+        self._escaped = False
+
+    @property
+    def state(self) -> EnvelopeState:
+        return self._state
+
+    @property
+    def payload(self) -> str | None:
+        return self._payload
+
+    @property
+    def complete(self) -> bool:
+        return self._state is EnvelopeState.COMPLETE
+
+    def feed(self, chunk: str) -> ScanEvent:
+        """Accept a chunk and return the resulting scanner event."""
+        if not isinstance(chunk, str):
+            raise TypeError("Envelope scanner chunks must be strings")
+        if self.complete:
+            if chunk:
+                raise EnvelopeError("Cannot feed data after envelope completion")
+            return ScanEvent(self._state, self._payload)
+
+        self._buffer += chunk
+        if self._state is EnvelopeState.SEEKING_START:
+            start_at = self._buffer.find(self.spec.start)
+            if start_at < 0:
+                self._buffer = (
+                    self._buffer[-(len(self.spec.start) - 1) :] if len(self.spec.start) > 1 else ""
+                )
+                return ScanEvent(self._state)
+            self._buffer = self._buffer[start_at + len(self.spec.start) :]
+            self._state = EnvelopeState.COLLECTING
+
+        end_at = _find_json_safe_delimiter(
+            self._buffer,
+            self.spec.end,
+            in_string=self._in_string,
+            escaped=self._escaped,
+        )
+        if end_at < 0:
+            safe_length = max(0, len(self._buffer) - len(self.spec.end) + 1)
+            self._append_payload(self._take(safe_length))
+            self._validate_incomplete_limits()
+            return ScanEvent(self._state)
+
+        self._append_payload(self._take(end_at))
+        envelope_size = len(self.spec.start) + self._payload_chars + len(self.spec.end)
+        if envelope_size > self._max_envelope_chars:
+            raise EnvelopeError(
+                f"Envelope exceeds configured limit of {self._max_envelope_chars} characters"
+            )
+        self._take(len(self.spec.end))
+        self._payload = "".join(self._payload_parts)
+        self._buffer = ""
+        self._state = EnvelopeState.COMPLETE
+        return ScanEvent(self._state, self._payload)
+
+    def finalize(self) -> str:
+        """Return the payload, or raise when the envelope is incomplete."""
+        if self._payload is None:
+            raise EnvelopeError("No complete envelope was found")
+        return self._payload
+
+    def _validate_incomplete_limits(self) -> None:
+        envelope_size = len(self.spec.start) + self._payload_chars + len(self._buffer)
+        if envelope_size > self._max_envelope_chars:
+            raise EnvelopeError(
+                f"Envelope exceeds configured limit of {self._max_envelope_chars} characters"
+            )
+
+    def _append_payload(self, text: str) -> None:
+        if not text:
+            return
+        self._payload_parts.append(text)
+        self._payload_chars += len(text)
+        if self._payload_chars > self._max_payload_chars:
+            raise EnvelopeError(
+                f"Envelope payload exceeds configured limit of {self._max_payload_chars} characters"
+            )
+        self._in_string, self._escaped = _json_string_state(
+            text,
+            in_string=self._in_string,
+            escaped=self._escaped,
+        )
+
+    def _take(self, length: int) -> str:
+        value = self._buffer[:length]
+        self._buffer = self._buffer[length:]
+        return value
+
+
+def _find_json_safe_delimiter(
+    text: str,
+    delimiter: str,
+    *,
+    in_string: bool = False,
+    escaped: bool = False,
+) -> int:
+    """Find a delimiter outside JSON strings, including escaped string content."""
+    for index, character in enumerate(text):
+        if in_string:
+            if escaped:
+                escaped = False
+            elif character == "\\":
+                escaped = True
+            elif character == '"':
+                in_string = False
+        elif character == '"':
+            in_string = True
+        elif text.startswith(delimiter, index):
+            return index
+    return -1
+
+
+def _json_string_state(text: str, *, in_string: bool, escaped: bool) -> tuple[bool, bool]:
+    for character in text:
+        if in_string:
+            if escaped:
+                escaped = False
+            elif character == "\\":
+                escaped = True
+            elif character == '"':
+                in_string = False
+        elif character == '"':
+            in_string = True
+    return in_string, escaped

xstructured/envelope/spec.py

@@ -0,0 +1,25 @@
+"""Envelope delimiter specification."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from xstructured.core.errors import EnvelopeError
+
+
+@dataclass(frozen=True, slots=True)
+class EnvelopeSpec:
+    """Opening and closing delimiters surrounding a structured payload."""
+
+    start: str = "<xstructured>"
+    end: str = "</xstructured>"
+
+    def __post_init__(self) -> None:
+        if not self.start or not self.end:
+            raise EnvelopeError("Envelope delimiters must be non-empty")
+        if self.start == self.end:
+            raise EnvelopeError("Envelope delimiters must differ")
+
+    def wrap(self, payload: str) -> str:
+        """Wrap *payload* with this specification's delimiters."""
+        return f"{self.start}{payload}{self.end}"

xstructured/langchain/__init__.py

@@ -0,0 +1,10 @@
+"""LangChain-facing xstructured integration."""
+
+from .result import XStructuredResult
+from .runnable import XStructuredRunnable, with_xstructured_output
+
+__all__ = [
+    "XStructuredResult",
+    "XStructuredRunnable",
+    "with_xstructured_output",
+]

xstructured/langchain/_support.py

@@ -0,0 +1,19 @@
+"""Internal helpers shared by the LangChain integration modules."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from langchain_core.messages import BaseMessage
+
+
+def output_text(output: Any) -> str:
+    """Extract the text content of a wrapped or repair Runnable's raw output."""
+    if isinstance(output, str):
+        return output
+    if isinstance(output, BaseMessage):
+        return output.text
+    raise TypeError(
+        "The wrapped Runnable must return strings or LangChain "
+        f"BaseMessage values, not {type(output).__name__}"
+    )

xstructured/langchain/repair.py

@@ -0,0 +1,129 @@
+"""Optional, bounded LLM-assisted repair for otherwise-unparseable output.
+
+Recovery (`xstructured.core.RecoveryConfig`) only ever changes which
+substring of a response is treated as JSON; it never rewrites syntax (see
+ADR 0003). Repair is a separate, strictly opt-in escape hatch: it only runs
+when the caller supplies a *repair* `Runnable` to `with_xstructured_output`,
+and it is bounded by `RepairConfig.max_attempts` so it can never loop
+indefinitely. Every repaired response is re-validated by the same
+`StructuredParser` -- the same schema(s) and the same recovery rules -- as
+the original response, so repair can change what an *invalid* response was,
+but never weaken what counts as *valid*.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any, Generic, TypeVar
+
+from langchain_core.runnables import Runnable, RunnableConfig
+
+from xstructured.core import ParseError, ParseResult, RecoveryError, RepairConfig, RepairError
+from xstructured.parser import StructuredParser
+
+from ._support import output_text
+
+T = TypeVar("T")
+
+_REPAIR_PREAMBLE = (
+    "The following response failed schema validation and must be corrected. "
+    "Return only a corrected response; do not add commentary or explanation."
+)
+
+
+class Repairer(Generic[T]):
+    """Bounded, opt-in retry loop that asks a Runnable to fix invalid output."""
+
+    def __init__(
+        self,
+        parser: StructuredParser[T],
+        repair_runnable: Runnable[Any, Any],
+        config: RepairConfig,
+        instructions: str,
+    ) -> None:
+        self._parser = parser
+        self._repair_runnable = repair_runnable
+        self._config = config
+        self._instructions = instructions
+
+    def repair(
+        self,
+        text: str,
+        error: ParseError,
+        *,
+        config: RunnableConfig | None = None,
+    ) -> ParseResult[T]:
+        """Synchronously retry parsing, invoking the repair Runnable each attempt."""
+        if not self._config.enabled:
+            raise error
+        current_text, current_error = text, error
+        for attempt in range(1, self._config.max_attempts + 1):
+            prompt = self._build_prompt(current_text, current_error, attempt)
+            response = self._repair_runnable.invoke(prompt, config=config)
+            current_text = output_text(response)
+            try:
+                return replace(
+                    self._parser.parse(current_text),
+                    repaired=True,
+                    repair_attempt_count=attempt,
+                )
+            except ParseError as new_error:
+                current_error = new_error
+        raise self._exhausted(current_error)
+
+    async def arepair(
+        self,
+        text: str,
+        error: ParseError,
+        *,
+        config: RunnableConfig | None = None,
+    ) -> ParseResult[T]:
+        """Asynchronously retry parsing, invoking the repair Runnable each attempt."""
+        if not self._config.enabled:
+            raise error
+        current_text, current_error = text, error
+        for attempt in range(1, self._config.max_attempts + 1):
+            prompt = self._build_prompt(current_text, current_error, attempt)
+            response = await self._repair_runnable.ainvoke(prompt, config=config)
+            current_text = output_text(response)
+            try:
+                return replace(
+                    self._parser.parse(current_text),
+                    repaired=True,
+                    repair_attempt_count=attempt,
+                )
+            except ParseError as new_error:
+                current_error = new_error
+        raise self._exhausted(current_error)
+
+    def _build_prompt(self, text: str, error: ParseError, attempt: int) -> str:
+        messages = _error_messages(error)
+        lines = [
+            _REPAIR_PREAMBLE,
+            "",
+            self._instructions,
+            "",
+            f"Repair attempt {attempt} of {self._config.max_attempts}.",
+            "Validation errors:",
+            *(f"- {message}" for message in messages),
+            "",
+            "Invalid response:",
+            text,
+        ]
+        return "\n".join(lines)
+
+    def _exhausted(self, error: ParseError) -> RepairError:
+        return RepairError(
+            "LLM-assisted repair did not produce schema-valid output within "
+            f"the configured attempt budget ({self._config.max_attempts})",
+            error.text,
+            error,
+            attempt_count=self._config.max_attempts,
+            repair_errors=_error_messages(error),
+        )
+
+
+def _error_messages(error: ParseError) -> tuple[str, ...]:
+    if isinstance(error, RecoveryError) and error.attempts:
+        return error.attempts
+    return (str(error),)

xstructured/langchain/result.py

@@ -0,0 +1,41 @@
+"""Result type returned by the LangChain integration."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import Any, Generic, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True, slots=True)
+class XStructuredResult(Generic[T]):
+    """Natural language, validated data, and the unmodified model response."""
+
+    content: str
+    structured: T
+    raw: Any
+    raw_text: str
+    json_text: str
+    recovered: bool = False
+    metadata: Mapping[str, Any] = field(default_factory=lambda: MappingProxyType({}))
+    schema_name: str | None = None
+    """Which named schema matched, when wrapped with multiple named schemas.
+
+    ``None`` when `with_xstructured_output` was called with a single schema
+    target.
+    """
+    repaired: bool = False
+    """Whether the optional, bounded LLM repair fallback produced this value.
+
+    Always ``False`` unless a *repair* `Runnable` was supplied to
+    `with_xstructured_output` and conservative recovery was exhausted first.
+    """
+    repair_attempt_count: int = 0
+
+    @property
+    def text(self) -> str:
+        """Alias for the natural-language content."""
+        return self.content