dtxt 0.1.0__py3-none-any.whl

dtxt/__init__.py

@@ -0,0 +1,23 @@
+"""dtxt: schema-centric bidirectional conversion between text and structured data."""
+
+from . import backends
+from ._config import configure
+from .d2t import render
+from .infer import InferError, infer_schema
+from .roundtrip import RoundtripResult, check_roundtrip
+from .schema import Schema
+from .t2d import ParseError, parse, parse_many
+
+__all__ = [
+    "InferError",
+    "ParseError",
+    "RoundtripResult",
+    "Schema",
+    "backends",
+    "check_roundtrip",
+    "configure",
+    "infer_schema",
+    "parse",
+    "parse_many",
+    "render",
+]

dtxt/_config.py

@@ -0,0 +1,55 @@
+"""Global backend configuration, one slot per public function.
+
+``configure()`` sets the default; each function's own ``backend=``
+argument overrides it for that single call.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .backends.base import Backend
+
+
+@dataclass
+class _Config:
+    infer: Backend | None = None
+    parse: Backend | None = None
+    render: Backend | None = None
+
+
+_config = _Config()
+
+
+def configure(
+    *,
+    infer: Backend | None = None,
+    parse: Backend | None = None,
+    render: Backend | None = None,
+) -> None:
+    """Set the default backend used by ``infer_schema`` / ``parse`` / ``render``."""
+    if infer is not None:
+        _config.infer = infer
+    if parse is not None:
+        _config.parse = parse
+    if render is not None:
+        _config.render = render
+
+
+def resolve_backend(function: str, override: Backend | None) -> Backend:
+    if override is not None:
+        return override
+    configured: Backend | None = getattr(_config, function)
+    if configured is None:
+        raise RuntimeError(
+            f"No backend configured for '{function}'. "
+            f"Call dtxt.configure({function}=...) or pass backend=... explicitly."
+        )
+    return configured
+
+
+def reset() -> None:
+    """Clear all configured backends. Mainly useful between tests."""
+    _config.infer = None
+    _config.parse = None
+    _config.render = None

dtxt/_util.py

@@ -0,0 +1,18 @@
+"""Small helpers shared across the T2D / D2T / infer pipelines."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def extract_json(raw: str) -> Any:
+    """Parse a JSON value out of a model response, tolerating code fences."""
+    text = raw.strip()
+    if text.startswith("```"):
+        text = text.strip("`")
+        if "\n" in text:
+            first_line, rest = text.split("\n", 1)
+            if first_line.strip().isalpha():
+                text = rest
+    return json.loads(text)

dtxt/backends/__init__.py

@@ -0,0 +1,33 @@
+"""Backend implementations for dtxt.
+
+Only the mock backend is imported eagerly. API/local backends
+(``Anthropic``, ``OpenAI``, ``LlamaCpp``) live in optional extras; their
+modules -- and the underlying SDK packages -- are imported lazily on first
+attribute access, so importing ``dtxt`` never requires them to be
+installed.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+from .base import Backend
+from .mock import MockBackend
+
+__all__ = ["Backend", "MockBackend", "Anthropic", "OpenAI", "LlamaCpp"]
+
+_LAZY: dict[str, tuple[str, str]] = {
+    "Anthropic": (".anthropic", "Anthropic"),
+    "OpenAI": (".openai", "OpenAI"),
+    "LlamaCpp": (".llamacpp", "LlamaCpp"),
+}
+
+
+def __getattr__(name: str) -> Any:
+    try:
+        module_name, attr_name = _LAZY[name]
+    except KeyError:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+    module = importlib.import_module(module_name, __name__)
+    return getattr(module, attr_name)

dtxt/backends/anthropic.py

@@ -0,0 +1,114 @@
+"""Anthropic API backend: structured output via forced tool use.
+
+The ``anthropic`` package is an optional extra (``dtxt[anthropic]``) and is
+imported lazily, only when a client actually needs to be built.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .base import TOOL_CALLING
+
+_INSTALL_HINT = "pip install dtxt[anthropic]"
+_TOOL_NAME = "emit_result"
+
+
+def _import_anthropic() -> Any:
+    try:
+        import anthropic
+    except ImportError as exc:
+        raise ImportError(
+            "the 'anthropic' package is required to use dtxt.backends.Anthropic. "
+            f"Install it with: {_INSTALL_HINT}"
+        ) from exc
+    return anthropic
+
+
+def _build_request_kwargs(
+    model: str, prompt: str, schema: dict[str, Any] | None, max_tokens: int
+) -> dict[str, Any]:
+    kwargs: dict[str, Any] = {
+        "model": model,
+        "max_tokens": max_tokens,
+        "messages": [{"role": "user", "content": prompt}],
+    }
+    if schema is not None:
+        kwargs["tools"] = [
+            {
+                "name": _TOOL_NAME,
+                "description": "Emit the result as structured data matching the schema.",
+                "input_schema": schema,
+            }
+        ]
+        kwargs["tool_choice"] = {"type": "tool", "name": _TOOL_NAME}
+    return kwargs
+
+
+def _extract_result(message: Any, schema: dict[str, Any] | None) -> str:
+    if schema is not None:
+        for block in message.content:
+            if getattr(block, "type", None) == "tool_use" and getattr(block, "name", None) == (
+                _TOOL_NAME
+            ):
+                return json.dumps(block.input)
+        raise RuntimeError("Anthropic response did not include the expected tool_use block")
+    return "".join(
+        block.text for block in message.content if getattr(block, "type", None) == "text"
+    )
+
+
+class Anthropic:
+    """Backend for the Anthropic Messages API.
+
+    Structured output is obtained by wrapping the JSON Schema as a single
+    tool and forcing ``tool_choice`` onto it, so ``tool_use.input`` is the
+    extracted object. This is not treated as ``constrained_decoding``: the
+    API does not guarantee full schema conformance the way grammar-based
+    decoding does, so dtxt still runs it through the retry + validation
+    loop.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        api_key: str | None = None,
+        max_tokens: int = 4096,
+        client: Any | None = None,
+        async_client: Any | None = None,
+    ) -> None:
+        self._model = model
+        self._api_key = api_key
+        self._max_tokens = max_tokens
+        self._client = client
+        self._async_client = async_client
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            anthropic = _import_anthropic()
+            kwargs: dict[str, Any] = {"api_key": self._api_key} if self._api_key else {}
+            self._client = anthropic.Anthropic(**kwargs)
+        return self._client
+
+    def _get_async_client(self) -> Any:
+        if self._async_client is None:
+            anthropic = _import_anthropic()
+            kwargs: dict[str, Any] = {"api_key": self._api_key} if self._api_key else {}
+            self._async_client = anthropic.AsyncAnthropic(**kwargs)
+        return self._async_client
+
+    @property
+    def capabilities(self) -> set[str]:
+        return {TOOL_CALLING}
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        kwargs = _build_request_kwargs(self._model, prompt, schema, self._max_tokens)
+        message = self._get_client().messages.create(**kwargs)
+        return _extract_result(message, schema)
+
+    async def agenerate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        kwargs = _build_request_kwargs(self._model, prompt, schema, self._max_tokens)
+        message = await self._get_async_client().messages.create(**kwargs)
+        return _extract_result(message, schema)

dtxt/backends/base.py

@@ -0,0 +1,27 @@
+"""Backend abstraction: the interface every dtxt backend implements."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+CONSTRAINED_DECODING = "constrained_decoding"
+JSON_MODE = "json_mode"
+TOOL_CALLING = "tool_calling"
+
+
+@runtime_checkable
+class Backend(Protocol):
+    """Anything that can turn a prompt into text, optionally schema-guided.
+
+    ``capabilities`` tells the caller which execution path to take:
+
+    - ``constrained_decoding``: the backend enforces the JSON schema at the
+      grammar level, so callers may skip syntactic validation and only run
+      semantic checks.
+    - otherwise: callers must retry and validate the output themselves.
+    """
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str: ...
+
+    @property
+    def capabilities(self) -> set[str]: ...

dtxt/backends/llamacpp.py

@@ -0,0 +1,126 @@
+"""llama.cpp backend: local GGUF models with GBNF-constrained decoding.
+
+``llama-cpp-python`` is a heavy, optional extra (``dtxt[llamacpp]``) and is
+imported lazily. Generation constrains JSON output at the grammar level
+via ``response_format``. Schema constructs GBNF can't reliably express
+(``format``, ``pattern``, deep nesting) are stripped from the
+grammar-facing copy of the schema; the original schema is left untouched
+for dtxt's normal post-hoc jsonschema validation. That is the two-stage
+split: what the grammar guarantees during decoding, and what gets checked
+afterwards.
+"""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from .base import CONSTRAINED_DECODING
+
+_INSTALL_HINT = "pip install dtxt[llamacpp]"
+
+DEFAULT_MAX_GRAMMAR_DEPTH = 6
+DEFAULT_PROMPT_TEMPLATE = "{prompt}"
+
+_UNGRAMMARABLE_KEYWORDS = ("format", "pattern")
+
+
+def _import_llama_cpp() -> Any:
+    try:
+        import llama_cpp
+    except ImportError as exc:
+        raise ImportError(
+            "the 'llama-cpp-python' package is required to use dtxt.backends.LlamaCpp. "
+            f"Install it with: {_INSTALL_HINT}"
+        ) from exc
+    return llama_cpp
+
+
+def grammar_safe_schema(
+    schema: dict[str, Any], *, max_depth: int = DEFAULT_MAX_GRAMMAR_DEPTH
+) -> dict[str, Any]:
+    """Return a copy of ``schema`` with GBNF-unfriendly constructs removed.
+
+    Drops ``format``/``pattern`` everywhere, and replaces ``object``/
+    ``array`` schemas past ``max_depth`` nested levels with an
+    unconstrained placeholder of the same type. The caller is expected to
+    still validate the original schema post-hoc.
+    """
+    return cast(dict[str, Any], _strip(schema, depth=0, max_depth=max_depth))
+
+
+def _strip(node: Any, *, depth: int, max_depth: int) -> Any:
+    if not isinstance(node, dict):
+        return node
+
+    if depth > max_depth:
+        node_type = node.get("type")
+        if node_type in ("object", "array"):
+            return {"type": node_type}
+        return dict(node)
+
+    stripped = {key: value for key, value in node.items() if key not in _UNGRAMMARABLE_KEYWORDS}
+    if "properties" in stripped:
+        stripped["properties"] = {
+            name: _strip(prop, depth=depth + 1, max_depth=max_depth)
+            for name, prop in stripped["properties"].items()
+        }
+    if "items" in stripped:
+        stripped["items"] = _strip(stripped["items"], depth=depth + 1, max_depth=max_depth)
+    return stripped
+
+
+class LlamaCpp:
+    """Backend for local GGUF models via ``llama-cpp-python``.
+
+    Small local models are prompt-sensitive, so the prompt can be wrapped
+    in a model-specific chat template by passing ``prompt_template`` (a
+    ``str.format`` template with a single ``{prompt}`` placeholder).
+
+    In-process inference assumes a single stream: batches are processed
+    sequentially (no ``agenerate``), so llama.cpp's prompt cache is reused
+    across calls sharing a prefix instead of contending for one model
+    instance in parallel.
+    """
+
+    def __init__(
+        self,
+        model_path: str,
+        *,
+        n_ctx: int = 4096,
+        max_tokens: int = 1024,
+        max_grammar_depth: int = DEFAULT_MAX_GRAMMAR_DEPTH,
+        prompt_template: str = DEFAULT_PROMPT_TEMPLATE,
+        llama: Any | None = None,
+        **llama_kwargs: Any,
+    ) -> None:
+        self._model_path = model_path
+        self._n_ctx = n_ctx
+        self._max_tokens = max_tokens
+        self._max_grammar_depth = max_grammar_depth
+        self._prompt_template = prompt_template
+        self._llama_kwargs = llama_kwargs
+        self._llama = llama
+
+    def _get_llama(self) -> Any:
+        if self._llama is None:
+            llama_cpp = _import_llama_cpp()
+            self._llama = llama_cpp.Llama(
+                model_path=self._model_path, n_ctx=self._n_ctx, **self._llama_kwargs
+            )
+        return self._llama
+
+    @property
+    def capabilities(self) -> set[str]:
+        return {CONSTRAINED_DECODING}
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        llama = self._get_llama()
+        kwargs: dict[str, Any] = {
+            "messages": [{"role": "user", "content": self._prompt_template.format(prompt=prompt)}],
+            "max_tokens": self._max_tokens,
+        }
+        if schema is not None:
+            safe_schema = grammar_safe_schema(schema, max_depth=self._max_grammar_depth)
+            kwargs["response_format"] = {"type": "json_object", "schema": safe_schema}
+        response = llama.create_chat_completion(**kwargs)
+        return str(response["choices"][0]["message"]["content"])

dtxt/backends/mock.py

@@ -0,0 +1,58 @@
+"""A deterministic, in-process backend used for unit tests."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable, Iterator
+from typing import Any
+
+_TYPE_PLACEHOLDERS: dict[str, Any] = {
+    "string": "",
+    "integer": 0,
+    "number": 0.0,
+    "boolean": False,
+    "array": [],
+    "object": {},
+}
+
+
+def _placeholder_for(schema: dict[str, Any]) -> dict[str, Any]:
+    properties = schema.get("properties", {})
+    return {
+        name: _TYPE_PLACEHOLDERS.get(prop.get("type"), None) for name, prop in properties.items()
+    }
+
+
+class MockBackend:
+    """No network, no randomness: responses are canned or computed locally.
+
+    Every call is recorded in ``calls`` so tests can assert on the prompts
+    a caller produced.
+    """
+
+    def __init__(
+        self,
+        responses: list[str] | None = None,
+        *,
+        generate_fn: Callable[[str, dict[str, Any] | None], str] | None = None,
+        capabilities: set[str] | None = None,
+    ) -> None:
+        self._responses: Iterator[str] | None = iter(responses) if responses is not None else None
+        self._generate_fn = generate_fn
+        self._capabilities = capabilities if capabilities is not None else set()
+        self.calls: list[tuple[str, dict[str, Any] | None]] = []
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        self.calls.append((prompt, schema))
+        if self._generate_fn is not None:
+            return self._generate_fn(prompt, schema)
+        if self._responses is not None:
+            try:
+                return next(self._responses)
+            except StopIteration as exc:
+                raise RuntimeError("MockBackend ran out of canned responses") from exc
+        return json.dumps(_placeholder_for(schema)) if schema is not None else ""
+
+    @property
+    def capabilities(self) -> set[str]:
+        return self._capabilities

dtxt/backends/openai.py

@@ -0,0 +1,96 @@
+"""OpenAI API backend: structured outputs via ``response_format``.
+
+The ``openai`` package is an optional extra (``dtxt[openai]``) and is
+imported lazily, only when a client actually needs to be built.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .base import JSON_MODE
+
+_INSTALL_HINT = "pip install dtxt[openai]"
+
+
+def _import_openai() -> Any:
+    try:
+        import openai
+    except ImportError as exc:
+        raise ImportError(
+            "the 'openai' package is required to use dtxt.backends.OpenAI. "
+            f"Install it with: {_INSTALL_HINT}"
+        ) from exc
+    return openai
+
+
+def _build_request_kwargs(model: str, prompt: str, schema: dict[str, Any] | None) -> dict[str, Any]:
+    kwargs: dict[str, Any] = {
+        "model": model,
+        "messages": [{"role": "user", "content": prompt}],
+    }
+    if schema is not None:
+        kwargs["response_format"] = {
+            "type": "json_schema",
+            "json_schema": {"name": "dtxt_result", "schema": schema},
+        }
+    return kwargs
+
+
+def _extract_result(response: Any) -> str:
+    content = response.choices[0].message.content
+    if content is None:
+        raise RuntimeError("OpenAI response contained no content")
+    return str(content)
+
+
+class OpenAI:
+    """Backend for the OpenAI Chat Completions API.
+
+    Uses ``response_format={"type": "json_schema", ...}`` (structured
+    outputs) when a schema is given. This is not treated as
+    ``constrained_decoding``: OpenAI's guarantee is on JSON syntax, not on
+    every schema keyword dtxt supports, so dtxt still runs it through the
+    retry + validation loop.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        api_key: str | None = None,
+        client: Any | None = None,
+        async_client: Any | None = None,
+    ) -> None:
+        self._model = model
+        self._api_key = api_key
+        self._client = client
+        self._async_client = async_client
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            openai = _import_openai()
+            kwargs: dict[str, Any] = {"api_key": self._api_key} if self._api_key else {}
+            self._client = openai.OpenAI(**kwargs)
+        return self._client
+
+    def _get_async_client(self) -> Any:
+        if self._async_client is None:
+            openai = _import_openai()
+            kwargs: dict[str, Any] = {"api_key": self._api_key} if self._api_key else {}
+            self._async_client = openai.AsyncOpenAI(**kwargs)
+        return self._async_client
+
+    @property
+    def capabilities(self) -> set[str]:
+        return {JSON_MODE}
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        kwargs = _build_request_kwargs(self._model, prompt, schema)
+        response = self._get_client().chat.completions.create(**kwargs)
+        return _extract_result(response)
+
+    async def agenerate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        kwargs = _build_request_kwargs(self._model, prompt, schema)
+        response = await self._get_async_client().chat.completions.create(**kwargs)
+        return _extract_result(response)

dtxt/d2t.py

@@ -0,0 +1,53 @@
+"""D2T: object -> text."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ._config import resolve_backend
+from .backends.base import Backend
+from .prompts import build_d2t_prompt
+from .schema import Schema
+
+
+def _field_guidance(schema: Schema) -> str:
+    lines: list[str] = []
+    for name in schema.properties:
+        parts: list[str] = []
+        description = schema.field_description(name)
+        if description:
+            parts.append(description)
+        examples = schema.field_examples(name)
+        if examples:
+            parts.append(f"examples: {examples}")
+        style = schema.field_style(name)
+        if style:
+            parts.append(f"style: {style}")
+        if parts:
+            lines.append(f"- {name}: {'; '.join(parts)}")
+    return "\n".join(lines) if lines else "(none)"
+
+
+def render(
+    obj: dict[str, Any],
+    schema: Schema,
+    *,
+    style: str | None = None,
+    backend: Backend | None = None,
+) -> str:
+    """Convert ``obj`` into text, guided by ``schema``'s ``x-dtxt-*`` metadata.
+
+    ``style`` overrides the schema's own ``x-dtxt-style`` (its root-level
+    style hint) for this call; per-field style hints still apply on top of
+    either one.
+    """
+    resolved = resolve_backend("render", backend)
+    json_schema = schema.to_json_schema()
+    effective_style = style if style is not None else schema.style
+    prompt = build_d2t_prompt(
+        obj,
+        json_schema,
+        style=effective_style or "(none)",
+        field_guidance=_field_guidance(schema),
+    )
+    return resolved.generate(prompt)

dtxt/infer.py

@@ -0,0 +1,96 @@
+"""infer_schema: schema inference via sampling + merge.
+
+Texts are sampled in small batches (kind to small context windows); each
+batch yields a candidate schema, and a field is kept in the merged schema
+only if it appears in at least ``min_coverage`` of the candidates.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from typing import Any
+
+from ._config import resolve_backend
+from ._util import extract_json
+from .backends.base import Backend
+from .prompts import build_infer_prompt
+from .schema import Schema
+
+DEFAULT_BATCH_SIZE = 5
+DEFAULT_MIN_COVERAGE = 0.6
+
+
+class InferError(Exception):
+    """Raised when schema inference cannot produce a usable schema."""
+
+
+def _batched(items: list[str], size: int) -> list[list[str]]:
+    return [items[i : i + size] for i in range(0, len(items), size)]
+
+
+def _candidate_schema(batch: list[str], backend: Backend) -> dict[str, Any] | None:
+    prompt = build_infer_prompt(batch)
+    raw = backend.generate(prompt)
+    try:
+        candidate = extract_json(raw)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(candidate, dict) or not isinstance(candidate.get("properties"), dict):
+        return None
+    return candidate
+
+
+def _merge_candidates(candidates: list[dict[str, Any]], min_coverage: float) -> dict[str, Any]:
+    n = len(candidates)
+    field_types: dict[str, Counter[str]] = {}
+    field_count: Counter[str] = Counter()
+
+    for candidate in candidates:
+        for name, field_schema in candidate["properties"].items():
+            field_count[name] += 1
+            field_types.setdefault(name, Counter())[field_schema.get("type", "string")] += 1
+
+    properties: dict[str, Any] = {}
+    required: list[str] = []
+    for name, count in field_count.items():
+        coverage = count / n
+        if coverage < min_coverage:
+            continue
+        most_common_type, _ = field_types[name].most_common(1)[0]
+        properties[name] = {"type": most_common_type}
+        if coverage >= 1.0:
+            required.append(name)
+
+    return {"type": "object", "properties": properties, "required": sorted(required)}
+
+
+def infer_schema(
+    texts: list[str],
+    *,
+    backend: Backend | None = None,
+    batch_size: int = DEFAULT_BATCH_SIZE,
+    min_coverage: float = DEFAULT_MIN_COVERAGE,
+) -> Schema:
+    """Infer a JSON Schema shared by ``texts``."""
+    if not texts:
+        raise InferError("cannot infer a schema from an empty text collection")
+
+    resolved = resolve_backend("infer", backend)
+    batches = _batched(texts, batch_size)
+
+    candidates = [
+        candidate
+        for candidate in (_candidate_schema(batch, resolved) for batch in batches)
+        if candidate is not None
+    ]
+    if not candidates:
+        raise InferError("backend did not return any usable candidate schema")
+
+    merged = _merge_candidates(candidates, min_coverage)
+    if not merged["properties"]:
+        raise InferError(
+            f"no fields met the min_coverage={min_coverage} threshold across "
+            f"{len(candidates)} candidate schema(s)"
+        )
+    return Schema(merged)

dtxt/prompts.py

@@ -0,0 +1,101 @@
+"""Prompt templates for T2D, D2T, and schema inference.
+
+Kept as plain module constants (rather than inline f-strings in the
+pipeline modules) so a caller or a backend can override them -- e.g. a
+small local model with different prompt sensitivities -- without touching
+``t2d.py`` / ``d2t.py`` / ``infer.py``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+T2D_TEMPLATE = """\
+You convert unstructured text into a single JSON object that strictly follows the given JSON Schema.
+
+# JSON Schema
+{schema}
+
+# Text
+{text}
+
+Respond with ONLY the JSON object. Do not include markdown fences or commentary.
+If a field's value cannot be determined from the text, set it to null."""
+
+T2D_RETRY_TEMPLATE = """\
+{base_prompt}
+
+# Previous attempt
+{previous_output}
+
+# Validation errors
+{errors}
+
+Fix the JSON object so it satisfies the schema and passes validation.
+Respond with ONLY the corrected JSON object."""
+
+D2T_TEMPLATE = """\
+You convert a JSON object into natural, fluent text that expresses every non-null field.
+
+# JSON Schema
+{schema}
+
+# Overall style
+{style}
+
+# Field guidance
+{field_guidance}
+
+# Object
+{obj}
+
+Write the text now. Do not include the JSON, labels, or commentary -- only the resulting text."""
+
+INFER_TEMPLATE = """\
+You infer a JSON Schema that describes the common structure shared by the following texts.
+
+# Texts
+{texts}
+
+Respond with ONLY a JSON Schema object (type "object", with "properties" and "required").
+Do not include markdown fences or commentary."""
+
+
+def build_t2d_prompt(text: str, schema: dict[str, Any], *, template: str = T2D_TEMPLATE) -> str:
+    return template.format(schema=json.dumps(schema, ensure_ascii=False, indent=2), text=text)
+
+
+def build_t2d_retry_prompt(
+    base_prompt: str,
+    previous_output: str,
+    errors: list[str],
+    *,
+    template: str = T2D_RETRY_TEMPLATE,
+) -> str:
+    return template.format(
+        base_prompt=base_prompt,
+        previous_output=previous_output,
+        errors="\n".join(f"- {error}" for error in errors),
+    )
+
+
+def build_d2t_prompt(
+    obj: dict[str, Any],
+    schema: dict[str, Any],
+    *,
+    style: str = "(none)",
+    field_guidance: str = "(none)",
+    template: str = D2T_TEMPLATE,
+) -> str:
+    return template.format(
+        schema=json.dumps(schema, ensure_ascii=False, indent=2),
+        obj=json.dumps(obj, ensure_ascii=False, indent=2),
+        style=style,
+        field_guidance=field_guidance,
+    )
+
+
+def build_infer_prompt(texts: list[str], *, template: str = INFER_TEMPLATE) -> str:
+    joined = "\n\n".join(f"[{i + 1}] {text}" for i, text in enumerate(texts))
+    return template.format(texts=joined)

dtxt/roundtrip.py

@@ -0,0 +1,54 @@
+"""round-trip verification: ``parse(render(obj)) ≈ obj``.
+
+Treated as a first-class feature: this is the main way to tell whether a
+schema + backend pair actually preserves information through D2T then T2D.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .backends.base import Backend
+from .d2t import render
+from .schema import Schema
+from .t2d import parse
+
+
+@dataclass
+class RoundtripResult:
+    original: dict[str, Any]
+    rendered_text: str
+    reparsed: dict[str, Any]
+    mismatches: dict[str, tuple[Any, Any]] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return not self.mismatches
+
+
+def check_roundtrip(
+    obj: dict[str, Any],
+    schema: Schema,
+    *,
+    render_backend: Backend | None = None,
+    parse_backend: Backend | None = None,
+) -> RoundtripResult:
+    """Render ``obj`` to text, parse it back, and diff against ``obj``.
+
+    Comparison is restricted to ``schema``'s declared fields, since ``parse``
+    only ever populates those.
+    """
+    text = render(obj, schema, backend=render_backend)
+    reparsed = parse(text, schema, backend=parse_backend)
+
+    mismatches: dict[str, tuple[Any, Any]] = {}
+    for key in schema.properties:
+        original_value = obj.get(key)
+        reparsed_value = reparsed.get(key)
+        if original_value != reparsed_value:
+            mismatches[key] = (original_value, reparsed_value)
+
+    return RoundtripResult(
+        original=obj, rendered_text=text, reparsed=reparsed, mismatches=mismatches
+    )

dtxt/schema.py

@@ -0,0 +1,116 @@
+"""Schema: the JSON-Schema-compatible internal representation used across dtxt."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Optional
+
+import jsonschema
+from pydantic import BaseModel, create_model
+
+_EXT_DESCRIPTION = "x-dtxt-description"
+_EXT_EXAMPLES = "x-dtxt-examples"
+_EXT_STYLE = "x-dtxt-style"
+
+_JSON_TYPE_TO_PY: dict[str, Any] = {
+    "string": str,
+    "integer": int,
+    "number": float,
+    "boolean": bool,
+    "array": list,
+    "object": dict,
+}
+
+
+class Schema:
+    """A JSON-Schema-compatible schema.
+
+    D2T description metadata (description, examples, style hints) is carried
+    as ``x-dtxt-*`` extension keywords on field schemas so it round-trips
+    through plain JSON Schema tooling.
+    """
+
+    def __init__(self, json_schema: dict[str, Any]) -> None:
+        self._json_schema = copy.deepcopy(json_schema)
+
+    def __repr__(self) -> str:
+        return f"Schema({self._json_schema!r})"
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Schema):
+            return NotImplemented
+        return self._json_schema == other._json_schema
+
+    @classmethod
+    def from_pydantic(cls, model: type[BaseModel]) -> Schema:
+        return cls(model.model_json_schema())
+
+    @classmethod
+    def from_dict(cls, json_schema: dict[str, Any]) -> Schema:
+        return cls(json_schema)
+
+    def to_json_schema(self) -> dict[str, Any]:
+        return copy.deepcopy(self._json_schema)
+
+    def to_pydantic(self, name: str = "DtxtModel") -> type[BaseModel]:
+        """Build a best-effort Pydantic model from the top-level properties.
+
+        This covers the common flat-record case; deeply nested or
+        format-constrained schemas keep their full fidelity in
+        ``to_json_schema()`` instead.
+        """
+        fields: dict[str, Any] = {}
+        required = set(self.required)
+        for field_name, field_schema in self.properties.items():
+            py_type = _JSON_TYPE_TO_PY.get(field_schema.get("type"), Any)
+            if field_name in required:
+                fields[field_name] = (py_type, ...)
+            else:
+                fields[field_name] = (Optional[py_type], None)  # noqa: UP045
+        return create_model(name, **fields)
+
+    @property
+    def properties(self) -> dict[str, Any]:
+        return dict(self._json_schema.get("properties", {}))
+
+    @property
+    def required(self) -> list[str]:
+        return list(self._json_schema.get("required", []))
+
+    @property
+    def style(self) -> str | None:
+        """Schema-wide D2T style hint (``x-dtxt-style`` on the schema root).
+
+        Distinct from ``field_style()``, which is per-field. ``render()``'s
+        ``style=`` argument overrides this for a single call.
+        """
+        result = self._json_schema.get(_EXT_STYLE)
+        return result if isinstance(result, str) else None
+
+    def field_description(self, field_name: str) -> str | None:
+        result = self.properties.get(field_name, {}).get(_EXT_DESCRIPTION)
+        return result if isinstance(result, str) else None
+
+    def field_examples(self, field_name: str) -> list[Any]:
+        result = self.properties.get(field_name, {}).get(_EXT_EXAMPLES, [])
+        return list(result) if isinstance(result, list) else []
+
+    def field_style(self, field_name: str) -> str | None:
+        result = self.properties.get(field_name, {}).get(_EXT_STYLE)
+        return result if isinstance(result, str) else None
+
+    def validate(self, obj: dict[str, Any]) -> None:
+        jsonschema.validate(instance=obj, schema=self._json_schema)
+
+    def is_valid(self, obj: dict[str, Any]) -> bool:
+        try:
+            self.validate(obj)
+        except jsonschema.ValidationError:
+            return False
+        return True
+
+    def iter_errors(self, obj: dict[str, Any]) -> list[str]:
+        validator_cls = jsonschema.validators.validator_for(self._json_schema)
+        validator_cls.check_schema(self._json_schema)
+        validator = validator_cls(self._json_schema)
+        return [error.message for error in validator.iter_errors(obj)]

dtxt/t2d.py

@@ -0,0 +1,166 @@
+"""T2D: text -> schema-conformant object."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any
+
+from ._config import resolve_backend
+from ._util import extract_json
+from .backends.base import Backend
+from .prompts import build_t2d_prompt, build_t2d_retry_prompt
+from .schema import Schema
+
+DEFAULT_MAX_RETRIES = 3
+DEFAULT_MAX_CONCURRENCY = 8
+
+
+class ParseError(Exception):
+    """Raised when a backend fails to produce a schema-conformant object."""
+
+
+def parse(
+    text: str,
+    schema: Schema,
+    *,
+    backend: Backend | None = None,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+) -> dict[str, Any]:
+    """Convert ``text`` into an object conforming to ``schema``.
+
+    Missing or unextractable fields are represented as ``None``.
+    """
+    resolved = resolve_backend("parse", backend)
+    json_schema = schema.to_json_schema()
+    prompt = build_t2d_prompt(text, json_schema)
+
+    # A constrained-decoding backend only guarantees the grammar-facing part
+    # of the schema (see backends/llamacpp.py's two-stage split); keywords
+    # like `format`/`pattern` still need this retry + validation loop, so
+    # `max_retries` applies uniformly regardless of backend capabilities.
+    attempts = max_retries
+
+    last_error: Exception = ParseError("backend produced no output")
+    raw = resolved.generate(prompt, schema=json_schema)
+    for attempt in range(attempts):
+        is_last_attempt = attempt == attempts - 1
+        try:
+            obj = extract_json(raw)
+        except json.JSONDecodeError as exc:
+            last_error = exc
+            if is_last_attempt:
+                break
+            raw = resolved.generate(
+                build_t2d_retry_prompt(prompt, raw, [f"invalid JSON: {exc}"]),
+                schema=json_schema,
+            )
+            continue
+
+        errors = schema.iter_errors(obj)
+        if not errors:
+            return dict(obj)
+        last_error = ValueError("; ".join(errors))
+        if is_last_attempt:
+            break
+        raw = resolved.generate(build_t2d_retry_prompt(prompt, raw, errors), schema=json_schema)
+
+    raise ParseError(
+        f"failed to parse a schema-conformant object after {attempts} attempt(s): {last_error}"
+    ) from last_error
+
+
+def parse_many(
+    texts: list[str],
+    schema: Schema,
+    *,
+    backend: Backend | None = None,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    max_concurrency: int = DEFAULT_MAX_CONCURRENCY,
+) -> list[dict[str, Any]]:
+    """Batch version of ``parse``.
+
+    When the resolved backend exposes an ``agenerate`` coroutine (API
+    backends such as Anthropic/OpenAI), texts are parsed concurrently via
+    asyncio, bounded by ``max_concurrency`` to avoid tripping rate limits.
+    Otherwise falls back to sequential ``parse`` calls (e.g. llama.cpp,
+    which assumes a single in-process stream).
+    """
+    resolved = resolve_backend("parse", backend)
+    if not texts:
+        return []
+    if getattr(resolved, "agenerate", None) is not None:
+        return asyncio.run(_parse_many_async(texts, schema, resolved, max_retries, max_concurrency))
+    return [parse(text, schema, backend=resolved, max_retries=max_retries) for text in texts]
+
+
+async def _parse_many_async(
+    texts: list[str],
+    schema: Schema,
+    backend: Any,
+    max_retries: int,
+    max_concurrency: int,
+) -> list[dict[str, Any]]:
+    semaphore = asyncio.Semaphore(max_concurrency)
+
+    async def _bounded(text: str) -> dict[str, Any]:
+        async with semaphore:
+            return await _parse_async(text, schema, backend, max_retries)
+
+    # `return_exceptions=True` lets every text finish (success or failure)
+    # instead of raising on the first failure and leaving other in-flight
+    # requests to be torn down as dangling tasks.
+    results: list[dict[str, Any] | BaseException] = await asyncio.gather(
+        *(_bounded(text) for text in texts), return_exceptions=True
+    )
+
+    failures = [(i, r) for i, r in enumerate(results) if isinstance(r, BaseException)]
+    if failures:
+        first_index, first_error = failures[0]
+        raise ParseError(
+            f"parse_many failed for {len(failures)}/{len(texts)} text(s); "
+            f"first failure at index {first_index}: {first_error}"
+        ) from first_error
+
+    return [result for result in results if isinstance(result, dict)]
+
+
+async def _parse_async(
+    text: str,
+    schema: Schema,
+    backend: Any,
+    max_retries: int,
+) -> dict[str, Any]:
+    json_schema = schema.to_json_schema()
+    prompt = build_t2d_prompt(text, json_schema)
+    attempts = max_retries
+
+    last_error: Exception = ParseError("backend produced no output")
+    raw = await backend.agenerate(prompt, schema=json_schema)
+    for attempt in range(attempts):
+        is_last_attempt = attempt == attempts - 1
+        try:
+            obj = extract_json(raw)
+        except json.JSONDecodeError as exc:
+            last_error = exc
+            if is_last_attempt:
+                break
+            raw = await backend.agenerate(
+                build_t2d_retry_prompt(prompt, raw, [f"invalid JSON: {exc}"]),
+                schema=json_schema,
+            )
+            continue
+
+        errors = schema.iter_errors(obj)
+        if not errors:
+            return dict(obj)
+        last_error = ValueError("; ".join(errors))
+        if is_last_attempt:
+            break
+        raw = await backend.agenerate(
+            build_t2d_retry_prompt(prompt, raw, errors), schema=json_schema
+        )
+
+    raise ParseError(
+        f"failed to parse a schema-conformant object after {attempts} attempt(s): {last_error}"
+    ) from last_error

dtxt-0.1.0.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.4
+Name: dtxt
+Version: 0.1.0
+Summary: Schema-centric bidirectional conversion between text and structured data
+Author: pillyshi
+Author-email: pillyshi <pillyshi21@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Dist: pydantic>=2.0
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: dtxt[openai,anthropic,llamacpp] ; extra == 'all'
+Requires-Dist: anthropic>=0.40 ; extra == 'anthropic'
+Requires-Dist: llama-cpp-python>=0.3 ; extra == 'llamacpp'
+Requires-Dist: openai>=1.0 ; extra == 'openai'
+Requires-Python: >=3.10
+Project-URL: Homepage, https://github.com/pillyshi/dtxt
+Project-URL: Repository, https://github.com/pillyshi/dtxt
+Project-URL: Issues, https://github.com/pillyshi/dtxt/issues
+Provides-Extra: all
+Provides-Extra: anthropic
+Provides-Extra: llamacpp
+Provides-Extra: openai
+Description-Content-Type: text/markdown
+
+# dtxt
+
+Schema-centric bidirectional conversion between text and structured data.
+
+`dtxt` is not related to [`dtx`](https://pypi.org/project/dtx/) (an AI red-teaming tool).
+
+## Core features
+
+1. **Schema inference** (`infer_schema`): derive a schema from a collection of texts
+2. **T2D** (`parse`): convert text into a schema-conformant object
+3. **D2T** (`render`): convert an object into text
+4. **Round-trip verification** (`check_roundtrip`): check that
+   `parse(render(obj)) ≈ obj` for a given schema and backend
+
+## Install
+
+```bash
+pip install dtxt              # core only
+pip install dtxt[anthropic]   # + Anthropic backend
+pip install dtxt[openai]      # + OpenAI backend
+pip install dtxt[llamacpp]    # + local GGUF models via llama.cpp
+pip install dtxt[all]         # everything
+```
+
+The core package depends only on `pydantic` and `jsonschema`. Backends are
+optional extras, imported lazily.
+
+## Usage
+
+```python
+import dtxt
+from dtxt import Schema
+from dtxt.backends import MockBackend
+
+schema = Schema({
+    "type": "object",
+    "properties": {
+        "name": {"type": "string", "x-dtxt-description": "the person's full name"},
+        "age": {"type": "integer"},
+    },
+    "required": ["name", "age"],
+})
+
+# Backends can be set globally per function...
+dtxt.configure(parse=MockBackend(), render=MockBackend())
+
+# ...or overridden per call via backend=.
+obj = dtxt.parse("Alice is 30 years old.", schema, backend=MockBackend())
+text = dtxt.render({"name": "Alice", "age": 30}, schema)
+
+result = dtxt.check_roundtrip({"name": "Alice", "age": 30}, schema)
+result.ok  # True if parse(render(obj)) == obj on every schema field
+```
+
+Swap `MockBackend` for a real one:
+
+```python
+dtxt.configure(
+    infer=dtxt.backends.Anthropic("claude-sonnet-4-6"),
+    parse=dtxt.backends.LlamaCpp("model.gguf", n_ctx=8192),
+    render=dtxt.backends.Anthropic("claude-sonnet-4-6"),
+)
+```
+
+`Anthropic` uses forced tool use to get structured output; `OpenAI` uses
+`response_format={"type": "json_schema", ...}`; `LlamaCpp` constrains
+decoding at the grammar level via GBNF. None of them guarantee full schema
+conformance on their own:
+
+- Anthropic/OpenAI guarantee valid JSON syntax, not every schema keyword.
+- `LlamaCpp` strips constructs GBNF can't reliably express (`format`,
+  `pattern`, deeply nested objects/arrays) from the grammar-facing schema;
+  the original schema is still checked afterwards.
+
+So all three go through dtxt's retry + validation loop the same way.
+`parse_many` runs concurrently via asyncio for Anthropic/OpenAI, bounded by
+`max_concurrency` (default 8) to avoid tripping rate limits; `LlamaCpp`
+processes it sequentially in-process so its prompt cache stays warm. A
+partial batch failure raises one `ParseError` naming how many texts failed
+and the first failing index, rather than aborting on the first error.
+
+Style is controllable at both the schema and call level:
+
+```python
+schema = Schema({
+    "type": "object",
+    "properties": {"name": {"type": "string"}},
+    "required": ["name"],
+    "x-dtxt-style": "formal, third person",  # schema-wide default
+})
+dtxt.render(obj, schema)                       # uses "formal, third person"
+dtxt.render(obj, schema, style="casual, upbeat")  # overrides it for this call
+```
+
+## Status
+
+Early development (`0.0.x`). M1-M5 of the milestone plan are implemented:
+`Schema`, `parse` / `parse_many` (asyncio-parallel + bounded concurrency
+for API backends), `render` (with schema-level and per-call style
+control), `infer_schema` (sampling + merge, `min_coverage`),
+`check_roundtrip`, `configure`, a mock backend for testing, and the
+Anthropic / OpenAI / llama.cpp backends. Not yet done: publishing to PyPI
+as `0.1.0` -- see `CLAUDE.md` for the milestone plan.
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+uv run mypy src/
+```

dtxt-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+dtxt/__init__.py,sha256=e84pUVjy851AnETN4ddW6TuyyDpFaZf5d31xupGy2Eg,548
+dtxt/_config.py,sha256=GtbtiYmhKR_2_HriLPD_UkUuHTu3_xlka_Jrr1V8dmE,1413
+dtxt/_util.py,sha256=iELcu6nCa8foaRHjvX8F43vTx8itGkvr8nbr4LqTp1c,516
+dtxt/backends/__init__.py,sha256=8ygdU7X_qLFr7KwX4oUC0eAJyn1rzhono0UYupmfNAU,1002
+dtxt/backends/anthropic.py,sha256=sRopjVAledL_rNq3_eFYHt2N_oXZV4BkAi3HMYyDp9Y,3963
+dtxt/backends/base.py,sha256=eLP4emV_7vhd0Ud_pJW4gCpgqvxKUuuEIc3BYkfdcic,857
+dtxt/backends/llamacpp.py,sha256=afIJ0j5jMdEbpm9QUK1yIek6Hl2i93h24hWfX6PJr20,4596
+dtxt/backends/mock.py,sha256=d8sbCqqncN9nw9laXIfGXwENu-bywmsjkYaRzHSTPd4,1909
+dtxt/backends/openai.py,sha256=9Y4gBx09ebQ_GyAaAnUBpcOTks1iGHzDAY5cpbR3nGw,3172
+dtxt/d2t.py,sha256=1tnB4ENQRhPKg1P6sCNWmptevU2cs1CERbe664AeitI,1579
+dtxt/infer.py,sha256=YaGxu6X_cKohGVN84Jvgz18UWWGSj5sY1Kpj0_MbrrM,3111
+dtxt/prompts.py,sha256=n51sElLY2qYWwCd_QGkGyLt7izpQETZN7tIYsXtlFoU,2645
+dtxt/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dtxt/roundtrip.py,sha256=mXxsDECWg-OgmVez5YdT4reaHsHXhuFzwdL7EimP2tM,1565
+dtxt/schema.py,sha256=2xYF3op_EJNlALPeTiegN3_ZKrm5eBBjqrOhvvs-PoY,4063
+dtxt/t2d.py,sha256=4xA3r-aYyuIqON0FTUZq0K7dd_U0uH5x6qrmRwo60HI,5625
+dtxt-0.1.0.dist-info/licenses/LICENSE,sha256=CsKg5iZ5CavN1hLssUVZLR3KwXDysPkEzHHAE3-uwa0,1065
+dtxt-0.1.0.dist-info/WHEEL,sha256=WvwXFgRajeoYkfRVmDhkP4Qlqo31Mk687zIO2QQoFmw,80
+dtxt-0.1.0.dist-info/METADATA,sha256=-ATOj-yjV_rc1tRLPdYi1wIG5vIZOBaaJeddcWLVbWU,5087
+dtxt-0.1.0.dist-info/RECORD,,

dtxt-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.7
+Root-Is-Purelib: true
+Tag: py3-none-any

dtxt-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pillyshi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.