dtxt 0.1.0__tar.gz

dtxt-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pillyshi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dtxt-0.1.0/PKG-INFO

@@ -0,0 +1,144 @@
+Metadata-Version: 2.4
+Name: dtxt
+Version: 0.1.0
+Summary: Schema-centric bidirectional conversion between text and structured data
+Author: pillyshi
+Author-email: pillyshi <pillyshi21@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Dist: pydantic>=2.0
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: dtxt[openai,anthropic,llamacpp] ; extra == 'all'
+Requires-Dist: anthropic>=0.40 ; extra == 'anthropic'
+Requires-Dist: llama-cpp-python>=0.3 ; extra == 'llamacpp'
+Requires-Dist: openai>=1.0 ; extra == 'openai'
+Requires-Python: >=3.10
+Project-URL: Homepage, https://github.com/pillyshi/dtxt
+Project-URL: Repository, https://github.com/pillyshi/dtxt
+Project-URL: Issues, https://github.com/pillyshi/dtxt/issues
+Provides-Extra: all
+Provides-Extra: anthropic
+Provides-Extra: llamacpp
+Provides-Extra: openai
+Description-Content-Type: text/markdown
+
+# dtxt
+
+Schema-centric bidirectional conversion between text and structured data.
+
+`dtxt` is not related to [`dtx`](https://pypi.org/project/dtx/) (an AI red-teaming tool).
+
+## Core features
+
+1. **Schema inference** (`infer_schema`): derive a schema from a collection of texts
+2. **T2D** (`parse`): convert text into a schema-conformant object
+3. **D2T** (`render`): convert an object into text
+4. **Round-trip verification** (`check_roundtrip`): check that
+   `parse(render(obj)) ≈ obj` for a given schema and backend
+
+## Install
+
+```bash
+pip install dtxt              # core only
+pip install dtxt[anthropic]   # + Anthropic backend
+pip install dtxt[openai]      # + OpenAI backend
+pip install dtxt[llamacpp]    # + local GGUF models via llama.cpp
+pip install dtxt[all]         # everything
+```
+
+The core package depends only on `pydantic` and `jsonschema`. Backends are
+optional extras, imported lazily.
+
+## Usage
+
+```python
+import dtxt
+from dtxt import Schema
+from dtxt.backends import MockBackend
+
+schema = Schema({
+    "type": "object",
+    "properties": {
+        "name": {"type": "string", "x-dtxt-description": "the person's full name"},
+        "age": {"type": "integer"},
+    },
+    "required": ["name", "age"],
+})
+
+# Backends can be set globally per function...
+dtxt.configure(parse=MockBackend(), render=MockBackend())
+
+# ...or overridden per call via backend=.
+obj = dtxt.parse("Alice is 30 years old.", schema, backend=MockBackend())
+text = dtxt.render({"name": "Alice", "age": 30}, schema)
+
+result = dtxt.check_roundtrip({"name": "Alice", "age": 30}, schema)
+result.ok  # True if parse(render(obj)) == obj on every schema field
+```
+
+Swap `MockBackend` for a real one:
+
+```python
+dtxt.configure(
+    infer=dtxt.backends.Anthropic("claude-sonnet-4-6"),
+    parse=dtxt.backends.LlamaCpp("model.gguf", n_ctx=8192),
+    render=dtxt.backends.Anthropic("claude-sonnet-4-6"),
+)
+```
+
+`Anthropic` uses forced tool use to get structured output; `OpenAI` uses
+`response_format={"type": "json_schema", ...}`; `LlamaCpp` constrains
+decoding at the grammar level via GBNF. None of them guarantee full schema
+conformance on their own:
+
+- Anthropic/OpenAI guarantee valid JSON syntax, not every schema keyword.
+- `LlamaCpp` strips constructs GBNF can't reliably express (`format`,
+  `pattern`, deeply nested objects/arrays) from the grammar-facing schema;
+  the original schema is still checked afterwards.
+
+So all three go through dtxt's retry + validation loop the same way.
+`parse_many` runs concurrently via asyncio for Anthropic/OpenAI, bounded by
+`max_concurrency` (default 8) to avoid tripping rate limits; `LlamaCpp`
+processes it sequentially in-process so its prompt cache stays warm. A
+partial batch failure raises one `ParseError` naming how many texts failed
+and the first failing index, rather than aborting on the first error.
+
+Style is controllable at both the schema and call level:
+
+```python
+schema = Schema({
+    "type": "object",
+    "properties": {"name": {"type": "string"}},
+    "required": ["name"],
+    "x-dtxt-style": "formal, third person",  # schema-wide default
+})
+dtxt.render(obj, schema)                       # uses "formal, third person"
+dtxt.render(obj, schema, style="casual, upbeat")  # overrides it for this call
+```
+
+## Status
+
+Early development (`0.0.x`). M1-M5 of the milestone plan are implemented:
+`Schema`, `parse` / `parse_many` (asyncio-parallel + bounded concurrency
+for API backends), `render` (with schema-level and per-call style
+control), `infer_schema` (sampling + merge, `min_coverage`),
+`check_roundtrip`, `configure`, a mock backend for testing, and the
+Anthropic / OpenAI / llama.cpp backends. Not yet done: publishing to PyPI
+as `0.1.0` -- see `CLAUDE.md` for the milestone plan.
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+uv run mypy src/
+```

dtxt-0.1.0/README.md

@@ -0,0 +1,112 @@
+# dtxt
+
+Schema-centric bidirectional conversion between text and structured data.
+
+`dtxt` is not related to [`dtx`](https://pypi.org/project/dtx/) (an AI red-teaming tool).
+
+## Core features
+
+1. **Schema inference** (`infer_schema`): derive a schema from a collection of texts
+2. **T2D** (`parse`): convert text into a schema-conformant object
+3. **D2T** (`render`): convert an object into text
+4. **Round-trip verification** (`check_roundtrip`): check that
+   `parse(render(obj)) ≈ obj` for a given schema and backend
+
+## Install
+
+```bash
+pip install dtxt              # core only
+pip install dtxt[anthropic]   # + Anthropic backend
+pip install dtxt[openai]      # + OpenAI backend
+pip install dtxt[llamacpp]    # + local GGUF models via llama.cpp
+pip install dtxt[all]         # everything
+```
+
+The core package depends only on `pydantic` and `jsonschema`. Backends are
+optional extras, imported lazily.
+
+## Usage
+
+```python
+import dtxt
+from dtxt import Schema
+from dtxt.backends import MockBackend
+
+schema = Schema({
+    "type": "object",
+    "properties": {
+        "name": {"type": "string", "x-dtxt-description": "the person's full name"},
+        "age": {"type": "integer"},
+    },
+    "required": ["name", "age"],
+})
+
+# Backends can be set globally per function...
+dtxt.configure(parse=MockBackend(), render=MockBackend())
+
+# ...or overridden per call via backend=.
+obj = dtxt.parse("Alice is 30 years old.", schema, backend=MockBackend())
+text = dtxt.render({"name": "Alice", "age": 30}, schema)
+
+result = dtxt.check_roundtrip({"name": "Alice", "age": 30}, schema)
+result.ok  # True if parse(render(obj)) == obj on every schema field
+```
+
+Swap `MockBackend` for a real one:
+
+```python
+dtxt.configure(
+    infer=dtxt.backends.Anthropic("claude-sonnet-4-6"),
+    parse=dtxt.backends.LlamaCpp("model.gguf", n_ctx=8192),
+    render=dtxt.backends.Anthropic("claude-sonnet-4-6"),
+)
+```
+
+`Anthropic` uses forced tool use to get structured output; `OpenAI` uses
+`response_format={"type": "json_schema", ...}`; `LlamaCpp` constrains
+decoding at the grammar level via GBNF. None of them guarantee full schema
+conformance on their own:
+
+- Anthropic/OpenAI guarantee valid JSON syntax, not every schema keyword.
+- `LlamaCpp` strips constructs GBNF can't reliably express (`format`,
+  `pattern`, deeply nested objects/arrays) from the grammar-facing schema;
+  the original schema is still checked afterwards.
+
+So all three go through dtxt's retry + validation loop the same way.
+`parse_many` runs concurrently via asyncio for Anthropic/OpenAI, bounded by
+`max_concurrency` (default 8) to avoid tripping rate limits; `LlamaCpp`
+processes it sequentially in-process so its prompt cache stays warm. A
+partial batch failure raises one `ParseError` naming how many texts failed
+and the first failing index, rather than aborting on the first error.
+
+Style is controllable at both the schema and call level:
+
+```python
+schema = Schema({
+    "type": "object",
+    "properties": {"name": {"type": "string"}},
+    "required": ["name"],
+    "x-dtxt-style": "formal, third person",  # schema-wide default
+})
+dtxt.render(obj, schema)                       # uses "formal, third person"
+dtxt.render(obj, schema, style="casual, upbeat")  # overrides it for this call
+```
+
+## Status
+
+Early development (`0.0.x`). M1-M5 of the milestone plan are implemented:
+`Schema`, `parse` / `parse_many` (asyncio-parallel + bounded concurrency
+for API backends), `render` (with schema-level and per-call style
+control), `infer_schema` (sampling + merge, `min_coverage`),
+`check_roundtrip`, `configure`, a mock backend for testing, and the
+Anthropic / OpenAI / llama.cpp backends. Not yet done: publishing to PyPI
+as `0.1.0` -- see `CLAUDE.md` for the milestone plan.
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+uv run mypy src/
+```

dtxt-0.1.0/pyproject.toml

@@ -0,0 +1,72 @@
+[project]
+name = "dtxt"
+version = "0.1.0"
+description = "Schema-centric bidirectional conversion between text and structured data"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "pillyshi", email = "pillyshi21@gmail.com" }
+]
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pydantic>=2.0",
+    "jsonschema>=4.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/pillyshi/dtxt"
+Repository = "https://github.com/pillyshi/dtxt"
+Issues = "https://github.com/pillyshi/dtxt/issues"
+
+[project.optional-dependencies]
+openai = ["openai>=1.0"]
+anthropic = ["anthropic>=0.40"]
+llamacpp = ["llama-cpp-python>=0.3"]
+all = ["dtxt[openai,anthropic,llamacpp]"]
+
+[build-system]
+requires = ["uv_build>=0.11.7,<0.12.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "mypy>=1.11",
+    "pytest>=8.0",
+    "ruff>=0.6",
+    "types-jsonschema>=4.0",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.mypy]
+strict = true
+packages = ["dtxt"]
+mypy_path = "src"
+explicit_package_bases = true
+
+[[tool.mypy.overrides]]
+module = ["anthropic.*", "openai.*", "llama_cpp.*"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "integration: tests that call a real LLM backend (excluded by default)",
+]
+addopts = "-m 'not integration'"

dtxt-0.1.0/src/dtxt/__init__.py

@@ -0,0 +1,23 @@
+"""dtxt: schema-centric bidirectional conversion between text and structured data."""
+
+from . import backends
+from ._config import configure
+from .d2t import render
+from .infer import InferError, infer_schema
+from .roundtrip import RoundtripResult, check_roundtrip
+from .schema import Schema
+from .t2d import ParseError, parse, parse_many
+
+__all__ = [
+    "InferError",
+    "ParseError",
+    "RoundtripResult",
+    "Schema",
+    "backends",
+    "check_roundtrip",
+    "configure",
+    "infer_schema",
+    "parse",
+    "parse_many",
+    "render",
+]

dtxt-0.1.0/src/dtxt/_config.py

@@ -0,0 +1,55 @@
+"""Global backend configuration, one slot per public function.
+
+``configure()`` sets the default; each function's own ``backend=``
+argument overrides it for that single call.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .backends.base import Backend
+
+
+@dataclass
+class _Config:
+    infer: Backend | None = None
+    parse: Backend | None = None
+    render: Backend | None = None
+
+
+_config = _Config()
+
+
+def configure(
+    *,
+    infer: Backend | None = None,
+    parse: Backend | None = None,
+    render: Backend | None = None,
+) -> None:
+    """Set the default backend used by ``infer_schema`` / ``parse`` / ``render``."""
+    if infer is not None:
+        _config.infer = infer
+    if parse is not None:
+        _config.parse = parse
+    if render is not None:
+        _config.render = render
+
+
+def resolve_backend(function: str, override: Backend | None) -> Backend:
+    if override is not None:
+        return override
+    configured: Backend | None = getattr(_config, function)
+    if configured is None:
+        raise RuntimeError(
+            f"No backend configured for '{function}'. "
+            f"Call dtxt.configure({function}=...) or pass backend=... explicitly."
+        )
+    return configured
+
+
+def reset() -> None:
+    """Clear all configured backends. Mainly useful between tests."""
+    _config.infer = None
+    _config.parse = None
+    _config.render = None

dtxt-0.1.0/src/dtxt/_util.py

@@ -0,0 +1,18 @@
+"""Small helpers shared across the T2D / D2T / infer pipelines."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def extract_json(raw: str) -> Any:
+    """Parse a JSON value out of a model response, tolerating code fences."""
+    text = raw.strip()
+    if text.startswith("```"):
+        text = text.strip("`")
+        if "\n" in text:
+            first_line, rest = text.split("\n", 1)
+            if first_line.strip().isalpha():
+                text = rest
+    return json.loads(text)

dtxt-0.1.0/src/dtxt/backends/__init__.py

@@ -0,0 +1,33 @@
+"""Backend implementations for dtxt.
+
+Only the mock backend is imported eagerly. API/local backends
+(``Anthropic``, ``OpenAI``, ``LlamaCpp``) live in optional extras; their
+modules -- and the underlying SDK packages -- are imported lazily on first
+attribute access, so importing ``dtxt`` never requires them to be
+installed.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+from .base import Backend
+from .mock import MockBackend
+
+__all__ = ["Backend", "MockBackend", "Anthropic", "OpenAI", "LlamaCpp"]
+
+_LAZY: dict[str, tuple[str, str]] = {
+    "Anthropic": (".anthropic", "Anthropic"),
+    "OpenAI": (".openai", "OpenAI"),
+    "LlamaCpp": (".llamacpp", "LlamaCpp"),
+}
+
+
+def __getattr__(name: str) -> Any:
+    try:
+        module_name, attr_name = _LAZY[name]
+    except KeyError:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+    module = importlib.import_module(module_name, __name__)
+    return getattr(module, attr_name)

dtxt-0.1.0/src/dtxt/backends/anthropic.py

@@ -0,0 +1,114 @@
+"""Anthropic API backend: structured output via forced tool use.
+
+The ``anthropic`` package is an optional extra (``dtxt[anthropic]``) and is
+imported lazily, only when a client actually needs to be built.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .base import TOOL_CALLING
+
+_INSTALL_HINT = "pip install dtxt[anthropic]"
+_TOOL_NAME = "emit_result"
+
+
+def _import_anthropic() -> Any:
+    try:
+        import anthropic
+    except ImportError as exc:
+        raise ImportError(
+            "the 'anthropic' package is required to use dtxt.backends.Anthropic. "
+            f"Install it with: {_INSTALL_HINT}"
+        ) from exc
+    return anthropic
+
+
+def _build_request_kwargs(
+    model: str, prompt: str, schema: dict[str, Any] | None, max_tokens: int
+) -> dict[str, Any]:
+    kwargs: dict[str, Any] = {
+        "model": model,
+        "max_tokens": max_tokens,
+        "messages": [{"role": "user", "content": prompt}],
+    }
+    if schema is not None:
+        kwargs["tools"] = [
+            {
+                "name": _TOOL_NAME,
+                "description": "Emit the result as structured data matching the schema.",
+                "input_schema": schema,
+            }
+        ]
+        kwargs["tool_choice"] = {"type": "tool", "name": _TOOL_NAME}
+    return kwargs
+
+
+def _extract_result(message: Any, schema: dict[str, Any] | None) -> str:
+    if schema is not None:
+        for block in message.content:
+            if getattr(block, "type", None) == "tool_use" and getattr(block, "name", None) == (
+                _TOOL_NAME
+            ):
+                return json.dumps(block.input)
+        raise RuntimeError("Anthropic response did not include the expected tool_use block")
+    return "".join(
+        block.text for block in message.content if getattr(block, "type", None) == "text"
+    )
+
+
+class Anthropic:
+    """Backend for the Anthropic Messages API.
+
+    Structured output is obtained by wrapping the JSON Schema as a single
+    tool and forcing ``tool_choice`` onto it, so ``tool_use.input`` is the
+    extracted object. This is not treated as ``constrained_decoding``: the
+    API does not guarantee full schema conformance the way grammar-based
+    decoding does, so dtxt still runs it through the retry + validation
+    loop.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        api_key: str | None = None,
+        max_tokens: int = 4096,
+        client: Any | None = None,
+        async_client: Any | None = None,
+    ) -> None:
+        self._model = model
+        self._api_key = api_key
+        self._max_tokens = max_tokens
+        self._client = client
+        self._async_client = async_client
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            anthropic = _import_anthropic()
+            kwargs: dict[str, Any] = {"api_key": self._api_key} if self._api_key else {}
+            self._client = anthropic.Anthropic(**kwargs)
+        return self._client
+
+    def _get_async_client(self) -> Any:
+        if self._async_client is None:
+            anthropic = _import_anthropic()
+            kwargs: dict[str, Any] = {"api_key": self._api_key} if self._api_key else {}
+            self._async_client = anthropic.AsyncAnthropic(**kwargs)
+        return self._async_client
+
+    @property
+    def capabilities(self) -> set[str]:
+        return {TOOL_CALLING}
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        kwargs = _build_request_kwargs(self._model, prompt, schema, self._max_tokens)
+        message = self._get_client().messages.create(**kwargs)
+        return _extract_result(message, schema)
+
+    async def agenerate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str:
+        kwargs = _build_request_kwargs(self._model, prompt, schema, self._max_tokens)
+        message = await self._get_async_client().messages.create(**kwargs)
+        return _extract_result(message, schema)

dtxt-0.1.0/src/dtxt/backends/base.py

@@ -0,0 +1,27 @@
+"""Backend abstraction: the interface every dtxt backend implements."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+CONSTRAINED_DECODING = "constrained_decoding"
+JSON_MODE = "json_mode"
+TOOL_CALLING = "tool_calling"
+
+
+@runtime_checkable
+class Backend(Protocol):
+    """Anything that can turn a prompt into text, optionally schema-guided.
+
+    ``capabilities`` tells the caller which execution path to take:
+
+    - ``constrained_decoding``: the backend enforces the JSON schema at the
+      grammar level, so callers may skip syntactic validation and only run
+      semantic checks.
+    - otherwise: callers must retry and validate the output themselves.
+    """
+
+    def generate(self, prompt: str, *, schema: dict[str, Any] | None = None) -> str: ...
+
+    @property
+    def capabilities(self) -> set[str]: ...