tstring-core 0.1.0__tar.gz

tstring_core-0.1.0/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.3
+Name: tstring-core
+Version: 0.1.0
+Summary: Shared runtime primitives for PEP 750 structural template renderers
+Keywords: pep750,t-strings,template-strings,runtime,structured-data
+Author: Koudai Aono
+Author-email: Koudai Aono <koxudaxi@gmail.com>
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: tstring-bindings>=0.1.0
+Maintainer: Koudai Aono
+Maintainer-email: Koudai Aono <koxudaxi@gmail.com>
+Requires-Python: >=3.14
+Project-URL: Homepage, https://github.com/koxudaxi/tstring-structured-data
+Project-URL: Repository, https://github.com/koxudaxi/tstring-structured-data
+Project-URL: Documentation, https://github.com/koxudaxi/tstring-structured-data/blob/main/tstring-core/README.md
+Project-URL: Changelog, https://github.com/koxudaxi/tstring-structured-data/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/koxudaxi/tstring-structured-data/issues
+Description-Content-Type: text/markdown
+
+# tstring-core
+
+Shared Python compatibility layer for the Rust-first JSON, TOML, and YAML
+t-string backends.
+
+Requires Python 3.14+.
+
+This package depends on `tstring-bindings`, a native PyO3 extension. On
+supported platforms, install from prebuilt wheels. Other environments require a
+local Rust 1.94.0 toolchain build.
+
+## What It Provides
+
+- shared error categories re-exported from the Rust bindings
+- compatibility helper APIs for tokenization, spans, diagnostics, and slots
+- a stable import surface for the Python wrapper packages and tests
+
+## What It Does Not Provide
+
+- JSON grammar rules
+- TOML grammar rules
+- YAML grammar rules
+- backend-specific representability policies
+
+Those responsibilities now live in the Rust workspace under `rust/`.
+
+## Runtime Contract
+
+Each backend exposed through the Python packages follows the same high-level
+pipeline:
+
+1. validate that the input is a PEP 750 `Template`
+2. pass the template into the shared PyO3 bindings
+3. parse backend-specific structure in Rust
+4. run semantic checks in Rust
+5. render text and backend-native Python data
+6. materialize backend-native data from the same parsed/rendered structure
+
+The shared layer is also responsible for keeping the Python-facing exception and
+typing surface stable across the JSON, TOML, and YAML wrapper packages.
+
+## See also
+
+- [Project README](https://github.com/koxudaxi/tstring-structured-data#readme)
+- [Architecture](https://github.com/koxudaxi/tstring-structured-data/blob/main/docs/architecture.md)

tstring_core-0.1.0/README.md

@@ -0,0 +1,45 @@
+# tstring-core
+
+Shared Python compatibility layer for the Rust-first JSON, TOML, and YAML
+t-string backends.
+
+Requires Python 3.14+.
+
+This package depends on `tstring-bindings`, a native PyO3 extension. On
+supported platforms, install from prebuilt wheels. Other environments require a
+local Rust 1.94.0 toolchain build.
+
+## What It Provides
+
+- shared error categories re-exported from the Rust bindings
+- compatibility helper APIs for tokenization, spans, diagnostics, and slots
+- a stable import surface for the Python wrapper packages and tests
+
+## What It Does Not Provide
+
+- JSON grammar rules
+- TOML grammar rules
+- YAML grammar rules
+- backend-specific representability policies
+
+Those responsibilities now live in the Rust workspace under `rust/`.
+
+## Runtime Contract
+
+Each backend exposed through the Python packages follows the same high-level
+pipeline:
+
+1. validate that the input is a PEP 750 `Template`
+2. pass the template into the shared PyO3 bindings
+3. parse backend-specific structure in Rust
+4. run semantic checks in Rust
+5. render text and backend-native Python data
+6. materialize backend-native data from the same parsed/rendered structure
+
+The shared layer is also responsible for keeping the Python-facing exception and
+typing surface stable across the JSON, TOML, and YAML wrapper packages.
+
+## See also
+
+- [Project README](https://github.com/koxudaxi/tstring-structured-data#readme)
+- [Architecture](https://github.com/koxudaxi/tstring-structured-data/blob/main/docs/architecture.md)

tstring_core-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[project]
+name = "tstring-core"
+version = "0.1.0"
+description = "Shared runtime primitives for PEP 750 structural template renderers"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Koudai Aono", email = "koxudaxi@gmail.com" }]
+maintainers = [{ name = "Koudai Aono", email = "koxudaxi@gmail.com" }]
+requires-python = ">=3.14"
+dependencies = ["tstring-bindings>=0.1.0"]
+keywords = ["pep750", "t-strings", "template-strings", "runtime", "structured-data"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://github.com/koxudaxi/tstring-structured-data"
+Repository = "https://github.com/koxudaxi/tstring-structured-data"
+Documentation = "https://github.com/koxudaxi/tstring-structured-data/blob/main/tstring-core/README.md"
+Changelog = "https://github.com/koxudaxi/tstring-structured-data/blob/main/CHANGELOG.md"
+Issues = "https://github.com/koxudaxi/tstring-structured-data/issues"
+
+[build-system]
+requires = ["uv_build>=0.10.9,<0.11.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.11.0",
+    "ty>=0.0.23",
+]
+
+[tool.uv.sources]
+tstring-bindings = { workspace = true }
+
+[tool.uv.build-backend]
+source-exclude = [".DS_Store"]
+wheel-exclude = [".DS_Store"]
+
+[tool.ty.environment]
+extra-paths = ["../rust/python-bindings/python"]
+
+[tool.ruff]
+target-version = "py314"
+
+[tool.ruff.lint]
+select = ["B", "E", "F", "I"]

tstring_core-0.1.0/src/tstring_core/__init__.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import warnings
+
+from ._diagnostics import Diagnostic, DiagnosticSeverity
+from ._errors import (
+    TemplateError,
+    TemplateParseError,
+    TemplateSemanticError,
+    UnrepresentableValueError,
+)
+from ._nodes import TemplateNode
+from ._slots import FragmentGroup, Slot, SlotContext
+from ._spans import SourcePosition, SourceSpan
+from ._tokens import (
+    InterpolationToken,
+    StaticTextToken,
+    StreamItem,
+    TemplateToken,
+    flatten_template,
+    tokenize_template,
+)
+from ._types import JsonValue, StructuredData, TomlValue, YamlKey, YamlValue
+from ._values import RenderResult, ValueKind
+
+_DEPRECATED_ROOT_EXPORTS = frozenset(
+    {"ParserFirstBackend", "ReturnMode", "render_with_backend"}
+)
+
+
+def __getattr__(name: str) -> object:
+    if name not in _DEPRECATED_ROOT_EXPORTS:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+    from . import _render
+
+    value = getattr(_render, name)
+    warnings.warn(
+        f"tstring_core.{name} is deprecated and will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    globals()[name] = value
+    return value
+
+
+__all__ = [
+    "Diagnostic",
+    "DiagnosticSeverity",
+    "FragmentGroup",
+    "InterpolationToken",
+    "JsonValue",
+    "RenderResult",
+    "Slot",
+    "SlotContext",
+    "SourcePosition",
+    "SourceSpan",
+    "StaticTextToken",
+    "StreamItem",
+    "TemplateError",
+    "TemplateNode",
+    "TemplateParseError",
+    "TemplateSemanticError",
+    "TomlValue",
+    "TemplateToken",
+    "UnrepresentableValueError",
+    "ValueKind",
+    "YamlKey",
+    "YamlValue",
+    "StructuredData",
+    "flatten_template",
+    "tokenize_template",
+]

tstring_core-0.1.0/src/tstring_core/_conformance.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import json
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TypedDict, cast
+
+from ._types import JsonValue
+
+
+class _ConformanceProfile(TypedDict):
+    manifest_path: str
+
+
+class _ConformanceProfileIndex(TypedDict):
+    supported_profiles: list[str]
+    profiles: dict[str, _ConformanceProfile]
+
+
+class _ConformanceProvenance(TypedDict):
+    source: str
+    snapshot: str
+
+
+class _ConformanceCaseRecord(TypedDict, total=False):
+    case_id: str
+    spec_ref: str
+    expected: str
+    execution_layer: str
+    input_path: str
+    note: str
+    expected_json_path: str
+    classification: str
+
+
+class _ConformanceManifest(TypedDict):
+    spec_title: str
+    claim_status: str
+    provenance: _ConformanceProvenance
+    cases: list[_ConformanceCaseRecord]
+
+
+@dataclass(frozen=True)
+class ConformanceCase:
+    format_name: str
+    case_id: str
+    spec_ref: str
+    expected: str
+    execution_layer: str
+    input_path: str
+    note: str
+    expected_json_path: str | None = None
+    classification: str | None = None
+
+    def input_text(self) -> str:
+        with self.base_dir.joinpath(self.input_path).open(
+            "r", encoding="utf-8", newline=""
+        ) as handle:
+            return handle.read()
+
+    def expected_json(self) -> JsonValue | None:
+        if self.expected_json_path is None:
+            return None
+        return cast(
+            JsonValue,
+            json.loads(
+                self.base_dir.joinpath(self.expected_json_path).read_text(
+                    encoding="utf-8"
+                )
+            ),
+        )
+
+    @property
+    def base_dir(self) -> Path:
+        return _repo_root() / "conformance" / self.format_name
+
+
+@dataclass(frozen=True)
+class ConformanceSuite:
+    format_name: str
+    profile: str
+    spec_title: str
+    claim_status: str
+    source: str
+    snapshot: str
+    cases: tuple[ConformanceCase, ...]
+
+    def iter_cases(self, layer: str) -> tuple[ConformanceCase, ...]:
+        return tuple(
+            case for case in self.cases if case.execution_layer in {layer, "both"}
+        )
+
+
+def load_conformance_suite(format_name: str, profile: str) -> ConformanceSuite:
+    format_root = _repo_root() / "conformance" / format_name
+    profile_index_path = format_root / "profiles.toml"
+    profile_index = cast(
+        _ConformanceProfileIndex,
+        tomllib.loads(profile_index_path.read_text(encoding="utf-8")),
+    )
+    supported_profiles = tuple(profile_index["supported_profiles"])
+    if profile not in supported_profiles:
+        supported_profile_list = ", ".join(repr(value) for value in supported_profiles)
+        raise ValueError(
+            f"Unsupported {format_name} conformance profile {profile!r}. "
+            f"Supported profiles: {supported_profile_list}."
+        )
+    manifest_path = format_root / profile_index["profiles"][profile]["manifest_path"]
+    manifest = cast(
+        _ConformanceManifest,
+        tomllib.loads(manifest_path.read_text(encoding="utf-8")),
+    )
+    return ConformanceSuite(
+        format_name=format_name,
+        profile=profile,
+        spec_title=manifest["spec_title"],
+        claim_status=manifest["claim_status"],
+        source=manifest["provenance"]["source"],
+        snapshot=manifest["provenance"]["snapshot"],
+        cases=tuple(
+            ConformanceCase(format_name=format_name, **case)
+            for case in manifest["cases"]
+        ),
+    )
+
+
+def _repo_root() -> Path:
+    return Path(__file__).resolve().parents[3]

tstring_core-0.1.0/src/tstring_core/_diagnostics.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from ._spans import SourceSpan
+
+
+class DiagnosticSeverity(str, Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+
+@dataclass(frozen=True, slots=True)
+class Diagnostic:
+    code: str
+    message: str
+    span: SourceSpan | None = None
+    severity: DiagnosticSeverity = DiagnosticSeverity.ERROR

tstring_core-0.1.0/src/tstring_core/_errors.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from tstring_bindings import (
+    TemplateError,
+    TemplateParseError,
+    TemplateSemanticError,
+    UnrepresentableValueError,
+)
+
+__all__ = [
+    "TemplateError",
+    "TemplateParseError",
+    "TemplateSemanticError",
+    "UnrepresentableValueError",
+]

tstring_core-0.1.0/src/tstring_core/_nodes.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ._diagnostics import Diagnostic
+from ._spans import SourceSpan
+
+
+@dataclass(slots=True)
+class TemplateNode:
+    span: SourceSpan
+    diagnostics: list[Diagnostic] = field(default_factory=list, kw_only=True)

tstring_core-0.1.0/src/tstring_core/_render.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from string.templatelib import Template
+from typing import Literal, Protocol, TypeIs, TypeVar, overload
+
+from ._errors import TemplateParseError
+from ._tokens import tokenize_template
+from ._types import StructuredData
+from ._values import RenderResult
+
+ReturnMode = Literal["data", "text"]
+NodeT = TypeVar("NodeT")
+DataT = TypeVar("DataT", bound=StructuredData)
+
+
+class ParserFirstBackend(Protocol[NodeT, DataT]):
+    def parse(self, template: Template) -> NodeT: ...
+
+    def analyze(self, node: NodeT) -> None: ...
+
+    def render(self, node: NodeT) -> RenderResult[DataT]: ...
+
+
+@overload
+def render_with_backend(
+    template: Template,
+    *,
+    api_name: str,
+    backend: ParserFirstBackend[NodeT, DataT],
+    return_mode: Literal["data"],
+) -> DataT: ...
+
+
+@overload
+def render_with_backend(
+    template: Template,
+    *,
+    api_name: str,
+    backend: ParserFirstBackend[NodeT, DataT],
+    return_mode: Literal["text"],
+) -> str: ...
+
+
+def render_with_backend(
+    template: Template,
+    *,
+    api_name: str,
+    backend: ParserFirstBackend[NodeT, DataT],
+    return_mode: ReturnMode,
+) -> DataT | str:
+    _ensure_template(template, api_name)
+    tokenize_template(template)
+
+    node = backend.parse(template)
+    backend.analyze(node)
+    rendered = backend.render(node)
+
+    if return_mode == "text":
+        return rendered.text
+    return rendered.data
+
+
+def _is_template(value: object) -> TypeIs[Template]:
+    return isinstance(value, Template)
+
+
+def _ensure_template(template: object, api_name: str) -> None:
+    if _is_template(template):
+        return
+
+    message = (
+        f"{api_name} require a PEP 750 Template object. "
+        f"Got {type(template).__name__} instead."
+    )
+    raise TypeError(message)
+
+
+__all__ = [
+    "ParserFirstBackend",
+    "ReturnMode",
+    "TemplateParseError",
+    "render_with_backend",
+]

tstring_core-0.1.0/src/tstring_core/_slots.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class SlotContext(str, Enum):
+    VALUE = "value"
+    KEY = "key"
+    STRING_FRAGMENT = "string_fragment"
+    UNSUPPORTED = "unsupported"
+
+
+@dataclass(slots=True)
+class Slot:
+    id: int
+
+
+@dataclass(slots=True)
+class FragmentGroup:
+    start_slot: int
+    end_slot: int
+    start_offset: int
+    end_offset: int

tstring_core-0.1.0/src/tstring_core/_spans.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class SourcePosition:
+    token_index: int
+    offset: int
+
+
+@dataclass(frozen=True, slots=True)
+class SourceSpan:
+    start: SourcePosition
+    end: SourcePosition
+
+    @classmethod
+    def point(cls, token_index: int, offset: int) -> "SourceSpan":
+        position = SourcePosition(token_index=token_index, offset=offset)
+        return cls(start=position, end=position)
+
+    @classmethod
+    def between(cls, start: SourcePosition, end: SourcePosition) -> "SourceSpan":
+        return cls(start=start, end=end)
+
+    def extend(self, end: SourcePosition) -> "SourceSpan":
+        return SourceSpan(start=self.start, end=end)
+
+    def merge(self, other: "SourceSpan") -> "SourceSpan":
+        return SourceSpan(start=self.start, end=other.end)

tstring_core-0.1.0/src/tstring_core/_tokens.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from string.templatelib import Interpolation, Template
+from typing import Literal, cast
+
+from ._spans import SourcePosition, SourceSpan
+
+TemplateTokenKind = Literal["text", "interpolation"]
+StreamItemKind = Literal["char", "interpolation", "eof"]
+
+
+@dataclass(frozen=True, slots=True)
+class StaticTextToken:
+    text: str
+    token_index: int
+    span: SourceSpan
+    kind: TemplateTokenKind = "text"
+
+
+@dataclass(frozen=True, slots=True)
+class InterpolationToken:
+    interpolation: Interpolation
+    interpolation_index: int
+    token_index: int
+    span: SourceSpan
+    kind: TemplateTokenKind = "interpolation"
+
+    @property
+    def expression(self) -> str:
+        return self.interpolation.expression or f"slot {self.interpolation_index}"
+
+
+TemplateToken = StaticTextToken | InterpolationToken
+
+
+@dataclass(frozen=True, slots=True)
+class StreamItem:
+    kind: StreamItemKind
+    value: str | Interpolation | None
+    span: SourceSpan
+    interpolation_index: int | None = None
+
+    @property
+    def char(self) -> str | None:
+        return cast("str | None", self.value) if self.kind == "char" else None
+
+    @property
+    def interpolation(self) -> Interpolation | None:
+        return (
+            cast("Interpolation | None", self.value)
+            if self.kind == "interpolation"
+            else None
+        )
+
+
+def tokenize_template(template: Template) -> list[TemplateToken]:
+    tokens: list[TemplateToken] = []
+    token_index = 0
+
+    for interpolation_index, interpolation in enumerate(template.interpolations):
+        text = template.strings[interpolation_index]
+        if text:
+            tokens.append(
+                StaticTextToken(
+                    text=text,
+                    token_index=token_index,
+                    span=SourceSpan.between(
+                        SourcePosition(token_index=token_index, offset=0),
+                        SourcePosition(token_index=token_index, offset=len(text)),
+                    ),
+                )
+            )
+            token_index += 1
+
+        tokens.append(
+            InterpolationToken(
+                interpolation=interpolation,
+                interpolation_index=interpolation_index,
+                token_index=token_index,
+                span=SourceSpan.point(token_index=token_index, offset=0),
+            )
+        )
+        token_index += 1
+
+    tail = template.strings[len(template.interpolations)]
+    if tail or not tokens:
+        tokens.append(
+            StaticTextToken(
+                text=tail,
+                token_index=token_index,
+                span=SourceSpan.between(
+                    SourcePosition(token_index=token_index, offset=0),
+                    SourcePosition(token_index=token_index, offset=len(tail)),
+                ),
+            )
+        )
+
+    return tokens
+
+
+def flatten_template(template: Template) -> list[StreamItem]:
+    items: list[StreamItem] = []
+
+    for token in tokenize_template(template):
+        if isinstance(token, StaticTextToken):
+            for offset, char in enumerate(token.text):
+                items.append(
+                    StreamItem(
+                        kind="char",
+                        value=char,
+                        span=SourceSpan.between(
+                            SourcePosition(
+                                token_index=token.token_index, offset=offset
+                            ),
+                            SourcePosition(
+                                token_index=token.token_index,
+                                offset=offset + 1,
+                            ),
+                        ),
+                    )
+                )
+            continue
+
+        items.append(
+            StreamItem(
+                kind="interpolation",
+                value=token.interpolation,
+                interpolation_index=token.interpolation_index,
+                span=token.span,
+            )
+        )
+
+    eof_span = items[-1].span if items else SourceSpan.point(token_index=0, offset=0)
+    items.append(StreamItem(kind="eof", value=None, span=eof_span))
+    return items

tstring_core-0.1.0/src/tstring_core/_types.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from datetime import date, datetime, time
+
+type JsonScalar = None | bool | int | float | str
+type JsonValue = JsonScalar | list[JsonValue] | dict[str, JsonValue]
+
+type TomlScalar = bool | int | float | str | date | time | datetime
+type TomlValue = TomlScalar | list[TomlValue] | dict[str, TomlValue]
+
+type YamlScalar = None | bool | int | float | str
+type YamlKey = YamlScalar | tuple[YamlKey, ...] | frozenset[tuple[YamlKey, YamlKey]]
+type YamlValue = YamlScalar | list[YamlValue] | dict[YamlKey, YamlValue]
+
+type StructuredData = JsonValue | TomlValue | YamlValue

tstring_core-0.1.0/src/tstring_core/_values.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Generic, TypeVar
+
+from ._types import StructuredData
+
+TData = TypeVar("TData", bound=StructuredData, covariant=True)
+
+
+class ValueKind(str, Enum):
+    TEXT = "text"
+    DATA = "data"
+
+
+@dataclass(frozen=True, slots=True)
+class RenderResult(Generic[TData]):
+    text: str
+    data: TData

tstring_core-0.1.0/src/tstring_core/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561 typing support.