gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/environment/schema/compiler.py

@@ -0,0 +1,615 @@
+"""One authored authority -> deterministic SQLite, runtime, docs and tools."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+from .validation import (
+    LANGUAGE,
+    ContractError,
+    Resolver,
+    canonical,
+    digest,
+    encoded,
+    fail,
+    members,
+    nonempty,
+    publication_digest,
+)
+
+COMPILER = "gateway-world-compiler/1"
+TARGET = "sqlite-json/1"
+GENERATED = (
+    "db/schema.sql",
+    "schema/runtime.json",
+    "schema/README.md",
+    "tool-schemas.json",
+)
+LOCK = "schema/lock.json"
+ROOT = "schema/world.json"
+RESERVED = ("_", "sqlite_", "gateway_", "benchmark_")
+FIELD_RESERVED = ("sqlite_", "gateway_", "benchmark_")
+
+
+def identifier(value: Any, location: str) -> str:
+    if (
+        not isinstance(value, str)
+        or not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*", value)
+        or value.lower().startswith(RESERVED)
+    ):
+        fail(
+            "identity",
+            "Use a SQL-safe name; _, sqlite_, gateway_, benchmark_ prefixes are reserved",
+            location,
+        )
+    return value
+
+
+def field_identifier(value: Any, location: str) -> str:
+    if (
+        not isinstance(value, str)
+        or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value)
+        or value.lower() in {"_key", "payload"}
+        or value.lower().startswith(FIELD_RESERVED)
+    ):
+        fail(
+            "identity",
+            "Use a SQL-safe payload name; _key, payload, sqlite_, gateway_, and benchmark_ are reserved",
+            location,
+        )
+    return value
+
+
+def quote(value: str) -> str:
+    return '"' + value.replace('"', '""') + '"'
+
+
+def json_pointer_parts(value: str, location: str, *, code: str) -> list[str]:
+    if not value.startswith("/"):
+        return [value]
+    parts = []
+    for part in value[1:].split("/"):
+        if re.search(r"~(?![01])", part):
+            fail(code, "Invalid JSON Pointer escape in field path", location)
+        decoded = part.replace("~1", "/").replace("~0", "~")
+        if '"' in decoded:
+            fail(code, "Quoted property names are unsupported in key paths", location)
+        parts.append(decoded)
+    return parts
+
+
+def field_schema(
+    schema: dict,
+    field: str,
+    location: str,
+    *,
+    code: str = "key",
+    nested: bool = True,
+) -> dict:
+    if field.startswith("/") and not nested:
+        fail(code, "Primary keys must use direct property names", location)
+    parts = json_pointer_parts(field, location, code=code)
+
+    def resolve(current: dict, remaining: list[str]) -> dict | None:
+        if not remaining:
+            return current
+        part, *rest = remaining
+        if current.get("type") == "object" and part in current.get("properties", {}):
+            return resolve(current["properties"][part], rest)
+        for keyword in ("allOf", "anyOf", "oneOf"):
+            matches = [
+                match for branch in current.get(keyword, []) if (match := resolve(branch, remaining)) is not None
+            ]
+            if len(matches) == 1 or (matches and all(canonical(match) == canonical(matches[0]) for match in matches)):
+                return matches[0]
+        return None
+
+    resolved = resolve(schema, parts)
+    if resolved is None:
+        fail(code, "Field paths must resolve through object properties", location)
+    return resolved
+
+
+def payload_value(value: dict, field: str) -> Any:
+    current: Any = value
+    for part in json_pointer_parts(field, field, code="key"):
+        if not isinstance(current, dict) or part not in current:
+            return None
+        current = current[part]
+    return current
+
+
+def _json_path(field: str) -> str:
+    parts = json_pointer_parts(field, field, code="key")
+    return "$" + "".join(f".{json.dumps(part, ensure_ascii=False)}" for part in parts)
+
+
+def bundle_mode(bundle: Path) -> dict | None:
+    """Markers never downgrade to legacy when an authority has disappeared."""
+    bundle = Path(bundle)
+    path = bundle / "gateway-env.toml"
+    # exists() is False for a dangling symlink; a broken authority must fail,
+    # never quietly run as an unconstrained legacy bundle.
+    markers = any((bundle / p).exists() or (bundle / p).is_symlink() for p in (ROOT, LOCK, "schema/runtime.json"))
+    try:
+        manifest = tomllib.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+    except (OSError, ValueError) as error:
+        raise ContractError("manifest", "Cannot parse gateway-env.toml") from error
+    declaration = manifest.get("schema")
+    if declaration is None and not markers:
+        return None
+    if declaration != {"language": LANGUAGE}:
+        fail(
+            "language",
+            f"Schema bundles require exactly [schema] language = '{LANGUAGE}'",
+            "gateway-env.toml/schema",
+        )
+    if (bundle / "gateway-api-world.toml").exists():
+        fail("bundle_mode", "Schema contracts currently support mock-tools bundles only")
+    env = manifest.get("environment", manifest.get("env", {}))
+    if not isinstance(env, dict) or env.get("kind", "mock-tools") != "mock-tools":
+        fail("bundle_mode", "Schema contracts currently support mock-tools bundles only")
+    db = manifest.get("db", {})
+    if not isinstance(db, dict) or db.get("engine", "sqlite") != "sqlite" or env.get("db", "sqlite") != "sqlite":
+        fail("target", "Schema worlds require SQLite")
+    if any(k in db for k in ("schema", "seed", "path")):
+        fail(
+            "unsupported",
+            "Schema mode uses fixed generated/initial paths; remove db.schema/seed/path",
+        )
+    if (bundle / "db/seed.sql").exists():
+        fail(
+            "raw_seed",
+            "Remove db/seed.sql; schema worlds seed through data/initial.json",
+        )
+    for p in (ROOT, LOCK, *GENERATED, "data/initial.json"):
+        target = bundle / p
+        if target.is_symlink() or not target.resolve().is_relative_to(bundle.resolve()):
+            fail(
+                "path",
+                "Bundle contract paths may not escape the bundle or be symlinks",
+                p,
+            )
+    return manifest
+
+
+def require_legacy_bundle(bundle: str | Path, runtime: str) -> None:
+    """Explicit rejection for alternate execution paths without store support."""
+    if bundle_mode(Path(bundle)) is not None:
+        fail(
+            "bundle_mode",
+            f"Schema-backed bundles are unsupported by {runtime}; use the SDK environment or mock-tools runner",
+        )
+
+
+def _key_fields(
+    fields: Any,
+    schema: dict,
+    location: str,
+    *,
+    required: bool = False,
+    nested: bool = True,
+    code: str = "key",
+) -> list[str]:
+    if (
+        not isinstance(fields, list)
+        or not fields
+        or any(not isinstance(k, str) for k in fields)
+        or len(set(fields)) != len(fields)
+    ):
+        fail(code, "Keys require a nonempty list of distinct property paths", location)
+    for field in fields:
+        prop = field_schema(schema, field, f"{location}/{field}", code=code, nested=nested)
+        types = prop.get("type", [])
+        types = [types] if isinstance(types, str) else types
+        if not set(types) or set(types) - {"string", "integer", "null"} or not set(types) & {"string", "integer"}:
+            fail(code, "Key paths must resolve to string/integer properties", location)
+        if any(k in prop for k in ("allOf", "oneOf", "anyOf")):
+            fail(code, "Composition is not supported on key fields", location)
+        if required and (field not in schema.get("required", []) or "null" in types):
+            fail(
+                code,
+                "Primary-key fields must be required and non-null",
+                f"{location}/{field}",
+            )
+    return fields
+
+
+@dataclass(frozen=True)
+class Contract:
+    metadata: dict
+    artifacts: dict[str, bytes]
+    lock: dict
+
+    @property
+    def target(self) -> dict:
+        return self.metadata["domain"]
+
+    @property
+    def entities(self) -> dict:
+        return self.metadata["entities"]
+
+    @property
+    def tools(self) -> dict:
+        return self.metadata["tools"]
+
+
+def build(bundle: Path) -> Contract:
+    bundle = Path(bundle)
+    manifest = bundle_mode(bundle)
+    if manifest is None:
+        fail("language", "Not a schema-backed bundle; declare [schema] language")
+    resolver = Resolver(bundle)
+    document = resolver.load("world.json")
+    annotation = document.get("x-gateway")
+    if not isinstance(annotation, dict):
+        fail("schema", "x-gateway must be an object", "world.json#/x-gateway")
+    members(
+        annotation,
+        {"version", "domain_version", "package", "entities", "tools"},
+        {"version", "domain_version", "package", "entities", "tools"},
+        "world.json#/x-gateway",
+    )
+    if type(annotation["version"]) is not int or annotation["version"] != 1:
+        fail(
+            "language",
+            "Only x-gateway version 1 is supported",
+            "world.json#/x-gateway/version",
+        )
+    nonempty(annotation["domain_version"], "world.json#/x-gateway/domain_version")
+    package = annotation["package"]
+    if not isinstance(package, dict):
+        fail("schema", "package must be an object", "world.json#/x-gateway/package")
+    members(package, {"id", "version"}, {"id", "version"}, "world.json#/x-gateway/package")
+    for key, value in package.items():
+        nonempty(value, f"world.json#/x-gateway/package/{key}")
+    entities = annotation["entities"]
+    if not isinstance(entities, dict) or not entities:
+        fail("schema", "Declare at least one entity", "world.json#/x-gateway/entities")
+    compiled = {}
+    seen = set()
+    for name, entity in sorted(entities.items()):
+        location = f"world.json#/x-gateway/entities/{name}"
+        identifier(name, location)
+        if name.lower() in seen:
+            fail("identity", "Entity names collide under SQLite case folding", location)
+        seen.add(name.lower())
+        members(
+            entity,
+            {"schema", "primary_key", "unique", "relationships"},
+            {"schema", "primary_key"},
+            location,
+        )
+        payload = resolver.resolve(entity["schema"], "world.json", f"{location}/schema")
+        if (
+            payload.get("type") != "object"
+            or payload.get("additionalProperties") is not False
+            or any(k in payload for k in ("allOf", "oneOf", "anyOf"))
+        ):
+            fail(
+                "entity",
+                "Entity roots must be direct closed objects (reuse through $ref, no inheritance)",
+                location,
+            )
+        field_names = set()
+        for field in payload.get("properties", {}):
+            field_identifier(field, f"{location}/schema/properties/{field}")
+            if field.lower() in {"payload"} | field_names:
+                fail(
+                    "identity",
+                    "Payload field collides with storage or another field",
+                    location,
+                )
+            field_names.add(field.lower())
+        primary = _key_fields(
+            entity["primary_key"],
+            payload,
+            f"{location}/primary_key",
+            required=True,
+            nested=False,
+        )
+        unique = entity.get("unique", [])
+        if not isinstance(unique, list):
+            fail("key", "unique must be a list of key lists", location)
+        constraints = {frozenset(primary)}
+        for key in unique:
+            _key_fields(key, payload, f"{location}/unique")
+            if frozenset(key) in constraints:
+                fail("key", "Duplicate key declaration", location)
+            constraints.add(frozenset(key))
+        relationships = entity.get("relationships", [])
+        if not isinstance(relationships, list):
+            fail("relationship", "relationships must be an array", location)
+        relation_fields = set()
+        for rel in relationships:
+            members(
+                rel,
+                {"fields", "target", "on_delete"},
+                {"fields", "target", "on_delete"},
+                f"{location}/relationships",
+            )
+            _key_fields(
+                rel["fields"],
+                payload,
+                f"{location}/relationships/fields",
+                code="relationship",
+            )
+            nonempty(rel["target"], f"{location}/relationships/target")
+            if rel["on_delete"] != "restrict":
+                fail(
+                    "unsupported",
+                    "Only restrict deletion is supported; delete dependents explicitly in one batch",
+                    location,
+                )
+            if tuple(rel["fields"]) in relation_fields:
+                fail("relationship", "A field tuple may have only one target", location)
+            relation_fields.add(tuple(rel["fields"]))
+        compiled[name] = {
+            "schema": payload,
+            "primary_key": primary,
+            "unique": sorted(unique),
+            "relationships": sorted(relationships, key=canonical),
+            "location": location,
+        }
+    for name, entity in compiled.items():
+        for rel in entity["relationships"]:
+            location = f"{entity['location']}/relationships"
+            target = compiled.get(rel["target"])
+            if target is None or len(target["primary_key"]) != len(rel["fields"]):
+                fail(
+                    "relationship",
+                    "Target must exist with matching primary-key arity",
+                    location,
+                )
+            for source, dest in zip(rel["fields"], target["primary_key"]):
+                source_type = field_schema(
+                    entity["schema"],
+                    source,
+                    location,
+                    code="relationship",
+                )["type"]
+                source_types = set(source_type if isinstance(source_type, list) else [source_type]) - {"null"}
+                target_type = target["schema"]["properties"][dest]["type"]
+                target_types = set(target_type if isinstance(target_type, list) else [target_type])
+                if source_types != target_types:
+                    fail(
+                        "relationship",
+                        "Relationship endpoint key types must match",
+                        location,
+                    )
+    tools = annotation["tools"]
+    if not isinstance(tools, dict):
+        fail("tool", "tools must be an object", "world.json#/x-gateway/tools")
+    compiled_tools = {}
+    for name, tool in sorted(tools.items()):
+        location = f"world.json#/x-gateway/tools/{name}"
+        if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]*", name):
+            fail("tool", "Invalid tool name", location)
+        members(tool, {"input", "output", "description"}, {"input", "output"}, location)
+        if "description" in tool and not isinstance(tool["description"], str):
+            fail("tool", "Tool description must be text", location)
+        inputs = resolver.resolve(tool["input"], "world.json", f"{location}/input")
+        outputs = resolver.resolve(tool["output"], "world.json", f"{location}/output")
+        if inputs.get("type") != "object":
+            fail("tool", "Tool inputs must resolve to an object schema", location)
+        compiled_tools[name] = {
+            "input_schema": inputs,
+            "output_schema": outputs,
+            "description": tool.get("description", name),
+        }
+    served = manifest.get("tools", {}).get("serve") if isinstance(manifest.get("tools", {}), dict) else None
+    if (
+        not isinstance(served, list)
+        or any(type(n) is not str for n in served)
+        or len(served) != len(set(served))
+        or set(served) != set(tools)
+    ):
+        fail(
+            "tool",
+            "Manifest tools.serve must exactly match authored tool contracts",
+            "gateway-env.toml/tools/serve",
+        )
+    resolver.check_documents()
+    inputs = {
+        name: {
+            "id": doc["$id"],
+            "hash": digest(encoded(doc)),
+            "publication_hash": publication_digest(doc),
+        }
+        for name, doc in sorted(resolver.documents.items())
+    }
+    # Domain contract excludes package/tool behavior; changing a mapping cannot change it.
+    domain = {
+        "id": document["$id"],
+        "version": annotation["domain_version"],
+        "hash": digest(encoded({"language": LANGUAGE, "target": TARGET, "entities": compiled})),
+    }
+    metadata = {
+        "language": LANGUAGE,
+        "compiler": COMPILER,
+        "target": TARGET,
+        "domain": domain,
+        "package": package,
+        "entities": compiled,
+        "tools": compiled_tools,
+    }
+    sql = _ddl(compiled)
+    docs = [
+        f"# {domain['id']} ({domain['version']})",
+        "",
+        "Generated; edit schema/world.json and compile. Defaults are annotations only.",
+        "",
+    ]
+    for name, entity in compiled.items():
+        docs.extend(
+            [
+                f"## {name}",
+                "",
+                f"Primary key: {', '.join(entity['primary_key'])}",
+                "",
+                "| Property | Type | Required |",
+                "| --- | --- | --- |",
+            ]
+        )
+        for field, prop in sorted(entity["schema"].get("properties", {}).items()):
+            required = field in entity["schema"].get("required", [])
+            docs.append(f"| {field} | {canonical(prop.get('type', 'composition'))} | " f"{required} |")
+        docs.extend(
+            [
+                "",
+                f"Unique: {canonical(entity['unique'])}",
+                f"Relationships: {canonical(entity['relationships'])}",
+                "",
+                "```json",
+                json.dumps(entity["schema"], sort_keys=True, ensure_ascii=False, indent=2),
+                "```",
+                "",
+            ]
+        )
+    docs.extend(
+        [
+            "## Tool contracts",
+            "",
+            "```json",
+            json.dumps(compiled_tools, sort_keys=True, ensure_ascii=False, indent=2),
+            "```",
+            "",
+        ]
+    )
+    artifacts = {
+        "db/schema.sql": sql.encode(),
+        "schema/runtime.json": encoded(metadata),
+        "schema/README.md": "\n".join(docs).encode(),
+        "tool-schemas.json": encoded({"tools": compiled_tools}),
+    }
+    lock = {
+        "language": LANGUAGE,
+        "annotations": 1,
+        "compiler": COMPILER,
+        "target": TARGET,
+        "domain": domain,
+        "package": package,
+        "inputs": inputs,
+        "tools_hash": digest(encoded(compiled_tools)),
+        "artifacts": {p: digest(b) for p, b in sorted(artifacts.items())},
+    }
+    return Contract(metadata, artifacts, lock)
+
+
+def _ddl(entities: dict) -> str:
+    lines = [
+        "-- Generated by gateway-world-compiler/1; do not edit.",
+        "PRAGMA foreign_keys = ON;",
+    ]
+    for name, entity in entities.items():
+        columns = [
+            '"_key" TEXT PRIMARY KEY NOT NULL',
+            '"payload" TEXT NOT NULL CHECK(json_valid("payload"))',
+        ]
+        for field, prop in sorted(entity["schema"].get("properties", {}).items()):
+            # JSON remains the source of truth. Projections make verifier SQL useful,
+            # and provide SQLite UNIQUE/FK enforcement without affinity coercion.
+            types = prop.get("type", [])
+            types = [types] if isinstance(types, str) else types
+            if set(types) & {"string", "integer", "number", "boolean"}:
+                columns.append(
+                    f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", '
+                    f"{quote_sql(_json_path(field))})) VIRTUAL"
+                )
+        projected = set(entity["schema"].get("properties", {}))
+        constrained = {field for fields in [entity["primary_key"], *entity["unique"]] for field in fields} | {
+            field for relationship in entity["relationships"] for field in relationship["fields"]
+        }
+        for field in sorted(constrained - projected):
+            columns.append(
+                f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", '
+                f"{quote_sql(_json_path(field))})) VIRTUAL"
+            )
+        for fields in [entity["primary_key"], *entity["unique"]]:
+            columns.append(f"UNIQUE ({', '.join(quote(f) for f in fields)})")
+        for rel in entity["relationships"]:
+            target = entities[rel["target"]]
+            columns.append(
+                f"FOREIGN KEY ({', '.join(quote(f) for f in rel['fields'])}) "
+                f"REFERENCES {quote(rel['target'])} "
+                f"({', '.join(quote(f) for f in target['primary_key'])}) "
+                "ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"
+            )
+        lines.append(f"CREATE TABLE {quote(name)} (\n  " + ",\n  ".join(columns) + "\n);")
+    return "\n".join(lines) + "\n"
+
+
+def quote_sql(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
+    """Generate artifacts; never overwrite authored handlers or seed data.
+
+    With ``snapshots`` (the default) the data snapshots are built too, once the
+    tree has its ``data/initial.json``: ``db/state.sqlite`` and one per scenario,
+    each skipped when its sidecar already names this contract and batch. That
+    build is the proof the data applies through the contract. Data the contract
+    refuses (an author changed the model and has not re-ingested yet) leaves no
+    snapshot behind at all; the refusal is reported where the data is next used
+    (`gateway worlds schema compile` reports it at once, see ``compile_world``)."""
+    bundle = Path(bundle)
+    contract = build(bundle)
+    for path, content in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
+        destination = bundle / path
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(content)
+    if snapshots and (bundle / "data/initial.json").is_file():
+        from .snapshot import build_all, discard_all
+
+        try:
+            build_all(bundle, contract)
+        except ContractError:
+            discard_all(bundle)
+    return contract
+
+
+def compile_world(bundle: str | Path) -> tuple[Contract, dict]:
+    """``compile_bundle`` plus the snapshots, refusing loudly when the data does not
+    apply through the contract: what the CLI runs, so an author sees the row."""
+    from .snapshot import build_all
+
+    contract = compile_bundle(bundle, snapshots=False)
+    return contract, build_all(bundle, contract)
+
+
+def check_bundle(bundle: str | Path) -> Contract:
+    """The contract, provided every generated artifact and every snapshot present in
+    the tree matches it; a snapshot's sidecar must name this contract and the
+    file's own sha256. Absent snapshots are not required: the legacy path serves."""
+    bundle = Path(bundle)
+    contract = build(bundle)
+    for path, expected in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
+        try:
+            matches = (bundle / path).read_bytes() == expected
+        except OSError:
+            matches = False
+        if not matches:
+            fail(
+                "lock_mismatch",
+                "Missing, edited or incompatible contract artifact; explicitly "
+                "recompile (state migration is not automatic)",
+                path,
+            )
+    from .snapshot import check_all
+
+    check_all(bundle, contract)
+    return contract
+
+
+def describe_bundle(bundle: str | Path) -> str:
+    return build(Path(bundle)).artifacts["schema/README.md"].decode()