gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/environment/schema/conform.py

@@ -0,0 +1,428 @@
+"""Conformance: does the served world answer the way the vendor's OpenAPI says it does?
+
+A world is 1:1 when every declared operation, called the way the vendor's client
+calls it, comes back with the status the vendor documents and a body the vendor's
+response schema accepts. This runs the world in-process, drives each operation
+with arguments taken from the world's own seeded rows (so lookups hit data), and
+validates each answer against the OpenAPI document TRM or the vendor supplied.
+
+    gateway worlds connector conform ./torchlight-world --openapi briefs/torchlight-openapi.yaml
+
+The validator here is deliberately lenient and standalone: vendor documents use
+the whole of OpenAPI 3.1 (``nullable``, ``readOnly``, ``example`` siblings,
+``discriminator``), which the world contract's strict validator refuses on
+purpose. What is checked: types (with ``null``), ``required``, ``properties``
+(unknown members allowed unless ``additionalProperties: false``), ``items``,
+``enum``, ``const``, ``allOf``/``anyOf``/``oneOf`` and ``$ref`` into the document.
+Formats are not checked — a mock that answers a UUID-shaped string where the
+vendor answers a UUID is not the failure this tool exists to find.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import threading
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .api import WorldState, load_api_settings, serve
+from .connector import Operation, load_spec
+from .validation import fail
+
+PIN = "conform"
+
+
+# ---------------------------------------------------------------- the document
+
+
+def load_openapi(path: str | Path) -> dict:
+    text = Path(path).read_text(encoding="utf-8")
+    if str(path).endswith((".yaml", ".yml")):
+        try:
+            import yaml  # optional at runtime; the SDK depends on it, the vendored server does not
+        except ImportError:  # pragma: no cover
+            fail("conform", "Reading a YAML document needs PyYAML; convert it to JSON", "/conform/openapi")
+        document = yaml.safe_load(text)
+    else:
+        document = json.loads(text)
+    if not isinstance(document, dict) or "paths" not in document:
+        fail("conform", f"{path} is not an OpenAPI document (no paths)", "/conform/openapi")
+    return document
+
+
+def server_prefixes(document: dict) -> list[str]:
+    """The path part of each server URL, longest first, so a world whose paths carry
+    the vendor's prefix (``/investigations-v2/breach/...``) still lines up."""
+    prefixes: list[str] = []
+    for server in document.get("servers") or []:
+        url = server.get("url", "") if isinstance(server, dict) else ""
+        for name, variable in ((server.get("variables") or {}) if isinstance(server, dict) else {}).items():
+            url = url.replace("{" + name + "}", str(variable.get("default", "")))
+        path = urllib.parse.urlsplit(url).path.rstrip("/") if "://" in url else url.rstrip("/")
+        if path and path not in prefixes:
+            prefixes.append(path)
+    return sorted(prefixes, key=len, reverse=True)
+
+
+def _canonical(path: str) -> str:
+    """Placeholder names differ between the world and the document; shapes must not."""
+    out, depth = [], 0
+    for char in path:
+        if char == "{":
+            depth += 1
+            out.append("{}")
+        elif char == "}":
+            depth -= 1
+        elif depth == 0:
+            out.append(char)
+    return "".join(out)
+
+
+def find_operation(document: dict, prefixes: list[str], operation: Operation) -> tuple[str, dict] | None:
+    """The document's path item + method for a world operation, prefix-tolerant."""
+    candidates = [operation.path] + [operation.path[len(p) :] for p in prefixes if operation.path.startswith(p + "/")]
+    wanted = {_canonical(c) for c in candidates}
+    for path, item in (document.get("paths") or {}).items():
+        if _canonical(path) in wanted and isinstance(item, dict) and operation.method.lower() in item:
+            return path, item[operation.method.lower()]
+    return None
+
+
+# ---------------------------------------------------------------- lenient validation
+
+
+class Checker:
+    def __init__(self, document: dict):
+        self.document = document
+
+    def deref(self, schema: Any, seen: tuple[str, ...] = ()) -> dict:
+        while isinstance(schema, dict) and "$ref" in schema:
+            ref = schema["$ref"]
+            if ref in seen or not ref.startswith("#/"):
+                return {}
+            seen = seen + (ref,)
+            node: Any = self.document
+            for part in ref[2:].split("/"):
+                part = part.replace("~1", "/").replace("~0", "~")
+                node = node.get(part) if isinstance(node, dict) else None
+                if node is None:
+                    return {}
+            siblings = {k: v for k, v in schema.items() if k != "$ref"}
+            schema = {**node, **siblings} if isinstance(node, dict) else {}
+        return schema if isinstance(schema, dict) else {}
+
+    def check(self, schema: Any, value: Any, path: str = "$") -> list[str]:
+        schema = self.deref(schema)
+        if not schema:
+            return []
+        problems: list[str] = []
+        types = schema.get("type")
+        types = [types] if isinstance(types, str) else list(types or [])
+        if schema.get("nullable") and "null" not in types:
+            types.append("null")
+        actual = {
+            type(None): "null",
+            bool: "boolean",
+            str: "string",
+            dict: "object",
+            list: "array",
+            int: "integer",
+            float: "number",
+        }.get(type(value), "unknown")
+        if types and not (
+            actual in types
+            or (actual == "integer" and "number" in types)
+            or (actual == "number" and "integer" in types and float(value).is_integer())
+        ):
+            return [f"{path}: expected {' | '.join(types)}, got {actual}"]
+        if value is None:
+            return problems
+        if "enum" in schema and value not in schema["enum"]:
+            problems.append(f"{path}: {value!r} is not one of the enum")
+        if "const" in schema and value != schema["const"]:
+            problems.append(f"{path}: differs from const {schema['const']!r}")
+        for keyword in ("allOf", "anyOf", "oneOf"):
+            branches = schema.get(keyword) or []
+            if not branches:
+                continue
+            results = [self.check(branch, value, path) for branch in branches]
+            passing = sum(1 for r in results if not r)
+            if keyword == "allOf" and passing != len(branches):
+                problems.extend(p for r in results for p in r)
+            elif keyword == "anyOf" and passing == 0:
+                problems.append(f"{path}: matches no anyOf branch ({'; '.join(results[0][:2])})")
+            elif keyword == "oneOf" and passing != 1:
+                problems.append(f"{path}: matches {passing} oneOf branches, not exactly one")
+        if isinstance(value, dict):
+            properties = schema.get("properties") or {}
+            for name in schema.get("required") or []:
+                if name not in value:
+                    problems.append(f"{path}.{name}: required by the vendor, missing")
+            for name, member in value.items():
+                if name in properties:
+                    problems.extend(self.check(properties[name], member, f"{path}.{name}"))
+                elif schema.get("additionalProperties") is False:
+                    problems.append(f"{path}.{name}: not a property the vendor documents")
+                elif isinstance(schema.get("additionalProperties"), dict):
+                    problems.extend(self.check(schema["additionalProperties"], member, f"{path}.{name}"))
+        if isinstance(value, list) and isinstance(schema.get("items"), dict):
+            for index, item in enumerate(value[:50]):
+                problems.extend(self.check(schema["items"], item, f"{path}[{index}]"))
+        return problems
+
+    def example(self, schema: Any, depth: int = 0) -> Any:
+        """A body the vendor's request schema accepts, from its examples or its shape."""
+        schema = self.deref(schema)
+        for key in ("example", "default"):
+            if key in schema:
+                return schema[key]
+        if isinstance(schema.get("examples"), list) and schema["examples"]:
+            return schema["examples"][0]
+        if "enum" in schema and schema["enum"]:
+            return schema["enum"][0]
+        for keyword in ("allOf", "anyOf", "oneOf"):
+            if schema.get(keyword):
+                merged: dict = {}
+                for branch in schema[keyword] if keyword == "allOf" else schema[keyword][:1]:
+                    value = self.example(branch, depth + 1)
+                    if isinstance(value, dict):
+                        merged.update(value)
+                    else:
+                        return value
+                return merged
+        types = schema.get("type")
+        kind = types if isinstance(types, str) else next((t for t in (types or []) if t != "null"), None)
+        if kind == "object" or (kind is None and "properties" in schema):
+            if depth > 6:
+                return {}
+            return {
+                name: self.example(prop, depth + 1)
+                for name, prop in (schema.get("properties") or {}).items()
+                if name in (schema.get("required") or [])
+            }
+        if kind == "array":
+            return (
+                [self.example(schema["items"], depth + 1)]
+                if isinstance(schema.get("items"), dict) and depth < 6
+                else []
+            )
+        if kind == "integer":
+            return int(schema.get("minimum", 1))
+        if kind == "number":
+            return float(schema.get("minimum", 1))
+        if kind == "boolean":
+            return True
+        if kind == "string":
+            fmt = schema.get("format")
+            if fmt == "date-time":
+                return "2026-01-01T00:00:00Z"
+            if fmt == "date":
+                return "2026-01-01"
+            if fmt == "uuid":
+                return "00000000-0000-4000-8000-000000000001"
+            if fmt == "email":
+                return "sample@example.invalid"
+            return "sample"
+        return None
+
+
+# ---------------------------------------------------------------- the run
+
+
+@dataclass
+class Outcome:
+    operation: str
+    method: str
+    path: str
+    documented: bool
+    status: int | None = None
+    expected: list[str] = field(default_factory=list)
+    schema: str = "unchecked"  # ok | violations | no-schema | unchecked
+    problems: list[str] = field(default_factory=list)
+    request: dict = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return (
+            self.documented
+            and self.status is not None
+            and str(self.status) in self.expected
+            and self.schema in ("ok", "no-schema")
+        )
+
+
+@dataclass
+class Report:
+    world: str
+    outcomes: list[Outcome]
+
+    @property
+    def summary(self) -> dict:
+        checked = [o for o in self.outcomes if o.documented]
+        return {
+            "operations": len(self.outcomes),
+            "documented": len(checked),
+            "conforming": sum(1 for o in checked if o.ok),
+            "status_mismatch": sum(1 for o in checked if o.status is not None and str(o.status) not in o.expected),
+            "schema_violations": sum(1 for o in checked if o.schema == "violations"),
+            "undocumented": sum(1 for o in self.outcomes if not o.documented),
+        }
+
+    def to_json(self) -> dict:
+        return {"world": self.world, "summary": self.summary, "operations": [o.__dict__ for o in self.outcomes]}
+
+
+def _credentials(base: str, spec, settings) -> dict[str, str]:
+    """Headers that get the conformance run through the world's own auth, using the pin."""
+    if spec.scheme == "none":
+        return {}
+    if spec.scheme == "header":
+        return {spec.auth_name or "": PIN}
+    if spec.scheme == "bearer":
+        return {"Authorization": f"Bearer {PIN}"}
+    if spec.scheme == "basic":
+        return {"Authorization": "Basic " + base64.b64encode(PIN.encode()).decode()}
+    if spec.scheme == "query":
+        return {}
+    form = urllib.parse.urlencode(
+        {"grant_type": "client_credentials", "client_id": "conform", "client_secret": PIN}
+    ).encode()
+    prefix = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
+    request = urllib.request.Request(base + prefix + (settings.token_path or "/oauth/token"), data=form, method="POST")
+    with urllib.request.urlopen(request, timeout=10) as reply:
+        return {"Authorization": "Bearer " + json.loads(reply.read())["access_token"]}
+
+
+def _hints(spec, settings) -> dict[str, tuple[str, str]]:
+    """Where a placeholder's values live, learned from every operation that filters on it:
+    ``{recordId}`` on a handler route resolves through the read route that maps it to ``id``."""
+    hints: dict[str, tuple[str, str]] = {}
+    for operation in spec.operations.values():
+        if not operation.entity:
+            continue
+        for arg, path in settings.for_operation(operation.name).filter.items():
+            hints.setdefault(arg, (operation.entity, path))
+    return hints
+
+
+def _first_value(rows: list[dict], path: str) -> Any:
+    for row in rows:
+        found: Any = row
+        for part in path.split("."):
+            found = found.get(part) if isinstance(found, dict) else None
+        if found is not None and not isinstance(found, (dict, list)):
+            return found
+    return None
+
+
+def _sample_args(
+    operation: Operation, spec, settings, state: WorldState, checker: Checker, documented: dict | None
+) -> tuple[dict[str, str], dict[str, str]]:
+    """Path arguments from the world's own rows — the operation's declared ``sample``
+    first (a value a validating handler accepts), then this operation's filter, then any
+    operation's filter on the same placeholder, then the entity's primary key — else the
+    document's examples. The point is to hit data, not to probe 404s."""
+    op = settings.for_operation(operation.name)
+    hints = _hints(spec, settings)
+    args: dict[str, str] = {}
+    params: dict[str, str] = {}
+    documented_params = {p.get("name"): p for p in (documented or {}).get("parameters", []) if isinstance(p, dict)}
+    for name in operation.params:
+        if name in op.sample:
+            params[name] = op.sample[name]
+    for name in operation.placeholders:
+        value = op.sample.get(name)
+        if value is None and name in op.filter and operation.entity:
+            value = _first_value(state.rows(operation.entity), op.filter[name])
+        if value is None and name in hints:
+            entity, path = hints[name]
+            value = _first_value(state.rows(entity), path)
+        if value is None and operation.entity:
+            key = state.contract.entities[operation.entity]["primary_key"]
+            if len(key) == 1:
+                value = _first_value(state.rows(operation.entity), key[0])
+        if value is None and name in documented_params:
+            value = checker.example(documented_params[name].get("schema") or {})
+        args[name] = str(value if value is not None else "1")
+    return args, params
+
+
+def conform(world: str | Path, openapi: list[str | Path], *, max_operations: int | None = None) -> Report:
+    root = Path(world)
+    spec = load_spec(root)
+    settings = load_api_settings(root, spec)
+    documents = [load_openapi(p) for p in openapi]
+    if not documents:
+        fail("conform", "Give at least one --openapi document to conform against", "/conform/openapi")
+    server = serve(root, port=0, host="127.0.0.1", api_key=PIN)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    base = f"http://127.0.0.1:{server.server_port}"
+    prefix = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
+    state: WorldState = server.world_state  # type: ignore[attr-defined]
+    outcomes: list[Outcome] = []
+    try:
+        headers = _credentials(base, spec, settings)
+        for operation in list(spec.operations.values())[:max_operations]:
+            located = None
+            checker = None
+            for document in documents:
+                found = find_operation(document, server_prefixes(document), operation)
+                if found:
+                    located, checker = found, Checker(document)
+                    break
+            outcome = Outcome(operation.name, operation.method, operation.path, documented=located is not None)
+            if located is None or checker is None:
+                outcomes.append(outcome)
+                continue
+            _, documented = located
+            responses = documented.get("responses") or {}
+            outcome.expected = [code for code in responses if code.startswith("2")] or ["200"]
+            args, params = _sample_args(operation, spec, settings, state, checker, documented)
+            if spec.scheme == "query":
+                params[spec.auth_name or ""] = PIN
+            path = operation.path
+            for name, value in args.items():
+                path = path.replace("{" + name + "}", urllib.parse.quote(value, safe=""))
+            url = base + prefix + path + (f"?{urllib.parse.urlencode(params)}" if params else "")
+            body = None
+            content = ((documented.get("requestBody") or {}).get("content") or {}).get("application/json") or {}
+            if operation.method in ("POST", "PUT", "PATCH"):
+                body = content.get("example") or checker.example(content.get("schema") or {}) or {}
+            outcome.request = {"url": prefix + path, "args": args, **({"body": body} if body is not None else {})}
+            data = json.dumps(body).encode() if body is not None else None
+            request = urllib.request.Request(
+                url,
+                method=operation.method,
+                data=data,
+                headers={**headers, **({"Content-Type": "application/json"} if data else {})},
+            )
+            try:
+                with urllib.request.urlopen(request, timeout=15) as reply:
+                    status, raw = reply.status, reply.read()
+            except urllib.error.HTTPError as error:
+                status, raw = error.code, error.read()
+            outcome.status = status
+            answer = responses.get(str(status)) or responses.get("default") or {}
+            schema = ((answer.get("content") or {}).get("application/json") or {}).get("schema")
+            if schema is None:
+                outcome.schema = "no-schema"
+            else:
+                try:
+                    value = json.loads(raw) if raw else None
+                except json.JSONDecodeError:
+                    value = None
+                    outcome.problems.append("$: body is not JSON")
+                outcome.problems.extend(checker.check(schema, value))
+                outcome.schema = "violations" if outcome.problems else "ok"
+            outcomes.append(outcome)
+    finally:
+        server.shutdown()
+        server.server_close()
+        state.close()
+    return Report(str(root), outcomes)