gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/environment/schema/connector.py

@@ -0,0 +1,777 @@
+"""Custom connections: describe a vendor API once, capture what the agent's
+operations return, and seed the world from those captures through the schema.
+
+The same ``connector.toml`` serves the CLI here, the world's own API
+(``gateway worlds serve``) and the platform's custom source later, so a
+connection is described once::
+
+    [connector]
+    slug = "spycloud"
+    base_url = "https://api.spycloud.io/investigations-v2"
+
+    [auth]
+    scheme = "header"            # header | bearer | basic | query | none
+    name = "x-api-key"           # header or query parameter that carries the secret
+    secret_env = "SPYCLOUD_API_KEY"
+    headers = { "X-App-Token" = "SPYCLOUD_APP_TOKEN" }   # more credential headers, each named by an env var
+
+    [capture]
+    requests_per_minute = 30     # pacing between requests; providers ask us to be sparing
+    timeout_seconds = 30
+    max_pages = 5
+    cursor_param = "cursor"      # optional: cursor pagination, request side
+    cursor_field = "cursor"      # optional: cursor pagination, response side ('' or null ends)
+
+    [redact]
+    hash = ["email", "password"] # replaced by "sha256:<hex>" salted with the slug, at any depth
+    drop = ["ssn"]               # removed at any depth
+
+    [[operations]]
+    name = "get_records_by_email"
+    method = "GET"
+    path = "/breach/data/emails/{email}"
+    params = ["since", "until", "severity"]
+    entity = "spycloud_records"  # optional: rows land in this entity on ingest
+    results = "results"          # dotted path to the row array; "" means the body is the row
+    [operations.ingest]
+    drop = ["channels"]          # vendor fields the entity does not carry (they are modelled elsewhere)
+    [[operations.ingest.explode]]
+    field = "channels"           # a nested array on the row becomes rows of a child entity
+    entity = "spycloud_memberships"
+    carry = { user_id = "id" }   # child field = parent row field
+    take = { channel_id = "id" } # child field = nested item field
+
+Captures are written under ``captures/<operation>/`` with the secret stripped
+from the recorded request, a redacted body, and a manifest; every response is
+recorded, including errors, because the vendor's error bodies are part of what
+the world must reproduce. ``ingest`` turns the successful captures into the
+world's ``data/initial.json`` batch, proving it applies through the contract in
+a scratch database before the file is written. Nothing here imports beyond the
+standard library, so the credential-free bridge can load it.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import os
+import re
+import shutil
+import tempfile
+import time
+import tomllib
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from .compiler import check_bundle, compile_bundle
+from .store import SQLiteStore
+from .validation import fail
+
+SPEC = "connector.toml"
+CAPTURES = "captures"
+MANIFEST = "manifest.json"
+SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
+DEFAULT_TOKEN_PATH = "/oauth/token"
+METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
+SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
+NAME = re.compile(r"^[a-z][a-z0-9_]*$")
+#: Vendor-named things — query parameters, path arguments, redacted fields — keep the
+#: vendor's casing (``startDate``, ``Limit``); only our own operation names are snake_case.
+FIELD = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
+PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
+MAX_BODY = 8 * 1024 * 1024
+_PAGING_NAMES = ("limit", "offset", "page", "page_size", "cursor")
+RECORDED_HEADERS = ("content-type", "retry-after", "x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-reset")
+
+Opener = Callable[[urllib.request.Request, float], "Response"]
+
+
+@dataclass(frozen=True)
+class Response:
+    status: int
+    headers: dict[str, str]
+    body: bytes
+
+
+@dataclass(frozen=True)
+class Explode:
+    field: str
+    entity: str
+    carry: dict[str, str]
+    take: dict[str, str]
+
+
+@dataclass(frozen=True)
+class Operation:
+    name: str
+    method: str
+    path: str
+    params: tuple[str, ...]
+    entity: str | None
+    results: str
+    drop: tuple[str, ...] = ()
+    explode: tuple[Explode, ...] = ()
+
+    @property
+    def placeholders(self) -> tuple[str, ...]:
+        return tuple(PLACEHOLDER.findall(self.path))
+
+
+@dataclass(frozen=True)
+class Spec:
+    slug: str
+    base_url: str
+    scheme: str
+    auth_name: str | None
+    secret_env: str | None
+    extra_headers: dict[str, str]  # header name -> env var holding its value
+    #: oauth2 only: where the client-credentials grant is exchanged (absolute, or a
+    #: path under base_url), the env var naming the client id, and the scope asked for.
+    token_url: str | None
+    client_id_env: str | None
+    scope: str | None
+    requests_per_minute: int
+    timeout_seconds: float
+    max_pages: int
+    cursor_param: str | None
+    cursor_field: str | None
+    cursor_header: str | None
+    hash_fields: frozenset[str]
+    drop_fields: frozenset[str]
+    round_fields: dict[str, int] = field(default_factory=dict)  # numeric field -> decimals kept
+    operations: dict[str, Operation] = field(default_factory=dict)
+
+    def operation(self, name: str) -> Operation:
+        if name not in self.operations:
+            fail("operation", f"Unknown operation; choose one of {', '.join(sorted(self.operations))}", "/capture/op")
+        return self.operations[name]
+
+
+# ---------------------------------------------------------------- spec
+
+
+def _table(value: Any, location: str) -> dict:
+    if not isinstance(value, dict):
+        fail("spec", "Expected a table", location)
+    return value
+
+
+def _string(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
+    if key not in table:
+        if required:
+            fail("spec", f"Missing {key}", location)
+        return default
+    value = table[key]
+    if not isinstance(value, str) or not value.strip():
+        fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
+    return value.strip()
+
+
+def _paths(table: dict, key: str, location: str) -> tuple[str, ...]:
+    """Redaction names: a bare field name matches at any depth; a dotted path
+    (``results.*.email``, ``*`` for list items) matches that place only, so the
+    catalog's ``assets`` count map keeps its ``email`` key while records lose theirs."""
+    value = table.get(key, [])
+    if not isinstance(value, list) or not all(
+        isinstance(v, str) and all(part == "*" or FIELD.match(part) for part in v.split(".")) and v for v in value
+    ):
+        fail("spec", f"{key} must be a list of field names or dotted paths", f"{location}/{key}")
+    return tuple(value)
+
+
+def _names(table: dict, key: str, location: str) -> tuple[str, ...]:
+    value = table.get(key, [])
+    if not isinstance(value, list) or not all(isinstance(v, str) and FIELD.match(v) for v in value):
+        fail("spec", f"{key} must be a list of snake_case names", f"{location}/{key}")
+    if len(set(value)) != len(value):
+        fail("spec", f"{key} repeats a name", f"{location}/{key}")
+    return tuple(value)
+
+
+def _positive(table: dict, key: str, location: str, default: int | float, *, zero_ok: bool = False) -> int | float:
+    value = table.get(key, default)
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0 or (value == 0 and not zero_ok):
+        fail("spec", f"{key} must be a positive number", f"{location}/{key}")
+    return value
+
+
+def parse_spec(text: str) -> Spec:
+    try:
+        raw = tomllib.loads(text)
+    except tomllib.TOMLDecodeError as error:
+        fail("spec", f"connector.toml is not valid TOML: {error}", "/connector.toml")
+    connector = _table(raw.get("connector"), "/connector")
+    slug = _string(connector, "slug", "/connector")
+    if not SLUG.match(slug):
+        fail("spec", "slug must be lowercase letters, digits and hyphens", "/connector/slug")
+    base_url = _string(connector, "base_url", "/connector")
+    parsed = urllib.parse.urlsplit(base_url)
+    if parsed.scheme not in ("https", "http") or not parsed.netloc or parsed.query or parsed.fragment:
+        fail("spec", "base_url must be an http(s) origin with an optional path and no query", "/connector/base_url")
+
+    auth = _table(raw.get("auth", {"scheme": "none"}), "/auth")
+    scheme = _string(auth, "scheme", "/auth")
+    if scheme not in SCHEMES:
+        fail("spec", f"auth.scheme must be one of {', '.join(SCHEMES)}", "/auth/scheme")
+    auth_name = _string(auth, "name", "/auth", required=scheme in ("header", "query"))
+    secret_env = _string(auth, "secret_env", "/auth", required=scheme != "none")
+    if scheme == "none" and (auth_name or secret_env):
+        fail("spec", "auth.scheme none takes no name or secret_env", "/auth")
+    token_url = _string(auth, "token_url", "/auth", default=DEFAULT_TOKEN_PATH, required=False)
+    client_id_env = _string(auth, "client_id_env", "/auth", required=scheme == "oauth2")
+    scope = _string(auth, "scope", "/auth", required=False)
+    if scheme != "oauth2" and (client_id_env or scope or "token_url" in auth):
+        fail("spec", "token_url, client_id_env and scope belong to auth.scheme oauth2", "/auth")
+    if scheme == "oauth2" and not (token_url or "").startswith(("/", "https://", "http://")):
+        fail("spec", "auth.token_url is a path under base_url or an absolute URL", "/auth/token_url")
+    extra_headers = auth.get("headers", {})
+    if not isinstance(extra_headers, dict) or not all(
+        isinstance(k, str) and k.strip() and isinstance(v, str) and v.strip() for k, v in extra_headers.items()
+    ):
+        fail("spec", "auth.headers maps header names to environment variable names", "/auth/headers")
+    if auth_name and scheme == "header" and any(k.lower() == auth_name.lower() for k in extra_headers):
+        fail("spec", "auth.headers repeats the secret header", "/auth/headers")
+
+    capture = _table(raw.get("capture", {}), "/capture")
+    cursor_param = _string(capture, "cursor_param", "/capture", required=False)
+    cursor_field = _string(capture, "cursor_field", "/capture", required=False)
+    cursor_header = _string(capture, "cursor_header", "/capture", required=False)
+    if cursor_field and cursor_header:
+        fail("spec", "The next cursor is read from a body field or a response header, not both", "/capture")
+    if bool(cursor_param) != bool(cursor_field or cursor_header):
+        fail("spec", "cursor_param comes with cursor_field or cursor_header", "/capture")
+
+    redact = _table(raw.get("redact", {}), "/redact")
+    hash_fields = frozenset(_paths(redact, "hash", "/redact"))
+    drop_fields = frozenset(_paths(redact, "drop", "/redact"))
+    rounding = redact.get("round", {})
+    if not isinstance(rounding, dict) or not all(
+        isinstance(k, str) and FIELD.match(k) and isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= 10
+        for k, v in rounding.items()
+    ):
+        fail("spec", "redact.round maps numeric field names to the decimals kept (0-10)", "/redact/round")
+    round_fields = dict(rounding)
+    if (hash_fields & drop_fields) or (set(round_fields) & (hash_fields | drop_fields)):
+        fail("spec", "A field is hashed, dropped or rounded, never two of those", "/redact")
+
+    operations: dict[str, Operation] = {}
+    listed = raw.get("operations", [])
+    if not isinstance(listed, list) or not listed:
+        fail("spec", "Declare at least one [[operations]] entry", "/operations")
+    for index, entry in enumerate(listed):
+        location = f"/operations/{index}"
+        table = _table(entry, location)
+        name = _string(table, "name", location)
+        if not NAME.match(name):
+            fail("spec", "operation name must be snake_case", f"{location}/name")
+        if name in operations:
+            fail("spec", f"operation {name} is declared twice", f"{location}/name")
+        method = _string(table, "method", location, default="GET", required=False).upper()
+        if method not in METHODS:
+            fail("spec", f"method must be one of {', '.join(METHODS)}", f"{location}/method")
+        path = _string(table, "path", location)
+        if not path.startswith("/") or "?" in path or "#" in path:
+            fail("spec", "path must start with / and carry no query", f"{location}/path")
+        params = _names(table, "params", location)
+        entity = _string(table, "entity", location, required=False)
+        results = table.get("results", "")
+        if not isinstance(results, str):
+            fail("spec", "results must be a dotted path string", f"{location}/results")
+        for placeholder in PLACEHOLDER.findall(path):
+            if placeholder in params:
+                fail("spec", f"{placeholder} is both a path argument and a query parameter", location)
+        drop, explode = _ingest_projection(table.get("ingest", {}), f"{location}/ingest")
+        if (drop or explode) and not entity:
+            fail("spec", "ingest projection needs an entity to land rows in", f"{location}/ingest")
+        operations[name] = Operation(name, method, path, params, entity, results.strip("."), drop, explode)
+
+    return Spec(
+        slug=slug,
+        base_url=base_url.rstrip("/"),
+        scheme=scheme,
+        auth_name=auth_name,
+        secret_env=secret_env,
+        extra_headers=dict(extra_headers),
+        token_url=token_url if scheme == "oauth2" else None,
+        client_id_env=client_id_env if scheme == "oauth2" else None,
+        scope=scope if scheme == "oauth2" else None,
+        requests_per_minute=int(_positive(capture, "requests_per_minute", "/capture", 30)),
+        timeout_seconds=float(_positive(capture, "timeout_seconds", "/capture", 30)),
+        max_pages=int(_positive(capture, "max_pages", "/capture", 5)),
+        cursor_param=cursor_param,
+        cursor_field=cursor_field,
+        cursor_header=cursor_header,
+        hash_fields=hash_fields,
+        drop_fields=drop_fields,
+        round_fields=round_fields,
+        operations=operations,
+    )
+
+
+def _mapping(table: dict, key: str, location: str) -> dict[str, str]:
+    value = table.get(key, {})
+    if not isinstance(value, dict) or not all(
+        isinstance(k, str) and isinstance(v, str) and k and v for k, v in value.items()
+    ):
+        fail("spec", f"{key} maps child field names to source field names", f"{location}/{key}")
+    return dict(value)
+
+
+def _ingest_projection(raw: Any, location: str) -> tuple[tuple[str, ...], tuple[Explode, ...]]:
+    table = _table(raw, location)
+    drop = _names(table, "drop", location)
+    explode: list[Explode] = []
+    listed = table.get("explode", [])
+    if not isinstance(listed, list):
+        fail("spec", "explode must be a list of tables", f"{location}/explode")
+    for index, entry in enumerate(listed):
+        where = f"{location}/explode/{index}"
+        item = _table(entry, where)
+        field_name = _string(item, "field", where)
+        entity = _string(item, "entity", where)
+        carry = _mapping(item, "carry", where)
+        take = _mapping(item, "take", where)
+        if not carry and not take:
+            fail("spec", "explode needs carry or take, or the child rows would be empty", where)
+        if set(carry) & set(take):
+            fail("spec", "a child field is either carried from the parent or taken from the item", where)
+        explode.append(Explode(field_name, entity, carry, take))
+    return drop, tuple(explode)
+
+
+def load_spec(world: str | Path) -> Spec:
+    path = Path(world) / SPEC
+    if not path.is_file():
+        fail("spec", f"No {SPEC} in the world; describe the connection first", f"/{SPEC}")
+    return parse_spec(path.read_text(encoding="utf-8"))
+
+
+# ---------------------------------------------------------------- requests
+
+
+def _secret_value(spec: Spec, secret: str | None) -> str:
+    if secret is not None:
+        return secret
+    value = os.environ.get(spec.secret_env or "")
+    if not value:
+        fail("secret", f"Set {spec.secret_env} to the connection's credential", "/auth/secret_env")
+    return value
+
+
+def _secret(spec: Spec, secret: str | None, opener: "Opener | None" = None) -> str | None:
+    """The value the requests carry: the credential itself, or for oauth2 the
+    access token minted from it."""
+    if spec.scheme == "none":
+        return None
+    if spec.scheme == "oauth2":
+        return mint_token(spec, secret=secret, opener=opener)
+    return _secret_value(spec, secret)
+
+
+def build_request(
+    spec: Spec,
+    operation: Operation,
+    args: dict[str, str],
+    params: dict[str, str],
+    *,
+    secret: str | None,
+) -> tuple[urllib.request.Request, str]:
+    """The request to send and the URL to record, which never carries the secret."""
+    missing = [name for name in operation.placeholders if name not in args]
+    if missing:
+        fail("capture", f"Missing path argument(s): {', '.join(missing)}", "/capture/args")
+    extra = [name for name in args if name not in operation.placeholders]
+    if extra:
+        fail("capture", f"Unknown path argument(s): {', '.join(extra)}", "/capture/args")
+    unknown = [name for name in params if name not in operation.params and name != spec.cursor_param]
+    if unknown and spec.cursor_field and any(u for u in unknown):
+        # A next-page URL may carry the vendor's own paging parameters; they are not ours to refuse.
+        unknown = [name for name in unknown if name not in _PAGING_NAMES]
+    if unknown:
+        fail("capture", f"Operation {operation.name} takes no parameter(s): {', '.join(unknown)}", "/capture/params")
+    path = PLACEHOLDER.sub(lambda m: urllib.parse.quote(args[m.group(1)], safe=""), operation.path)
+    query = [(key, value) for key, value in params.items()]
+    recorded = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
+    headers = {"Accept": "application/json", "User-Agent": f"gateway-connector/{spec.slug}"}
+    for header, variable in spec.extra_headers.items():
+        value = os.environ.get(variable)
+        if not value:
+            fail("secret", f"Set {variable} for the {header} header", "/auth/headers")
+        headers[header] = value
+    if spec.scheme == "header":
+        headers[spec.auth_name or ""] = secret or ""
+    elif spec.scheme in ("bearer", "oauth2"):
+        headers["Authorization"] = f"Bearer {secret}"
+    elif spec.scheme == "basic":
+        headers["Authorization"] = "Basic " + base64.b64encode((secret or "").encode()).decode()
+    elif spec.scheme == "query":
+        query.append((spec.auth_name or "", secret or ""))
+    url = spec.base_url + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
+    return urllib.request.Request(url, method=operation.method, headers=headers), recorded
+
+
+def token_endpoint(spec: Spec) -> str:
+    """Where an oauth2 connection exchanges its client credentials."""
+    url = spec.token_url or DEFAULT_TOKEN_PATH
+    return url if url.startswith(("https://", "http://")) else spec.base_url + url
+
+
+def mint_token(spec: Spec, *, secret: str | None = None, opener: "Opener | None" = None) -> str:
+    """Exchange the client credentials for an access token (RFC 6749 §4.4).
+
+    The client id and secret come from the environment the way every other
+    credential does; the token is what the captures then send as a bearer.
+    Nothing about the exchange is recorded, so the secret never reaches disk."""
+    if spec.scheme != "oauth2":
+        fail("secret", "Only an oauth2 connection mints tokens", "/auth/scheme")
+    client_id = os.environ.get(spec.client_id_env or "")
+    if not client_id:
+        fail("secret", f"Set {spec.client_id_env} to the connection's client id", "/auth/client_id_env")
+    client_secret = _secret_value(spec, secret)
+    form = {"grant_type": "client_credentials", "client_id": client_id, "client_secret": client_secret}
+    if spec.scope:
+        form["scope"] = spec.scope
+    request = urllib.request.Request(
+        token_endpoint(spec),
+        method="POST",
+        data=urllib.parse.urlencode(form).encode(),
+        headers={
+            "Accept": "application/json",
+            "Content-Type": "application/x-www-form-urlencoded",
+            "User-Agent": f"gateway-connector/{spec.slug}",
+        },
+    )
+    response = (opener or _open)(request, spec.timeout_seconds)
+    try:
+        body = json.loads(response.body.decode("utf-8")) if response.body else {}
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        body = {}
+    token = body.get("access_token") if isinstance(body, dict) else None
+    if not (200 <= response.status < 300) or not isinstance(token, str) or not token:
+        fail("secret", f"Token endpoint answered {response.status} without an access_token", "/auth/token_url")
+    return token
+
+
+def _open(request: urllib.request.Request, timeout: float) -> Response:
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as reply:
+            return Response(reply.status, {k.lower(): v for k, v in reply.headers.items()}, reply.read(MAX_BODY + 1))
+    except urllib.error.HTTPError as error:
+        return Response(error.code, {k.lower(): v for k, v in error.headers.items()}, error.read(MAX_BODY + 1))
+    except (urllib.error.URLError, TimeoutError, OSError) as error:
+        fail("transport", f"Could not reach the connection: {error}", "/capture")
+
+
+# ---------------------------------------------------------------- redaction
+
+
+def _named(key: str, here: str, names: frozenset[str]) -> bool:
+    return key in names or here in names
+
+
+def _digest(slug: str, item: Any) -> Any:
+    """The hash of a value, keeping its shape: a list of strings hashes element by
+    element (an array stays an array, so the world's contract still accepts the row);
+    objects are left alone — name their fields instead."""
+    if isinstance(item, list):
+        return [_digest(slug, element) for element in item]
+    if isinstance(item, dict) or item is None:
+        return item
+    return f"sha256:{hashlib.sha256(f'{slug}:{item}'.encode()).hexdigest()}"
+
+
+def redact(value: Any, spec: Spec, path: str = "") -> Any:
+    """Hash, drop or round the named fields; everything else is untouched.
+
+    A bare name applies at any depth; a dotted path applies at that place only, with
+    ``*`` standing for a list item."""
+    if isinstance(value, dict):
+        out = {}
+        for key, item in value.items():
+            here = f"{path}.{key}" if path else key
+            if _named(key, here, spec.drop_fields):
+                continue
+            if _named(key, here, spec.hash_fields) and item is not None:
+                out[key] = _digest(spec.slug, item)
+            elif key in spec.round_fields and isinstance(item, (int, float)) and not isinstance(item, bool):
+                # Coordinates and the like: keep the shape and the type, lose the precision.
+                out[key] = round(item, spec.round_fields[key])
+            else:
+                out[key] = redact(item, spec, here)
+        return out
+    if isinstance(value, list):
+        return [redact(item, spec, f"{path}.*" if path else "*") for item in value]
+    return value
+
+
+def _dig(body: Any, path: str) -> Any:
+    for part in [p for p in path.split(".") if p]:
+        if not isinstance(body, dict) or part not in body:
+            return None
+        body = body[part]
+    return body
+
+
+# ---------------------------------------------------------------- capture
+
+
+def capture(
+    world: str | Path,
+    op: str,
+    args: dict[str, str] | None = None,
+    params: dict[str, str] | None = None,
+    *,
+    secret: str | None = None,
+    max_pages: int | None = None,
+    opener: Opener = _open,
+    sleep: Callable[[float], None] = time.sleep,
+) -> list[Path]:
+    """Run one operation against the live connection and record every page.
+
+    Returns the capture files written. Pagination follows the spec's cursor;
+    a non-2xx reply is recorded and ends the run, since error bodies are
+    evidence too.
+    """
+    root = Path(world)
+    spec = load_spec(root)
+    operation = spec.operation(op)
+    args = dict(args or {})
+    params = dict(params or {})
+    credential = _secret(spec, secret, opener)
+    pages = max_pages if max_pages is not None else spec.max_pages
+    if pages < 1:
+        fail("capture", "max_pages must be at least 1", "/capture/max_pages")
+    pause = 60.0 / spec.requests_per_minute
+    written: list[Path] = []
+    cursor: str | None = None
+    for page in range(pages):
+        if page:
+            sleep(pause)
+        page_params = dict(params)
+        if cursor and cursor.startswith(("https://", "http://")):
+            # A vendor that hands back the next page as a URL (DRF's ``next``): follow its
+            # query, never its host — the page must come from the connection described.
+            following = urllib.parse.urlsplit(cursor)
+            if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
+                fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
+            page_params = {k: v for k, v in urllib.parse.parse_qsl(following.query) if k != (spec.auth_name or "")}
+        elif cursor:
+            page_params[spec.cursor_param or ""] = cursor
+        request, recorded = build_request(spec, operation, args, page_params, secret=credential)
+        response = opener(request, spec.timeout_seconds)
+        if len(response.body) > MAX_BODY:
+            fail("capture", "Response exceeds 8 MiB; narrow the request", "/capture")
+        try:
+            body: Any = json.loads(response.body.decode("utf-8")) if response.body else None
+            raw_text = None
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            body, raw_text = None, response.body.decode("utf-8", "replace")[:4096]
+        redacted = redact(body, spec)
+        stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+        record = {
+            "connector": spec.slug,
+            "operation": operation.name,
+            "captured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            "request": {"method": operation.method, "url": recorded, "args": args, "params": page_params},
+            "status": response.status,
+            "headers": {
+                k: v
+                for k, v in response.headers.items()
+                if k in RECORDED_HEADERS or (spec.cursor_header and k == spec.cursor_header.lower())
+            },
+            "body": redacted,
+            **({"text": raw_text} if raw_text is not None else {}),
+            "page": page + 1,
+        }
+        encoded = json.dumps(record, sort_keys=True, indent=1, ensure_ascii=False).encode("utf-8")
+        record_path = root / CAPTURES / operation.name / f"{stamp}-p{page + 1}.json"
+        record_path.parent.mkdir(parents=True, exist_ok=True)
+        record_path.write_bytes(encoded)
+        _append_manifest(root, record_path, record, hashlib.sha256(encoded).hexdigest())
+        written.append(record_path)
+        if not (200 <= response.status < 300) or not (spec.cursor_field or spec.cursor_header):
+            break
+        if spec.cursor_header:
+            cursor = response.headers.get(spec.cursor_header.lower())
+        else:
+            cursor = _dig(body, spec.cursor_field or "")
+        if not isinstance(cursor, str) or not cursor:
+            break
+    return written
+
+
+def _append_manifest(root: Path, path: Path, record: dict, sha256: str) -> None:
+    manifest_path = root / CAPTURES / MANIFEST
+    entries = json.loads(manifest_path.read_text()) if manifest_path.is_file() else []
+    entries.append(
+        {
+            "file": path.relative_to(root).as_posix(),
+            "operation": record["operation"],
+            "status": record["status"],
+            "captured_at": record["captured_at"],
+            "sha256": sha256,
+        }
+    )
+    manifest_path.write_text(json.dumps(entries, indent=1) + "\n")
+
+
+# ---------------------------------------------------------------- ingest
+
+
+def _key(contract: Any, entity: str, row: dict) -> tuple | None:
+    fields = contract.entities.get(entity, {}).get("primary_key", [])
+    if not fields or any(field not in row for field in fields):
+        return None
+    return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
+
+
+def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list[dict]:
+    """Insert what is new, update what changed, skip what the world already holds."""
+    manifest_path = root / CAPTURES / MANIFEST
+    if not manifest_path.is_file():
+        fail("ingest", "No captures to ingest; run capture first", f"/{CAPTURES}")
+    held: dict[tuple[str, tuple], dict] = {}
+    for mutation in prior:
+        if mutation.get("op") in ("insert", "update") and isinstance(mutation.get("value"), dict):
+            key = _key(contract, mutation["entity"], mutation["value"])
+            if key is not None:
+                held[(mutation["entity"], key)] = mutation["value"]
+    mutations: list[dict] = []
+    seen: dict[tuple[str, str], str] = {}
+    for entry in json.loads(manifest_path.read_text()):
+        if not (200 <= entry["status"] < 300):
+            continue
+        operation = spec.operations.get(entry["operation"])
+        if operation is None or not operation.entity:
+            continue
+        path = root / entry["file"]
+        if not path.is_file():
+            fail("ingest", f"Capture listed in the manifest is missing: {entry['file']}", f"/{CAPTURES}")
+        encoded = path.read_bytes()
+        if hashlib.sha256(encoded).hexdigest() != entry["sha256"]:
+            fail("ingest", f"Capture changed since it was recorded: {entry['file']}", f"/{CAPTURES}")
+        record = json.loads(encoded)
+        rows = _dig(record["body"], operation.results) if operation.results else record["body"]
+        if rows is None:
+            continue
+        if isinstance(rows, dict):
+            rows = [rows]
+        if not isinstance(rows, list):
+            fail("ingest", f"results path {operation.results!r} is not an array in {entry['file']}", f"/{CAPTURES}")
+        for raw_row in rows:
+            if not isinstance(raw_row, dict):
+                fail("ingest", f"A row in {entry['file']} is not an object", f"/{CAPTURES}")
+            for entity, row in _project(operation, raw_row, entry["file"]):
+                identity = hashlib.sha256(json.dumps(row, sort_keys=True).encode()).hexdigest()
+                token = (entity, identity)
+                if token in seen:
+                    continue
+                seen[token] = entry["file"]
+                key = _key(contract, entity, row)
+                existing = held.get((entity, key)) if key is not None else None
+                if existing == row:
+                    continue
+                mutation = {
+                    "op": "update" if existing is not None else "insert",
+                    "entity": entity,
+                    "value": row,
+                    "source": {
+                        "record": f"{spec.slug}:{operation.name}:{identity}",
+                        "evidence": [{"reference": f"capture:{entry['file']}", "digest": entry["sha256"]}],
+                    },
+                }
+                if existing is not None:
+                    fields = contract.entities[entity]["primary_key"]
+                    mutation["key"] = {field: row[field] for field in fields}
+                mutations.append(mutation)
+    return mutations
+
+
+def _project(operation: Operation, raw: dict, file: str) -> list[tuple[str, dict]]:
+    """The entity rows one vendor row yields: children exploded first, then the parent without its dropped fields."""
+    out: list[tuple[str, dict]] = []
+    for explode in operation.explode:
+        items = raw.get(explode.field)
+        if items is None:
+            continue
+        if not isinstance(items, list):
+            fail("ingest", f"{explode.field} is not an array in {file}", f"/{CAPTURES}")
+        for item in items:
+            if not isinstance(item, dict):
+                fail("ingest", f"An item of {explode.field} in {file} is not an object", f"/{CAPTURES}")
+            child = {name: raw.get(source) for name, source in explode.carry.items()}
+            child.update({name: item.get(source) for name, source in explode.take.items()})
+            out.append((explode.entity, child))
+    parent = {k: v for k, v in raw.items() if k not in operation.drop}
+    assert operation.entity is not None
+    out.append((operation.entity, parent))
+    return out
+
+
+def ingest(world: str | Path, *, batch_id: str | None = None) -> dict:
+    """Seed the world from its captures, gated by the contract.
+
+    The existing ``data/initial.json`` mutations are kept and the captured rows
+    appended as a new batch version. The batch is applied in a scratch copy of
+    the world first; only a batch the store accepts is written back, and the
+    world is recompiled and checked so the lock reflects the new state.
+    """
+    root = Path(world)
+    spec = load_spec(root)
+    contract = compile_bundle(root)
+    initial_path = root / "data/initial.json"
+    previous = json.loads(initial_path.read_text()) if initial_path.is_file() else None
+    prior_mutations = list(previous["mutations"]) if previous else []
+    prior_id = previous["id"] if previous else f"{contract.metadata['package']['id']}.initial.v0"
+    version = int(re.search(r"\.v(\d+)$", prior_id).group(1)) + 1 if re.search(r"\.v(\d+)$", prior_id) else 1
+    fresh = _mutations(root, spec, contract, prior_mutations)
+    if not fresh:
+        if not prior_mutations:
+            fail(
+                "ingest",
+                "No successful capture maps to an entity; give the operation an entity and results path",
+                f"/{CAPTURES}",
+            )
+        counts = _counts(root, contract)
+        return {"batch": prior_id, "mutations": len(prior_mutations), "entity_counts": counts, "changed": False}
+    batch = {
+        "id": batch_id or f"{contract.metadata['package']['id']}.initial.v{version}",
+        "target": contract.target,
+        "mapping": {"id": f"connector-capture:{spec.slug}", "version": "1"},
+        "coverage": "partial",
+        "mutations": prior_mutations + fresh,
+    }
+    from .._materialize import materialize_template  # only ingest needs the parent package
+
+    with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
+        staged = Path(scratch) / "world"
+        shutil.copytree(
+            root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", "state.sqlite", "state.json")
+        )
+        (staged / "data/initial.json").write_text(json.dumps(batch, sort_keys=True, indent=1) + "\n")
+        compile_bundle(staged)
+        database = Path(scratch) / "world.db"
+        materialize_template(staged, database)
+        with SQLiteStore.open(database, check_bundle(staged)) as store:
+            store.audit()
+            counts = {entity: len(store.list(entity)) for entity in contract.entities}
+    initial_path.write_text(json.dumps(batch, sort_keys=True, indent=1) + "\n")
+    compile_bundle(root)
+    check_bundle(root)
+    return {"batch": batch["id"], "mutations": len(batch["mutations"]), "entity_counts": counts, "changed": True}
+
+
+def _counts(root: Path, contract: Any) -> dict[str, int]:
+    from .._materialize import materialize_template
+
+    with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
+        database = Path(scratch) / "world.db"
+        materialize_template(root, database)
+        with SQLiteStore.open(database, contract) as store:
+            return {entity: len(store.list(entity)) for entity in contract.entities}