keel-context 0.1.0__py3-none-any.whl

keel/__init__.py

@@ -0,0 +1,28 @@
+"""Keel: a context layer for AI agents in vertical SaaS.
+
+Domain events in, a bitemporal context graph per tenant out,
+shared by every agent through five tools.
+"""
+
+from keel.cartridge import Cartridge, load_cartridge
+from keel.envelope import Event
+from keel.handlers import HandlerRegistry
+from keel.ingest import Ingestor
+from keel.scopes import Scope
+from keel.store.memory import MemoryStore
+from keel.store.sqlite import SqliteStore
+from keel.tools import Tools
+
+__all__ = [
+    "Cartridge",
+    "Event",
+    "HandlerRegistry",
+    "Ingestor",
+    "MemoryStore",
+    "Scope",
+    "SqliteStore",
+    "Tools",
+    "load_cartridge",
+]
+
+__version__ = "0.1.0"

keel/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Event adapters: anything that yields `Event`s.
+
+Shipped: JsonlAdapter (keel.ingest) for development, PostgresTableAdapter
+(table + polling with a cursor), webhook `build_app` (FastAPI, push).
+Planned: KafkaAdapter, SQSAdapter, Postgres CDC.
+Each adapter only produces envelopes; it never understands payloads.
+"""

keel/adapters/postgres.py

@@ -0,0 +1,108 @@
+"""Table + polling adapter: your SaaS writes envelopes to an outbox table, Keel polls it.
+
+Optional extra for the driver: `uv sync --extra postgres` (psycopg). Any DB-API 2.0
+connection works; the tests run it on sqlite3.
+
+Expected table (names are configurable):
+
+    CREATE TABLE keel_events (
+        seq          bigserial PRIMARY KEY,   -- the cursor: assigned at insert, read in order
+        id           text NOT NULL,
+        tenant       text NOT NULL,
+        type         text NOT NULL,
+        subject      text NOT NULL,
+        occurred_at  timestamptz NOT NULL,
+        recorded_at  timestamptz NOT NULL DEFAULT now(),
+        source       text NOT NULL,
+        payload      jsonb NOT NULL DEFAULT '{}'
+    );
+
+Usage:
+
+    import psycopg
+    from keel.adapters.postgres import PostgresTableAdapter
+
+    adapter = PostgresTableAdapter(lambda: psycopg.connect(DSN), cursor_path=".keel/cursor")
+    ingestor.ingest(adapter)                   # one pass over everything after the cursor
+    ingestor.ingest(adapter.follow(interval=5))  # or poll forever
+
+The cursor is a text file holding the last `seq` the consumer took, so a restart
+continues where it stopped. Delivery is at-least-once: the cursor advances after
+the consumer has taken an event, so a crash mid-event re-delivers it.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable, Iterator
+from pathlib import Path
+from typing import Any
+
+from keel.envelope import Event
+
+Connect = Callable[[], Any]  # returns a DB-API 2.0 connection
+
+
+class PostgresTableAdapter:
+    def __init__(
+        self,
+        connect: Connect,
+        table: str = "keel_events",
+        cursor_path: str | Path = ".keel/cursor",
+        cursor_column: str = "seq",
+        batch_size: int = 500,
+        placeholder: str = "%s",  # psycopg uses %s; sqlite3 uses ?
+    ) -> None:
+        self.connect = connect
+        self.cursor_path = Path(cursor_path)
+        self.cursor_column = cursor_column
+        self.batch_size = batch_size
+        # ponytail: a serial cursor can skip a row that commits late under concurrent
+        # writers; use one writer per table, or a recorded_at cursor with a lag, if that bites.
+        self.sql = (
+            f"SELECT * FROM {table} WHERE {cursor_column} > {placeholder} "
+            f"ORDER BY {cursor_column} LIMIT {placeholder}"
+        )
+
+    @property
+    def position(self) -> int:
+        try:
+            return int(self.cursor_path.read_text())
+        except FileNotFoundError:
+            return 0
+
+    def _save(self, seq: int) -> None:
+        self.cursor_path.parent.mkdir(parents=True, exist_ok=True)
+        self.cursor_path.write_text(str(seq))
+
+    def __iter__(self) -> Iterator[Event]:
+        """One pass: every row after the cursor, in order, then stop."""
+        conn = self.connect()
+        try:
+            while True:
+                cur = conn.cursor()
+                cur.execute(self.sql, (self.position, self.batch_size))
+                cols = [d[0] for d in cur.description]
+                rows = cur.fetchall()
+                for row in rows:
+                    rec = dict(zip(cols, row))
+                    seq = rec.pop(self.cursor_column)
+                    if isinstance(rec.get("payload"), str):
+                        rec["payload"] = json.loads(rec["payload"])
+                    yield Event.model_validate(rec)
+                    self._save(seq)  # runs once the consumer asks for the next event
+                if len(rows) < self.batch_size:
+                    return
+        finally:
+            conn.close()
+
+    def follow(self, interval: float = 5.0) -> Iterator[Event]:
+        """Poll forever. `ingestor.ingest(adapter.follow())` never returns."""
+        while True:
+            n = 0
+            for ev in self:
+                n += 1
+                yield ev
+            if n == 0:
+                time.sleep(interval)

keel/adapters/webhook.py

@@ -0,0 +1,38 @@
+"""Webhook adapter: HTTP in, envelope validated, ingested. Optional extra: `uv sync --extra webhook`.
+
+    from keel.adapters.webhook import build_app
+    app = build_app(ingestor, token="shared-secret")   # then: uvicorn yourmodule:app
+
+POST /events with one envelope or a list of them and `Authorization: Bearer <token>`.
+Unlike the pull adapters, this one pushes into the ingestor: the sender drives the loop.
+"""
+
+from __future__ import annotations
+
+import hmac
+from typing import Any
+
+from keel.envelope import Event
+from keel.ingest import Ingestor
+
+
+def build_app(ingestor: Ingestor, token: str | None = None, strict: bool = False):
+    """`token=None` accepts any caller; use it only on localhost."""
+    try:
+        from fastapi import Depends, FastAPI, Header, HTTPException
+    except ImportError as exc:
+        raise SystemExit("Webhook adapter needs the optional dependency: uv sync --extra webhook") from exc
+
+    app = FastAPI(title=f"keel:{ingestor.cartridge.name}")
+
+    def auth(authorization: str | None = Header(default=None)) -> None:
+        if token and not hmac.compare_digest(authorization or "", f"Bearer {token}"):
+            raise HTTPException(401, "bad or missing bearer token")
+
+    @app.post("/events", dependencies=[Depends(auth)])
+    def events(body: list[Event] | Event) -> dict[str, Any]:
+        batch = body if isinstance(body, list) else [body]
+        report = ingestor.ingest(batch, strict=strict)
+        return {"processed": report.processed, "unhandled": report.unhandled, "errors": report.errors}
+
+    return app

keel/cartridge.py

@@ -0,0 +1,151 @@
+"""Cartridge loader.
+
+A cartridge is a folder:
+
+    cartridges/fleet/
+      cartridge.yaml   ontology, event catalog, semantic rules, memory policies
+      handlers.py      a `registry = HandlerRegistry()` with one handler per event type
+
+The core loads the YAML, validates that every cataloged event has a
+handler and every relation uses declared entity kinds, and hands the
+registry to the ingestor. That is the whole contract between core and
+domain. If writing your second cartridge makes you touch the core, some
+domain leaked in; pull it back out.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from dataclasses import dataclass, field
+from datetime import timedelta
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from keel.envelope import Event, parse_iso
+from keel.handlers import HandlerRegistry
+from keel.rules import problems as rule_problems
+from keel.scopes import MemoryPolicy, Scope
+
+
+@dataclass
+class Cartridge:
+    name: str
+    path: Path
+    entities: dict[str, dict[str, Any]]  # kind -> {description, ref?}
+    relations: dict[str, dict[str, str]]  # rel -> {from, to}
+    events: dict[str, dict[str, Any]]  # type -> {subject, owner, payload: {...}}
+    rules: dict[str, Any] = field(default_factory=dict)  # semantic definitions, versioned
+    policy: MemoryPolicy = field(default_factory=MemoryPolicy.default)
+    registry: HandlerRegistry = field(default_factory=HandlerRegistry)
+
+    def validate(self) -> list[str]:
+        problems: list[str] = []
+        for rel, spec in self.relations.items():
+            for side in ("from", "to"):
+                kind = spec.get(side)
+                if kind not in self.entities:
+                    problems.append(f"relation '{rel}' {side}='{kind}' is not a declared entity kind")
+        for etype, spec in self.events.items():
+            if spec.get("subject") not in self.entities:
+                problems.append(f"event '{etype}' subject kind '{spec.get('subject')}' is not declared")
+            if etype not in self.registry.types():
+                problems.append(f"event '{etype}' has no handler in handlers.py")
+        for etype in self.registry.types() - set(self.events):
+            problems.append(f"handler for '{etype}' exists but the event is not in the catalog")
+        for name, rule in self.rules.items():
+            problems += rule_problems(name, rule)
+        return problems
+
+    def check_event(self, event: Event) -> list[str]:
+        """One event against the catalog: known type, right subject kind, declared payload fields
+        present with the declared type. Structural only: keys and types, never values."""
+        spec = self.events.get(event.type)
+        if spec is None:
+            return [f"unknown event type '{event.type}'"]
+        out = []
+        if event.subject_kind != spec.get("subject"):
+            out.append(f"'{event.type}' subject '{event.subject}' is not a '{spec.get('subject')}'")
+        for name, hint in (spec.get("payload") or {}).items():
+            kinds = {k.strip() for k in str(hint).split("|")}
+            if name not in event.payload:
+                if "null" not in kinds:
+                    out.append(f"'{event.type}' payload is missing '{name}' ({hint})")
+                continue
+            value = event.payload[name]
+            if value is None and "null" in kinds:
+                continue
+            if not any(_is(value, k) for k in kinds - {"null"}):
+                out.append(f"'{event.type}' payload '{name}' should be {hint}, got {type(value).__name__}")
+        return out
+
+
+def _is(value: Any, hint: str) -> bool:
+    """Does a payload value match a catalog type hint? Hints the core does not know are documentation."""
+    if hint == "string":
+        return isinstance(value, str)
+    if hint == "bool":
+        return isinstance(value, bool)
+    if hint == "int":
+        return isinstance(value, int) and not isinstance(value, bool)
+    if hint == "number":
+        return isinstance(value, (int, float)) and not isinstance(value, bool)
+    if hint in ("date", "datetime"):
+        try:
+            parse_iso(value)
+        except ValueError:
+            return False
+        return True
+    return True
+
+
+def _parse_ttl(v: Any) -> timedelta | None:
+    if v in (None, "never", "forever"):
+        return None
+    if isinstance(v, (int, float)):
+        return timedelta(days=float(v))
+    s = str(v).strip()
+    unit = s[-1]
+    n = float(s[:-1])
+    return {"d": timedelta(days=n), "h": timedelta(hours=n), "m": timedelta(minutes=n)}[unit]
+
+
+def _policy_from(raw: dict[str, Any] | None) -> MemoryPolicy:
+    base = MemoryPolicy.default()
+    if not raw:
+        return base
+    ttl = dict(base.ttl)
+    for k, v in (raw.get("ttl") or {}).items():
+        ttl[Scope(k)] = _parse_ttl(v)
+    req = {Scope(s) for s in raw.get("requires_evidence", [s.value for s in base.requires_evidence])}
+    return MemoryPolicy(ttl=ttl, requires_evidence=req)
+
+
+def _load_registry(path: Path) -> HandlerRegistry:
+    handlers_py = path / "handlers.py"
+    if not handlers_py.exists():
+        return HandlerRegistry()
+    spec = importlib.util.spec_from_file_location(f"keel_cartridge_{path.name}_handlers", handlers_py)
+    assert spec and spec.loader
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    reg = getattr(mod, "registry", None)
+    if not isinstance(reg, HandlerRegistry):
+        raise TypeError(f"{handlers_py} must define `registry = HandlerRegistry()`")
+    return reg
+
+
+def load_cartridge(path: str | Path) -> Cartridge:
+    path = Path(path)
+    raw = yaml.safe_load((path / "cartridge.yaml").read_text(encoding="utf-8")) or {}
+    return Cartridge(
+        name=raw.get("name", path.name),
+        path=path,
+        entities=raw.get("entities") or {},
+        relations=raw.get("relations") or {},
+        events=raw.get("events") or {},
+        rules=raw.get("rules") or {},
+        policy=_policy_from(raw.get("memory")),
+        registry=_load_registry(path),
+    )

keel/cli.py

@@ -0,0 +1,182 @@
+"""keel CLI: init, validate, ingest, replay, expire, export, context, serve."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+from keel.cartridge import load_cartridge
+from keel.envelope import Event
+from keel.ingest import Ingestor, IngestReport, JsonlAdapter
+from keel.store.memory import MemoryStore
+from keel.store.sqlite import SqliteStore
+from keel.tools import Tools
+
+app = typer.Typer(help="Keel: a context layer for AI agents in vertical SaaS.", no_args_is_help=True)
+
+DB = typer.Option(None, "--db", help="SQLite file for a durable graph. Without it the graph lives in memory and ends with the command.")
+
+CARTRIDGE_TEMPLATE = """name: {name}
+
+entities:
+  account:
+    description: The thing your product is about. Rename it.
+
+relations: {{}}
+
+events:
+  account.created:
+    subject: account
+    owner: core
+    payload:
+      id: string
+      name: string
+
+rules: {{}}
+
+memory:
+  ttl:
+    task: 14d
+    agent: 1d
+  requires_evidence: [global, tenant]
+"""
+
+HANDLERS_TEMPLATE = '''"""Handlers for the {name} cartridge. One per event type, a few lines each."""
+
+from keel.handlers import HandlerRegistry
+
+registry = HandlerRegistry()
+
+
+@registry.on("account.created")
+def account_created(event, store):
+    p = event.payload
+    store.upsert_entity(event.tenant, "account", p["id"], ref=event.source, name=p.get("name"))
+'''
+
+
+@app.command()
+def init(path: Path = typer.Argument(..., help="Folder for the new cartridge, e.g. cartridges/clinics")):
+    """Scaffold a new cartridge."""
+    path.mkdir(parents=True, exist_ok=False)
+    (path / "cartridge.yaml").write_text(CARTRIDGE_TEMPLATE.format(name=path.name), encoding="utf-8")
+    (path / "handlers.py").write_text(HANDLERS_TEMPLATE.format(name=path.name), encoding="utf-8")
+    typer.echo(f"created {path}/cartridge.yaml and {path}/handlers.py")
+
+
+@app.command()
+def validate(
+    cartridge: Path,
+    events: Path | None = typer.Option(None, help="Also check a .jsonl stream: unknown types, wrong subject kinds, payload shape."),
+):
+    """Check that the catalog, the ontology and the handlers agree; with --events, that a stream fits the catalog."""
+    c = load_cartridge(cartridge)
+    problems = c.validate()
+    checked = 0
+    if events:
+        for n, line in enumerate(events.read_text(encoding="utf-8").splitlines(), 1):
+            if not line.strip() or line.startswith("#"):
+                continue
+            try:
+                ev = Event.model_validate(json.loads(line))
+            except ValueError as exc:  # json and pydantic errors both subclass ValueError
+                errs = getattr(exc, "errors", None)  # pydantic: first error, with its field
+                detail = f"{'.'.join(map(str, errs()[0]['loc']))}: {errs()[0]['msg']}" if errs else str(exc)
+                problems.append(f"line {n}: bad envelope: {detail}")
+                continue
+            checked += 1
+            problems += [f"line {n}: {p}" for p in c.check_event(ev)]
+    if problems:
+        for p in problems:
+            typer.echo(f"  - {p}")
+        raise typer.Exit(code=1)
+    typer.echo(f"{c.name}: {len(c.entities)} entity kinds, {len(c.relations)} relations, {len(c.events)} events, all handled")
+    if events:
+        typer.echo(f"{events}: {checked} events match the catalog")
+
+
+def _run(cartridge: Path, events: Path | None, strict: bool = False, db: Path | None = None):
+    c = load_cartridge(cartridge)
+    store = SqliteStore(db) if db else MemoryStore()
+    ing = Ingestor(c, store)
+    report = ing.ingest(JsonlAdapter(events), strict=strict) if events else IngestReport()
+    return c, store, ing, report
+
+
+@app.command()
+def ingest(cartridge: Path, events: Path, strict: bool = False, db: Path | None = DB):
+    """Ingest a .jsonl event stream into the graph (in memory, or the --db file) and report."""
+    _, _, _, report = _run(cartridge, events, strict, db)
+    typer.echo(f"processed={report.processed} unhandled={len(report.unhandled)} errors={len(report.errors)}")
+    for eid, err in report.errors:
+        typer.echo(f"  {eid}: {err}")
+
+
+@app.command()
+def replay(cartridge: Path, events: Path, tenant: str, db: Path | None = DB):
+    """Rebuild one tenant's subgraph from the event log (recovery path). With --db, the file is wiped for that tenant first."""
+    _, _, ing, _ = _run(cartridge, None, db=db)
+    report = ing.replay(tenant, events=JsonlAdapter(events))
+    typer.echo(f"replayed tenant={tenant} processed={report.processed}")
+
+
+@app.command()
+def expire(tenant: str, db: Path = typer.Option(..., "--db", help="The durable graph to clean.")):
+    """Drop task/agent memory past its expiry for one tenant. Reads already ignore expired facts; run this from cron to reclaim space."""
+    n = SqliteStore(db).expire(tenant)
+    typer.echo(f"expired tenant={tenant} dropped={n}")
+
+
+@app.command()
+def export(
+    tenant: str,
+    db: Path = typer.Option(..., "--db", help="The durable graph to export from."),
+    out: Path | None = typer.Option(None, "--out", help="Write here instead of stdout."),
+):
+    """Audit trail for one tenant as JSONL: every entity, relation and fact, closed ones included, with provenance, scope, evidence and who corrected what."""
+    from dataclasses import asdict
+
+    from keel.tools import _ser
+
+    lines = (json.dumps({"record": type(o).__name__.lower(), **_ser(asdict(o))}, ensure_ascii=False) for o in SqliteStore(db).dump(tenant))
+    if out:
+        n = 0
+        with out.open("w", encoding="utf-8") as fh:
+            for line in lines:
+                fh.write(line + "\n")
+                n += 1
+        typer.echo(f"exported tenant={tenant} records={n} to {out}")
+    else:
+        for line in lines:
+            typer.echo(line)
+
+
+@app.command()
+def context(cartridge: Path, events: Path, subject: str, tenant: str | None = None):
+    """Ingest, then print what the `context` tool returns for a subject."""
+    c, store, ing, _ = _run(cartridge, events)
+    tenant = tenant or next(iter({e.tenant for e in ing.log}), "")
+    typer.echo(json.dumps(Tools(c, store, tenant).context(subject), indent=2, ensure_ascii=False))
+
+
+@app.command()
+def serve(
+    cartridge: Path,
+    tenant: str = typer.Option(..., help="The end-customer this server is bound to."),
+    events: Path | None = None,
+    db: Path | None = DB,
+):
+    """Serve the five tools over MCP (stdio). Requires `uv sync --extra mcp`. --events fills an in-memory graph; --db serves a durable one."""
+    from keel.mcp_server import build_server
+
+    c = load_cartridge(cartridge)
+    store = SqliteStore(db) if db else MemoryStore()
+    if events:
+        Ingestor(c, store).ingest(JsonlAdapter(events))
+    build_server(Tools(c, store, tenant)).run()
+
+
+if __name__ == "__main__":
+    app()

keel/envelope.py

@@ -0,0 +1,72 @@
+"""The standard event envelope.
+
+Every event, from any source, arrives in this shape. The core never reads
+`payload`; it only routes on `type` and keys on `tenant` and `subject`.
+A cartridge's handlers are the only code that understands the payload.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from hashlib import sha256
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def parse_iso(value: Any) -> datetime:
+    """ISO 8601 date or datetime to an aware UTC datetime. Accepts a trailing Z on Python 3.10 too."""
+    if isinstance(value, datetime):
+        d = value
+    else:
+        d = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
+
+
+class Event(BaseModel):
+    """A domain fact with time, from one of the SaaS's systems."""
+
+    id: str = Field(default="", description="The source's own id when it has one; else a content hash, so replay yields the same ids.")
+    tenant: str = Field(..., description="The end-customer this fact belongs to (the tenant of the tenant).")
+    type: str = Field(..., description="Dotted event type from the cartridge catalog, e.g. 'inspection.created'.")
+    subject: str = Field(..., description="Entity reference '<kind>:<id>' the fact is about, e.g. 'unit:MXK-4821'.")
+    occurred_at: datetime = Field(..., description="When it happened in the world (valid time).")
+    recorded_at: datetime = Field(default_factory=utcnow, description="When Keel learned it (transaction time).")
+    source: str = Field(..., description="System of record, e.g. 'vec.inspections' or 'webhook:telematics'.")
+    payload: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _content_hash_id(cls, data: Any) -> Any:
+        if isinstance(data, dict) and not data.get("id"):
+            fields = ("tenant", "type", "subject", "occurred_at", "source", "payload")
+            key = json.dumps([data.get(k) for k in fields], sort_keys=True, default=str)
+            data = {**data, "id": sha256(key.encode()).hexdigest()[:32]}
+        return data
+
+    @field_validator("subject")
+    @classmethod
+    def _subject_shape(cls, v: str) -> str:
+        if ":" not in v:
+            raise ValueError("subject must look like '<kind>:<id>'")
+        return v
+
+    @field_validator("occurred_at", "recorded_at")
+    @classmethod
+    def _aware(cls, v: datetime) -> datetime:
+        if v.tzinfo is None:
+            return v.replace(tzinfo=timezone.utc)
+        return v
+
+    @property
+    def subject_kind(self) -> str:
+        return self.subject.split(":", 1)[0]
+
+    @property
+    def subject_id(self) -> str:
+        return self.subject.split(":", 1)[1]

keel/handlers.py

@@ -0,0 +1,56 @@
+"""Handler registry.
+
+A handler is the only code that understands an event's payload. It maps
+the payload to store primitives in a few lines. The core routes on
+`event.type`; it never looks inside.
+
+    registry = HandlerRegistry()
+
+    @registry.on("inspection.created")
+    def inspection_created(event, store):
+        store.upsert_entity(event.tenant, "inspection", event.payload["id"], ref=event.source)
+        store.link(event.tenant, event.subject, "has_inspection", f"inspection:{event.payload['id']}", event.occurred_at)
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from keel.envelope import Event
+from keel.store.base import Store
+
+Handler = Callable[[Event, Store], None]
+
+
+class UnhandledEvent(Exception):
+    pass
+
+
+class HandlerRegistry:
+    def __init__(self) -> None:
+        self._handlers: dict[str, list[Handler]] = {}
+
+    def on(self, event_type: str) -> Callable[[Handler], Handler]:
+        def deco(fn: Handler) -> Handler:
+            self._handlers.setdefault(event_type, []).append(fn)
+            return fn
+
+        return deco
+
+    def register(self, event_type: str, fn: Handler) -> None:
+        self._handlers.setdefault(event_type, []).append(fn)
+
+    def types(self) -> set[str]:
+        return set(self._handlers)
+
+    def dispatch(self, event: Event, store: Store) -> int:
+        handlers = self._handlers.get(event.type)
+        if not handlers:
+            raise UnhandledEvent(f"no handler for event type '{event.type}'")
+        for h in handlers:
+            h(event, store)
+        return len(handlers)
+
+    def merge(self, other: HandlerRegistry) -> None:
+        for t, hs in other._handlers.items():
+            self._handlers.setdefault(t, []).extend(hs)