spatialagents 0.1.0__py3-none-any.whl

spatialagents/schema.py

@@ -0,0 +1,167 @@
+"""Answer models -> a JSON schema the provider's structured outputs accept.
+
+Structured outputs take a subset of JSON Schema: every object closed
+(`additionalProperties: false`), no numeric or length constraints, no external
+or recursive `$ref`, at most 24 optional properties and 16 unions per request.
+A model's JSON schema (pydantic's `model_json_schema()`) is reduced to that
+subset; the constraints it loses are still enforced, because every answer is
+validated locally against the real model before it is used. A schema that
+cannot fit the limits is not sent at all (`None`): the packet then asks for
+JSON in words, and local validation does the rest.
+
+Maps (`dict[str, X]`: verdicts keyed by marker or channel, modules keyed by
+name) have no closed form -- their keys are the packet's -- so a map is sent
+as a list of `{key, value}` entries and `decode` turns such a list back into
+the object the answer model expects before local validation. An answer that
+already gives the object is left alone.
+
+This is the mechanism only. Which answer model a packet kind has, and how a
+product's kinds share one schema per task, is the product's (Plexora's
+`harness/schema.py`).
+"""
+
+from __future__ import annotations
+
+import copy
+
+DROP = {"minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "multipleOf", "minLength",
+        "maxLength", "maxItems", "title", "discriminator", "examples", "uniqueItems", "minProperties",
+        "maxProperties", "propertyNames"}
+FORMATS = {"date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid"}
+MAX_OPTIONAL = 24
+MAX_UNIONS = 16
+
+
+#: What a map's entries are called in the provider schema.
+MAP_NOTE = "One entry per key, as {key, value}."
+
+
+def _map_value(node):
+    """The value schema of a map node (`additionalProperties: {...}` and no
+    named properties), else None."""
+    extra = node.get("additionalProperties") if isinstance(node, dict) else None
+    return extra if isinstance(extra, dict) and not node.get("properties") else None
+
+
+def _resolve(node, defs):
+    while isinstance(node, dict) and "$ref" in node:
+        name = node["$ref"].rsplit("/", 1)[-1]
+        if name not in defs:
+            raise ValueError(f"unresolvable $ref {node['$ref']}")
+        node = {**copy.deepcopy(defs[name]), **{k: v for k, v in node.items() if k != "$ref"}}
+    return node
+
+
+def _inline(node, defs, depth=0):
+    if depth > 24:
+        raise ValueError("schema nests too deeply (recursive?)")
+    if isinstance(node, list):
+        return [_inline(item, defs, depth + 1) for item in node]
+    if not isinstance(node, dict):
+        return node
+    if "$ref" in node:
+        return _inline(_resolve(node, defs), defs, depth + 1)
+    value = _map_value(node)
+    if value is not None:
+        described = " ".join(p for p in (node.get("description"), MAP_NOTE) if p)
+        return {"type": "array", "description": described,
+                "items": {"type": "object", "additionalProperties": False, "required": ["key", "value"],
+                          "properties": {"key": {"type": "string"},
+                                         "value": _inline(value, defs, depth + 1)}}}
+    out = {}
+    for key, value in node.items():
+        if key == "properties" and isinstance(value, dict):
+            # Field names, not keywords: a field called `pattern`, `title` or
+            # `format` is kept (dropping it left it `required` but unsayable).
+            out[key] = {name: _inline(sub, defs, depth + 1) for name, sub in value.items()}
+            continue
+        if key in DROP or key in ("$defs", "definitions"):
+            continue
+        if key == "format" and value not in FORMATS:
+            continue
+        if key == "minItems" and isinstance(value, int) and value > 1:
+            out[key] = 1
+            continue
+        if key == "pattern":
+            continue        # regex support is partial; local validation enforces it
+        if key == "default" and value is None:
+            continue
+        out[key] = _inline(value, defs, depth + 1) if key not in ("enum", "const", "required") else value
+    if out.get("type") == "object" or "properties" in out:
+        out["additionalProperties"] = False
+    return out
+
+
+def _count(node, tally):
+    if isinstance(node, list):
+        for item in node:
+            _count(item, tally)
+        return
+    if not isinstance(node, dict):
+        return
+    props = node.get("properties")
+    if isinstance(props, dict):
+        required = set(node.get("required") or ())
+        tally["optional"] += sum(1 for name in props if name not in required)
+    if "anyOf" in node or isinstance(node.get("type"), list):
+        tally["unions"] += 1
+    for value in node.values():
+        _count(value, tally)
+
+
+def structured(model_schema: dict) -> dict | None:
+    """The provider-ready schema, or None when it cannot fit the limits."""
+    try:
+        schema = _inline(model_schema, model_schema.get("$defs") or model_schema.get("definitions") or {})
+    except ValueError:
+        return None
+    tally = {"optional": 0, "unions": 0}
+    _count(schema, tally)
+    if tally["optional"] > MAX_OPTIONAL or tally["unions"] > MAX_UNIONS:
+        return None
+    return schema
+
+
+def _entries(value) -> bool:
+    return isinstance(value, list) and all(isinstance(e, dict) and set(e) <= {"key", "value"} and "key" in e
+                                           for e in value)
+
+
+def _decode(value, node, defs, depth=0):
+    if depth > 24 or not isinstance(node, dict):
+        return value
+    node = _resolve(node, defs)
+    if "anyOf" in node:
+        for branch in node["anyOf"]:
+            branch = _resolve(branch, defs)
+            if _map_value(branch) is not None and _entries(value):
+                return _decode(value, branch, defs, depth + 1)
+        for branch in node["anyOf"]:
+            branch = _resolve(branch, defs)
+            if isinstance(value, dict) and (branch.get("properties") or _map_value(branch) is not None):
+                return _decode(value, branch, defs, depth + 1)
+            if isinstance(value, list) and branch.get("type") == "array":
+                return _decode(value, branch, defs, depth + 1)
+        return value
+    inner = _map_value(node)
+    if inner is not None:
+        if _entries(value):
+            value = {str(e["key"]): e.get("value") for e in value}
+        if isinstance(value, dict):
+            return {k: _decode(v, inner, defs, depth + 1) for k, v in value.items()}
+        return value
+    props = node.get("properties")
+    if isinstance(props, dict) and isinstance(value, dict):
+        return {k: _decode(v, props[k], defs, depth + 1) if k in props else v for k, v in value.items()}
+    if node.get("type") == "array" and isinstance(value, list) and isinstance(node.get("items"), dict):
+        return [_decode(v, node["items"], defs, depth + 1) for v in value]
+    return value
+
+
+def decode(answer, model_schema: dict):
+    """An answer given against `structured(model_schema)` back in the answer
+    model's own shape: every map sent as `{key, value}` entries is an object
+    again. Anything else passes through unchanged."""
+    return _decode(answer, model_schema, model_schema.get("$defs") or model_schema.get("definitions") or {})
+
+

spatialagents/serialize.py

@@ -0,0 +1,88 @@
+"""Turning a tool's result into what goes to the model -- bounded.
+
+A model's context is the scarce resource. A result is serialized as compact
+JSON; when it would exceed the limit, its longest lists are halved until it
+fits, and the result says `truncated: true` with the paths that were cut --
+never a silently shorter answer. (The same rule as a product's MCP server
+applies to what it sends an external agent, so a result reads the same
+wherever it went.)
+
+`clean` is the default rendering of a value as JSON-able data: non-finite
+floats become null, bytes a size, numpy-style scalars their `.item()`. A
+backend whose results carry other types renders them in its own
+`ToolBackend.serialize`, and this module bounds what comes back.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from typing import Any
+
+#: Characters in one serialized tool result, unless the backend says otherwise.
+MAX_RESULT_CHARS = 60_000
+
+
+def clean(value):
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, dict):
+        return {str(k): clean(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [clean(v) for v in value]
+    if isinstance(value, (bytes, bytearray)):
+        return f"<{len(value)} bytes>"
+    if hasattr(value, "item") and callable(value.item):
+        try:
+            return clean(value.item())
+        except Exception:
+            return str(value)
+    return value
+
+
+def dumps(value) -> str:
+    return json.dumps(clean(value), separators=(",", ":"), ensure_ascii=False, default=str)
+
+
+def _lists(value, path=""):
+    if isinstance(value, dict):
+        for key, item in value.items():
+            yield from _lists(item, f"{path}.{key}" if path else str(key))
+    elif isinstance(value, list):
+        yield path, value
+        for index, item in enumerate(value[:20]):
+            yield from _lists(item, f"{path}[{index}]")
+
+
+def bound(value: Any, limit: int = MAX_RESULT_CHARS) -> str:
+    """Compact JSON of `value`, at most about `limit` characters."""
+    value = clean(value)
+    text = dumps(value)
+    if len(text) <= limit:
+        return text
+    cut = []
+    for _ in range(64):
+        candidates = [(len(dumps(items)), path, items) for path, items in _lists(value)
+                      if len(items) > 1]
+        if not candidates:
+            break
+        _size, path, items = max(candidates, key=lambda c: c[0])
+        keep = max(1, len(items) // 2)
+        dropped = len(items) - keep
+        del items[keep:]
+        cut.append({"path": path, "dropped": dropped})
+        if isinstance(value, dict):
+            value["truncated"] = True
+            value["_truncated"] = cut
+        text = dumps(value)
+        if len(text) <= limit:
+            return text
+    return text[:limit]
+
+
+def indented(value) -> str:
+    """`value` as JSON indented one space per level: what an offloaded result
+    is stored as, so a query matches a line that means something."""
+    if isinstance(value, str):
+        return value
+    return json.dumps(clean(value), indent=1, ensure_ascii=False, default=str)

spatialagents/sessions.py

@@ -0,0 +1,276 @@
+"""A session on disk: the record, its packets, its decisions, and who holds it.
+
+    <root>/<session_id>/
+        session.json      the record (atomic writes)
+        control.json      pause / stop / approvals, written by the person's routes
+        decisions.jsonl   every packet issued and every answer applied (a
+                          conversation's numbered transcript)
+        packets/<id>.json the outstanding (and recent) packets
+        packets/<id>/     their images, as sent
+        .lock             the process driving the session (pid)
+
+The root is the product's, one per kind of session (Plexora's
+`<data_root>/.agent/sessions/<kind>/`, its conversations'
+`<data_root>/.agent/ai/conversations/`). The thread that runs a session's bulk
+job and the tool calls answering its packets are in one process; `lock`
+serialises them. A second process that tries to drive the same session is
+refused (`conflict`) while the first is alive, and takes over when it is not.
+
+A product names itself in what a refused driver is told (`HOLDER`) and where an
+unknown session id's error points (`LIST_HINT`) by subclassing.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import threading
+import time
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+
+from spatialagents.errors import AgentError
+
+KEEP_SESSIONS = 20
+KEEP_DAYS = 60
+
+_LOCKS: dict = {}
+_GUARD = threading.Lock()
+
+
+def new_session_id(prefix="gs") -> str:
+    return f"{prefix}_{datetime.now(timezone.utc):%Y%m%dT%H%M%S}_{uuid.uuid4().hex[:6]}"
+
+
+def atomic_write(path: Path, text: str):
+    """`text` into `path`, whole or not at all."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
+    tmp.write_text(text, encoding="utf-8")
+    # Windows refuses to replace a file another thread has open for reading
+    # (WinError 5/32) -- the bulk job and an answering loop read session.json
+    # while the other writes it, and parallel sessions make that common. The
+    # reader holds it for microseconds, so a short retry always gets through.
+    for attempt in range(40):
+        try:
+            os.replace(tmp, path)
+            return
+        except PermissionError:
+            if attempt == 39:
+                raise
+            time.sleep(0.005 * (attempt + 1))
+
+
+def _pid_alive(pid) -> bool:
+    try:
+        pid = int(pid)
+    except (TypeError, ValueError):
+        return False
+    if pid == os.getpid():
+        return True
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError:
+        return False
+    return True
+
+
+class SessionBusy(Exception):
+    """`SessionStore.lock(timeout=...)` ran out: another thread holds the session."""
+
+
+class SessionStore:
+    """The sessions of one kind under one root."""
+
+    #: Who a second process trying to drive a held session is told holds it.
+    HOLDER = "another process"
+    #: Where an unknown session id's error points, if anywhere.
+    LIST_HINT: str | None = None
+
+    def __init__(self, kind: str, root: Path | str):
+        self.kind = kind
+        self.root = Path(root)
+
+    # -- paths -----------------------------------------------------------
+
+    def folder(self, session_id) -> Path:
+        if not str(session_id).replace("_", "").replace("-", "").isalnum():
+            raise AgentError("invalid_input", f"not a session id: {session_id!r}")
+        return self.root / str(session_id)
+
+    def exists(self, session_id) -> bool:
+        return (self.folder(session_id) / "session.json").is_file()
+
+    # -- the record ------------------------------------------------------
+
+    def create(self, record: dict) -> dict:
+        folder = self.folder(record["session_id"])
+        folder.mkdir(parents=True, exist_ok=False)
+        self.save(record)
+        self.claim(record["session_id"])
+        return record
+
+    def load(self, session_id) -> dict:
+        path = self.folder(session_id) / "session.json"
+        if not path.is_file():
+            raise AgentError("invalid_input", f"no {self.kind} session {session_id!r}",
+                             detail={"hint": self.LIST_HINT} if self.LIST_HINT else None)
+        return json.loads(path.read_text(encoding="utf-8"))
+
+    def save(self, record: dict):
+        record["updated_at"] = time.time()
+        atomic_write(self.folder(record["session_id"]) / "session.json",
+                json.dumps(record, default=str, ensure_ascii=False))
+
+    @contextmanager
+    def lock(self, session_id, *, timeout: float = -1):
+        """Exclusive access to one session's record within this process.
+
+        `timeout` (seconds) bounds the wait and raises `SessionBusy` when it
+        runs out; the default waits as long as it takes. Best-effort work
+        that runs while holding another lock (a bulk pass's progress, inside
+        an open image reader) must not wait: an answer drawing the next
+        packet holds this lock and waits for that reader."""
+        with _GUARD:
+            lock = _LOCKS.setdefault((str(self.root), str(session_id)), threading.RLock())
+        if not lock.acquire(timeout=timeout):
+            raise SessionBusy(session_id)
+        try:
+            yield
+        finally:
+            lock.release()
+
+    # -- ownership across processes ---------------------------------------
+
+    def claim(self, session_id):
+        """Take the session for this process, or refuse while another live
+        process holds it."""
+        path = self.folder(session_id) / ".lock"
+        if path.exists():
+            try:
+                holder = json.loads(path.read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                holder = {}
+            if holder.get("pid") != os.getpid() and _pid_alive(holder.get("pid")):
+                raise AgentError("conflict", f"{self.HOLDER} is driving this session",
+                                 detail={"holder_pid": holder.get("pid"),
+                                         "since": holder.get("since")}, retryable=True)
+        atomic_write(path, json.dumps({"pid": os.getpid(), "since": time.time()}))
+
+    def release(self, session_id):
+        path = self.folder(session_id) / ".lock"
+        try:
+            holder = json.loads(path.read_text(encoding="utf-8"))
+            if holder.get("pid") == os.getpid():
+                path.unlink()
+        except (OSError, ValueError):
+            pass
+
+    # -- control -----------------------------------------------------------
+
+    def control(self, session_id) -> dict:
+        path = self.folder(session_id) / "control.json"
+        if not path.is_file():
+            return {"paused": False, "paused_by": None, "locked_markers": []}
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return {"paused": False, "paused_by": None, "locked_markers": []}
+
+    def set_control(self, session_id, **changes) -> dict:
+        control = self.control(session_id)
+        control.update(changes)
+        atomic_write(self.folder(session_id) / "control.json", json.dumps(control))
+        return control
+
+    # -- packets and decisions ---------------------------------------------
+
+    def write_packet(self, session_id, packet: dict, images=()):
+        folder = self.folder(session_id) / "packets"
+        atomic_write(folder / f"{packet['packet_id']}.json", json.dumps(packet, default=str))
+        image_dir = folder / packet["packet_id"]
+        if image_dir.is_dir():
+            # A re-rendered packet replaces its images; none may linger.
+            for stale in image_dir.iterdir():
+                stale.unlink()
+        if images:
+            image_dir.mkdir(parents=True, exist_ok=True)
+        for index, (data, fmt) in enumerate(images):
+            (image_dir / f"{index}.{fmt}").write_bytes(data)
+
+    def read_packet(self, session_id, packet_id) -> tuple:
+        """(packet, [(bytes, format)])."""
+        folder = self.folder(session_id) / "packets"
+        path = folder / f"{packet_id}.json"
+        if not path.is_file():
+            raise AgentError("invalid_input", f"no packet {packet_id!r} in this session")
+        packet = json.loads(path.read_text(encoding="utf-8"))
+        images = []
+        image_dir = folder / packet_id
+        if image_dir.is_dir():
+            for file in sorted(image_dir.iterdir(), key=lambda p: int(p.stem)
+                               if p.stem.isdigit() else 0):
+                images.append((file.read_bytes(), file.suffix.lstrip(".")))
+        return packet, images
+
+    def log(self, session_id, record: dict):
+        path = self.folder(session_id) / "decisions.jsonl"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        line = json.dumps({"t": time.time(), **record}, default=str, ensure_ascii=False)
+        with open(path, "a", encoding="utf-8") as handle:
+            handle.write(line + "\n")
+
+    def decisions(self, session_id) -> list:
+        path = self.folder(session_id) / "decisions.jsonl"
+        if not path.is_file():
+            return []
+        out = []
+        for line in path.read_text(encoding="utf-8").splitlines():
+            try:
+                out.append(json.loads(line))
+            except ValueError:
+                continue
+        return out
+
+    def clear_packets(self, session_id):
+        folder = self.folder(session_id) / "packets"
+        if folder.is_dir():
+            shutil.rmtree(folder, ignore_errors=True)
+
+    # -- listing and retention ---------------------------------------------
+
+    def list(self, limit=20) -> list:
+        if not self.root.is_dir():
+            return []
+        records = []
+        for folder in self.root.iterdir():
+            path = folder / "session.json"
+            if not path.is_file():
+                continue
+            try:
+                record = json.loads(path.read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                continue
+            records.append(record)
+        records.sort(key=lambda r: r.get("created_at") or "", reverse=True)
+        return records[:limit]
+
+    def sweep(self, keep=KEEP_SESSIONS, days=KEEP_DAYS, now=None) -> int:
+        """Remove finished sessions past the newest `keep` and older than `days`."""
+        now = now or time.time()
+        records = self.list(limit=10_000)
+        removed = 0
+        for index, record in enumerate(records):
+            finished = record.get("state") in ("done", "cancelled", "failed", "rolled_back")
+            old = now - float(record.get("updated_at") or now) > days * 86400
+            if finished and (index >= keep or old):
+                shutil.rmtree(self.folder(record["session_id"]), ignore_errors=True)
+                removed += 1
+        return removed