gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/environment/schema/tasks.py

@@ -0,0 +1,525 @@
+"""Session tasks: a task supplied at open instead of read from the bundle.
+
+A bundled task is ``scenarios/<name>/`` in the version (instruction, seed
+overlay, ``checks/verify.py``). A session task is the same three things sent
+with the open, for one session, so one published api world serves any number
+of end users each with their own goal, starting rows and pass criteria.
+
+This module is the host's half of the contract whose other half is
+``gateway/packages/shared/src/domain/worldTasks.ts`` (the platform validates
+with Zod, the host validates again here: it trusts nothing it did not check).
+Keep the two in step by hand; the limits and the check grammar are the same.
+
+* :func:`parse_task_spec` — the spec, checked and normalised.
+* :func:`write_task_dir` — the spec on disk under the session
+  (``task.json``, ``instruction.md``, ``checks/verify.py``), where a grade
+  and a restart find it.
+* :func:`evaluate_assertions` — the declarative grader, pure Python over
+  the state dump, so a check cannot read outside its session.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+from .validation import fail
+
+# The caps, as the platform's WORLD_TASK_LIMITS.
+NAME_CHARS = 120
+INSTRUCTION_BYTES = 32 * 1024
+METADATA_BYTES = 4 * 1024
+SEED_ROWS = 5_000
+SEED_BYTES = 4 * 1024 * 1024
+CHECKS = 200
+PYTHON_SOURCE_BYTES = 64 * 1024
+PYTHON_TIMEOUT_S = 30.0
+RUBRIC_BYTES = 16 * 1024
+RUBRIC_CRITERIA = 50
+RUBRIC_CRITERION_BYTES = 2 * 1024
+TOOLS = 200
+TOOL_NAME_CHARS = 200
+
+SEED_MODES = ("append", "replace")
+ASSERTIONS = ("exists", "absent", "count", "all", "any")
+FIELD_OPS = ("eq", "neq", "in", "gt", "gte", "lt", "lte", "contains")
+_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
+_CHECK_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
+_CRITERION_ID = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*$")
+_TOOL_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]*$")
+
+TASK_DIR = "task"
+TASK_FILE = "task.json"
+
+
+def _bytes(value: Any) -> int:
+    text = value if isinstance(value, str) else json.dumps(value, separators=(",", ":"))
+    return len(text.encode("utf-8"))
+
+
+def _scalar(value: Any) -> bool:
+    return value is None or isinstance(value, (str, bool, int, float))
+
+
+@dataclass(frozen=True)
+class TaskSpec:
+    """A validated session task. ``raw`` is the normalised spec as a dict: what
+    the platform sent, defaults filled, and what is written to ``task.json``
+    and compared when an open is retried."""
+
+    instruction: str
+    name: str | None
+    seed: dict | None
+    grader: dict | None
+    metadata: dict
+    raw: dict
+    #: The tools the session may call, or None for every tool the world serves.
+    tools: tuple[str, ...] | None = None
+
+    @property
+    def grader_kind(self) -> str | None:
+        return self.grader["kind"] if self.grader else None
+
+    @property
+    def graded_on_platform(self) -> bool:
+        """A rubric task is judged by the platform's inference path, never here."""
+        return self.grader_kind == "rubric"
+
+    def allows_tool(self, tool: str) -> bool:
+        return self.tools is None or tool in self.tools
+
+    def entities(self) -> list[str]:
+        """Every entity the task names, seed rows and assertion checks alike, in
+        order of first mention. A world that defines none of a name cannot start
+        or grade this task: a seed row has no table, and a check over a table
+        that does not exist selects no rows, which an ``absent`` or ``count:
+        eq 0`` check would pass on a typo."""
+        named: list[str] = []
+        if self.seed is not None:
+            named.extend(self.seed["rows"])
+        if self.grader is not None and self.grader["kind"] == "assertions":
+            named.extend(check["entity"] for check in self.grader["checks"])
+        return list(dict.fromkeys(named))
+
+
+def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
+    """The entities ``spec`` names that ``defined`` (a world contract's) lacks."""
+    known = set(defined)
+    return [entity for entity in spec.entities() if entity not in known]
+
+
+def parse_task_spec(value: Any) -> TaskSpec:
+    """The spec checked for shape and size; every refusal names its path."""
+    if not isinstance(value, dict):
+        fail("task", "a task is a json object", "/task")
+    unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata"}
+    if unknown:
+        fail("task", f"unknown task field(s): {', '.join(sorted(unknown))}", "/task")
+    name = value.get("name")
+    if name is not None and (not isinstance(name, str) or not _NAME.match(name) or len(name) > NAME_CHARS):
+        fail("task", "a task name is letters, digits, _ . - (leading letter or digit)", "/task/name")
+    instruction = value.get("instruction")
+    if not isinstance(instruction, str) or not instruction:
+        fail("task", "a task needs an instruction", "/task/instruction")
+    if _bytes(instruction) > INSTRUCTION_BYTES:
+        fail("task", f"an instruction is at most {INSTRUCTION_BYTES} bytes", "/task/instruction")
+    seed = _parse_seed(value.get("seed"))
+    grader = _parse_grader(value.get("grader"))
+    tools = _parse_tools(value.get("tools"))
+    metadata = value.get("metadata")
+    if metadata is None:
+        metadata = {}
+    if not isinstance(metadata, dict) or any(not isinstance(k, str) or not k for k in metadata):
+        fail("task", "metadata is a json object with string keys", "/task/metadata")
+    if _bytes(metadata) > METADATA_BYTES:
+        fail("task", f"task metadata is at most {METADATA_BYTES} bytes of json", "/task/metadata")
+    raw: dict = {"instruction": instruction}
+    if name is not None:
+        raw["name"] = name
+    if seed is not None:
+        raw["seed"] = seed
+    if grader is not None:
+        raw["grader"] = grader
+    if tools is not None:
+        raw["tools"] = list(tools)
+    if metadata:
+        raw["metadata"] = metadata
+    return TaskSpec(
+        instruction=instruction, name=name, seed=seed, grader=grader, metadata=metadata, raw=raw, tools=tools
+    )
+
+
+def _parse_tools(tools: Any) -> tuple[str, ...] | None:
+    if tools is None:
+        return None
+    if not isinstance(tools, list) or not tools:
+        fail("task", "tools is a non-empty list of tool names", "/task/tools")
+    if len(tools) > TOOLS:
+        fail("task", f"a task allows at most {TOOLS} tools", "/task/tools")
+    for i, tool in enumerate(tools):
+        if not isinstance(tool, str) or not _TOOL_NAME.match(tool) or len(tool) > TOOL_NAME_CHARS:
+            fail("task", "a tool name is letters, digits, _ . : -", f"/task/tools/{i}")
+    if len(set(tools)) != len(tools):
+        fail("task", "a tool is listed once", "/task/tools")
+    return tuple(tools)
+
+
+def unknown_task_tools(spec: TaskSpec, served: Iterable[str]) -> list[str]:
+    """The tools ``spec`` allows that the world does not serve. A name the
+    world never had would make the allow-list smaller than the author meant
+    without anyone noticing, so it is refused at open."""
+    if spec.tools is None:
+        return []
+    known = set(served)
+    return [tool for tool in spec.tools if tool not in known]
+
+
+def _parse_seed(seed: Any) -> dict | None:
+    if seed is None:
+        return None
+    if not isinstance(seed, dict) or set(seed) - {"mode", "rows"}:
+        fail("task", "seed is {mode?: append|replace, rows: {entity: [row, ...]}}", "/task/seed")
+    mode = seed.get("mode", "append")
+    if mode not in SEED_MODES:
+        fail("task", f"seed mode must be one of {', '.join(SEED_MODES)}", "/task/seed/mode")
+    rows = seed.get("rows")
+    if not isinstance(rows, dict):
+        fail("task", "seed.rows is {entity: [row, ...]}", "/task/seed/rows")
+    total = 0
+    for entity, entity_rows in rows.items():
+        if not isinstance(entity, str) or not entity or len(entity) > 120:
+            fail("task", "an entity name is a string of up to 120 characters", "/task/seed/rows")
+        if not isinstance(entity_rows, list) or any(not isinstance(r, dict) for r in entity_rows):
+            fail("task", f"seed.rows.{entity} is a list of row objects", f"/task/seed/rows/{entity}")
+        total += len(entity_rows)
+    if total > SEED_ROWS:
+        fail("task", f"a task seed carries at most {SEED_ROWS} rows (got {total})", "/task/seed/rows")
+    if _bytes(rows) > SEED_BYTES:
+        fail("task", f"a task seed is at most {SEED_BYTES} bytes of json", "/task/seed/rows")
+    return {"mode": mode, "rows": rows}
+
+
+def _parse_grader(grader: Any) -> dict | None:
+    if grader is None:
+        return None
+    if not isinstance(grader, dict):
+        fail("task", "grader is a json object with a kind", "/task/grader")
+    kind = grader.get("kind")
+    if kind == "python":
+        if set(grader) - {"kind", "source", "timeoutSeconds"}:
+            fail("task", "a python grader is {kind, source, timeoutSeconds?}", "/task/grader")
+        source = grader.get("source")
+        if not isinstance(source, str) or not source:
+            fail("task", "a python grader needs its verify.py source", "/task/grader/source")
+        if _bytes(source) > PYTHON_SOURCE_BYTES:
+            fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", "/task/grader/source")
+        out: dict = {"kind": "python", "source": source}
+        timeout = grader.get("timeoutSeconds")
+        if timeout is not None:
+            if (
+                isinstance(timeout, bool)
+                or not isinstance(timeout, (int, float))
+                or not 0 < timeout <= PYTHON_TIMEOUT_S
+            ):
+                fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", "/task/grader/timeoutSeconds")
+            out["timeoutSeconds"] = float(timeout)
+        return out
+    if kind == "assertions":
+        if set(grader) - {"kind", "checks"}:
+            fail("task", "an assertions grader is {kind, checks}", "/task/grader")
+        checks = grader.get("checks")
+        if not isinstance(checks, list) or not checks:
+            fail("task", "an assertions grader needs at least one check", "/task/grader/checks")
+        if len(checks) > CHECKS:
+            fail("task", f"an assertions grader has at most {CHECKS} checks", "/task/grader/checks")
+        parsed = [_parse_check(check, f"/task/grader/checks/{i}") for i, check in enumerate(checks)]
+        # Every check owns one reward key, its id or its position; the keys are
+        # distinct as a set, so an explicit `check_2` cannot shadow the third
+        # check's positional one. Mirrors WorldTaskAssertionsGraderSchema.
+        seen: set[str] = set()
+        for i, check in enumerate(parsed):
+            key = check_key(check, i)
+            if key in seen:
+                if "id" in check:
+                    fail(
+                        "task",
+                        f"check id '{key}' is used twice (an id of the form check_<n> is also the positional key of check n)",
+                        f"/task/grader/checks/{i}/id",
+                    )
+                fail(
+                    "task",
+                    f"check {i} has no id and its positional key '{key}' is taken by another check's id",
+                    f"/task/grader/checks/{i}/id",
+                )
+            seen.add(key)
+        return {"kind": "assertions", "checks": parsed}
+    if kind == "rubric":
+        return _parse_rubric(grader)
+    fail("task", "grader.kind must be assertions, python or rubric", "/task/grader/kind")
+    raise AssertionError  # fail() never returns
+
+
+def _parse_rubric(grader: dict) -> dict:
+    """An LLM judge's brief. Checked here for shape only: the host stores it
+    with the task and never runs it; the platform grades it."""
+    if set(grader) - {"kind", "rubric", "criteria", "model"}:
+        fail("task", "a rubric grader is {kind, rubric, criteria?, model?}", "/task/grader")
+    rubric = grader.get("rubric")
+    if not isinstance(rubric, str) or not rubric:
+        fail("task", "a rubric grader needs its rubric text", "/task/grader/rubric")
+    if _bytes(rubric) > RUBRIC_BYTES:
+        fail("task", f"a rubric is at most {RUBRIC_BYTES} bytes", "/task/grader/rubric")
+    out: dict = {"kind": "rubric", "rubric": rubric}
+    criteria = grader.get("criteria")
+    if criteria is not None:
+        if not isinstance(criteria, list) or not criteria:
+            fail("task", "criteria is a non-empty list", "/task/grader/criteria")
+        if len(criteria) > RUBRIC_CRITERIA:
+            fail("task", f"a rubric has at most {RUBRIC_CRITERIA} criteria", "/task/grader/criteria")
+        parsed = []
+        seen: set[str] = set()
+        for i, criterion in enumerate(criteria):
+            at = f"/task/grader/criteria/{i}"
+            if not isinstance(criterion, dict) or set(criterion) - {"id", "description", "weight"}:
+                fail("task", "a criterion is {id, description, weight?}", at)
+            criterion_id = criterion.get("id")
+            if not isinstance(criterion_id, str) or not _CRITERION_ID.match(criterion_id) or len(criterion_id) > 120:
+                fail("task", "a criterion id is letters, digits, _ . -", f"{at}/id")
+            if criterion_id in seen:
+                fail("task", f"criterion id '{criterion_id}' is used twice", f"{at}/id")
+            seen.add(criterion_id)
+            description = criterion.get("description")
+            if not isinstance(description, str) or not description:
+                fail("task", "a criterion needs a description", f"{at}/description")
+            if _bytes(description) > RUBRIC_CRITERION_BYTES:
+                fail("task", f"a criterion is at most {RUBRIC_CRITERION_BYTES} bytes", f"{at}/description")
+            item: dict = {"id": criterion_id, "description": description}
+            weight = criterion.get("weight")
+            if weight is not None:
+                if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0:
+                    fail("task", "weight is a positive number", f"{at}/weight")
+                item["weight"] = float(weight)
+            parsed.append(item)
+        out["criteria"] = parsed
+    model = grader.get("model")
+    if model is not None:
+        if not isinstance(model, dict) or set(model) != {"provider", "model"}:
+            fail("task", "model is {provider, model}", "/task/grader/model")
+        provider, name = model["provider"], model["model"]
+        if not isinstance(provider, str) or not 0 < len(provider) <= 120:
+            fail("task", "model.provider is a string of up to 120 characters", "/task/grader/model/provider")
+        if not isinstance(name, str) or not 0 < len(name) <= 200:
+            fail("task", "model.model is a string of up to 200 characters", "/task/grader/model/model")
+        out["model"] = {"provider": provider, "model": name}
+    return out
+
+
+def check_key(check: dict, index: int) -> str:
+    """The reward key of a check: its id, else its position as ``check_<i>``."""
+    return check.get("id") or f"check_{index}"
+
+
+def _parse_check(check: Any, at: str) -> dict:
+    if not isinstance(check, dict):
+        fail("task", "a check is a json object", at)
+    unknown = set(check) - {"id", "weight", "entity", "where", "assert", "count", "field", "op", "value"}
+    if unknown:
+        fail("task", f"unknown check field(s): {', '.join(sorted(unknown))}", at)
+    out: dict = {}
+    check_id = check.get("id")
+    if check_id is not None:
+        if not isinstance(check_id, str) or not _CHECK_ID.match(check_id) or len(check_id) > 120:
+            fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
+        out["id"] = check_id
+    weight = check.get("weight", 1)
+    if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0 or weight == float("inf"):
+        fail("task", "weight is a positive number", f"{at}/weight")
+    out["weight"] = float(weight)
+    entity = check.get("entity")
+    if not isinstance(entity, str) or not entity or len(entity) > 120:
+        fail("task", "a check names an entity", f"{at}/entity")
+    out["entity"] = entity
+    where = check.get("where")
+    if where is not None:
+        if not isinstance(where, dict) or any(
+            not isinstance(k, str) or not k or not _scalar(v) for k, v in where.items()
+        ):
+            fail("task", "where is {field: scalar, ...}", f"{at}/where")
+        out["where"] = where
+    assertion = check.get("assert", "exists")
+    if assertion not in ASSERTIONS:
+        fail("task", f"assert must be one of {', '.join(ASSERTIONS)}", f"{at}/assert")
+    out["assert"] = assertion
+    count = check.get("count")
+    if assertion == "count":
+        if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
+            fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
+        for key, bound in count.items():
+            if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
+                fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
+        out["count"] = count
+    elif count is not None:
+        fail("task", "count only goes with assert: count", f"{at}/count")
+    field = check.get("field")
+    op = check.get("op", "eq")
+    value = check.get("value")
+    if assertion in ("all", "any"):
+        if not isinstance(field, str) or not field:
+            fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
+        if op not in FIELD_OPS:
+            fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
+        if "value" not in check:
+            fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
+        if op == "in":
+            if not isinstance(value, list) or any(not _scalar(v) for v in value):
+                fail("task", "op: in compares against a list of scalars", f"{at}/value")
+        elif not _scalar(value):
+            fail("task", f"op: {op} compares against one scalar", f"{at}/value")
+        out["field"], out["op"], out["value"] = field, op, value
+    elif field is not None or "value" in check or "op" in check:
+        fail("task", "field/op/value only go with assert: all or any", f"{at}/field")
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# On disk
+# --------------------------------------------------------------------------- #
+
+
+def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
+    """The task under the session: ``task.json`` (the whole spec),
+    ``instruction.md``, and for a python grader ``checks/verify.py`` where
+    the bundled layout has it, so the one verifier path runs both."""
+    root = Path(session_dir) / TASK_DIR
+    root.mkdir(parents=True, exist_ok=True)
+    (root / TASK_FILE).write_text(json.dumps(spec.raw, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    (root / "instruction.md").write_text(spec.instruction, encoding="utf-8")
+    if spec.grader_kind == "python":
+        checks = root / "checks"
+        checks.mkdir(exist_ok=True)
+        (checks / "verify.py").write_text(spec.grader["source"], encoding="utf-8")
+    return root
+
+
+def read_task_dir(session_dir: Path) -> TaskSpec | None:
+    """The task a session directory holds, or None for a bundled-task session."""
+    path = Path(session_dir) / TASK_DIR / TASK_FILE
+    if not path.is_file():
+        return None
+    return parse_task_spec(json.loads(path.read_text(encoding="utf-8")))
+
+
+# --------------------------------------------------------------------------- #
+# The assertions grader
+# --------------------------------------------------------------------------- #
+
+
+def _lookup(row: Any, path: str) -> tuple[bool, Any]:
+    """``(found, value)`` of a dotted path into a row."""
+    current = row
+    for part in path.split("."):
+        if not isinstance(current, dict) or part not in current:
+            return False, None
+        current = current[part]
+    return True, current
+
+
+def _same(actual: Any, expected: Any) -> bool:
+    """JSON equality: ``true`` is not ``1``, as it is not in the schema."""
+    if isinstance(actual, bool) or isinstance(expected, bool):
+        return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
+    return actual == expected
+
+
+def _matches_where(row: dict, where: dict | None) -> bool:
+    for path, expected in (where or {}).items():
+        found, actual = _lookup(row, path)
+        if not found or not _same(actual, expected):
+            return False
+    return True
+
+
+def _ordered(actual: Any, expected: Any) -> bool:
+    """Whether the two compare with ``<``: both numbers (not bools) or both strings."""
+    numbers = (int, float)
+    if isinstance(actual, bool) or isinstance(expected, bool):
+        return False
+    if isinstance(actual, numbers) and isinstance(expected, numbers):
+        return True
+    return isinstance(actual, str) and isinstance(expected, str)
+
+
+def _predicate(row: dict, field: str, op: str, expected: Any) -> bool:
+    found, actual = _lookup(row, field)
+    if op == "eq":
+        return found and _same(actual, expected)
+    if op == "neq":
+        return not found or not _same(actual, expected)
+    if op == "in":
+        return found and any(_same(actual, option) for option in expected)
+    if op == "contains":
+        if not found:
+            return False
+        if isinstance(actual, str):
+            return isinstance(expected, str) and expected in actual
+        if isinstance(actual, list):
+            return any(_same(item, expected) for item in actual)
+        return False
+    if not found or not _ordered(actual, expected):
+        return False
+    if op == "gt":
+        return actual > expected
+    if op == "gte":
+        return actual >= expected
+    if op == "lt":
+        return actual < expected
+    return actual <= expected  # lte
+
+
+def _rows_of(state: dict, entity: str) -> list[dict]:
+    """The rows of one entity from a state dump: ``{entity: {key: row}}`` (the
+    verifier's shape) or ``{entity: [row, ...]}``."""
+    held = state.get(entity)
+    if isinstance(held, dict):
+        return [row for row in held.values() if isinstance(row, dict)]
+    if isinstance(held, list):
+        return [row for row in held if isinstance(row, dict)]
+    return []
+
+
+def evaluate_check(check: dict, state: dict) -> bool:
+    selected = [row for row in _rows_of(state, check["entity"]) if _matches_where(row, check.get("where"))]
+    assertion = check["assert"]
+    if assertion == "exists":
+        return bool(selected)
+    if assertion == "absent":
+        return not selected
+    if assertion == "count":
+        bound = check["count"]
+        n = len(selected)
+        return all(
+            (key != "eq" or n == value) and (key != "gte" or n >= value) and (key != "lte" or n <= value)
+            for key, value in bound.items()
+        )
+    tests = (_predicate(row, check["field"], check["op"], check["value"]) for row in selected)
+    if assertion == "all":
+        return bool(selected) and all(tests)
+    return any(tests)  # any
+
+
+def evaluate_assertions(checks: list[dict], state: dict) -> dict:
+    """The grade of an assertions grader over a state dump: the weighted share
+    of checks that passed as ``reward``, each check's 0/1 under its id (or its
+    position) in ``rewards``. The runner's shape, as ``verify.py`` prints it."""
+    rewards: dict[str, float] = {}
+    total = 0.0
+    passed = 0.0
+    for i, check in enumerate(checks):
+        ok = evaluate_check(check, state)
+        rewards[check_key(check, i)] = 1.0 if ok else 0.0
+        total += check["weight"]
+        passed += check["weight"] if ok else 0.0
+    return {"reward": (passed / total) if total else None, "rewards": rewards}