agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/harness/understand.py

@@ -0,0 +1,169 @@
+"""Stage one: read an agent and produce its contract.
+
+The stage is the same whatever the agent is. What changes between a repository, a provider
+connection and a pasted definition is where the truth lives, and that comes from the source.
+
+It stays open after the first answer, because a contract is usually right on the second look and
+not the first. Correcting it is the next thing said, not a re-run.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Callable
+
+from .config import artifact_dir, load_skill, read_only_session
+from .contract import AgentContract
+from .session import Stage
+from .sources import AgentSource
+from .tools import CONTRACT_SERVER, contract_tools
+
+SKILL = "understand-agent"
+PROVIDER_IMPORT_PROFILE_PATH_ENV = "ALK_PROVIDER_IMPORT_PROFILE_PATH"
+
+
+def _provider_import_briefing() -> str:
+    """Load the sanitized external-provider definition prepared by the control process."""
+    configured = os.environ.get(PROVIDER_IMPORT_PROFILE_PATH_ENV, "").strip()
+    if not configured:
+        return ""
+    path = Path(configured)
+    try:
+        profile = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return ""
+    return (
+        "\n\n## Imported provider target (authoritative, read-only)\n\n"
+        "The submitted repository implements this target's environment-facing webhooks. "
+        "The externally hosted assistant definition below is equally authoritative for its "
+        "conversation, prompt, model, voice, and tool schemas. Reconcile both sources. Do not "
+        "invent behavior or tool inputs that conflict with the provider definition.\n\n"
+        f"```json\n{json.dumps(profile, indent=2, sort_keys=True)}\n```"
+    )
+
+
+def _eval_catalogue_briefing(available_evals: list[dict[str, Any]] | None) -> str:
+    """The eval catalogue grouped by modality, since the contract has not claimed one yet."""
+    grouped: dict[str, list[str]] = {}
+    for one in available_evals or []:
+        if not isinstance(one, dict):
+            continue
+        name = str(one.get("name") or "").strip()
+        if not name:
+            continue
+        keys = ", ".join(str(key) for key in one.get("required_keys") or [])
+        grouped.setdefault(
+            str(one.get("modality") or "any").strip().lower(), []
+        ).append(
+            "- {name}: {description}{keys}".format(
+                name=name,
+                description=str(one.get("description") or "").strip()[:240],
+                keys=f" [needs: {keys}]" if keys else "",
+            )
+        )
+    if not grouped:
+        return ""
+    sections = [
+        f"### Applies to {kind}\n\n" + "\n".join(sorted(lines))
+        for kind, lines in sorted(grouped.items())
+    ]
+    return (
+        "\n\n## Evals this platform can run on the finished calls\n\n"
+        "Record the ones this agent should be judged by in `chosen_evals`, by exact name. Each one "
+        "runs as a judge on every call of every scenario, so choose the few that tell you "
+        "something the scenarios' own deterministic checks cannot, and choose none rather than "
+        "padding.\n\n"
+        "Choose only from the section matching the `modality` you are about to record. The "
+        "platform refuses an eval belonging to another modality, so an eval for spoken calls "
+        "recorded against a chat agent costs the whole submission.\n\n"
+        "Two ways to choose wrongly, both common. An eval whose subject only exists in speech, "
+        "dead air, voicemail detection, voicemail handling, is meaningless for a chat agent and "
+        "worth having for a voice one. An eval named for a domain, misselling, advice authority, "
+        "lead qualification, claim intake, intake field accuracy, is only worth choosing when this "
+        "agent's own tools and constraints show it doing that work; choose it from the evidence in "
+        "front of you, never because its name sounds close to the agent's industry.\n\n"
+        + "\n\n".join(sections)
+    )
+
+
+def open_stage(
+    source: AgentSource,
+    *,
+    out: Path | None = None,
+    ask: Callable[..., Any] | None = None,
+    max_turns: int = 70,
+    available_evals: list[dict[str, Any]] | None = None,
+) -> tuple[Stage, Path]:
+    """A live understand-the-agent stage, and where it will write."""
+    destination = out or artifact_dir(source.name)
+    spec = read_only_session(
+        system_prompt=(
+            f"{load_skill(SKILL)}\n\n## This agent\n\n{source.briefing()}"
+            # A source-free connect-only target already carries this definition in its
+            # ProviderSource briefing.  Repository-backed provider imports still need the
+            # separately injected profile so the model can reconcile both sources of truth.
+            f"{_provider_import_briefing() if source.kind != 'provider' else ''}"
+            f"{_eval_catalogue_briefing(available_evals)}"
+        ),
+        cwd=source.workdir(),
+        servers={
+            **source.servers(),
+            CONTRACT_SERVER: contract_tools(
+                destination,
+                available_evals,
+                source_root=source.workdir() if source.kind == "repo" else None,
+            ),
+        },
+        extra_builtins=source.builtin_tools(),
+        max_turns=max_turns,
+    )
+    if ask is not None:
+        spec.permission_override = ask
+    return Stage(spec, name=SKILL), destination
+
+
+def opening(source: AgentSource) -> str:
+    # The name is only a label for the artifact folder, and saying so matters: told to "read
+    # the agent named verify_fix", a model went hunting the whole workspace for something
+    # called verify_fix instead of reading the path it was given.
+    return (
+        "Read this agent and produce its contract. Where it lives is in your briefing; "
+        f"{source.name!r} is only the label its artifacts are filed under, not something to "
+        "search for.\n\n"
+        "Work through the tools, their exact argument names and types, the constrained argument "
+        "values, the rules it enforces, and its data. Ask me if the source genuinely does not "
+        "settle something that changes what gets built. Call submit_contract when you are done."
+    )
+
+
+def load(destination: Path) -> AgentContract | None:
+    """The contract on disk, if the stage produced one."""
+    path = Path(destination) / "contract.json"
+    if not path.exists():
+        return None
+    return AgentContract.model_validate(json.loads(path.read_text(encoding="utf-8")))
+
+
+async def understand(
+    source: AgentSource,
+    *,
+    out: Path | None = None,
+    follow_ups: list[str] | None = None,
+    on_event: Callable[..., Any] | None = None,
+    ask: Callable[..., Any] | None = None,
+    max_turns: int = 70,
+) -> AgentContract | None:
+    """Run the stage start to finish and return the contract.
+
+    ``follow_ups`` are corrections applied in the same session, the scripted equivalent of an
+    operator typing them. ``ask`` handles clarifying questions; without it the model records what
+    it could not resolve in ``open_questions`` instead of blocking.
+    """
+    stage, destination = open_stage(source, out=out, ask=ask, max_turns=max_turns)
+    async with stage:
+        await stage.say(opening(source), on_event=on_event)
+        for follow_up in follow_ups or []:
+            await stage.say(follow_up, on_event=on_event)
+    return load(destination)

fi/alk/harness/voicemail_audio.py

@@ -0,0 +1,74 @@
+"""Which recorded mailbox greeting, if any, a voicemail scenario is heard through.
+
+Clips must greet without naming anybody, so one recording fits any persona. The catalogue is a
+local file and is not committed: absent, the mailbox speaks its greeting and the tone is generated.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+CATALOG = Path(__file__).with_name("data") / "voicemail" / "catalog.json"
+# A run may point somewhere else, so a deployment can serve these from object storage.
+CATALOG_ENV = "ALK_VOICEMAIL_CATALOG"
+
+
+def _entries() -> list[dict[str, Any]]:
+    path = Path(os.environ.get(CATALOG_ENV, "").strip() or CATALOG)
+    try:
+        body = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:  # noqa: BLE001 - no catalogue is the ordinary case, never an error
+        return []
+    return [one for one in body if isinstance(one, dict)] if isinstance(body, list) else []
+
+
+def _resolved(entry: dict[str, Any]) -> str:
+    """Where the audio actually is: a URL if the catalogue serves one, else a file beside us."""
+    url = str(entry.get("url") or "").strip()
+    if url:
+        return url
+    raw = str(entry.get("path") or "").strip()
+    if not raw:
+        return ""
+    here = Path(raw)
+    if here.is_file():
+        return str(here)
+    # Paths are relative to the repository root, so fall back to beside this module.
+    beside = CATALOG.parent / Path(str(entry.get("file_name") or here.name))
+    return str(beside) if beside.is_file() else ""
+
+
+def clip_for(style: str, language: str = "") -> dict[str, Any] | None:
+    """A clip whose style and language match, or None where the catalogue has nothing to offer.
+
+    Language matters as much as style: a scenario whose mailbox greets in Hindi cannot be served an
+    English recording, and no clip is the right answer there because the session speaks the greeting
+    itself in the language the scenario asked for.
+
+    Deterministic rather than random: the first matching entry wins, so two runs of the same suite
+    are heard through the same mailbox and a difference between them is never the audio.
+    """
+    wanted = str(style or "").strip().lower()
+    if not wanted:
+        return None
+    spoken = (str(language or "").strip().lower() or "en").split("-")[0]
+    for entry in _entries():
+        if str(entry.get("style") or "").strip().lower() != wanted:
+            continue
+        if str(entry.get("language") or "en").strip().lower().split("-")[0] != spoken:
+            continue
+        source = _resolved(entry)
+        if not source:
+            continue
+        return {
+            "source": source,
+            # A clip that ends with its own tone must not be given a second one.
+            "has_tone": bool(entry.get("has_tone")),
+            "id": str(entry.get("id") or ""),
+            # The greeting must reach the transcript, or the call reads as the agent talking to nobody.
+            "transcript": str(entry.get("transcript") or "").strip(),
+        }
+    return None

fi/alk/harness/world/__init__.py

@@ -0,0 +1,33 @@
+"""Generated worlds: a real data store behind an agent's tools.
+
+The pieces here are the parts that must be exact, so that what gets generated per agent stays
+small: the runtime a world executes on, the snapshot every scenario restores from, and the probe
+suite that decides whether a world is usable at all.
+"""
+
+from .kinds import WorldKind, register_kind, supported as supported_kinds
+from .probe import EDGE, HAPPY, SEQUENCE, ProbeReport, ProbeResult, dirty_state, probe
+from .runtime import Call, Db, GeneratedWorld, ToolError, WorldSpec
+from .snapshot import apply_overlay, read_manifest, restore, save
+
+__all__ = [
+    "Call",
+    "Db",
+    "EDGE",
+    "GeneratedWorld",
+    "HAPPY",
+    "ProbeReport",
+    "ProbeResult",
+    "WorldKind",
+    "dirty_state",
+    "register_kind",
+    "supported_kinds",
+    "SEQUENCE",
+    "ToolError",
+    "WorldSpec",
+    "apply_overlay",
+    "probe",
+    "read_manifest",
+    "restore",
+    "save",
+]

fi/alk/harness/world/errors.py

@@ -0,0 +1,68 @@
+"""What the hosted world handle raises when scenario code asks it for something it will not do.
+
+Every one of these is scenario code at fault, never the world's contents and never the agent
+under test: a `KeyError` from a missing table reads as a finding about data, and a bare
+`StoreError` from the postgres driver reads as an infrastructure fault. Neither is right for
+"you called `change` without saying which column `key` names," so the handle has its own
+vocabulary, and folds it under one base class for whoever has to route "scenario code misused
+the handle" to one outcome without naming all six.
+"""
+
+from __future__ import annotations
+
+
+class WorldError(RuntimeError):
+    """Scenario code asked the world handle for something it will not do."""
+
+
+class WorldReadOnly(WorldError):
+    """`put`, `change`, `drop` or `call` reached the handle `ready()` or `check()` were given.
+
+    Those two only ever observe a run. A check that could write would be able to change the very
+    thing it is grading, and nothing downstream could tell the difference between a check that
+    found a problem and one that quietly fixed it.
+    """
+
+
+class WorldReservedName(WorldError):
+    """Scenario code named the harness's own conformance canary.
+
+    That table exists to prove worlds are really isolated from each other, not to hold scenario
+    data, and it never appears in `state()` either.
+    """
+
+
+class WorldQueryRejected(WorldError):
+    """`query()` was handed something that is not one plain read.
+
+    The database's own read-only transaction is what actually stops a write; this is the
+    friendlier rejection in front of it, so a statement that was never going to be allowed fails
+    on a message naming the reason rather than a lock error three layers down.
+    """
+
+
+class WorldStateTooLarge(WorldError):
+    """`state()` reached a table whose row count, measured when the baseline was frozen, passed
+    the cap.
+
+    Measured once, at freeze time, by the provisioner — never recomputed here, so which tables
+    raise is fixed before a scenario ever runs and nothing a call does during one can move it.
+    """
+
+
+class WorldUnavailable(WorldError):
+    """The handle cannot do this, given how the world in front of it is built — not what it holds.
+
+    A postgres world whose `public` schema has no tables, and `call()` — which raises
+    unconditionally until the `http_tool` shim's wire format is pinned somewhere in the contracts
+    — are both this: nothing went wrong, the capability was never there.
+    """
+
+
+class WorldUsageError(WorldError):
+    """A `put`, `change` or `drop` cannot be carried out as asked.
+
+    Inserting into something that is not a table, or changing or dropping a record without
+    saying which column `key` names — hosted worlds cannot invent tables or guess a column, so
+    both are reported here rather than attempted.
+    """

fi/alk/harness/world/expectations.py

@@ -0,0 +1,91 @@
+"""What a world is expected to look like afterwards, and whether it does.
+
+Written once and used twice. The build stage declares a sequence and asserts the state it leaves
+behind; a scenario declares the state a conversation should leave behind. Those are the same
+question asked at two different scales, and if each had its own implementation they would drift
+until a check that passes the gate fails the run for reasons that have nothing to do with the
+agent.
+
+The shape is ``{"table.count": 3, "table.column": "value"}``: how many records there are, and
+whether a particular value is among them.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+COUNT = "count"
+
+
+def check_state(
+    state: Mapping[str, list[dict[str, Any]]], expected: Mapping[str, Any]
+) -> list[str]:
+    """Every expectation that does not hold, said in terms of what was found instead."""
+    failures: list[str] = []
+    for path, want in (expected or {}).items():
+        table, _, column = str(path).partition(".")
+        if table not in state:
+            failures.append(
+                f"{path}: no {table} in this world; it has "
+                f"{', '.join(sorted(state)) or 'nothing'}"
+            )
+            continue
+        rows = state[table]
+        if column in ("", COUNT):
+            if len(rows) != want:
+                failures.append(f"{path}: {len(rows)} rows, expected {want}")
+            continue
+        if rows and column not in rows[0]:
+            failures.append(
+                f"{path}: {table} has no {column}; its columns are "
+                f"{', '.join(sorted(rows[0]))}"
+            )
+            continue
+        present = {str(row.get(column)) for row in rows}
+        # A list means every one of these has to be somewhere, which is how an expectation about
+        # a basket of several items is naturally written. Compared as a single value it could
+        # never hold, and an expectation that cannot hold grades nothing while appearing to.
+        wanted = list(want) if isinstance(want, (list, tuple)) else [want]
+        absent = [value for value in wanted if str(value) not in present]
+        if absent:
+            found = ", ".join(sorted(present)[:6]) or "nothing"
+            failures.append(
+                f"{path}: no row has {column}="
+                + " or ".join(repr(value) for value in absent)
+                + f"; found {found}"
+            )
+    return failures
+
+
+def unresolvable(
+    state: Mapping[str, list[dict[str, Any]]], expected: Mapping[str, Any]
+) -> list[str]:
+    """Expectations that name a table or column the world does not have.
+
+    Separate from whether they hold, because they are a different kind of wrong. An expectation
+    that fails is a finding about the agent; one that names a table nobody built is a finding
+    about the expectation, and letting it through means grading a run against a typo.
+    """
+    problems: list[str] = []
+    for path in expected or {}:
+        table, _, column = str(path).partition(".")
+        if table not in state:
+            # Indexing a particular row is the most common way to write an expectation this
+            # cannot carry, and saying only "no such table" sends the reader looking for a
+            # spelling mistake instead of at the shape.
+            indexed = "[" in table
+            problems.append(
+                f"{path}: no table called {table!r}"
+                + (
+                    ". Expectations are about the whole table, not one row: use "
+                    "'table.count' for how many, or 'table.column' for a value that has to "
+                    "appear in some row."
+                    if indexed
+                    else ""
+                )
+            )
+        elif (
+            column not in ("", COUNT) and state[table] and column not in state[table][0]
+        ):
+            problems.append(f"{path}: {table} has no column {column!r}")
+    return problems