agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/live/_workers/langgraph_worker.py

@@ -0,0 +1,217 @@
+"""LangGraph lane worker (3D, manifest/factory path) — untrusted subprocess
+entry (P3-D1).
+
+Imports the caller's ``module:factory``, compiles the REAL graph against a
+real checkpoint store (MemorySaver or SqliteSaver in the run tempdir), runs
+the turn script via ``invoke`` on the same thread_id, and executes the
+cross-session probe (R§1 #6): session 1 injects via the persistence channel,
+the graph object is DISCARDED and REBUILT against the same checkpointer,
+session 2 asserts firing/containment on the same thread. End-state diffs of
+the checkpoint store are emitted as ``lane/end_state_diff`` (R§1 #14).
+
+IPC: see livekit_worker.py — same one-boot-line / JSONL-events contract.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+import traceback
+from typing import Any
+
+
+def _emit(channel: str, type_: str, payload: dict[str, Any]) -> None:
+    print(
+        json.dumps(
+            {"channel": channel, "type": type_, "payload": payload},
+            ensure_ascii=False,
+            default=str,
+        ),
+        flush=True,
+    )
+
+
+def _read_boot() -> dict[str, Any]:
+    line = sys.stdin.readline()
+    if not line.strip():
+        raise RuntimeError("missing boot message on stdin")
+    boot = json.loads(line)
+    if not isinstance(boot, dict) or boot.get("type") != "boot":
+        raise RuntimeError("first stdin line must be a boot message")
+    return boot
+
+
+def _capability_hash(framework: str, version: str) -> str:
+    return hashlib.sha256(f"{framework}:{version}".encode("utf-8")).hexdigest()
+
+
+def _package_paths(module: Any) -> list[str]:
+    """Filesystem roots of a framework package for traceback attribution.
+
+    langgraph (like livekit) ships as a NAMESPACE package: ``__file__`` is
+    None and the roots live on ``__path__`` instead.
+    """
+
+    file = getattr(module, "__file__", None)
+    if file:
+        return [os.path.dirname(file)]
+    return [str(path) for path in getattr(module, "__path__", None) or []]
+
+
+def _turn_input(turn: dict[str, Any]) -> Any:
+    if "input" in turn:
+        return turn["input"]
+    return {"messages": [{"role": "user", "content": str(turn.get("user") or "")}]}
+
+
+def _last_message_text(output: Any) -> str:
+    if isinstance(output, dict):
+        messages = output.get("messages")
+        if isinstance(messages, (list, tuple)) and messages:
+            last = messages[-1]
+            content = getattr(last, "content", None)
+            if content is None and isinstance(last, dict):
+                content = last.get("content")
+            if content is not None:
+                return str(content)
+        return str(output)
+    return str(output)
+
+
+def _run(boot: dict[str, Any]) -> None:
+    import importlib.metadata
+
+    import langgraph
+
+    version = importlib.metadata.version("langgraph")
+    package_paths = _package_paths(langgraph)
+    try:
+        import langchain_core
+
+        package_paths.extend(_package_paths(langchain_core))
+    except ImportError:
+        pass
+    _emit(
+        "lane",
+        "framework_ready",
+        {
+            "framework": "langgraph",
+            "framework_version": version,
+            "capability_hash": _capability_hash("langgraph", version),
+            "package_paths": package_paths,
+            "execution_model": "subprocess",
+        },
+    )
+    config = boot.get("config") or {}
+    factory_path = str(config.get("factory") or "")
+    module_name, _, attr = factory_path.partition(":")
+    if not module_name or not attr:
+        raise RuntimeError(f"factory must be 'module:attr', got {factory_path!r}")
+    factory = getattr(importlib.import_module(module_name), attr)
+
+    checkpointer_kind = str(config.get("checkpointer") or "memory")
+    if checkpointer_kind == "sqlite":
+        import sqlite3
+
+        from langgraph.checkpoint.sqlite import SqliteSaver
+
+        connection = sqlite3.connect("checkpoints.sqlite", check_same_thread=False)
+        checkpointer = SqliteSaver(connection)
+    else:
+        from langgraph.checkpoint.memory import MemorySaver
+
+        checkpointer = MemorySaver()
+
+    def _build_graph() -> Any:
+        try:
+            candidate = factory(checkpointer=checkpointer)
+        except TypeError:
+            candidate = factory()
+        if hasattr(candidate, "compile"):
+            candidate = candidate.compile(checkpointer=checkpointer)
+        return candidate
+
+    thread_id = str(config.get("thread_id") or "live-thread")
+    invoke_config = {"configurable": {"thread_id": thread_id}}
+
+    def _checkpoint_count() -> int | None:
+        try:
+            return sum(1 for _ in checkpointer.list(invoke_config))
+        except Exception:
+            return None
+
+    graph = _build_graph()
+    checkpoints_before = _checkpoint_count()
+    checks: list[bool] = []
+    turns = boot.get("turns") or []
+    for index, turn in enumerate(turns):
+        turn = turn or {}
+        text = str(turn.get("user") or "")
+        _emit("user", "message", {"turn": index, "text": text, "session": 1})
+        output = graph.invoke(_turn_input(turn), config=invoke_config)
+        reply = _last_message_text(output)
+        _emit("agent", "message", {"turn": index, "text": reply, "session": 1})
+        expect = turn.get("expect")
+        ok = bool(reply.strip())
+        if isinstance(expect, dict) and isinstance(expect.get("contains"), str):
+            ok = ok and expect["contains"].lower() in reply.lower()
+        checks.append(ok)
+
+    probe = config.get("probe")
+    if config.get("cross_session_probe") and isinstance(probe, dict):
+        inject = str(probe.get("inject") or "")
+        question = str(probe.get("question") or "What do you remember?")
+        if inject:
+            _emit("user", "message", {"session": 1, "text": inject, "probe": True})
+            graph.invoke(_turn_input({"user": inject}), config=invoke_config)
+        # Discard and REBUILD against the same checkpointer — the process
+        # crosses a real persistence boundary, not an in-memory alias.
+        del graph
+        graph = _build_graph()
+        _emit("user", "message", {"session": 2, "text": question, "probe": True})
+        output = graph.invoke(_turn_input({"user": question}), config=invoke_config)
+        reply = _last_message_text(output)
+        _emit("agent", "message", {"session": 2, "text": reply, "probe": True})
+        fired = True
+        if isinstance(probe.get("assert_contains"), str):
+            fired = probe["assert_contains"].lower() in reply.lower()
+        contained = True
+        if isinstance(probe.get("assert_not_contains"), str):
+            contained = probe["assert_not_contains"].lower() not in reply.lower()
+        _emit(
+            "lane",
+            "cross_session_probe",
+            {"probe_mode": "rebuilt", "fired": fired, "contained": contained},
+        )
+        checks.append(fired and contained)
+
+    checkpoints_after = _checkpoint_count()
+    _emit(
+        "lane",
+        "end_state_diff",
+        {
+            "checkpoint_store": checkpointer_kind,
+            "checkpoints_before": checkpoints_before,
+            "checkpoints_after": checkpoints_after,
+            "thread_id": thread_id,
+        },
+    )
+    passed = bool(checks) and all(checks)
+    _emit("lane", "verification", {"passed": passed, "checks": checks})
+
+
+def main() -> int:
+    boot = _read_boot()
+    try:
+        _run(boot)
+    except Exception:
+        _emit("lane", "worker_error", {"traceback": traceback.format_exc()})
+        traceback.print_exc(file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

fi/alk/live/_workers/livekit_worker.py

@@ -0,0 +1,207 @@
+"""LiveKit lane worker (3B rung 1) — untrusted subprocess entry (P3-D1).
+
+Boots a REAL ``livekit.agents.AgentSession`` and drives it with the
+first-party text-rung helper ``session.run(user_input=...)`` (LiveKit's own
+pytest surface) under a virtual-clock turn script with a deterministic
+scripted LLM (no transport, no credentials — P3-D3 rung 1).
+
+IPC: reads ONE boot JSON line on stdin; emits one JSON object per line on
+stdout: ``{"channel": "user"|"agent"|"tool"|"lane", "type": ..., "payload": ...}``.
+The handshake event is ``lane/framework_ready`` carrying
+``{framework, framework_version, capability_hash, package_paths}``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import sys
+import traceback
+from typing import Any
+
+
+def _emit(channel: str, type_: str, payload: dict[str, Any]) -> None:
+    print(
+        json.dumps(
+            {"channel": channel, "type": type_, "payload": payload},
+            ensure_ascii=False,
+            default=str,
+        ),
+        flush=True,
+    )
+
+
+def _read_boot() -> dict[str, Any]:
+    line = sys.stdin.readline()
+    if not line.strip():
+        raise RuntimeError("missing boot message on stdin")
+    boot = json.loads(line)
+    if not isinstance(boot, dict) or boot.get("type") != "boot":
+        raise RuntimeError("first stdin line must be a boot message")
+    return boot
+
+
+def _capability_hash(framework: str, version: str) -> str:
+    return hashlib.sha256(f"{framework}:{version}".encode("utf-8")).hexdigest()
+
+
+def _extract_reply(result: Any) -> str:
+    """Pull the last assistant text out of a session.run RunResult,
+    defensively across livekit-agents 1.x minor versions."""
+
+    texts: list[str] = []
+    for event in getattr(result, "events", None) or []:
+        item = getattr(event, "item", None)
+        if getattr(item, "role", None) != "assistant":
+            continue
+        text_content = getattr(item, "text_content", None)
+        if text_content:
+            texts.append(str(text_content))
+            continue
+        content = getattr(item, "content", None)
+        if isinstance(content, str):
+            texts.append(content)
+        elif isinstance(content, (list, tuple)):
+            texts.append(
+                " ".join(str(part) for part in content if isinstance(part, str))
+            )
+    return texts[-1] if texts else ""
+
+
+async def _run(boot: dict[str, Any]) -> None:
+    import importlib.metadata
+
+    import livekit
+    from livekit.agents import Agent, AgentSession
+    from livekit.agents import llm as lk_llm
+
+    version = importlib.metadata.version("livekit-agents")
+    _emit(
+        "lane",
+        "framework_ready",
+        {
+            "framework": "livekit-agents",
+            "framework_version": version,
+            "capability_hash": _capability_hash("livekit-agents", version),
+            # livekit is a NAMESPACE package: __file__ is None, roots are on
+            # __path__ (same fix as langgraph_worker._package_paths).
+            "package_paths": (
+                [os.path.dirname(livekit.__file__)]
+                if getattr(livekit, "__file__", None)
+                else [str(path) for path in getattr(livekit, "__path__", None) or []]
+            ),
+        },
+    )
+    rung = int(boot.get("rung") or 1)
+    if rung != 1:
+        raise RuntimeError(f"livekit worker implements rung 1 only, got {rung}")
+    config = boot.get("config") or {}
+    responses = [str(r) for r in (config.get("responses") or [])]
+    instructions = str(
+        config.get("instructions")
+        or "You are a concise, helpful voice agent under test."
+    )
+    expect = config.get("expect") if isinstance(config.get("expect"), dict) else {}
+    turns = boot.get("turns") or []
+
+    def _make_chunk(text: str) -> Any:
+        try:
+            return lk_llm.ChatChunk(
+                id="scripted",
+                delta=lk_llm.ChoiceDelta(role="assistant", content=text),
+            )
+        except TypeError:
+            return lk_llm.ChatChunk(
+                request_id="scripted",
+                choices=[
+                    lk_llm.Choice(
+                        delta=lk_llm.ChoiceDelta(role="assistant", content=text),
+                        index=0,
+                    )
+                ],
+            )
+
+    class _ScriptedStream(lk_llm.LLMStream):
+        def __init__(self, llm_obj: Any, *, chat_ctx: Any, tools: Any, conn_options: Any, text: str) -> None:
+            super().__init__(
+                llm_obj, chat_ctx=chat_ctx, tools=tools, conn_options=conn_options
+            )
+            self._text = text
+
+        async def _run(self) -> None:
+            self._event_ch.send_nowait(_make_chunk(self._text))
+
+    default_conn_options = getattr(lk_llm, "DEFAULT_API_CONNECT_OPTIONS", None)
+    if default_conn_options is None:
+        try:
+            from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS as default_conn_options
+        except ImportError:
+            default_conn_options = None
+
+    class _ScriptedLLM(lk_llm.LLM):
+        """Deterministic stub LLM node — rung 1 is credential-free (P3-D3)."""
+
+        def __init__(self) -> None:
+            super().__init__()
+            self._index = 0
+
+        @property
+        def model(self) -> str:
+            return "scripted-stub"
+
+        def chat(self, *, chat_ctx: Any, tools: Any = None, conn_options: Any = None, **kwargs: Any) -> Any:
+            if responses:
+                text = responses[self._index % len(responses)]
+            else:
+                text = "Acknowledged."
+            self._index += 1
+            return _ScriptedStream(
+                self,
+                chat_ctx=chat_ctx,
+                tools=tools or [],
+                conn_options=conn_options or default_conn_options,
+                text=text,
+            )
+
+    session = AgentSession(llm=_ScriptedLLM())
+    await session.start(Agent(instructions=instructions))
+    checks: list[bool] = []
+    try:
+        for index, turn in enumerate(turns):
+            text = str((turn or {}).get("user") or "")
+            _emit("user", "message", {"turn": index, "text": text})
+            result = await session.run(user_input=text)
+            reply = _extract_reply(result)
+            _emit("agent", "message", {"turn": index, "text": reply})
+            ok = bool(reply.strip())
+            contains = (turn or {}).get("expect", {}).get("contains") if isinstance((turn or {}).get("expect"), dict) else None
+            contains = contains or expect.get("contains")
+            if isinstance(contains, str):
+                ok = ok and contains.lower() in reply.lower()
+            checks.append(ok)
+    finally:
+        aclose = getattr(session, "aclose", None)
+        if aclose is not None:
+            try:
+                await aclose()
+            except Exception:
+                pass
+    passed = bool(checks) and all(checks)
+    _emit("lane", "verification", {"passed": passed, "checks": checks})
+
+
+def main() -> int:
+    boot = _read_boot()
+    try:
+        asyncio.run(_run(boot))
+    except Exception:
+        _emit("lane", "worker_error", {"traceback": traceback.format_exc()})
+        traceback.print_exc(file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

fi/alk/live/_workers/mcp_loopback_server.py

@@ -0,0 +1,46 @@
+"""Loopback MCP stdio server (P3-D6) — the shipped credential-free fixture.
+
+A REAL ``FastMCP`` server process with deterministic tools: credential-free
+but a genuinely separate process speaking the real protocol over the wire —
+that IS the live graduation (R§1 #12). Spawned by ``mcp_worker.py`` via the
+MCP SDK's own stdio transport; its stdio IS the MCP wire (it does not speak
+the lane JSONL protocol).
+"""
+
+from __future__ import annotations
+
+import sys
+
+SERVER_NAME = "agent-learning-loopback"
+SERVER_VERSION = "1.0.0"
+
+
+def main() -> int:
+    from mcp.server.fastmcp import FastMCP
+
+    server = FastMCP(SERVER_NAME)
+
+    @server.tool()
+    def echo(text: str) -> str:
+        """Echo the input text back verbatim."""
+
+        return text
+
+    @server.tool()
+    def add(a: float, b: float) -> float:
+        """Add two numbers deterministically."""
+
+        return a + b
+
+    @server.tool()
+    def sort_unique(items: list[str]) -> list[str]:
+        """Return the sorted, de-duplicated items (deterministic)."""
+
+        return sorted(set(items))
+
+    server.run("stdio")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

fi/alk/live/_workers/mcp_worker.py

@@ -0,0 +1,158 @@
+"""MCP lane worker (3E) — untrusted subprocess entry (P3-D1).
+
+The client side of the MCP lane: a REAL ``ClientSession`` over the SDK's
+stdio transport. It launches the target server (default: the shipped
+loopback fixture ``mcp_loopback_server.py``) as ITS OWN subprocess via
+``StdioServerParameters`` — the real protocol over the wire between two
+separate processes — lists tools (capability hash = sha256 of the sorted
+``list_tools()`` JSON, R§1 #11), runs the scenario's tool-call script with
+claim-level expectations, and emits the server-behavior snapshot stamp.
+
+IPC with the harness: see livekit_worker.py — same one-boot-line / JSONL
+contract on THIS process's stdio (the MCP wire is the server subprocess's
+stdio, owned by the SDK).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import sys
+import traceback
+from typing import Any
+
+
+def _emit(channel: str, type_: str, payload: dict[str, Any]) -> None:
+    print(
+        json.dumps(
+            {"channel": channel, "type": type_, "payload": payload},
+            ensure_ascii=False,
+            default=str,
+        ),
+        flush=True,
+    )
+
+
+def _read_boot() -> dict[str, Any]:
+    line = sys.stdin.readline()
+    if not line.strip():
+        raise RuntimeError("missing boot message on stdin")
+    boot = json.loads(line)
+    if not isinstance(boot, dict) or boot.get("type") != "boot":
+        raise RuntimeError("first stdin line must be a boot message")
+    return boot
+
+
+def _result_text(result: Any) -> str:
+    texts: list[str] = []
+    for block in getattr(result, "content", None) or []:
+        text = getattr(block, "text", None)
+        if text:
+            texts.append(str(text))
+    return "\n".join(texts)
+
+
+async def _run(boot: dict[str, Any]) -> None:
+    import importlib.metadata
+
+    import mcp as mcp_pkg
+    from mcp import ClientSession, StdioServerParameters
+    from mcp.client.stdio import stdio_client
+
+    version = importlib.metadata.version("mcp")
+    config = boot.get("config") or {}
+    command = [str(part) for part in (config.get("server_command") or [])]
+    if not command:
+        command = [
+            sys.executable,
+            os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp_loopback_server.py"),
+        ]
+    env_names = [str(name) for name in (config.get("server_env_names") or [])]
+    server_env = {name: os.environ[name] for name in env_names if name in os.environ}
+    # The server inherits the kit path so the loopback fixture imports clean.
+    if "PYTHONPATH" in os.environ:
+        server_env.setdefault("PYTHONPATH", os.environ["PYTHONPATH"])
+    if "PATH" in os.environ:
+        server_env.setdefault("PATH", os.environ["PATH"])
+
+    params = StdioServerParameters(
+        command=command[0], args=command[1:], env=server_env or None
+    )
+    async with stdio_client(params) as (read_stream, write_stream):
+        async with ClientSession(read_stream, write_stream) as session:
+            init = await session.initialize()
+            tools_result = await session.list_tools()
+            tool_summary = sorted(
+                (
+                    {
+                        "name": tool.name,
+                        "description": tool.description or "",
+                    }
+                    for tool in tools_result.tools
+                ),
+                key=lambda item: item["name"],
+            )
+            capability_hash = hashlib.sha256(
+                json.dumps(tool_summary, sort_keys=True).encode("utf-8")
+            ).hexdigest()
+            _emit(
+                "lane",
+                "framework_ready",
+                {
+                    "framework": "mcp",
+                    "framework_version": version,
+                    "capability_hash": capability_hash,
+                    "package_paths": [os.path.dirname(mcp_pkg.__file__)],
+                },
+            )
+            server_info = getattr(init, "serverInfo", None)
+            _emit(
+                "lane",
+                "server_snapshot",
+                {
+                    "server_name": getattr(server_info, "name", None),
+                    "server_version": getattr(server_info, "version", None),
+                    "capability_hash": capability_hash,
+                },
+            )
+            checks: list[bool] = []
+            for call in config.get("calls") or []:
+                call = call or {}
+                name = str(call.get("tool") or "")
+                arguments = call.get("arguments") or {}
+                _emit("tool", "tool_call", {"name": name, "arguments": arguments})
+                result = await session.call_tool(name, arguments)
+                text = _result_text(result)
+                ok = not bool(getattr(result, "isError", False))
+                # Claim-level rubric, tolerant of alternative trajectories
+                # (R§1 #11): the claim is about the answer, not the path.
+                expect = call.get("expect")
+                if isinstance(expect, dict) and isinstance(
+                    expect.get("contains"), str
+                ):
+                    ok = ok and expect["contains"].lower() in text.lower()
+                _emit(
+                    "tool",
+                    "tool_result",
+                    {"name": name, "ok": ok, "text": text[:2000]},
+                )
+                checks.append(ok)
+            passed = bool(checks) and all(checks)
+            _emit("lane", "verification", {"passed": passed, "checks": checks})
+
+
+def main() -> int:
+    boot = _read_boot()
+    try:
+        asyncio.run(_run(boot))
+    except Exception:
+        _emit("lane", "worker_error", {"traceback": traceback.format_exc()})
+        traceback.print_exc(file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())