korrel 0.1.0__py3-none-any.whl

korrel/__init__.py

@@ -0,0 +1,42 @@
+"""korrel: an OSS Python SDK for agent simulation.
+
+Define a multi-turn agent test once as a code-first Scenario, run it against a
+bring-your-own agent adapter, and score the transcript with a Rubric. Keys are
+read from the environment at call time and stored nowhere.
+"""
+
+from .adapter import AgentAdapter, adapter_from_provider
+from .persona import Persona
+from .rubric import Rubric, RewardFn, make_judge
+from .runtime import (
+    FailureCluster,
+    RunResult,
+    Transcript,
+    Turn,
+    run_scenario,
+)
+from .scenario import Scenario
+from .tools import MockTool
+from .types import Message, ToolCall, ToolFunction, ToolSchema
+
+__all__ = [
+    "Scenario",
+    "Persona",
+    "MockTool",
+    "Rubric",
+    "RewardFn",
+    "make_judge",
+    "run_scenario",
+    "RunResult",
+    "Transcript",
+    "Turn",
+    "FailureCluster",
+    "AgentAdapter",
+    "adapter_from_provider",
+    "Message",
+    "ToolCall",
+    "ToolFunction",
+    "ToolSchema",
+]
+
+__version__ = "0.1.0"

korrel/adapter.py

@@ -0,0 +1,38 @@
+"""AgentAdapter: the user-supplied agent under test.
+
+An adapter is any callable that takes the conversation so far as canonical
+``Message`` objects plus the available tool schemas, and returns the next
+assistant ``Message`` (which may carry ``tool_calls``). The user wires their
+own agent and their own keys here; Korrel holds none of them.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+from .types import Message, ToolSchema
+
+if TYPE_CHECKING:
+    from .providers import Provider
+
+
+@runtime_checkable
+class AgentAdapter(Protocol):
+    def __call__(
+        self, messages: list[Message], tools: list[ToolSchema]
+    ) -> Message:
+        ...
+
+
+def adapter_from_provider(provider: "Provider") -> AgentAdapter:
+    """Wrap a Provider as an AgentAdapter.
+
+    Returns a callable that delegates to ``provider.complete(messages,
+    tools=tools)``. The caller supplies their own provider and their own
+    keys; Korrel holds none.
+    """
+
+    def _adapter(messages: list[Message], tools: list[ToolSchema]) -> Message:
+        return provider.complete(messages, tools=tools)
+
+    return _adapter  # type: ignore[return-value]

korrel/cli.py

@@ -0,0 +1,375 @@
+"""korrel CLI entry point.
+
+Subcommands:
+  run <scenario.py>  Run a scenario module and print the result.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import sys
+import time
+import types
+from pathlib import Path
+from typing import Optional
+
+from .adapter import AgentAdapter
+from .runtime import RunResult, run_scenario
+from .scenario import Scenario
+
+
+# ---------------------------------------------------------------------------
+# Shared transcript writer (used by CLI and pytest plugin)
+# ---------------------------------------------------------------------------
+
+
+def _safe_filename_stem(scenario_id: str) -> str:
+    """Reduce a scenario id to a single safe path component.
+
+    A scenario id is author-controlled. Used raw as a filename it could carry
+    path separators or an absolute path and escape ``out_dir`` (security review
+    K9). ``Path(...).name`` strips any directory part on both POSIX and Windows;
+    the fallback covers ids that reduce to nothing usable (``""``, ``.``, ``..``).
+    """
+    stem = Path(scenario_id).name
+    if stem in ("", ".", ".."):
+        return "scenario"
+    return stem
+
+
+def write_transcript_for(result: RunResult, scenario_id: str, out_dir: Path) -> Path:
+    """Write ``result.transcript`` as JSON and return the file path.
+
+    The file is ``<out_dir>/<scenario_id>.transcript.json``. ``scenario_id`` is
+    reduced to a single safe path component first. ``out_dir`` is created if it
+    does not exist.
+    """
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / f"{_safe_filename_stem(scenario_id)}.transcript.json"
+    path.write_text(result.transcript.model_dump_json(), encoding="utf-8")
+    return path
+
+
+# ---------------------------------------------------------------------------
+# Module loader (shared with pytest plugin)
+# ---------------------------------------------------------------------------
+
+
+def load_module_from_path(file_path: Path) -> types.ModuleType:
+    """Import a Python source file by path and return the module object."""
+    spec = importlib.util.spec_from_file_location(file_path.stem, file_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load module from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)  # type: ignore[union-attr]
+    return module
+
+
+def load_scenario_and_adapter(
+    module: types.ModuleType,
+    scenario_attr: str,
+    adapter_attr: str,
+) -> tuple[Scenario, AgentAdapter]:
+    """Extract a Scenario and AgentAdapter from a loaded module."""
+    scenario = getattr(module, scenario_attr, None)
+    if scenario is None:
+        raise AttributeError(
+            f"Module {module.__name__!r} has no attribute {scenario_attr!r}. "
+            "Define a module-level `scenario = Scenario(...)` in the file."
+        )
+    if not isinstance(scenario, Scenario):
+        raise TypeError(
+            f"{scenario_attr!r} in {module.__name__!r} is not a Scenario "
+            f"(got {type(scenario).__name__})."
+        )
+
+    adapter = getattr(module, adapter_attr, None)
+    if adapter is None:
+        raise AttributeError(
+            f"Module {module.__name__!r} has no attribute {adapter_attr!r}. "
+            "Define a module-level `adapter = adapter_from_provider(...)` or "
+            "a custom callable in the file."
+        )
+    if not callable(adapter):
+        raise TypeError(
+            f"{adapter_attr!r} in {module.__name__!r} is not callable "
+            f"(got {type(adapter).__name__})."
+        )
+
+    return scenario, adapter  # type: ignore[return-value]
+
+
+def load_scenario_only(
+    module: types.ModuleType,
+    scenario_attr: str,
+) -> Scenario:
+    """Extract a Scenario from a loaded module (no adapter required)."""
+    scenario = getattr(module, scenario_attr, None)
+    if scenario is None:
+        raise AttributeError(
+            f"Module {module.__name__!r} has no attribute {scenario_attr!r}. "
+            "Define a module-level `scenario = Scenario(...)` in the file."
+        )
+    if not isinstance(scenario, Scenario):
+        raise TypeError(
+            f"{scenario_attr!r} in {module.__name__!r} is not a Scenario "
+            f"(got {type(scenario).__name__})."
+        )
+    return scenario
+
+
+# ---------------------------------------------------------------------------
+# `korrel export` command
+# ---------------------------------------------------------------------------
+
+_SUPPORTED_TARGETS = ("verifiers", "openenv")
+
+
+def _cmd_export(args: argparse.Namespace) -> int:
+    target: str = args.to
+    if target not in _SUPPORTED_TARGETS:
+        print(
+            f"error: unsupported export target {target!r}. "
+            f"Supported: {', '.join(_SUPPORTED_TARGETS)}.",
+            file=sys.stderr,
+        )
+        return 1
+
+    file_path = Path(args.file).resolve()
+    if not file_path.exists():
+        print(f"error: file not found: {file_path}", file=sys.stderr)
+        return 1
+
+    try:
+        module = load_module_from_path(file_path)
+    except Exception as exc:
+        print(f"error: could not load {file_path}: {exc}", file=sys.stderr)
+        return 1
+
+    if not args.scenario_attr.isidentifier():
+        print(
+            f"error: --scenario-attr must be a valid Python identifier, "
+            f"got {args.scenario_attr!r}.",
+            file=sys.stderr,
+        )
+        return 1
+
+    try:
+        scenario = load_scenario_only(module, scenario_attr=args.scenario_attr)
+    except (AttributeError, TypeError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    # scenario.id is author-controlled; reduce it to a single safe path component
+    # before using it as the default output directory name (security review K9:
+    # an id carrying separators or an absolute path could otherwise escape
+    # .korrel/export/). When --out is given the operator owns that path.
+    out_dir = (
+        Path(args.out)
+        if args.out
+        else Path(".korrel") / "export" / _safe_filename_stem(scenario.id)
+    )
+
+    if target == "verifiers":
+        # Import the artifact emitter (does not import verifiers itself).
+        from .exporters.verifiers import write_verifiers_env
+
+        produced = write_verifiers_env(
+            scenario,
+            out_dir,
+            scenario_source_path=file_path,
+            scenario_attr=args.scenario_attr,
+        )
+        pyproject = produced / "pyproject.toml"
+        env_module = _safe_filename_stem(scenario.id).replace("-", "_").replace(" ", "_")
+        env_module_py = produced / f"{env_module}.py"
+        scenario_py = produced / "_scenario.py"
+        print(f"{'target':<12}: verifiers")
+        print(f"{'scenario':<12}: {scenario.id}")
+        print(f"{'out_dir':<12}: {produced}")
+        print(f"{'pyproject':<12}: {pyproject}")
+        print(f"{'env_module':<12}: {env_module_py}")
+        print(f"{'scenario_src':<12}: {scenario_py}")
+
+    elif target == "openenv":
+        # Import the artifact emitter (does not import openenv-core itself).
+        from .exporters.openenv import write_openenv_env
+
+        produced = write_openenv_env(
+            scenario,
+            out_dir,
+            scenario_source_path=file_path,
+            scenario_attr=args.scenario_attr,
+        )
+        env_name = _safe_filename_stem(scenario.id).replace("_", "-")
+        env_module = env_name.replace("-", "_")
+        pyproject = produced / "pyproject.toml"
+        models_py = produced / "models.py"
+        env_py = produced / "server" / f"{env_module}_environment.py"
+        scenario_py = produced / "_scenario.py"
+        print(f"{'target':<12}: openenv")
+        print(f"{'scenario':<12}: {scenario.id}")
+        print(f"{'out_dir':<12}: {produced}")
+        print(f"{'pyproject':<12}: {pyproject}")
+        print(f"{'models':<12}: {models_py}")
+        print(f"{'environment':<12}: {env_py}")
+        print(f"{'scenario_src':<12}: {scenario_py}")
+
+    return 0
+
+
+# ---------------------------------------------------------------------------
+# `korrel run` command
+# ---------------------------------------------------------------------------
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    file_path = Path(args.file).resolve()
+    if not file_path.exists():
+        print(f"error: file not found: {file_path}", file=sys.stderr)
+        return 1
+
+    try:
+        module = load_module_from_path(file_path)
+    except Exception as exc:
+        print(f"error: could not load {file_path}: {exc}", file=sys.stderr)
+        return 1
+
+    try:
+        scenario, adapter = load_scenario_and_adapter(
+            module,
+            scenario_attr=args.scenario_attr,
+            adapter_attr=args.adapter_attr,
+        )
+    except (AttributeError, TypeError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    seed: Optional[int] = args.seed
+    t_start = time.monotonic()
+    result = run_scenario(scenario, adapter, seed=seed)
+    duration_s = time.monotonic() - t_start
+
+    out_dir = Path(args.out) if args.out else Path(".korrel")
+    transcript_path = write_transcript_for(result, scenario.id, out_dir)
+
+    # Print result summary. The "model calls" label is wider than the prior
+    # 10-char pad, so the pad is widened to 12 across every line to stay aligned.
+    status = "pass" if result.passed else "fail"
+    print(f"{'scenario':<12}: {scenario.id}")
+    print(f"{'score':<12}: {result.score:.4f}")
+    print(f"{'status':<12}: {status}")
+    print(f"{'model calls':<12}: {result.model_calls}")
+    if result.failed_functions:
+        print(f"{'failed':<12}: {', '.join(result.failed_functions)}")
+    if result.clusters:
+        cluster_strs = [f"{c.function}({c.signature})" for c in result.clusters]
+        print(f"{'clusters':<12}: {', '.join(cluster_strs)}")
+    print(f"{'transcript':<12}: {transcript_path}")
+
+    # Emit telemetry. Best-effort: errors are swallowed inside emit_run.
+    from .telemetry import emit_run
+
+    total_turns = len(result.transcript.turns)
+    emit_run(
+        scenario_count=1,
+        total_turns=total_turns,
+        pass_count=1 if result.passed else 0,
+        fail_count=0 if result.passed else 1,
+        duration_s=duration_s,
+    )
+
+    return 0 if result.passed else 1
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="korrel",
+        description="korrel: agent simulation SDK.",
+    )
+    sub = parser.add_subparsers(dest="command", metavar="COMMAND")
+
+    run_parser = sub.add_parser("run", help="Run a scenario module.")
+    run_parser.add_argument("file", metavar="SCENARIO_PY", help="Path to the scenario module.")
+    run_parser.add_argument(
+        "--out",
+        metavar="DIR",
+        default=None,
+        help="Directory for transcript output (default: .korrel/).",
+    )
+    run_parser.add_argument(
+        "--seed",
+        metavar="N",
+        type=int,
+        default=None,
+        help="Override the scenario seed.",
+    )
+    run_parser.add_argument(
+        "--scenario-attr",
+        metavar="NAME",
+        default="scenario",
+        help="Module attribute name for the Scenario (default: scenario).",
+    )
+    run_parser.add_argument(
+        "--adapter-attr",
+        metavar="NAME",
+        default="adapter",
+        help="Module attribute name for the AgentAdapter (default: adapter).",
+    )
+
+    export_parser = sub.add_parser(
+        "export",
+        help="Export a scenario as an RL training environment.",
+    )
+    export_parser.add_argument(
+        "file",
+        metavar="SCENARIO_PY",
+        help="Path to the scenario module.",
+    )
+    export_parser.add_argument(
+        "--to",
+        metavar="TARGET",
+        required=True,
+        help=f"Export target. Supported: {', '.join(_SUPPORTED_TARGETS)}.",
+    )
+    export_parser.add_argument(
+        "--out",
+        metavar="DIR",
+        default=None,
+        help="Output directory for the generated package (default: .korrel/export/<scenario-id>/).",
+    )
+    export_parser.add_argument(
+        "--scenario-attr",
+        metavar="NAME",
+        default="scenario",
+        help="Module attribute name for the Scenario (default: scenario).",
+    )
+
+    return parser
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(0)
+
+    if args.command == "run":
+        sys.exit(_cmd_run(args))
+    elif args.command == "export":
+        sys.exit(_cmd_export(args))
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

korrel/exporters/__init__.py

@@ -0,0 +1,14 @@
+"""korrel.exporters: optional export targets for Korrel scenarios.
+
+Each submodule implements an exporter for a specific RL training framework.
+None of them import the target framework at module load; imports are deferred
+to function call time so that ``import korrel`` works without the target
+framework installed.
+
+Available submodules:
+
+- ``korrel.exporters.verifiers``: export a Scenario as a verifiers
+  Environment (requires the ``verifiers`` optional extra, Python <3.14).
+- ``korrel.exporters.openenv``: export a Scenario as an OpenEnv environment
+  package (requires the ``openenv`` optional extra; openenv-core>=0.3.0).
+"""

korrel/exporters/_shared.py

@@ -0,0 +1,139 @@
+"""korrel.exporters._shared: conversion helpers shared across exporters.
+
+These helpers convert between the canonical Korrel message types and the flat
+message shapes used by downstream frameworks (verifiers, OpenEnv). None of them
+import a framework at definition time.
+
+Confirmed against: korrel canonical types as of v0.2 (src/korrel/types.py).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Optional
+
+
+def _field(obj: Any, key: str, default: Any = None) -> Any:
+    """Read a field from a message or tool call object.
+
+    The value may arrive as a pydantic object attribute or a plain dict entry.
+    Unlike ``getattr(...) or dict.get(...)``, this does not treat a falsy-but-
+    present value (an empty string content) as missing: only ``None`` or a
+    truly absent key falls back to ``default``.
+    """
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    value = getattr(obj, key, default)
+    return default if value is None else value
+
+
+def _content_to_str(content: Any) -> str:
+    """Narrow MessageContent (str | list[ContentPart]) to str.
+
+    Content may be a list of content parts (text, image, audio). Korrel
+    canonical content is Optional[str]. String content passes through; list
+    content is serialized as JSON per the spec (lossy edge: content-shape
+    narrowing documented in the mapping spec).
+    """
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    # List of content parts: join text parts, serialize non-text as JSON.
+    parts: list[str] = []
+    for part in content:
+        if isinstance(part, dict):
+            if part.get("type") == "text":
+                parts.append(part.get("text", ""))
+            else:
+                parts.append(json.dumps(part))
+        elif hasattr(part, "type"):
+            if getattr(part, "type", None) == "text":
+                parts.append(getattr(part, "text", ""))
+            else:
+                parts.append(
+                    json.dumps(
+                        part.model_dump() if hasattr(part, "model_dump") else str(part)
+                    )
+                )
+        else:
+            parts.append(str(part))
+    return "".join(parts)
+
+
+def _to_korrel_messages(messages: list[Any]) -> list[Any]:
+    """Convert a list of flat messages to korrel canonical Messages.
+
+    Field-by-field per the mapping spec (Section: Reward-function value shim):
+    - system/user/assistant without tool calls: content narrowed to str.
+    - assistant with tool calls: FLAT ToolCall{id,name,arguments} ->
+      NESTED korrel.ToolCall{id,type:"function",function:{name,arguments}}.
+    - tool result: ToolMessage{role:"tool",tool_call_id,content}.
+    ``arguments`` stays a JSON string on both sides.
+    """
+    from ..types import Message, ToolCall, ToolFunction
+
+    result: list[Message] = []
+    for msg in messages:
+        role = _field(msg, "role")
+        if role is None:
+            continue
+
+        if role == "system":
+            result.append(
+                Message(role="system", content=_content_to_str(_field(msg, "content")))
+            )
+
+        elif role == "user":
+            result.append(
+                Message(role="user", content=_content_to_str(_field(msg, "content")))
+            )
+
+        elif role == "assistant":
+            content = _field(msg, "content")
+            raw_tool_calls = _field(msg, "tool_calls")
+            korrel_tool_calls: Optional[list[ToolCall]] = None
+            if raw_tool_calls:
+                korrel_tool_calls = []
+                for tc in raw_tool_calls:
+                    tc_id = _field(tc, "id", "") or ""
+                    tc_name = _field(tc, "name", "") or ""
+                    tc_args = _field(tc, "arguments", "{}") or "{}"
+                    korrel_tool_calls.append(
+                        ToolCall(
+                            id=tc_id,
+                            type="function",
+                            function=ToolFunction(name=tc_name, arguments=tc_args),
+                        )
+                    )
+            result.append(
+                Message(
+                    role="assistant",
+                    content=_content_to_str(content) if content is not None else None,
+                    tool_calls=korrel_tool_calls or None,
+                )
+            )
+
+        elif role == "tool":
+            result.append(
+                Message(
+                    role="tool",
+                    content=_content_to_str(_field(msg, "content")),
+                    tool_call_id=_field(msg, "tool_call_id"),
+                )
+            )
+
+    return result
+
+
+def _sanitize_id_for_comment(scenario_id: str) -> str:
+    """Return a display-safe single-line label for use in a comment or header.
+
+    Strips leading/trailing whitespace, collapses newlines to a space, and
+    removes every occurrence of triple-double-quote so the label cannot close
+    or escape from any surrounding string region in the generated module.
+    The exact scenario id is always preserved separately via repr().
+    """
+    label = scenario_id.strip().replace("\n", " ").replace("\r", " ")
+    label = label.replace('"""', "")
+    return label