convy 0.1.0__py3-none-any.whl

convy/__init__.py

@@ -0,0 +1,53 @@
+"""convy: check conversational agents with a simulated user and a judge."""
+
+from convy.agent import (
+    Agent,
+    AgentFailure,
+    Answer,
+    Conversation,
+    Message,
+    NoUsage,
+    TimeLimited,
+    Usage,
+)
+from convy.bench import Bench, Journal, JsonlJournal, RunHeader
+from convy.dialog import NoVerdict, Transcript, Turn, Verdict
+from convy.env import Env
+from convy.http import HttpFailure, JsonAgent, JsonEndpoint, Tls
+from convy.model import Model, ModelFailure, Models, OpenAiModel
+from convy.report import Report, Runs
+from convy.scenario import Matching, Outcome, Scenario, Scenarios
+
+__all__ = [
+    "Agent",
+    "AgentFailure",
+    "Answer",
+    "Bench",
+    "Conversation",
+    "Env",
+    "HttpFailure",
+    "Journal",
+    "JsonAgent",
+    "JsonEndpoint",
+    "JsonlJournal",
+    "Matching",
+    "Message",
+    "Model",
+    "ModelFailure",
+    "Models",
+    "NoUsage",
+    "NoVerdict",
+    "OpenAiModel",
+    "Outcome",
+    "Report",
+    "RunHeader",
+    "Runs",
+    "Scenario",
+    "Scenarios",
+    "TimeLimited",
+    "Tls",
+    "Transcript",
+    "Turn",
+    "Usage",
+    "Verdict",
+]

convy/agent.py

@@ -0,0 +1,94 @@
+"""What convy needs from an agent, and the wrapper that limits the time of each of its steps."""
+
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from typing import Protocol
+
+from msgspec import Struct
+
+
+class Message(Struct, frozen=True):
+    """A message from the simulated user to the agent."""
+
+    text: str
+
+
+class Usage(Struct, frozen=True, tag="usage"):
+    """Tokens the agent spent on one answer."""
+
+    input: int
+    output: int
+
+
+class NoUsage(Struct, frozen=True, tag="no_usage"):
+    """The agent did not report the tokens it spent."""
+
+
+class Answer(Struct, frozen=True):
+    """The agent's answer to a message."""
+
+    text: str
+    usage: Usage | NoUsage = NoUsage()
+
+
+class AgentFailure(Exception):
+    """A turn of the agent failed."""
+
+
+class Conversation(Protocol):
+    """One dialogue with an agent."""
+
+    async def answer(self, message: Message) -> Answer:
+        """Answer the message, or raise `AgentFailure` with the reason."""
+        ...
+
+
+class Agent(Protocol):
+    """Anything convy can talk to."""
+
+    def conversation(self) -> AbstractAsyncContextManager[Conversation]:
+        """Open a new dialogue, independent of any other."""
+        ...
+
+
+class TimeLimited(Struct, frozen=True):
+    """The same agent, with a limit on the time of each step: opening a conversation, a turn,
+    closing it."""
+
+    agent: Agent
+    seconds: float
+
+    @asynccontextmanager
+    async def conversation(self) -> AsyncIterator[Conversation]:
+        limit = asyncio.timeout(self.seconds)
+        try:
+            async with limit, self.agent.conversation() as conversation:
+                limit.reschedule(None)  # turns have limits of their own
+                try:
+                    yield LimitedConversation(conversation, self.seconds)
+                finally:
+                    limit.reschedule(asyncio.get_running_loop().time() + self.seconds)
+        except TimeoutError:
+            if limit.expired():
+                raise AgentFailure(
+                    f"the conversation did not open or close in {self.seconds:g} s"
+                ) from None
+            raise
+
+
+class LimitedConversation(Struct, frozen=True):
+    """A conversation whose turns fail when they take longer than `seconds`."""
+
+    conversation: Conversation
+    seconds: float
+
+    async def answer(self, message: Message) -> Answer:
+        limit = asyncio.timeout(self.seconds)
+        try:
+            async with limit:
+                return await self.conversation.answer(message)
+        except TimeoutError:
+            if limit.expired():
+                raise AgentFailure(f"no answer in {self.seconds:g} s") from None
+            raise

convy/bench.py

@@ -0,0 +1,96 @@
+"""A run: every scenario played against an agent, each outcome written to a journal."""
+
+import asyncio
+from datetime import datetime
+from pathlib import Path
+from typing import Protocol
+
+import msgspec
+from msgspec import Struct
+
+from convy.agent import Agent
+from convy.model import Models
+from convy.scenario import Outcome, Scenario
+
+
+class RunHeader(Struct, frozen=True):
+    """What a run was: the agent and its version, the models, how many attempts were planned."""
+
+    agent: str
+    version: str
+    user: str
+    judge: str
+    attempts: int
+    planned: int
+    started: datetime
+
+
+class Journal(Protocol):
+    """Where outcomes go as soon as they are ready."""
+
+    def record(self, outcome: Outcome) -> None: ...
+
+
+class JsonlJournal(Struct, frozen=True):
+    """A run's file, `<directory>/<agent>/<start>.jsonl`: the header, then a line per attempt.
+
+    The file is created on the first `record`, so a run that never got an outcome leaves no file.
+    """
+
+    directory: Path
+    header: RunHeader
+
+    def path(self) -> Path:
+        name = self.header.started.strftime("%Y-%m-%dT%H-%M-%S.%f")  # unique per run
+        return self.directory / self.header.agent / f"{name}.jsonl"
+
+    def record(self, outcome: Outcome) -> None:
+        path = self.path()
+        lines = [msgspec.json.encode(outcome)]
+        if not path.exists():
+            path.parent.mkdir(parents=True, exist_ok=True)
+            lines.insert(0, msgspec.json.encode(self.header))
+        with path.open("ab") as file:
+            file.write(b"".join(line + b"\n" for line in lines))
+
+
+class Bench(Struct, frozen=True):
+    """Scenarios to play, the models that play the user and judge, and how to play them."""
+
+    scenarios: tuple[Scenario, ...]
+    models: Models
+    attempts: int = 1
+    parallel: int = 4
+
+    def __post_init__(self) -> None:
+        if self.attempts < 1:
+            raise ValueError(f"attempts must be at least 1, got {self.attempts}")
+        if self.parallel < 1:
+            raise ValueError(f"parallel must be at least 1, got {self.parallel}")
+
+    def planned(self) -> int:
+        return len(self.scenarios) * self.attempts
+
+    async def run(self, agent: Agent, journal: Journal) -> tuple[Outcome, ...]:
+        """Play every scenario `attempts` times, at most `parallel` conversations at once."""
+        limit = asyncio.Semaphore(self.parallel)
+        async with asyncio.TaskGroup() as group:
+            tasks = [
+                group.create_task(self.played(scenario, attempt, agent, journal, limit))
+                for scenario in self.scenarios
+                for attempt in range(1, self.attempts + 1)
+            ]
+        return tuple(task.result() for task in tasks)
+
+    async def played(
+        self,
+        scenario: Scenario,
+        attempt: int,
+        agent: Agent,
+        journal: Journal,
+        limit: asyncio.Semaphore,
+    ) -> Outcome:
+        async with limit:
+            outcome = await scenario.outcome(agent, self.models, attempt)
+        journal.record(outcome)
+        return outcome

convy/cli.py

@@ -0,0 +1,196 @@
+"""The `convy` command: `init`, `run` and `report`. It prints, and it picks the exit code."""
+
+import asyncio
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from msgspec import Struct
+from pydantic import BaseModel, Field, ValidationError
+from pydantic_settings import (
+    BaseSettings,
+    CliApp,
+    CliPositionalArg,
+    CliSubCommand,
+    SettingsConfigDict,
+)
+
+from convy.agent import NoUsage, TimeLimited, Usage
+from convy.bench import Bench, Journal, JsonlJournal, RunHeader
+from convy.dialog import NoVerdict, Verdict
+from convy.fakes import FakeModel, MemoryJournal
+from convy.model import Models
+from convy.project import Project, ProjectAgent
+from convy.scenario import Matching, Outcome, Scenario
+
+SMOKE = Scenario(
+    id="smoke",
+    max_turns=2,
+    instructions="Greet the assistant and ask what it can help with, then thank it.",
+    claims=("The agent answered the greeting",),
+)
+
+
+class Printed(Struct, frozen=True):
+    """A journal that also prints each outcome as it is recorded."""
+
+    journal: Journal
+
+    def record(self, outcome: Outcome) -> None:
+        self.journal.record(outcome)
+        match outcome.verdict:
+            case Verdict(passed=True):
+                mark = "✓"
+            case Verdict():
+                mark = "✗"
+            case NoVerdict():
+                mark = "?"
+        seconds = sum(turn.seconds for turn in outcome.transcript.turns)
+        print(f"  {mark} {outcome.scenario} #{outcome.attempt}  {seconds:.1f} s  {outcome.stop}")
+
+
+class Invalid(Struct, frozen=True):
+    """Settings that failed validation, as the command shows them: where and what, never the
+    values read, since they may be keys."""
+
+    error: ValidationError
+
+    def show(self) -> None:
+        for found in self.error.errors(include_input=False, include_url=False):
+            field = ".".join(str(part) for part in found["loc"])
+            print(f"error: {self.error.title}: {field}: {found['msg']}", file=sys.stderr)
+
+
+class Rebuilt(Struct, frozen=True):
+    """A project's report, written again from its journals, as the command shows it."""
+
+    project: Project
+
+    def show(self) -> None:
+        for path in self.project.report():
+            print(
+                f"skipped unreadable lines or a whole journal: {self.project.runs() / path}",
+                file=sys.stderr,
+            )
+        print(f"report: {self.project.page()}")
+
+
+class InitCommand(BaseModel):
+    path: CliPositionalArg[Path] = Path(".")
+
+    def cli_cmd(self) -> None:
+        created = Project(self.path).init()
+        for path in created:
+            print(f"created {path}")
+        print(
+            "nothing to create: every file exists"
+            if not created
+            else "next: convy run echo --smoke"
+        )
+
+
+class RunCommand(BaseModel):
+    agents: CliPositionalArg[list[str]]
+    k: int = Field(1, ge=1, description="attempts per scenario")
+    scenarios: str = Field("*", description="only scenarios whose id matches this mask")
+    parallel: int = Field(4, ge=1, description="conversations at once")
+    turn_timeout: float = Field(
+        600,
+        gt=0,
+        description="seconds for each step of the agent: opening a conversation, a turn, closing",
+    )
+    smoke: bool = Field(False, description="check the connection to the agent; no models called")
+
+    def cli_cmd(self) -> None:
+        project = Project(Path.cwd())
+        try:
+            agents = [project.agent(name) for name in self.agents]
+            benches = [self.bench(project) for _ in agents]
+        except ValidationError as error:  # settings in the project's files
+            Invalid(error).show()
+            raise SystemExit(2) from None
+        except Exception as error:  # anything else wrong in the project's files
+            print(f"error: {error}", file=sys.stderr)
+            raise SystemExit(2) from None
+        failed = False
+        for loaded, bench in zip(agents, benches, strict=True):
+            outcomes = asyncio.run(self.played(project, bench, loaded))
+            failed |= any(o.stop in ("agent_failure", "model_failure") for o in outcomes)
+        if not self.smoke:
+            Rebuilt(project).show()
+        raise SystemExit(int(failed))
+
+    def bench(self, project: Project) -> Bench:
+        """A bench for one agent, with models of its own: a fake model's replies run on across its
+        calls, and a project's models are built fresh by running `models.py`."""
+        if self.smoke:
+            user = FakeModel("Hello! What can you help me with?", "Thank you!")
+            return Bench((SMOKE,), Models(user, FakeModel('{"pass": true, "reason": "smoke"}')))
+        scenarios = tuple(Matching(project.scenarios(), self.scenarios))
+        return Bench(scenarios, project.models(), self.k, self.parallel)
+
+    async def played(
+        self, project: Project, bench: Bench, loaded: ProjectAgent
+    ) -> tuple[Outcome, ...]:
+        agent = TimeLimited(loaded.agent, self.turn_timeout)
+        if self.smoke:
+            print(f"{loaded.name}: checking the connection")
+            (outcome,) = await bench.run(agent, MemoryJournal())
+            self.smoked(outcome)
+            return (outcome,)
+        header = RunHeader(
+            agent=loaded.name,
+            version=loaded.version,
+            user=bench.models.user.name,
+            judge=bench.models.judge.name,
+            attempts=bench.attempts,
+            planned=bench.planned(),
+            started=datetime.now().astimezone(),
+        )
+        print(f"{loaded.name}: {len(bench.scenarios)} scenarios, {bench.attempts} attempts each")
+        return await bench.run(agent, Printed(JsonlJournal(project.runs(), header)))
+
+    def smoked(self, outcome: Outcome) -> None:
+        for turn in outcome.transcript.turns:
+            match turn.answer.usage:
+                case Usage(input=spent, output=produced):
+                    tokens = f", {spent} / {produced} tokens"
+                case NoUsage():
+                    tokens = ""
+            print(f"  agent ({turn.seconds:.1f} s{tokens}): {turn.answer.text[:200]}")
+        match outcome:
+            case Outcome(stop="agent_failure", verdict=Verdict(reason=reason)):
+                print(f"connection failed: {reason}")
+            case _:
+                print("connection works")
+
+
+class ReportCommand(BaseModel):
+    def cli_cmd(self) -> None:
+        Rebuilt(Project(Path.cwd())).show()
+
+
+class Convy(BaseSettings):
+    """Check conversational agents with a simulated user and a judge."""
+
+    model_config = SettingsConfigDict(
+        cli_prog_name="convy",
+        cli_kebab_case=True,
+        cli_implicit_flags=True,
+        cli_enforce_required=True,
+    )
+    init: CliSubCommand[InitCommand] = Field(description="create a project with examples")
+    run: CliSubCommand[RunCommand] = Field(description="play scenarios against agents")
+    report: CliSubCommand[ReportCommand] = Field(description="rebuild results/index.html")
+
+    def cli_cmd(self) -> None:
+        CliApp.run_subcommand(self)
+
+
+def main(argv: list[str] | None = None) -> None:
+    """The command's entry point."""
+    try:
+        CliApp.run(Convy, cli_args=argv)  # None: sys.argv[1:]
+    except ValidationError as error:
+        Invalid(error).show()
+        raise SystemExit(2) from None

convy/dialog.py

@@ -0,0 +1,136 @@
+"""The dialogue: its record, the simulated user who drives it, and the judge who rates it."""
+
+import json
+
+from msgspec import Struct
+
+from convy.agent import Answer, Message
+from convy.model import Model, ModelFailure
+
+STOP = "###STOP###"
+
+OPENING = "(The assistant is waiting for your first message.)"
+
+USER_PROMPT = """You are playing a user who writes to an assistant. Follow the instructions below.
+
+Rules:
+- Write one message at a time, as a real person does in a chat.
+- Do not give away the whole of the instructions at once: say only what the current step needs.
+- Do not make up facts that are not in the instructions. If you are asked about something they do
+  not cover, say that you do not know.
+- Keep to the persona from the instructions until the end of the conversation.
+- Write in the language of the instructions.
+- When the goal is reached, or it is clear that it cannot be reached, answer exactly {stop}
+
+Instructions:
+{instructions}"""
+
+JUDGE_PROMPT = """Below are a dialogue between a user and an agent, and a list of claims about it.
+Check every claim strictly against the text of the dialogue.
+Answer with JSON only: {{"pass": true or false, "reason": "a short explanation"}}
+"pass" is true only if every claim holds. Write the reason in the language of the claims.
+
+Dialogue:
+{dialogue}
+
+Claims:
+{claims}"""
+
+
+class Turn(Struct, frozen=True):
+    """A message, the agent's answer, and how many seconds the answer took."""
+
+    message: Message
+    answer: Answer
+    seconds: float
+
+
+class Transcript(Struct, frozen=True):
+    """The record of a dialogue. It never changes: adding a turn gives a new transcript."""
+
+    turns: tuple[Turn, ...] = ()
+
+    def with_turn(self, turn: Turn) -> "Transcript":
+        return Transcript((*self.turns, turn))
+
+    def as_text(self) -> str:
+        """The dialogue as text, for the judge."""
+        return "\n".join(f"User: {t.message.text}\nAgent: {t.answer.text}" for t in self.turns)
+
+    def as_chat(self) -> list[dict[str, str]]:
+        """The dialogue for the simulated user: its messages are `assistant`, the agent's `user`."""
+        chat = []
+        for turn in self.turns:
+            chat.append({"role": "assistant", "content": turn.message.text})
+            chat.append({"role": "user", "content": turn.answer.text})
+        return chat
+
+
+class Finished(Struct, frozen=True):
+    """The simulated user has nothing more to say."""
+
+
+class SimulatedUser(Struct, frozen=True):
+    """A model that plays a person following a scenario's instructions."""
+
+    model: Model
+    instructions: str
+
+    async def next(self, transcript: Transcript) -> Message | Finished:
+        system = USER_PROMPT.format(stop=STOP, instructions=self.instructions)
+        messages = [
+            {"role": "system", "content": system},
+            {"role": "user", "content": OPENING},
+            *transcript.as_chat(),
+        ]
+        text = (await self.model.reply(messages)).strip()
+        if STOP in text:
+            return Finished()
+        if not text:
+            raise ModelFailure("the simulated user gave an empty message")
+        return Message(text)
+
+
+class Verdict(Struct, frozen=True, tag="verdict"):
+    """The judge's decision on an attempt."""
+
+    passed: bool
+    reason: str
+
+
+class NoVerdict(Struct, frozen=True, tag="no_verdict"):
+    """There is no verdict, because one of convy's models failed."""
+
+    error: str
+
+
+class Judge(Struct, frozen=True):
+    """A model that decides whether a dialogue meets a scenario's claims."""
+
+    model: Model
+
+    async def verdict(self, transcript: Transcript, claims: tuple[str, ...]) -> Verdict:
+        prompt = JUDGE_PROMPT.format(
+            dialogue=transcript.as_text(), claims="\n".join(f"- {claim}" for claim in claims)
+        )
+        return self.parsed(await self.model.reply([{"role": "user", "content": prompt}]))
+
+    def parsed(self, text: str) -> Verdict:
+        """The first JSON object with "pass" in the judge's answer, even with text around it.
+
+        An answer without one, or with a "pass" that is not true or false, is a failure of the
+        judge's model, like no answer at all."""
+        decoder = json.JSONDecoder()
+        start = text.find("{")
+        while start != -1:
+            try:
+                found, end = decoder.raw_decode(text, start)
+            except json.JSONDecodeError:
+                start = text.find("{", start + 1)
+                continue
+            if isinstance(found, dict) and "pass" in found:
+                if type(found["pass"]) is not bool:  # "true" or 1 is not a verdict
+                    raise ModelFailure(f'the judge\'s "pass" is not true or false: {text[:200]}')
+                return Verdict(found["pass"], str(found.get("reason", "")))
+            start = text.find("{", end)  # skip the whole object, nested braces included
+        raise ModelFailure(f"the judge did not answer with JSON: {text[:200]}")

convy/env.py

@@ -0,0 +1,16 @@
+"""Settings from `.env` and the environment."""
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Env(BaseSettings):
+    """The base class for settings in a project's `models.py` and `agents/*.py`.
+
+    It reads `.env` in the current directory and then the environment, ignores variables it does not
+    declare, and cannot be changed after it is read. Its errors do not show the values read, since
+    they may be keys.
+    """
+
+    model_config = SettingsConfigDict(
+        env_file=".env", extra="ignore", frozen=True, hide_input_in_errors=True
+    )

convy/fakes.py

@@ -0,0 +1,106 @@
+"""Simple working stand-ins for convy's interfaces, for tests: yours and convy's own."""
+
+import asyncio
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
+
+from msgspec import Struct
+
+from convy.agent import Answer, Conversation, Message, Usage
+from convy.model import ModelFailure
+from convy.scenario import Outcome
+
+type Step = str | Answer | BaseException
+
+
+class FakeAgent:
+    """An agent that answers from a list of steps.
+
+    A step is a string, an `Answer`, or an exception to raise on that turn. After the last step the
+    agent repeats it. Every conversation starts the list over. Each answer takes `delay` seconds.
+    `opened` counts the conversations opened, `peak` the most that were open at once.
+    """
+
+    def __init__(self, *steps: Step, delay: float = 0):
+        if not steps:
+            raise ValueError("FakeAgent needs at least one step")
+        self.steps = steps
+        self.delay = delay
+        self.opened = 0
+        self.peak = 0
+        self.open = 0
+
+    @asynccontextmanager
+    async def conversation(self) -> AsyncIterator[Conversation]:
+        self.opened += 1
+        self.open += 1
+        self.peak = max(self.peak, self.open)
+        try:
+            yield FakeConversation(self.steps, self.delay)
+        finally:
+            self.open -= 1
+
+
+class FakeConversation:
+    """A conversation of `FakeAgent`."""
+
+    def __init__(self, steps: Sequence[Step], delay: float):
+        self.steps = steps
+        self.delay = delay
+        self.turns = 0
+
+    async def answer(self, message: Message) -> Answer:
+        step = self.steps[min(self.turns, len(self.steps) - 1)]
+        self.turns += 1
+        await asyncio.sleep(self.delay)  # even at 0, other conversations get to run
+        if isinstance(step, BaseException):
+            raise step
+        return Answer(step) if isinstance(step, str) else step
+
+
+class Echo(Struct, frozen=True):
+    """An agent that answers with the user's message; its tokens are the message's length."""
+
+    @asynccontextmanager
+    async def conversation(self) -> AsyncIterator[Conversation]:
+        yield EchoConversation()
+
+
+class EchoConversation(Struct, frozen=True):
+    """A conversation of `Echo`."""
+
+    async def answer(self, message: Message) -> Answer:
+        length = len(message.text)
+        return Answer(message.text, Usage(input=length, output=length))
+
+
+class FakeModel:
+    """A model that replies from a list: a step is a string, or an exception to raise.
+
+    The list runs across all calls, and the last step repeats. Its `name` is `"fake"`.
+    """
+
+    name = "fake"
+
+    def __init__(self, *steps: str | ModelFailure):
+        if not steps:
+            raise ValueError("FakeModel needs at least one step")
+        self.steps = steps
+        self.calls: list[list[dict[str, str]]] = []
+
+    async def reply(self, messages: list[dict[str, str]]) -> str:
+        step = self.steps[min(len(self.calls), len(self.steps) - 1)]
+        self.calls.append(list(messages))
+        if isinstance(step, ModelFailure):
+            raise step
+        return step
+
+
+class MemoryJournal:
+    """A journal that keeps outcomes in memory, in `outcomes`."""
+
+    def __init__(self) -> None:
+        self.outcomes: list[Outcome] = []
+
+    def record(self, outcome: Outcome) -> None:
+        self.outcomes.append(outcome)