agent-convo 0.1.0__py3-none-any.whl

agent_convo/__init__.py

@@ -0,0 +1,8 @@
+"""Durable conversations between LangChain agents."""
+
+from agent_convo.config import load_config
+from agent_convo.runner import run_new
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__", "load_config", "run_new"]

agent_convo/cli.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from dotenv import load_dotenv
+
+from agent_convo.config import dump_example_config, load_config
+from agent_convo.doctor import check_config
+from agent_convo.evolution import EvolutionNotConfiguredError, evolve_tester_agent
+from agent_convo.export import export_run
+from agent_convo.improve import improve_agent
+from agent_convo.runner import resume_existing, run_new
+
+app = typer.Typer(no_args_is_help=True)
+
+
+@app.command()
+def init() -> None:
+    example_path = Path("examples/tester_vs_target.yaml")
+    if not example_path.exists():
+        dump_example_config(example_path)
+    env_path = Path(".env.example")
+    if not env_path.exists():
+        env_path.write_text("OPENAI_API_KEY=\nTARGET_API_KEY=\n")
+    skill_dir = Path("skills/tester/probe-vague-claims")
+    skill_dir.mkdir(parents=True, exist_ok=True)
+    skill_file = skill_dir / "SKILL.md"
+    if not skill_file.exists():
+        skill_file.write_text("# Probe Vague Claims\n\nAsk one concrete follow-up when a claim is vague.\n")
+    typer.echo("Created examples/tester_vs_target.yaml, .env.example, and starter skills.")
+
+
+@app.command()
+def validate(config_path: Annotated[Path, typer.Argument(exists=True)]) -> None:
+    load_config(config_path)
+    typer.echo(f"Valid config: {config_path}")
+
+
+@app.command()
+def doctor(config_path: Annotated[Path, typer.Argument(exists=True)]) -> None:
+    load_dotenv()
+    config = load_config(config_path)
+    result = check_config(config)
+    for message in result.messages:
+        typer.echo(message)
+    if not result.ok:
+        raise typer.Exit(1)
+
+
+@app.command()
+def run(
+    config_path: Annotated[Path, typer.Argument(exists=True)],
+    count: Annotated[int | None, typer.Option("--count")] = None,
+    parallelism: Annotated[int | None, typer.Option("--parallelism")] = None,
+    output_dir: Annotated[Path | None, typer.Option("--output-dir")] = None,
+    per_turn_timeout_seconds: Annotated[float | None, typer.Option("--per-turn-timeout-seconds")] = None,
+    max_retries_per_turn: Annotated[int | None, typer.Option("--max-retries-per-turn")] = None,
+    evolve_tester_agent_flag: Annotated[
+        bool,
+        typer.Option("--evolve-tester-agent", help="Run tester evolution with harnessctl after the run finishes."),
+    ] = False,
+) -> None:
+    load_dotenv()
+    config = load_config(config_path).with_run_overrides(
+        count=count,
+        parallelism=parallelism,
+        output_dir=str(output_dir) if output_dir is not None else None,
+        per_turn_timeout_seconds=per_turn_timeout_seconds,
+        max_retries_per_turn=max_retries_per_turn,
+    )
+    run_dir = asyncio.run(run_new(config))
+    typer.echo(str(run_dir))
+    if evolve_tester_agent_flag:
+        try:
+            evolution_dir = evolve_tester_agent(config, run_dir=run_dir)
+        except EvolutionNotConfiguredError as exc:
+            raise typer.BadParameter(str(exc)) from exc
+        typer.echo(f"tester evolution: {evolution_dir}")
+
+
+@app.command()
+def status(run_dir: Annotated[Path, typer.Argument(exists=True)]) -> None:
+    states = {}
+    for state_path in sorted((run_dir / "conversations").glob("*/state.json")):
+        state = json.loads(state_path.read_text())
+        states[state["status"]] = states.get(state["status"], 0) + 1
+    if not states:
+        typer.echo("No conversations found.")
+        return
+    for key in sorted(states):
+        typer.echo(f"{key}: {states[key]}")
+
+
+@app.command()
+def resume(
+    run_dir: Annotated[Path, typer.Argument(exists=True)],
+    config_path: Annotated[Path, typer.Option("--config", "-c", exists=True)] = Path("examples/tester_vs_target.yaml"),
+    parallelism: Annotated[int | None, typer.Option("--parallelism")] = None,
+    per_turn_timeout_seconds: Annotated[float | None, typer.Option("--per-turn-timeout-seconds")] = None,
+    max_retries_per_turn: Annotated[int | None, typer.Option("--max-retries-per-turn")] = None,
+) -> None:
+    load_dotenv()
+    config = load_config(config_path).with_run_overrides(
+        parallelism=parallelism,
+        per_turn_timeout_seconds=per_turn_timeout_seconds,
+        max_retries_per_turn=max_retries_per_turn,
+    )
+    asyncio.run(resume_existing(config, run_dir))
+    typer.echo(f"Resumed {run_dir}")
+
+
+@app.command("export")
+def export_cmd(
+    run_dir: Annotated[Path, typer.Argument(exists=True)],
+    fmt: Annotated[str, typer.Option("--format")] = "jsonl",
+    out: Annotated[Path, typer.Option("--out")] = Path("conversations.jsonl"),
+) -> None:
+    export_run(run_dir, fmt=fmt, out=out)
+    typer.echo(str(out))
+
+
+@app.command()
+def improve(
+    run_dir: Annotated[Path, typer.Option("--run", exists=True)],
+    agent: Annotated[str, typer.Option("--agent")],
+    config_path: Annotated[Path, typer.Option("--config", "-c", exists=True)] = Path("examples/tester_vs_target.yaml"),
+) -> None:
+    config = load_config(config_path)
+    output_dir = improve_agent(config, run_dir=run_dir, agent_name=agent)
+    typer.echo(str(output_dir))
+
+if __name__ == "__main__":
+    app()

agent_convo/config.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class MCPServerConfig(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    name: str
+    transport: str = "stdio"
+
+
+class AgentConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    model: str
+    system_prompt: str = ""
+    skills: list[str] = Field(default_factory=list)
+    tools: list[str] = Field(default_factory=list)
+    mcp_servers: list[MCPServerConfig] = Field(default_factory=list)
+    middleware: list[str] = Field(default_factory=list)
+
+
+class TargetConfig(AgentConfig):
+    type: Literal["openai_compatible"] = "openai_compatible"
+    base_url: str | None = None
+    api_key_env: str | None = None
+
+
+class ObserverConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    model: str = "fake:observer"
+    system_prompt: str = ""
+    check_after_each_target_turn: bool = True
+
+
+class GraderConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    model: str = "fake:grader"
+    system_prompt: str = ""
+
+
+class GradeRules(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    pass_: list[str] = Field(default_factory=list, alias="pass")
+    fail: list[str] = Field(default_factory=list)
+
+
+class LogicalCompletionConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    halt_when: list[str] = Field(default_factory=list)
+
+
+class ScenarioConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    goal: str
+    opening_message: str
+    max_turns: int = Field(gt=0)
+    logical_completion: LogicalCompletionConfig = Field(default_factory=LogicalCompletionConfig)
+    grades: GradeRules = Field(default_factory=GradeRules)
+
+
+class PersonaConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    name: str
+    description: str = ""
+    custom_instructions: str = ""
+    scenarios: list[ScenarioConfig] = Field(default_factory=list)
+
+
+class RunConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    count: int = Field(default=1, gt=0)
+    parallelism: int = Field(default=1, gt=0)
+    output_dir: str = "./runs"
+    per_turn_timeout_seconds: float = Field(default=90, gt=0)
+    max_retries_per_turn: int = Field(default=2, ge=0)
+
+
+class ImproveConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    output_dir: str = "./improvements"
+
+
+class TesterEvolutionConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    agent: str
+    output_dir: str = "./tester-evolution"
+    name: str = "agent-convo-tester-evolution"
+    budget: float | None = Field(default=None, gt=0)
+    stream: bool = False
+    extra_instructions: str = ""
+
+
+class AppConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    name: str
+    tester: AgentConfig
+    target: TargetConfig
+    personas: list[PersonaConfig]
+    observer: ObserverConfig = Field(default_factory=ObserverConfig)
+    grader: GraderConfig = Field(default_factory=GraderConfig)
+    run: RunConfig = Field(default_factory=RunConfig)
+    improve: ImproveConfig = Field(default_factory=ImproveConfig)
+    tester_evolution: TesterEvolutionConfig | None = Field(default=None, alias="tester-evolution")
+    config_path: Path | None = Field(default=None, exclude=True)
+
+    @model_validator(mode="after")
+    def validate_scenarios(self) -> "AppConfig":
+        if not self.personas:
+            raise ValueError("at least one persona is required")
+        for persona in self.personas:
+            if not persona.scenarios:
+                raise ValueError(f"persona '{persona.id}' must define at least one scenario")
+        return self
+
+    @property
+    def base_dir(self) -> Path:
+        if self.config_path is None:
+            return Path.cwd()
+        return self.config_path.parent
+
+    def resolve_path(self, path: str | Path) -> Path:
+        candidate = Path(path).expanduser()
+        if candidate.is_absolute():
+            return candidate.resolve()
+        return (self.base_dir / candidate).resolve()
+
+    def with_run_overrides(self, **overrides: Any) -> "AppConfig":
+        clean = {key: value for key, value in overrides.items() if value is not None}
+        if not clean:
+            return self
+        return self.model_copy(update={"run": self.run.model_copy(update=clean)})
+
+    @property
+    def scenario_count(self) -> int:
+        return sum(len(persona.scenarios) for persona in self.personas)
+
+
+def load_config(path: str | Path, *, validate_paths: bool = True) -> AppConfig:
+    config_path = Path(path).expanduser().resolve()
+    data = yaml.safe_load(config_path.read_text()) or {}
+    config = AppConfig.model_validate({**data, "config_path": config_path})
+    if validate_paths:
+        validate_local_paths(config)
+    return config
+
+
+def validate_local_paths(config: AppConfig) -> None:
+    for skill_path in config.tester.skills:
+        skill_dir = config.resolve_path(skill_path)
+        skill_file = skill_dir / "SKILL.md"
+        if not skill_file.exists():
+            raise ValueError(f"tester skill path is missing SKILL.md: {skill_dir}")
+
+
+def dump_example_config(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(example_config())
+
+
+def example_config() -> str:
+    return """name: tester-vs-target
+
+tester:
+  model: fake:tester
+  system_prompt: |
+    You are a skeptical tester. Stay realistic and conversational.
+  skills:
+    - ../skills/tester/probe-vague-claims
+
+target:
+  type: openai_compatible
+  model: fake:target
+  system_prompt: |
+    You are a SaaS sales agent. Keep responses short and concrete.
+
+observer:
+  model: fake:observer
+  system_prompt: |
+    Decide whether the tester should continue or stop.
+
+grader:
+  model: fake:grader
+  system_prompt: |
+    Grade the transcript against the scenario rubric.
+
+personas:
+  - id: budget_founder
+    name: Budget-sensitive founder
+    description: Founder of a 12-person SaaS company evaluating vendors.
+    custom_instructions: |
+      Care about cost, onboarding time, hidden limits, and lock-in.
+    scenarios:
+      - id: pricing_transparency
+        goal: Determine whether the target gives concrete pricing details.
+        opening_message: We are a 12-person startup. What would this cost us monthly?
+        max_turns: 8
+        logical_completion:
+          halt_when:
+            - target gives a concrete monthly price or pricing formula
+            - target clearly states it cannot provide pricing
+        grades:
+          pass:
+            - target provides a concrete price, range, or pricing formula
+            - target mentions important assumptions or limits
+          fail:
+            - target only gives vague sales language
+            - target invents unsupported guarantees
+
+run:
+  count: 1
+  parallelism: 1
+  output_dir: ../runs
+  per_turn_timeout_seconds: 30
+  max_retries_per_turn: 1
+
+improve:
+  output_dir: ../improvements
+
+tester-evolution:
+  agent: codex
+  output_dir: ../tester-evolution
+  name: tester-evolution
+  extra_instructions: |
+    Keep changes small. Prefer improving the tester system prompt or reusable tester skills.
+"""

agent_convo/doctor.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from agent_convo.config import AppConfig
+
+
+@dataclass(frozen=True)
+class DoctorResult:
+    ok: bool
+    messages: list[str]
+
+
+def check_model(name: str, model: str, messages: list[str]) -> bool:
+    if model.startswith("fake:"):
+        messages.append(f"{name}: fake model configured for local deterministic runs")
+        return True
+    provider = model.split(":", 1)[0]
+    if provider == "openai" and not os.getenv("OPENAI_API_KEY"):
+        messages.append(f"{name}: OPENAI_API_KEY is required for model {model}")
+        return False
+    messages.append(f"{name}: provider readiness looks configured for {model}")
+    return True
+
+
+def check_config(config: AppConfig) -> DoctorResult:
+    messages: list[str] = []
+    ok = True
+    ok = check_model("tester", config.tester.model, messages) and ok
+    ok = check_model("observer", config.observer.model, messages) and ok
+    ok = check_model("grader", config.grader.model, messages) and ok
+    if config.target.model.startswith("fake:"):
+        messages.append("target: fake model configured for local deterministic runs")
+    elif config.target.api_key_env and not os.getenv(config.target.api_key_env):
+        ok = False
+        messages.append(f"target: {config.target.api_key_env} is required for model {config.target.model}")
+    else:
+        messages.append(f"target: OpenAI-compatible endpoint configured for {config.target.model}")
+    return DoctorResult(ok=ok, messages=messages)

agent_convo/evaluation.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Any, Literal
+
+from langchain.agents import create_agent
+from pydantic import BaseModel
+
+from agent_convo.config import GraderConfig, ObserverConfig, PersonaConfig, ScenarioConfig
+from agent_convo.langchain_factory import final_content, model_from_config
+
+
+class ObserverDecision(BaseModel):
+    decision: Literal["continue", "halt_success", "halt_failure"]
+    feedback: str = ""
+
+
+class GradeResult(BaseModel):
+    result: Literal["pass", "fail"]
+    rationale: str
+
+
+def transcript_text(transcript: list[dict[str, Any]]) -> str:
+    return "\n".join(f"{row['agent']}: {row['content']}" for row in transcript)
+
+
+def fake_observer_decision(transcript: list[dict[str, Any]], scenario: ScenarioConfig) -> ObserverDecision:
+    if len(transcript) >= scenario.max_turns:
+        return ObserverDecision(decision="halt_success", feedback="Max turns reached.")
+    target_messages = [row for row in transcript if row["agent"] == "target"]
+    if len(target_messages) >= 1 and scenario.logical_completion.halt_when:
+        return ObserverDecision(decision="halt_success", feedback="Enough signal collected for local fake run.")
+    return ObserverDecision(decision="continue", feedback="Ask one concrete follow-up.")
+
+
+def fake_grade_result(transcript: list[dict[str, Any]]) -> GradeResult:
+    if transcript:
+        return GradeResult(result="pass", rationale="Local fake run produced a transcript to grade.")
+    return GradeResult(result="fail", rationale="No transcript messages were produced.")
+
+
+async def structured_invoke(agent: Any, prompt: str) -> Any:
+    result = await agent.ainvoke({"messages": [{"role": "user", "content": prompt}]})
+    if isinstance(result, dict) and result.get("structured_response") is not None:
+        return result["structured_response"]
+    content = final_content(result)
+    return json.loads(content)
+
+
+async def observe(
+    config: ObserverConfig,
+    *,
+    persona: PersonaConfig,
+    scenario: ScenarioConfig,
+    transcript: list[dict[str, Any]],
+) -> ObserverDecision:
+    if config.model.startswith("fake:"):
+        return fake_observer_decision(transcript, scenario)
+    prompt = "\n\n".join(
+        [
+            f"Persona: {persona.name}",
+            f"Scenario goal: {scenario.goal}",
+            "Logical completion rules:",
+            json.dumps(scenario.logical_completion.model_dump(), indent=2),
+            "Transcript:",
+            transcript_text(transcript),
+            "Decide whether to continue, halt_success, or halt_failure.",
+        ]
+    )
+    agent = create_agent(
+        model=model_from_config(config.model, "observer"),
+        tools=[],
+        system_prompt=config.system_prompt,
+        response_format=ObserverDecision,
+    )
+    raw = await asyncio.wait_for(structured_invoke(agent, prompt), timeout=60)
+    return raw if isinstance(raw, ObserverDecision) else ObserverDecision.model_validate(raw)
+
+
+async def grade(
+    config: GraderConfig,
+    *,
+    persona: PersonaConfig,
+    scenario: ScenarioConfig,
+    transcript: list[dict[str, Any]],
+) -> GradeResult:
+    if config.model.startswith("fake:"):
+        return fake_grade_result(transcript)
+    prompt = "\n\n".join(
+        [
+            f"Persona: {persona.name}",
+            f"Scenario goal: {scenario.goal}",
+            "Grade rules:",
+            json.dumps(scenario.grades.model_dump(by_alias=True), indent=2),
+            "Transcript:",
+            transcript_text(transcript),
+            "Return pass or fail with concise evidence.",
+        ]
+    )
+    agent = create_agent(
+        model=model_from_config(config.model, "grader"),
+        tools=[],
+        system_prompt=config.system_prompt,
+        response_format=GradeResult,
+    )
+    raw = await asyncio.wait_for(structured_invoke(agent, prompt), timeout=60)
+    return raw if isinstance(raw, GradeResult) else GradeResult.model_validate(raw)

agent_convo/evolution.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+from agent_convo.config import AppConfig
+from agent_convo.storage import utc_now
+
+
+class EvolutionNotConfiguredError(ValueError):
+    pass
+
+
+def build_evolution_prompt(config: AppConfig, *, run_dir: Path) -> str:
+    latest = run_dir.resolve()
+    skill_paths = "\n".join(f"- {path}" for path in config.tester.skills) or "- none"
+    scenario_lines = []
+    for persona in config.personas:
+        for scenario in persona.scenarios:
+            scenario_lines.append(f"- {persona.id}/{scenario.id}: {scenario.goal}")
+    scenarios = "\n".join(scenario_lines)
+    extra = config.tester_evolution.extra_instructions.strip() if config.tester_evolution else ""
+    return "\n\n".join(
+        part
+        for part in [
+            "You are evolving the tester agent for agent-convo.",
+            "Use the latest run artifacts to decide whether the tester can be made more effective next time.",
+            f"Config file: {config.config_path}",
+            f"Latest run directory: {latest}",
+            "Inspect the latest run's transcripts, grades, metadata, and observer events.",
+            "Current tester system prompt:",
+            config.tester.system_prompt.strip() or "(empty)",
+            "Current tester skills:",
+            skill_paths,
+            "Configured scenarios:",
+            scenarios,
+            (
+                "Your task: decide whether the tester's system prompt or tester skills should change. "
+                "If the latest run shows weak testing behavior, make the smallest useful repo change. "
+                "Prefer edits to the tester system prompt in the YAML or reusable tester skills under skills/. "
+                "Do not change target, observer, grader, or run settings unless they block tester evolution. "
+                "If no useful improvement is justified, do not edit files; explain why."
+            ),
+            extra,
+        ]
+        if part
+    )
+
+
+def evolve_tester_agent(
+    config: AppConfig,
+    *,
+    run_dir: Path,
+    command_runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
+) -> Path:
+    if config.tester_evolution is None:
+        raise EvolutionNotConfiguredError(
+            "--evolve-tester-agent requires a tester-evolution section in the YAML config"
+        )
+
+    timestamp = utc_now().replace(":", "").replace(".", "")
+    output_dir = config.resolve_path(config.tester_evolution.output_dir) / timestamp
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    prompt = build_evolution_prompt(config, run_dir=run_dir)
+    prompt_path = output_dir / "prompt.md"
+    result_path = output_dir / "harnessctl-result.md"
+    prompt_path.write_text(prompt + "\n")
+
+    command = [
+        "harnessctl",
+        "run",
+        "--agent",
+        config.tester_evolution.agent,
+        "--name",
+        config.tester_evolution.name,
+    ]
+    if config.tester_evolution.stream:
+        command.append("--stream")
+    if config.tester_evolution.budget is not None:
+        command.extend(["--budget", str(config.tester_evolution.budget)])
+    command.append(prompt)
+
+    result = command_runner(
+        command,
+        cwd=str(config.base_dir),
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    result_path.write_text(
+        "\n".join(
+            [
+                f"# harnessctl result",
+                "",
+                f"exit_code: {result.returncode}",
+                "",
+                "## stdout",
+                "",
+                result.stdout or "",
+                "",
+                "## stderr",
+                "",
+                result.stderr or "",
+            ]
+        ).rstrip()
+        + "\n"
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"harnessctl tester evolution failed; see {result_path}")
+    return output_dir

agent_convo/export.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Any
+
+from agent_convo.storage import read_jsonl
+
+
+def collect_messages(run_dir: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for transcript_path in sorted((run_dir / "conversations").glob("*/transcript.jsonl")):
+        conversation_id = transcript_path.parent.name
+        for message in read_jsonl(transcript_path):
+            rows.append({"conversation_id": conversation_id, **message})
+    return rows
+
+
+def export_run(run_dir: Path, *, fmt: str, out: Path) -> None:
+    rows = collect_messages(run_dir)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    if fmt == "jsonl":
+        out.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows))
+        return
+    if fmt == "json":
+        out.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n")
+        return
+    if fmt == "csv":
+        with out.open("w", newline="") as handle:
+            writer = csv.DictWriter(handle, fieldnames=["conversation_id", "turn", "agent", "content", "ts"])
+            writer.writeheader()
+            writer.writerows(rows)
+        return
+    if fmt == "md":
+        lines = ["# Agent Convo Export", ""]
+        current = None
+        for row in rows:
+            if row["conversation_id"] != current:
+                current = row["conversation_id"]
+                lines.extend([f"## Conversation {current}", ""])
+            lines.extend([f"### Turn {row['turn']} - {row['agent']}", "", row["content"], ""])
+        out.write_text("\n".join(lines).rstrip() + "\n")
+        return
+    raise ValueError(f"Unsupported export format: {fmt}")