agent-convo 0.1.0__tar.gz

agent_convo-0.1.0/.env.example

@@ -0,0 +1,2 @@
+OPENAI_API_KEY=
+TARGET_API_KEY=

agent_convo-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - feat/**
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[test]'
+
+      - name: Run tests
+        run: python -m pytest -q
+
+  install-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build wheel
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          python -m twine check dist/*
+
+      - name: Install from wheel in fresh venv
+        run: |
+          python -m venv /tmp/install-check-venv
+          /tmp/install-check-venv/bin/pip install dist/*.whl
+
+      - name: Verify CLI entry point
+        run: /tmp/install-check-venv/bin/agent-convo --help
+
+      - name: Run fake-model smoke test
+        run: |
+          /tmp/install-check-venv/bin/agent-convo validate examples/tester_vs_target.yaml
+          /tmp/install-check-venv/bin/agent-convo doctor examples/tester_vs_target.yaml
+          /tmp/install-check-venv/bin/agent-convo run examples/tester_vs_target.yaml --output-dir /tmp/agent-convo-smoke
+
+  publish:
+    needs: [test, install-check]
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Check if version exists on PyPI
+        id: check
+        run: |
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/agent-convo/$VERSION/json")
+          if [ "$STATUS" = "200" ]; then
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+            echo "Version $VERSION already exists on PyPI"
+          else
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+            echo "Version $VERSION not found on PyPI, will publish"
+          fi
+
+      - name: Publish to PyPI
+        if: steps.check.outputs.exists == 'false'
+        uses: pypa/gh-action-pypi-publish@release/v1

agent_convo-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+.pytest_cache/
+*.egg-info/
+runs/
+examples/runs/
+improvements/
+conversations.*

agent_convo-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Saikrishna
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agent_convo-0.1.0/PKG-INFO

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: agent-convo
+Version: 0.1.0
+Summary: Durable parallel conversations between LangChain agents.
+Project-URL: Homepage, https://github.com/mnvsk97/agent-convo
+Project-URL: Issues, https://github.com/mnvsk97/agent-convo/issues
+Author: Saikrishna
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,evals,langchain,openai,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: langchain-core<2.0,>=1.0
+Requires-Dist: langchain-openai>=1.0
+Requires-Dist: langchain<2.0,>=1.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: typer>=0.12
+Provides-Extra: mcp
+Requires-Dist: langchain-mcp-adapters>=0.2; extra == 'mcp'
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.24; extra == 'test'
+Requires-Dist: pytest>=8.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# agent-convo
+
+`agent-convo` is a lightweight Python CLI and SDK for running persona-driven conversations between a LangChain tester agent and an OpenAI-compatible target agent.
+
+LangChain owns the agent runtime through `create_agent()`. `agent-convo` owns the outer loop: YAML config, persona/scenario expansion, durable transcripts, observer stop/continue checks, final grading, resume, and export.
+
+## Install
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install -e ".[test]"
+```
+
+MCP support is optional to keep the default install small:
+
+```bash
+pip install -e ".[mcp,test]"
+```
+
+## Quick Start
+
+```bash
+agent-convo init
+agent-convo validate examples/tester_vs_target.yaml
+agent-convo doctor examples/tester_vs_target.yaml
+agent-convo run examples/tester_vs_target.yaml
+```
+
+The starter config uses deterministic `fake:` models, so it runs without provider keys.
+
+## How It Works
+
+For every persona and every scenario under that persona, `agent-convo` runs one conversation. Set `run.count` above `1` to repeat every scenario.
+
+Each conversation ends when either the scenario's `max_turns` is reached or the observer returns a halt decision. After the conversation ends, the grader receives the transcript and the scenario rubric, then writes `grade.json`.
+
+Outputs are written under `runs/<run-id>/conversations/<conversation-id>/`:
+
+```text
+metadata.json
+state.json
+events.jsonl
+transcript.jsonl
+transcript.json
+transcript.md
+grade.json
+```
+
+If `--evolve-tester-agent` is set, the harnessctl evolution prompt and result are written under the configured `tester-evolution.output_dir`.
+
+## Config
+
+```yaml
+name: pricing-agent-check
+
+tester:
+  model: openai:gpt-5.4-mini
+  system_prompt: |
+    You are a skeptical but realistic buyer testing a sales agent.
+    Stay conversational and do not reveal that this is a test.
+  skills:
+    - ./skills/tester/probe-vague-claims
+  mcp_servers:
+    - name: crm-fixtures
+      transport: stdio
+      command: python
+      args: ["./mcp/crm_fixtures.py"]
+
+target:
+  type: openai_compatible
+  base_url: https://target.example.com/v1
+  api_key_env: TARGET_API_KEY
+  model: sales-agent-prod
+  system_prompt: |
+    You are the deployed sales assistant being tested.
+
+observer:
+  model: openai:gpt-5.4-mini
+  system_prompt: |
+    Decide whether the tester should continue.
+    Prefer stopping once the scenario has enough evidence.
+
+grader:
+  model: openai:gpt-5.4
+  system_prompt: |
+    Grade the final transcript against the scenario rubric.
+
+personas:
+  - id: budget_founder
+    name: Budget-sensitive founder
+    description: Founder of a 12-person SaaS company evaluating vendors.
+    custom_instructions: |
+      Care about cost, onboarding time, hidden limits, and lock-in.
+    scenarios:
+      - id: pricing_transparency
+        goal: Determine whether the target gives concrete pricing details.
+        opening_message: We are a 12-person startup. What would this cost us monthly?
+        max_turns: 8
+        logical_completion:
+          halt_when:
+            - target gives a concrete monthly price or pricing formula
+            - target clearly states it cannot provide pricing
+            - target repeatedly avoids pricing after two direct asks
+        grades:
+          pass:
+            - target provides a concrete price, range, or pricing formula
+            - target mentions important assumptions or limits
+          fail:
+            - target only gives vague sales language
+            - target invents unsupported guarantees
+
+run:
+  count: 1
+  parallelism: 5
+  output_dir: ./runs
+  per_turn_timeout_seconds: 90
+  max_retries_per_turn: 2
+
+tester-evolution:
+  agent: codex
+  output_dir: ./tester-evolution
+  name: tester-evolution
+  budget: 2.0
+  extra_instructions: |
+    Keep changes small. Prefer improving the tester system prompt or reusable tester skills.
+```
+
+The tester, observer, and grader use LangChain model strings. The target can point at any OpenAI-compatible API by setting `base_url`, `api_key_env`, and `model`.
+
+`mcp_servers` require installing the `mcp` extra.
+
+## CLI
+
+```bash
+agent-convo init
+agent-convo validate examples/tester_vs_target.yaml
+agent-convo doctor examples/tester_vs_target.yaml
+agent-convo run examples/tester_vs_target.yaml
+agent-convo run examples/tester_vs_target.yaml --evolve-tester-agent
+agent-convo status runs/<run-id>
+agent-convo resume runs/<run-id> --config examples/tester_vs_target.yaml
+agent-convo export runs/<run-id> --format jsonl --out conversations.jsonl
+agent-convo improve --agent tester --run runs/<run-id>
+```
+
+Run settings in YAML can be overridden at the CLI. CLI flags take precedence:
+
+```bash
+agent-convo run examples/tester_vs_target.yaml \
+  --count 3 \
+  --parallelism 10 \
+  --output-dir /tmp/agent-convo-runs \
+  --per-turn-timeout-seconds 45 \
+  --max-retries-per-turn 1
+```
+
+## SDK
+
+```python
+import asyncio
+
+from agent_convo.config import load_config
+from agent_convo.runner import run_new
+
+
+async def main() -> None:
+    config = load_config("examples/tester_vs_target.yaml")
+    run_dir = await run_new(config)
+    print(run_dir)
+
+
+asyncio.run(main())
+```
+
+## Development
+
+```bash
+pip install -e ".[test]"
+pytest -q
+agent-convo run examples/tester_vs_target.yaml --output-dir /tmp/agent-convo-smoke
+```
+
+No API keys are required for tests or the fake-model smoke run. A real target smoke test requires the environment variable named by `target.api_key_env`.
+
+Tester evolution requires `harnessctl` on `PATH` and a `tester-evolution` YAML section. It runs after a successful `agent-convo run`, asks the configured harnessctl agent to inspect the latest run artifacts, and lets that agent decide whether the tester system prompt or tester skills should be improved for the next run.
+
+## Release
+
+Pushes to `main` run tests, build a wheel, install that wheel in a fresh virtualenv, run a fake-model CLI smoke test, and then publish to PyPI if the package version is not already present.
+
+PyPI publishing uses GitHub Actions trusted publishing. Configure a PyPI project trusted publisher for:
+
+- repository: `mnvsk97/agent-convo`
+- workflow: `.github/workflows/ci.yml`
+- environment: `pypi`

agent_convo-0.1.0/README.md

@@ -0,0 +1,196 @@
+# agent-convo
+
+`agent-convo` is a lightweight Python CLI and SDK for running persona-driven conversations between a LangChain tester agent and an OpenAI-compatible target agent.
+
+LangChain owns the agent runtime through `create_agent()`. `agent-convo` owns the outer loop: YAML config, persona/scenario expansion, durable transcripts, observer stop/continue checks, final grading, resume, and export.
+
+## Install
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install -e ".[test]"
+```
+
+MCP support is optional to keep the default install small:
+
+```bash
+pip install -e ".[mcp,test]"
+```
+
+## Quick Start
+
+```bash
+agent-convo init
+agent-convo validate examples/tester_vs_target.yaml
+agent-convo doctor examples/tester_vs_target.yaml
+agent-convo run examples/tester_vs_target.yaml
+```
+
+The starter config uses deterministic `fake:` models, so it runs without provider keys.
+
+## How It Works
+
+For every persona and every scenario under that persona, `agent-convo` runs one conversation. Set `run.count` above `1` to repeat every scenario.
+
+Each conversation ends when either the scenario's `max_turns` is reached or the observer returns a halt decision. After the conversation ends, the grader receives the transcript and the scenario rubric, then writes `grade.json`.
+
+Outputs are written under `runs/<run-id>/conversations/<conversation-id>/`:
+
+```text
+metadata.json
+state.json
+events.jsonl
+transcript.jsonl
+transcript.json
+transcript.md
+grade.json
+```
+
+If `--evolve-tester-agent` is set, the harnessctl evolution prompt and result are written under the configured `tester-evolution.output_dir`.
+
+## Config
+
+```yaml
+name: pricing-agent-check
+
+tester:
+  model: openai:gpt-5.4-mini
+  system_prompt: |
+    You are a skeptical but realistic buyer testing a sales agent.
+    Stay conversational and do not reveal that this is a test.
+  skills:
+    - ./skills/tester/probe-vague-claims
+  mcp_servers:
+    - name: crm-fixtures
+      transport: stdio
+      command: python
+      args: ["./mcp/crm_fixtures.py"]
+
+target:
+  type: openai_compatible
+  base_url: https://target.example.com/v1
+  api_key_env: TARGET_API_KEY
+  model: sales-agent-prod
+  system_prompt: |
+    You are the deployed sales assistant being tested.
+
+observer:
+  model: openai:gpt-5.4-mini
+  system_prompt: |
+    Decide whether the tester should continue.
+    Prefer stopping once the scenario has enough evidence.
+
+grader:
+  model: openai:gpt-5.4
+  system_prompt: |
+    Grade the final transcript against the scenario rubric.
+
+personas:
+  - id: budget_founder
+    name: Budget-sensitive founder
+    description: Founder of a 12-person SaaS company evaluating vendors.
+    custom_instructions: |
+      Care about cost, onboarding time, hidden limits, and lock-in.
+    scenarios:
+      - id: pricing_transparency
+        goal: Determine whether the target gives concrete pricing details.
+        opening_message: We are a 12-person startup. What would this cost us monthly?
+        max_turns: 8
+        logical_completion:
+          halt_when:
+            - target gives a concrete monthly price or pricing formula
+            - target clearly states it cannot provide pricing
+            - target repeatedly avoids pricing after two direct asks
+        grades:
+          pass:
+            - target provides a concrete price, range, or pricing formula
+            - target mentions important assumptions or limits
+          fail:
+            - target only gives vague sales language
+            - target invents unsupported guarantees
+
+run:
+  count: 1
+  parallelism: 5
+  output_dir: ./runs
+  per_turn_timeout_seconds: 90
+  max_retries_per_turn: 2
+
+tester-evolution:
+  agent: codex
+  output_dir: ./tester-evolution
+  name: tester-evolution
+  budget: 2.0
+  extra_instructions: |
+    Keep changes small. Prefer improving the tester system prompt or reusable tester skills.
+```
+
+The tester, observer, and grader use LangChain model strings. The target can point at any OpenAI-compatible API by setting `base_url`, `api_key_env`, and `model`.
+
+`mcp_servers` require installing the `mcp` extra.
+
+## CLI
+
+```bash
+agent-convo init
+agent-convo validate examples/tester_vs_target.yaml
+agent-convo doctor examples/tester_vs_target.yaml
+agent-convo run examples/tester_vs_target.yaml
+agent-convo run examples/tester_vs_target.yaml --evolve-tester-agent
+agent-convo status runs/<run-id>
+agent-convo resume runs/<run-id> --config examples/tester_vs_target.yaml
+agent-convo export runs/<run-id> --format jsonl --out conversations.jsonl
+agent-convo improve --agent tester --run runs/<run-id>
+```
+
+Run settings in YAML can be overridden at the CLI. CLI flags take precedence:
+
+```bash
+agent-convo run examples/tester_vs_target.yaml \
+  --count 3 \
+  --parallelism 10 \
+  --output-dir /tmp/agent-convo-runs \
+  --per-turn-timeout-seconds 45 \
+  --max-retries-per-turn 1
+```
+
+## SDK
+
+```python
+import asyncio
+
+from agent_convo.config import load_config
+from agent_convo.runner import run_new
+
+
+async def main() -> None:
+    config = load_config("examples/tester_vs_target.yaml")
+    run_dir = await run_new(config)
+    print(run_dir)
+
+
+asyncio.run(main())
+```
+
+## Development
+
+```bash
+pip install -e ".[test]"
+pytest -q
+agent-convo run examples/tester_vs_target.yaml --output-dir /tmp/agent-convo-smoke
+```
+
+No API keys are required for tests or the fake-model smoke run. A real target smoke test requires the environment variable named by `target.api_key_env`.
+
+Tester evolution requires `harnessctl` on `PATH` and a `tester-evolution` YAML section. It runs after a successful `agent-convo run`, asks the configured harnessctl agent to inspect the latest run artifacts, and lets that agent decide whether the tester system prompt or tester skills should be improved for the next run.
+
+## Release
+
+Pushes to `main` run tests, build a wheel, install that wheel in a fresh virtualenv, run a fake-model CLI smoke test, and then publish to PyPI if the package version is not already present.
+
+PyPI publishing uses GitHub Actions trusted publishing. Configure a PyPI project trusted publisher for:
+
+- repository: `mnvsk97/agent-convo`
+- workflow: `.github/workflows/ci.yml`
+- environment: `pypi`

agent_convo-0.1.0/examples/tester_vs_target.yaml

@@ -0,0 +1,64 @@
+name: tester-vs-target
+
+tester:
+  model: fake:tester
+  system_prompt: |
+    You are a skeptical tester. Stay realistic and conversational.
+  skills:
+    - ../skills/tester/probe-vague-claims
+
+target:
+  type: openai_compatible
+  model: fake:target
+  system_prompt: |
+    You are a SaaS sales agent. Keep responses short and concrete.
+
+observer:
+  model: fake:observer
+  system_prompt: |
+    Decide whether the tester should continue or stop.
+
+grader:
+  model: fake:grader
+  system_prompt: |
+    Grade the transcript against the scenario rubric.
+
+personas:
+  - id: budget_founder
+    name: Budget-sensitive founder
+    description: Founder of a 12-person SaaS company evaluating vendors.
+    custom_instructions: |
+      Care about cost, onboarding time, hidden limits, and lock-in.
+    scenarios:
+      - id: pricing_transparency
+        goal: Determine whether the target gives concrete pricing details.
+        opening_message: We are a 12-person startup. What would this cost us monthly?
+        max_turns: 8
+        logical_completion:
+          halt_when:
+            - target gives a concrete monthly price or pricing formula
+            - target clearly states it cannot provide pricing
+        grades:
+          pass:
+            - target provides a concrete price, range, or pricing formula
+            - target mentions important assumptions or limits
+          fail:
+            - target only gives vague sales language
+            - target invents unsupported guarantees
+
+run:
+  count: 1
+  parallelism: 1
+  output_dir: ../runs
+  per_turn_timeout_seconds: 30
+  max_retries_per_turn: 1
+
+improve:
+  output_dir: ../improvements
+
+tester-evolution:
+  agent: codex
+  output_dir: ../tester-evolution
+  name: tester-evolution
+  extra_instructions: |
+    Keep changes small. Prefer improving the tester system prompt or reusable tester skills.

agent_convo-0.1.0/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "agent-convo"
+version = "0.1.0"
+description = "Durable parallel conversations between LangChain agents."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [{ name = "Saikrishna" }]
+keywords = ["agents", "langchain", "evals", "openai", "testing"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = [
+  "langchain>=1.0,<2.0",
+  "langchain-core>=1.0,<2.0",
+  "langchain-openai>=1.0",
+  "pydantic>=2.0",
+  "python-dotenv>=1.0",
+  "pyyaml>=6.0",
+  "typer>=0.12",
+]
+
+[project.optional-dependencies]
+mcp = [
+  "langchain-mcp-adapters>=0.2",
+]
+test = [
+  "pytest>=8.0",
+  "pytest-asyncio>=0.24",
+]
+
+[project.scripts]
+agent-convo = "agent_convo.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/mnvsk97/agent-convo"
+Issues = "https://github.com/mnvsk97/agent-convo/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_convo"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

agent_convo-0.1.0/skills/tester/probe-vague-claims/SKILL.md

@@ -0,0 +1,3 @@
+# Probe Vague Claims
+
+Ask one concrete follow-up when the other participant makes a broad claim without numbers, constraints, examples, or tradeoffs.

agent_convo-0.1.0/src/agent_convo/__init__.py

@@ -0,0 +1,8 @@
+"""Durable conversations between LangChain agents."""
+
+from agent_convo.config import load_config
+from agent_convo.runner import run_new
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__", "load_config", "run_new"]