agentsynth-ai 0.2.0__py3-none-any.whl

agentsynth/__init__.py

@@ -0,0 +1,168 @@
+"""AgentSynth: generate synthetic agent trajectories and score them with an
+LLM-as-Judge loop. Runs offline against a mock by default.
+
+    >>> from agentsynth import AgentTrajectoryGenerator, TrajectoryEvaluator
+    >>> gen = AgentTrajectoryGenerator()
+    >>> traj = gen.generate("What's the weather in Paris?")
+    >>> ev = TrajectoryEvaluator().evaluate(traj)
+    >>> ev.overall, ev.passed
+"""
+
+from __future__ import annotations
+
+__version__ = "0.2.0"
+
+from .benchmarks import (
+    BUILTIN_CASES,
+    BenchmarkCase,
+    BenchmarkReport,
+    agentsynth_model,
+    compare_models,
+    prompted_model,
+    report_table_md,
+    run_benchmark,
+)
+from .dedup import DedupResult, decontaminate, dedup_trajectories
+from .environments import (
+    CompositeEnvironment,
+    Environment,
+    MCPEnvironment,
+    PythonSandbox,
+    SQLEnvironment,
+)
+from .evaluator import TrajectoryEvaluator
+from .exporters import (
+    load_jsonl,
+    save_dataset,
+    to_adp,
+    to_jsonl,
+    to_sharegpt,
+)
+from .generator import AgentTrajectoryGenerator
+from .hub import dataset_card, prepare_dataset_dir, push_dataset
+from .metrics import compute_dataset_metrics, diversity_score
+from .pipelines import Recipe, RunResult, load_recipe, make_environment, run_recipe
+from .preferences import (
+    PreferencePair,
+    build_preference_pairs,
+    load_dpo_jsonl,
+    to_dpo_jsonl,
+)
+from .schemas import (
+    DEFAULT_RUBRIC_WEIGHTS,
+    RUBRIC_DIMENSIONS,
+    EvalResult,
+    RubricScores,
+    ToolSpec,
+    Trajectory,
+    TrajectoryStep,
+)
+from .tasks import SEED_TASKS, SeedTask, sample_tasks
+from .training import build_dpo_dataset, build_sft_dataset, to_dpo_records, to_sft_records
+from .utils import (
+    DEFAULT_TOOL_CATALOG,
+    LLMClient,
+    PythonREPL,
+    default_tool_catalog,
+    parse_tool_catalog,
+)
+from .verification import (
+    RUBRIC_PRESETS,
+    EnsembleEvaluator,
+    ExecutionVerifier,
+    ExpectedAnswerVerifier,
+    SafetyVerifier,
+    ToolArgVerifier,
+    VerificationResult,
+    Verifier,
+    batch_verify,
+    get_rubric,
+    rubric_names,
+    verify_trajectory,
+)
+
+__all__ = [
+    "__version__",
+    # schemas
+    "Trajectory",
+    "TrajectoryStep",
+    "ToolSpec",
+    "RubricScores",
+    "EvalResult",
+    "RUBRIC_DIMENSIONS",
+    "DEFAULT_RUBRIC_WEIGHTS",
+    # utils
+    "parse_tool_catalog",
+    "default_tool_catalog",
+    "DEFAULT_TOOL_CATALOG",
+    "PythonREPL",
+    "LLMClient",
+    # core
+    "AgentTrajectoryGenerator",
+    "TrajectoryEvaluator",
+    # metrics
+    "compute_dataset_metrics",
+    "diversity_score",
+    # exporters
+    "to_jsonl",
+    "to_sharegpt",
+    "to_adp",
+    "save_dataset",
+    "load_jsonl",
+    # environments
+    "Environment",
+    "CompositeEnvironment",
+    "SQLEnvironment",
+    "PythonSandbox",
+    "MCPEnvironment",
+    # tasks
+    "SeedTask",
+    "SEED_TASKS",
+    "sample_tasks",
+    # pipelines
+    "Recipe",
+    "RunResult",
+    "run_recipe",
+    "load_recipe",
+    "make_environment",
+    # verification
+    "Verifier",
+    "VerificationResult",
+    "verify_trajectory",
+    "batch_verify",
+    "ExecutionVerifier",
+    "ToolArgVerifier",
+    "SafetyVerifier",
+    "ExpectedAnswerVerifier",
+    "EnsembleEvaluator",
+    "RUBRIC_PRESETS",
+    "get_rubric",
+    "rubric_names",
+    # preferences (DPO)
+    "PreferencePair",
+    "build_preference_pairs",
+    "to_dpo_jsonl",
+    "load_dpo_jsonl",
+    # dedup / decontamination
+    "dedup_trajectories",
+    "decontaminate",
+    "DedupResult",
+    # training data prep
+    "to_sft_records",
+    "to_dpo_records",
+    "build_sft_dataset",
+    "build_dpo_dataset",
+    # benchmark
+    "BenchmarkCase",
+    "BenchmarkReport",
+    "BUILTIN_CASES",
+    "run_benchmark",
+    "compare_models",
+    "agentsynth_model",
+    "prompted_model",
+    "report_table_md",
+    # hub
+    "dataset_card",
+    "prepare_dataset_dir",
+    "push_dataset",
+]

agentsynth/benchmarks/__init__.py

@@ -0,0 +1,37 @@
+"""A self-contained function-calling benchmark, plus BFCL and τ-bench adapters."""
+
+from __future__ import annotations
+
+from .bfcl import SAMPLE_BFCL, bfcl_case, load_bfcl, sample_cases
+from .tau_bench import run_tau_bench, tau_bench_available
+from .tool_calling import (
+    BUILTIN_CASES,
+    BenchmarkCase,
+    BenchmarkReport,
+    CaseResult,
+    agentsynth_model,
+    compare_models,
+    prompted_model,
+    report_table_md,
+    run_benchmark,
+)
+
+__all__ = [
+    "BenchmarkCase",
+    "CaseResult",
+    "BenchmarkReport",
+    "BUILTIN_CASES",
+    "run_benchmark",
+    "compare_models",
+    "agentsynth_model",
+    "prompted_model",
+    "report_table_md",
+    # BFCL
+    "load_bfcl",
+    "bfcl_case",
+    "sample_cases",
+    "SAMPLE_BFCL",
+    # tau-bench
+    "run_tau_bench",
+    "tau_bench_available",
+]

agentsynth/benchmarks/bfcl.py

@@ -0,0 +1,123 @@
+"""Load Berkeley Function-Calling Leaderboard (BFCL) cases into the harness.
+
+BFCL ships JSONL: a questions file (`id`, `question`, `function`) and a
+possible-answers file (`id`, `ground_truth`). This converts the pair into
+`BenchmarkCase`s so any `model_fn` can be scored with `run_benchmark`.
+
+The scoring here is a simplified tool + argument match (arguments accept the list
+of allowed values BFCL gives), not BFCL's full AST checker. Use it for a quick
+before/after signal; use BFCL's own scorer for the leaderboard.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+from .tool_calling import BenchmarkCase
+
+
+def _user_query(question: Any) -> str:
+    turns = question[0] if question and isinstance(question[0], list) else question
+    for msg in turns or []:
+        if isinstance(msg, dict) and msg.get("role") == "user":
+            return str(msg.get("content", ""))
+    return str(question)
+
+
+def _normalize_tools(functions: Any) -> List[Dict[str, Any]]:
+    out: List[Dict[str, Any]] = []
+    for fn in functions or []:
+        params = dict(fn.get("parameters") or {})
+        if params.get("type") == "dict":  # BFCL uses "dict"; JSON Schema uses "object"
+            params["type"] = "object"
+        out.append(
+            {"name": fn.get("name"), "description": fn.get("description", ""), "parameters": params}
+        )
+    return out
+
+
+def bfcl_case(
+    question_rec: Dict[str, Any], answer_rec: Optional[Dict[str, Any]] = None
+) -> Optional[BenchmarkCase]:
+    """Convert one BFCL question (+ its ground truth) into a BenchmarkCase."""
+    ground_truth = (answer_rec or {}).get("ground_truth") or []
+    if not ground_truth or not isinstance(ground_truth[0], dict) or not ground_truth[0]:
+        return None
+    first = ground_truth[0]
+    expected_tool = next(iter(first))
+    expected_args = first[expected_tool] if isinstance(first[expected_tool], dict) else {}
+    return BenchmarkCase(
+        id=str(question_rec.get("id", "")),
+        query=_user_query(question_rec.get("question")),
+        expected_tool=expected_tool,
+        expected_args=expected_args,
+        tools=_normalize_tools(question_rec.get("function")),
+    )
+
+
+def _read_jsonl(path: str) -> List[Dict[str, Any]]:
+    with open(path, encoding="utf-8") as fh:
+        return [json.loads(line) for line in fh if line.strip()]
+
+
+def load_bfcl(questions_path: str, answers_path: Optional[str] = None) -> List[BenchmarkCase]:
+    """Load BFCL question + possible-answer JSONL files into BenchmarkCases."""
+    questions = _read_jsonl(questions_path)
+    answers: Dict[str, Dict[str, Any]] = {}
+    if answers_path:
+        answers = {str(a.get("id")): a for a in _read_jsonl(answers_path)}
+    cases: List[BenchmarkCase] = []
+    for q in questions:
+        case = bfcl_case(q, answers.get(str(q.get("id"))))
+        if case is not None:
+            cases.append(case)
+    return cases
+
+
+# A couple of BFCL-format records, for tests and an offline demo.
+SAMPLE_BFCL: List[Tuple[Dict[str, Any], Dict[str, Any]]] = [
+    (
+        {
+            "id": "simple_0",
+            "question": [[{"role": "user", "content": "What is the weather in Paris?"}]],
+            "function": [
+                {
+                    "name": "get_weather",
+                    "description": "Get the current weather for a city.",
+                    "parameters": {
+                        "type": "dict",
+                        "properties": {"city": {"type": "string"}},
+                        "required": ["city"],
+                    },
+                }
+            ],
+        },
+        {"id": "simple_0", "ground_truth": [{"get_weather": {"city": ["Paris", "paris"]}}]},
+    ),
+    (
+        {
+            "id": "simple_1",
+            "question": [[{"role": "user", "content": "Compute 15 factorial."}]],
+            "function": [
+                {
+                    "name": "math_factorial",
+                    "description": "Compute the factorial of a number.",
+                    "parameters": {
+                        "type": "dict",
+                        "properties": {"n": {"type": "integer"}},
+                        "required": ["n"],
+                    },
+                }
+            ],
+        },
+        {"id": "simple_1", "ground_truth": [{"math_factorial": {"n": [15]}}]},
+    ),
+]
+
+
+def sample_cases() -> List[BenchmarkCase]:
+    return [c for c in (bfcl_case(q, a) for q, a in SAMPLE_BFCL) if c is not None]
+
+
+__all__ = ["bfcl_case", "load_bfcl", "sample_cases", "SAMPLE_BFCL"]

agentsynth/benchmarks/tau_bench.py

@@ -0,0 +1,71 @@
+"""Run τ-bench against a model.
+
+Unlike BFCL, τ-bench is multi-turn and agentic — a user simulator plus a domain
+database — so it can't be scored from single tool calls. This is a thin bridge to
+the official `tau-bench` package: it builds the environment and the package's
+tool-calling agent and reports the pass rate.
+
+Point `model` at your fine-tuned model served behind an OpenAI-compatible endpoint
+(vLLM, TGI, …). Install the harness with:
+
+    pip install git+https://github.com/sierra-research/tau-bench
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+
+def tau_bench_available() -> bool:
+    try:
+        import tau_bench  # noqa: F401
+
+        return True
+    except Exception:
+        return False
+
+
+def run_tau_bench(
+    model: str,
+    provider: str = "openai",
+    env_name: str = "retail",
+    user_model: str = "gpt-4o",
+    user_provider: str = "openai",
+    task_split: str = "test",
+    task_ids: Optional[List[int]] = None,
+) -> Dict[str, Any]:
+    """Run τ-bench and return `{env, n, pass_rate, avg_reward}`.
+
+    Requires the `tau-bench` package and API keys for the agent + user-simulator models.
+    """
+    try:
+        from tau_bench.agents.tool_calling_agent import ToolCallingAgent
+        from tau_bench.envs import get_env
+    except Exception as exc:
+        raise ImportError(
+            "tau-bench is not installed: "
+            "pip install git+https://github.com/sierra-research/tau-bench"
+        ) from exc
+
+    env = get_env(
+        env_name,
+        user_strategy="llm",
+        user_model=user_model,
+        user_provider=user_provider,
+        task_split=task_split,
+    )
+    agent = ToolCallingAgent(
+        tools_info=env.tools_info, wiki=env.wiki, model=model, provider=provider
+    )
+    ids = task_ids if task_ids is not None else list(range(len(env.tasks)))
+    rewards = [float(getattr(agent.solve(env=env, task_index=i), "reward", 0.0)) for i in ids]
+    n = len(rewards)
+    return {
+        "env": env_name,
+        "n": n,
+        "pass_rate": round(sum(1 for r in rewards if r >= 0.999) / n, 4) if n else 0.0,
+        "avg_reward": round(sum(rewards) / n, 4) if n else 0.0,
+    }
+
+
+__all__ = ["tau_bench_available", "run_tau_bench"]

agentsynth/benchmarks/tool_calling.py

@@ -0,0 +1,275 @@
+"""A small function-calling benchmark.
+
+Each case gives a query and the full tool catalog; the model has to pick the right
+tool and supply sane arguments. Scoring is deliberately simple — tool match plus a
+lenient arg check — so it's easy to reason about and produces a clear before/after
+table. A "model" here is any callable `(query, tools) -> (tool_name, tool_args)`,
+so you can score a real LLM, a fine-tuned model, or AgentSynth's own generator.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from pydantic import BaseModel, Field
+
+from ..schemas import ToolSpec
+from ..utils import DEFAULT_TOOL_CATALOG, extract_json, parse_tool_catalog
+
+ModelFn = Callable[[str, List[ToolSpec]], Tuple[Optional[str], Dict[str, Any]]]
+
+
+class BenchmarkCase(BaseModel):
+    id: str
+    query: str
+    expected_tool: str
+    expected_args: Dict[str, Any] = Field(default_factory=dict)  # value None => key presence only
+    # The full default catalog (with parameter schemas), so the model must both pick the
+    # right tool and fill its arguments.
+    tools: List[Dict[str, Any]] = Field(default_factory=lambda: list(DEFAULT_TOOL_CATALOG))
+
+    def tool_specs(self) -> List[ToolSpec]:
+        return parse_tool_catalog(self.tools)
+
+
+class CaseResult(BaseModel):
+    id: str
+    expected_tool: str
+    predicted_tool: Optional[str]
+    tool_ok: bool
+    args_ok: bool
+    score: float
+
+
+class BenchmarkReport(BaseModel):
+    n: int
+    tool_accuracy: float
+    arg_accuracy: float
+    score: float
+    results: List[CaseResult] = Field(default_factory=list)
+
+
+BUILTIN_CASES: List[BenchmarkCase] = [
+    BenchmarkCase(
+        id="weather_paris",
+        query="What's the weather in Paris right now?",
+        expected_tool="get_weather",
+        expected_args={"city": "Paris"},
+    ),
+    BenchmarkCase(
+        id="weather_tokyo",
+        query="Is it raining in Tokyo today?",
+        expected_tool="get_weather",
+        expected_args={"city": "Tokyo"},
+    ),
+    BenchmarkCase(
+        id="math_mult",
+        query="What is 23 times 7 plus 4?",
+        expected_tool="calculator",
+        expected_args={"expression": None},
+    ),
+    BenchmarkCase(
+        id="math_tip",
+        query="Calculate an 18% tip on a $54 bill.",
+        expected_tool="calculator",
+        expected_args={"expression": None},
+    ),
+    BenchmarkCase(
+        id="search_news",
+        query="Find recent news about open-source AI agents.",
+        expected_tool="web_search",
+        expected_args={"query": None},
+    ),
+    BenchmarkCase(
+        id="search_fact",
+        query="Search the web for the population of Vietnam.",
+        expected_tool="web_search",
+        expected_args={"query": None},
+    ),
+    BenchmarkCase(
+        id="file_csv",
+        query="Read the file data/report.csv and summarize it.",
+        expected_tool="read_file",
+        expected_args={"path": None},
+    ),
+    BenchmarkCase(
+        id="file_notes",
+        query="Open notes.md and list the action items.",
+        expected_tool="read_file",
+        expected_args={"path": None},
+    ),
+    BenchmarkCase(
+        id="sql_revenue",
+        query="Query the database for total revenue by region.",
+        expected_tool="sql_query",
+        expected_args={"query": None},
+    ),
+    BenchmarkCase(
+        id="sql_count",
+        query="Run a SQL query to count rows in the sales table.",
+        expected_tool="sql_query",
+        expected_args={"query": None},
+    ),
+    BenchmarkCase(
+        id="email_launch",
+        query="Send an email to team@example.com about the launch.",
+        expected_tool="send_email",
+        expected_args={"to": None},
+    ),
+    BenchmarkCase(
+        id="email_summary",
+        query="Email a summary of the report to alex@example.com.",
+        expected_tool="send_email",
+        expected_args={"to": None},
+    ),
+]
+
+
+def _args_ok(expected: Dict[str, Any], predicted: Dict[str, Any]) -> bool:
+    for key, value in (expected or {}).items():
+        if key not in (predicted or {}):
+            return False
+        if value is None:
+            continue  # key presence is enough
+        pred = str(predicted[key]).strip().lower()
+        if isinstance(value, (list, tuple)):  # any acceptable value (BFCL-style)
+            if pred not in [str(v).strip().lower() for v in value]:
+                return False
+        elif pred != str(value).strip().lower():
+            return False
+    return True
+
+
+def _mean(values: List[float]) -> float:
+    return round(sum(values) / len(values), 4) if values else 0.0
+
+
+def run_benchmark(
+    model_fn: ModelFn, cases: Optional[List[BenchmarkCase]] = None
+) -> BenchmarkReport:
+    """Score a model on the function-calling cases.
+
+    `model_fn(query, tools)` returns the `(tool_name, tool_args)` the model would call.
+    """
+    cases = cases or BUILTIN_CASES
+    results: List[CaseResult] = []
+    for case in cases:
+        try:
+            predicted_tool, predicted_args = model_fn(case.query, case.tool_specs())
+        except Exception:
+            predicted_tool, predicted_args = None, {}
+        tool_ok = predicted_tool == case.expected_tool
+        args_ok = tool_ok and _args_ok(case.expected_args, predicted_args or {})
+        score = 1.0 if (tool_ok and args_ok) else (0.5 if tool_ok else 0.0)
+        results.append(
+            CaseResult(
+                id=case.id,
+                expected_tool=case.expected_tool,
+                predicted_tool=predicted_tool,
+                tool_ok=tool_ok,
+                args_ok=args_ok,
+                score=score,
+            )
+        )
+    return BenchmarkReport(
+        n=len(results),
+        tool_accuracy=_mean([1.0 if r.tool_ok else 0.0 for r in results]),
+        arg_accuracy=_mean([1.0 if r.args_ok else 0.0 for r in results]),
+        score=_mean([r.score for r in results]),
+        results=results,
+    )
+
+
+def compare_models(
+    before: ModelFn, after: ModelFn, cases: Optional[List[BenchmarkCase]] = None
+) -> Dict[str, Any]:
+    """Run two models on the same cases and return a before/after comparison."""
+    cases = cases or BUILTIN_CASES
+    before_report = run_benchmark(before, cases)
+    after_report = run_benchmark(after, cases)
+    return {
+        "before": before_report,
+        "after": after_report,
+        "delta_tool_accuracy": round(after_report.tool_accuracy - before_report.tool_accuracy, 4),
+        "delta_score": round(after_report.score - before_report.score, 4),
+        "n": before_report.n,
+    }
+
+
+def agentsynth_model(generator: Any, mode: str = "single_agent") -> ModelFn:
+    """Adapt an AgentTrajectoryGenerator into a benchmark model: it takes the first
+    tool call the generated trajectory makes."""
+
+    def model_fn(query: str, tools: List[ToolSpec]) -> Tuple[Optional[str], Dict[str, Any]]:
+        traj = generator.generate(query, tools=tools, mode=mode)
+        calls = traj.tool_calls()
+        if not calls:
+            return None, {}
+        return calls[0].tool_name, calls[0].tool_args or {}
+
+    return model_fn
+
+
+def prompted_model(complete_fn: Callable[[str], str]) -> ModelFn:
+    """Turn a text-completion function `(prompt) -> text` into a benchmark model.
+
+    It asks the model for a single JSON tool call and parses the reply, so it works
+    with any instruction-following model (a base or fine-tuned HF model, etc.).
+    """
+
+    def model_fn(query: str, tools: List[ToolSpec]) -> Tuple[Optional[str], Dict[str, Any]]:
+        tool_json = json.dumps(
+            [
+                {"name": t.name, "description": t.description, "parameters": t.parameters}
+                for t in tools
+            ]
+        )
+        prompt = (
+            "You can call exactly one tool to help the user.\n"
+            f"Tools (JSON): {tool_json}\n\n"
+            f"User: {query}\n"
+            'Respond with ONLY a JSON object: {"tool": "<tool name>", "args": {<arguments>}}'
+        )
+        parsed = extract_json(complete_fn(prompt))
+        if isinstance(parsed, dict):
+            args = parsed.get("args")
+            return parsed.get("tool"), args if isinstance(args, dict) else {}
+        return None, {}
+
+    return model_fn
+
+
+def report_table_md(comparison: Dict[str, Any]) -> str:
+    """Render a before/after comparison as a markdown table."""
+    before = comparison["before"]
+    after = comparison["after"]
+
+    def pct(x: float) -> str:
+        return f"{x * 100:.1f}%"
+
+    lines = [
+        f"Function-calling benchmark ({comparison['n']} cases)",
+        "",
+        "| Metric | Before | After | Δ |",
+        "| --- | --- | --- | --- |",
+        f"| Tool accuracy | {pct(before.tool_accuracy)} | {pct(after.tool_accuracy)} "
+        f"| {pct(comparison['delta_tool_accuracy'])} |",
+        f"| Arg accuracy | {pct(before.arg_accuracy)} | {pct(after.arg_accuracy)} | |",
+        f"| Overall score | {pct(before.score)} | {pct(after.score)} "
+        f"| {pct(comparison['delta_score'])} |",
+    ]
+    return "\n".join(lines)
+
+
+__all__ = [
+    "BenchmarkCase",
+    "CaseResult",
+    "BenchmarkReport",
+    "BUILTIN_CASES",
+    "run_benchmark",
+    "compare_models",
+    "agentsynth_model",
+    "prompted_model",
+    "report_table_md",
+]