assayer 0.1.0__py3-none-any.whl

assayer/cli/main.py

@@ -0,0 +1,190 @@
+import asyncio
+import sys
+
+import click
+import httpx
+
+from assayer.config import get_api_key, set_api_key, show_config
+
+_KNOWN_MODELS: dict[str, list[str]] = {
+    "openai": ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "o1-mini"],
+    "anthropic": ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5-20251001"],
+    "gemini": ["gemini-1.5-pro", "gemini-1.5-flash"],
+    "ollama": ["ollama/llama3", "ollama/mistral", "ollama/phi3"],
+}
+
+
+@click.group()
+def cli() -> None:
+    pass
+
+
+@cli.command()
+@click.argument("prompt", required=False)
+@click.option("--models", required=True, help="Comma-separated model identifiers.")
+@click.option(
+    "--prompt-file",
+    type=click.Path(exists=True),
+    help="Path to a .txt prompt file.",
+)
+@click.option(
+    "--var",
+    multiple=True,
+    metavar="KEY=VALUE",
+    help="Template variables, repeatable.",
+)
+@click.option("--system", default=None, help="System prompt applied to all models.")
+@click.option("--temperature", type=float, default=None, help="Sampling temperature.")
+@click.option("--max-tokens", type=int, default=None, help="Max output tokens.")
+@click.option("--output", default=None, help="Save results to file (.json or .csv).")
+@click.option("--score", is_flag=True, default=False, help="Show similarity matrix.")
+@click.option("--judge", default=None, help="Model to use as judge.")
+@click.option(
+    "--judge-criteria", default=None, help="Comma-separated evaluation criteria."
+)
+def run(
+    prompt: str | None,
+    models: str,
+    prompt_file: str | None,
+    var: tuple[str, ...],
+    system: str | None,
+    temperature: float | None,
+    max_tokens: int | None,
+    output: str | None,
+    score: bool,
+    judge: str | None,
+    judge_criteria: str | None,
+) -> None:
+    if prompt_file:
+        with open(prompt_file) as f:
+            prompt_text = f.read().strip()
+    elif prompt:
+        prompt_text = prompt
+    else:
+        click.echo("Provide a prompt or --prompt-file.", err=True)
+        sys.exit(1)
+
+    if var:
+        variables: dict[str, str] = {}
+        for item in var:
+            if "=" not in item:
+                click.echo(
+                    f"Invalid --var format: {item!r}. Expected KEY=VALUE.", err=True
+                )
+                sys.exit(1)
+            key, _, value = item.partition("=")
+            variables[key.strip()] = value
+        try:
+            prompt_text = prompt_text.format_map(variables)
+        except KeyError as exc:
+            click.echo(f"Missing template variable: {exc}", err=True)
+            sys.exit(1)
+
+    from assayer.exporter import export
+    from assayer.judge import run_judge
+    from assayer.renderer import render_run
+    from assayer.runner import run_all
+    from assayer.scorer import compute_similarity
+
+    model_list = [m.strip() for m in models.split(",") if m.strip()]
+    results = asyncio.run(
+        run_all(
+            prompt_text,
+            model_list,
+            system=system,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+    )
+    similarity = compute_similarity(results) if score else None
+
+    criteria = (
+        [c.strip() for c in judge_criteria.split(",")] if judge_criteria else None
+    )
+    judge_result = (
+        asyncio.run(run_judge(prompt_text, results, judge, criteria)) if judge else None
+    )
+
+    render_run(prompt_text, results, similarity=similarity, judge_result=judge_result)
+
+    if output:
+        export(results, output)
+        click.echo(f"Results saved to {output}")
+
+
+@cli.group()
+def models_cmd() -> None:
+    pass
+
+
+cli.add_command(models_cmd, name="models")
+
+
+@models_cmd.command(name="list")
+def models_list() -> None:
+    for provider, names in _KNOWN_MODELS.items():
+        click.echo(f"\n{provider}")
+        for name in names:
+            click.echo(f"  {name}")
+
+
+@models_cmd.command(name="check")
+@click.argument("provider", required=False)
+def models_check(provider: str | None) -> None:
+    if provider == "ollama":
+        _check_ollama()
+        return
+
+    keys = {
+        "openai": "OPENAI_API_KEY",
+        "anthropic": "ANTHROPIC_API_KEY",
+        "gemini": "GEMINI_API_KEY",
+    }
+    for name, env_var in keys.items():
+        value = get_api_key(env_var)
+        status = "set" if value else "not set"
+        symbol = "+" if value else "-"
+        click.echo(f"  [{symbol}] {name}: {env_var} {status}")
+
+
+def _check_ollama() -> None:
+    try:
+        response = httpx.get("http://localhost:11434/api/tags", timeout=3.0)
+        response.raise_for_status()
+        data = response.json()
+        local_models: list[str] = [m["name"] for m in data.get("models", [])]
+        click.echo("Ollama is running.")
+        if local_models:
+            click.echo("Local models:")
+            for m in local_models:
+                click.echo(f"  ollama/{m}")
+        else:
+            click.echo("No local models found.")
+    except httpx.ConnectError:
+        click.echo("Ollama is not running at localhost:11434.", err=True)
+    except Exception as exc:
+        click.echo(f"Ollama check failed: {exc}", err=True)
+
+
+@cli.group()
+def config() -> None:
+    pass
+
+
+@config.command(name="set")
+@click.argument("key")
+@click.argument("value")
+def config_set(key: str, value: str) -> None:
+    set_api_key(key, value)
+    click.echo(f"{key} saved.")
+
+
+@config.command(name="show")
+def config_show() -> None:
+    data = show_config()
+    for key, value in data.items():
+        if value:
+            masked = value[:8] + "..." if len(value) > 8 else value
+            click.echo(f"  {key}: {masked}")
+        else:
+            click.echo(f"  {key}: not set")

assayer/config.py

@@ -0,0 +1,30 @@
+import json
+import os
+from pathlib import Path
+
+_CONFIG_PATH = Path.home() / ".assayer" / "config.json"
+
+_KNOWN_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY")
+
+
+def get_api_key(name: str) -> str | None:
+    value = os.environ.get(name)
+    if value:
+        return value
+    if _CONFIG_PATH.exists():
+        data: dict[str, str] = json.loads(_CONFIG_PATH.read_text())
+        return data.get(name)
+    return None
+
+
+def set_api_key(name: str, value: str) -> None:
+    _CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    data: dict[str, str] = {}
+    if _CONFIG_PATH.exists():
+        data = json.loads(_CONFIG_PATH.read_text())
+    data[name] = value
+    _CONFIG_PATH.write_text(json.dumps(data, indent=2))
+
+
+def show_config() -> dict[str, str | None]:
+    return {key: get_api_key(key) for key in _KNOWN_KEYS}

assayer/exporter.py

@@ -0,0 +1,32 @@
+import csv
+import json
+from pathlib import Path
+
+from assayer.models import ModelResult
+
+
+def _to_dict(result: ModelResult) -> dict:
+    return {
+        "model": result.model,
+        "output": result.output,
+        "tokens_input": result.tokens_input,
+        "tokens_output": result.tokens_output,
+        "latency_seconds": result.latency_seconds,
+        "cost_usd": result.cost_usd,
+        "error": result.error,
+    }
+
+
+def export(results: list[ModelResult], path: str) -> None:
+    dest = Path(path)
+    records = [_to_dict(r) for r in results]
+
+    if dest.suffix.lower() == ".csv":
+        with dest.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=list(records[0].keys()))
+            writer.writeheader()
+            writer.writerows(records)
+    else:
+        dest.write_text(
+            json.dumps(records, indent=2, ensure_ascii=False), encoding="utf-8"
+        )

assayer/judge.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+import re
+import sys
+from dataclasses import dataclass, field
+
+from assayer.models import ModelResult
+from assayer.runner import _make_provider
+
+_PROMPT_TEMPLATE = """\
+You are evaluating outputs from multiple language models for the same prompt.
+
+Original prompt: {prompt}
+
+Outputs:
+{outputs}
+{criteria_section}
+Return JSON only:
+{{
+  "winner": "model name",
+  "reasoning": "one paragraph",
+  "scores": {{"model_name": {{"criterion": score_1_to_5}}}}
+}}"""
+
+
+@dataclass
+class JudgeResult:
+    winner: str
+    reasoning: str
+    scores: dict[str, dict[str, float]] = field(default_factory=dict)
+
+
+def _build_prompt(
+    prompt: str, results: list[ModelResult], criteria: list[str] | None
+) -> str:
+    labels = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    outputs_block = "\n".join(
+        f"Model {labels[i]} ({r.model}): {r.output}"
+        for i, r in enumerate(results)
+        if not r.error
+    )
+
+    if criteria:
+        criteria_section = f"\nEvaluate on these criteria: {', '.join(criteria)}\n"
+    else:
+        criteria_section = ""
+
+    return _PROMPT_TEMPLATE.format(
+        prompt=prompt,
+        outputs=outputs_block,
+        criteria_section=criteria_section,
+    )
+
+
+def _parse_response(text: str) -> JudgeResult:
+    stripped = re.sub(r"^```(?:json)?\s*", "", text.strip(), flags=re.IGNORECASE)
+    stripped = re.sub(r"\s*```$", "", stripped)
+    data = json.loads(stripped)
+    return JudgeResult(
+        winner=data["winner"],
+        reasoning=data["reasoning"],
+        scores=data.get("scores", {}),
+    )
+
+
+async def run_judge(
+    prompt: str,
+    results: list[ModelResult],
+    judge_model: str,
+    criteria: list[str] | None = None,
+) -> JudgeResult | None:
+    valid = [r for r in results if not r.error and r.output]
+    if len(valid) < 2:
+        print("Judge skipped: fewer than 2 successful results.", file=sys.stderr)
+        return None
+
+    judge_prompt = _build_prompt(prompt, valid, criteria)
+    provider = _make_provider(judge_model)
+
+    try:
+        result = await provider.run(judge_prompt)
+    except Exception as exc:
+        print(f"Judge call failed: {exc}", file=sys.stderr)
+        return None
+
+    if result.error:
+        print(f"Judge call failed: {result.error}", file=sys.stderr)
+        return None
+
+    try:
+        return _parse_response(result.output)
+    except Exception as exc:
+        print(f"Judge response could not be parsed: {exc}", file=sys.stderr)
+        return None

assayer/models.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ModelResult:
+    model: str
+    output: str
+    tokens_input: int
+    tokens_output: int
+    latency_seconds: float
+    cost_usd: float
+    error: str | None = None

assayer/providers/anthropic.py

@@ -0,0 +1,73 @@
+import time
+
+import litellm
+
+from assayer.config import get_api_key
+from assayer.models import ModelResult
+from assayer.providers.base import BaseProvider
+
+
+class AnthropicProvider(BaseProvider):
+    def __init__(self, model: str) -> None:
+        self.model = model
+
+    async def run(
+        self,
+        prompt: str,
+        system: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+    ) -> ModelResult:
+        api_key = get_api_key("ANTHROPIC_API_KEY")
+        if not api_key:
+            return ModelResult(
+                model=self.model,
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=0.0,
+                cost_usd=0.0,
+                error="ANTHROPIC_API_KEY is not set",
+            )
+
+        messages: list[dict[str, str]] = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+
+        kwargs: dict = {
+            "model": f"anthropic/{self.model}",
+            "messages": messages,
+            "api_key": api_key,
+        }
+        if temperature is not None:
+            kwargs["temperature"] = temperature
+        if max_tokens is not None:
+            kwargs["max_tokens"] = max_tokens
+
+        start = time.perf_counter()
+        try:
+            response = await litellm.acompletion(**kwargs)
+        except Exception as exc:
+            return ModelResult(
+                model=self.model,
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=time.perf_counter() - start,
+                cost_usd=0.0,
+                error=str(exc),
+            )
+
+        latency = time.perf_counter() - start
+        usage = response.usage
+        cost = litellm.completion_cost(completion_response=response)
+
+        return ModelResult(
+            model=self.model,
+            output=response.choices[0].message.content or "",
+            tokens_input=usage.prompt_tokens,
+            tokens_output=usage.completion_tokens,
+            latency_seconds=latency,
+            cost_usd=cost,
+        )

assayer/providers/base.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from assayer.models import ModelResult
+
+
+class BaseProvider(ABC):
+    model: str
+
+    @abstractmethod
+    async def run(
+        self,
+        prompt: str,
+        system: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+    ) -> ModelResult: ...

assayer/providers/gemini.py

@@ -0,0 +1,73 @@
+import time
+
+import litellm
+
+from assayer.config import get_api_key
+from assayer.models import ModelResult
+from assayer.providers.base import BaseProvider
+
+
+class GeminiProvider(BaseProvider):
+    def __init__(self, model: str) -> None:
+        self.model = model
+
+    async def run(
+        self,
+        prompt: str,
+        system: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+    ) -> ModelResult:
+        api_key = get_api_key("GEMINI_API_KEY")
+        if not api_key:
+            return ModelResult(
+                model=self.model,
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=0.0,
+                cost_usd=0.0,
+                error="GEMINI_API_KEY is not set",
+            )
+
+        messages: list[dict[str, str]] = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+
+        kwargs: dict = {
+            "model": f"gemini/{self.model}",
+            "messages": messages,
+            "api_key": api_key,
+        }
+        if temperature is not None:
+            kwargs["temperature"] = temperature
+        if max_tokens is not None:
+            kwargs["max_tokens"] = max_tokens
+
+        start = time.perf_counter()
+        try:
+            response = await litellm.acompletion(**kwargs)
+        except Exception as exc:
+            return ModelResult(
+                model=self.model,
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=time.perf_counter() - start,
+                cost_usd=0.0,
+                error=str(exc),
+            )
+
+        latency = time.perf_counter() - start
+        usage = response.usage
+        cost = litellm.completion_cost(completion_response=response)
+
+        return ModelResult(
+            model=self.model,
+            output=response.choices[0].message.content or "",
+            tokens_input=usage.prompt_tokens,
+            tokens_output=usage.completion_tokens,
+            latency_seconds=latency,
+            cost_usd=cost,
+        )

assayer/providers/ollama.py

@@ -0,0 +1,76 @@
+import time
+
+import httpx
+
+from assayer.models import ModelResult
+from assayer.providers.base import BaseProvider
+
+_BASE_URL = "http://localhost:11434"
+
+
+async def is_running() -> bool:
+    try:
+        async with httpx.AsyncClient(timeout=3.0) as client:
+            response = await client.get(_BASE_URL)
+            return response.status_code == 200
+    except httpx.ConnectError:
+        return False
+
+
+class OllamaProvider(BaseProvider):
+    def __init__(self, model: str) -> None:
+        self.model = model.removeprefix("ollama/")
+
+    async def run(
+        self,
+        prompt: str,
+        system: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+    ) -> ModelResult:
+        messages: list[dict[str, str]] = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+
+        body: dict = {"model": self.model, "messages": messages, "stream": False}
+        if temperature is not None:
+            body.setdefault("options", {})["temperature"] = temperature
+        if max_tokens is not None:
+            body.setdefault("options", {})["num_predict"] = max_tokens
+
+        start = time.perf_counter()
+        try:
+            async with httpx.AsyncClient(timeout=60.0) as client:
+                response = await client.post(f"{_BASE_URL}/api/chat", json=body)
+                response.raise_for_status()
+                data = response.json()
+        except httpx.ConnectError:
+            return ModelResult(
+                model=f"ollama/{self.model}",
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=time.perf_counter() - start,
+                cost_usd=0.0,
+                error="Ollama is not running at localhost:11434",
+            )
+        except Exception as exc:
+            return ModelResult(
+                model=f"ollama/{self.model}",
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=time.perf_counter() - start,
+                cost_usd=0.0,
+                error=str(exc),
+            )
+
+        return ModelResult(
+            model=f"ollama/{self.model}",
+            output=data["message"]["content"],
+            tokens_input=data.get("prompt_eval_count", 0),
+            tokens_output=data.get("eval_count", 0),
+            latency_seconds=time.perf_counter() - start,
+            cost_usd=0.0,
+        )

assayer/providers/openai.py

@@ -0,0 +1,69 @@
+import time
+
+import litellm
+
+from assayer.config import get_api_key
+from assayer.models import ModelResult
+from assayer.providers.base import BaseProvider
+
+
+class OpenAIProvider(BaseProvider):
+    def __init__(self, model: str) -> None:
+        self.model = model
+
+    async def run(
+        self,
+        prompt: str,
+        system: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+    ) -> ModelResult:
+        api_key = get_api_key("OPENAI_API_KEY")
+        if not api_key:
+            return ModelResult(
+                model=self.model,
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=0.0,
+                cost_usd=0.0,
+                error="OPENAI_API_KEY is not set",
+            )
+
+        messages: list[dict[str, str]] = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+
+        kwargs: dict = {"model": self.model, "messages": messages, "api_key": api_key}
+        if temperature is not None:
+            kwargs["temperature"] = temperature
+        if max_tokens is not None:
+            kwargs["max_tokens"] = max_tokens
+
+        start = time.perf_counter()
+        try:
+            response = await litellm.acompletion(**kwargs)
+        except Exception as exc:
+            return ModelResult(
+                model=self.model,
+                output="",
+                tokens_input=0,
+                tokens_output=0,
+                latency_seconds=time.perf_counter() - start,
+                cost_usd=0.0,
+                error=str(exc),
+            )
+
+        latency = time.perf_counter() - start
+        usage = response.usage
+        cost = litellm.completion_cost(completion_response=response)
+
+        return ModelResult(
+            model=self.model,
+            output=response.choices[0].message.content or "",
+            tokens_input=usage.prompt_tokens,
+            tokens_output=usage.completion_tokens,
+            latency_seconds=latency,
+            cost_usd=cost,
+        )

assayer/renderer.py

@@ -0,0 +1,109 @@
+from rich.columns import Columns
+from rich.console import Console
+from rich.panel import Panel
+from rich.rule import Rule
+from rich.table import Table
+from rich.text import Text
+
+from assayer.judge import JudgeResult
+from assayer.models import ModelResult
+
+console = Console()
+
+
+def _stats_line(result: ModelResult) -> str:
+    parts = [
+        f"{result.tokens_input}in / {result.tokens_output}out tokens",
+        f"{result.latency_seconds:.2f}s",
+    ]
+    if result.cost_usd > 0:
+        parts.append(f"${result.cost_usd:.6f}")
+    return " | ".join(parts)
+
+
+def _make_panel(result: ModelResult) -> Panel:
+    if result.error:
+        content = Text(result.error, style="red")
+        subtitle = None
+    else:
+        content = Text(result.output)
+        subtitle = _stats_line(result)
+
+    return Panel(
+        content,
+        title=f"[bold]{result.model}[/bold]",
+        subtitle=subtitle,
+        expand=True,
+    )
+
+
+def render_run(
+    prompt: str,
+    results: list[ModelResult],
+    similarity: dict[tuple[str, str], float] | None = None,
+    judge_result: JudgeResult | None = None,
+) -> None:
+    header = prompt if len(prompt) <= 80 else prompt[:77] + "..."
+    console.print(Rule(f"[bold]{header}[/bold]"))
+
+    panels = [_make_panel(r) for r in results]
+    console.print(Columns(panels, equal=True, expand=True))
+
+    if similarity is not None:
+        render_similarity_matrix(results, similarity)
+
+    if judge_result is not None:
+        render_judge(judge_result)
+
+
+def render_similarity_matrix(
+    results: list[ModelResult],
+    similarity: dict[tuple[str, str], float],
+) -> None:
+    models = [r.model for r in results]
+    table = Table(title="Similarity Matrix", show_header=True)
+    table.add_column("", style="bold")
+    for m in models:
+        table.add_column(m, justify="center")
+
+    for row_model in models:
+        cells: list[str] = []
+        for col_model in models:
+            if row_model == col_model:
+                cells.append("1.000")
+            else:
+                key = (
+                    (row_model, col_model)
+                    if (row_model, col_model) in similarity
+                    else (col_model, row_model)
+                )
+                cells.append(f"{similarity.get(key, 0.0):.3f}")
+        table.add_row(row_model, *cells)
+
+    console.print(table)
+
+
+def render_judge(judge_result: JudgeResult) -> None:
+    console.print(Rule("[bold]Judge[/bold]"))
+    console.print(f"[bold green]Winner:[/bold green] {judge_result.winner}")
+    console.print(f"[dim]{judge_result.reasoning}[/dim]")
+
+    if not judge_result.scores:
+        return
+
+    all_criteria: list[str] = []
+    for criteria_map in judge_result.scores.values():
+        for c in criteria_map:
+            if c not in all_criteria:
+                all_criteria.append(c)
+
+    table = Table(title="Scores", show_header=True)
+    table.add_column("Model", style="bold")
+    for c in all_criteria:
+        table.add_column(c, justify="center")
+
+    for model, criteria_map in judge_result.scores.items():
+        row = [str(criteria_map.get(c, "")) for c in all_criteria]
+        table.add_row(model, *row)
+
+    console.print(table)

assayer/runner.py

@@ -0,0 +1,65 @@
+import asyncio
+
+from assayer.models import ModelResult
+from assayer.providers.anthropic import AnthropicProvider
+from assayer.providers.base import BaseProvider
+from assayer.providers.gemini import GeminiProvider
+from assayer.providers.ollama import OllamaProvider
+from assayer.providers.openai import OpenAIProvider
+
+
+_TIMEOUT = 30.0
+
+
+def _make_provider(model: str) -> BaseProvider:
+    if model.startswith("ollama/"):
+        return OllamaProvider(model)
+    if model.startswith("claude-"):
+        return AnthropicProvider(model)
+    if model.startswith("gemini-"):
+        return GeminiProvider(model)
+    return OpenAIProvider(model)
+
+
+async def _run_one(
+    model: str,
+    prompt: str,
+    system: str | None,
+    temperature: float | None,
+    max_tokens: int | None,
+) -> ModelResult:
+    provider = _make_provider(model)
+    try:
+        return await asyncio.wait_for(
+            provider.run(
+                prompt,
+                system=system,
+                temperature=temperature,
+                max_tokens=max_tokens,
+            ),
+            timeout=_TIMEOUT,
+        )
+    except TimeoutError:
+        return ModelResult(
+            model=model,
+            output="",
+            tokens_input=0,
+            tokens_output=0,
+            latency_seconds=30.0,
+            cost_usd=0.0,
+            error="Request timed out after 30 seconds",
+        )
+
+
+async def run_all(
+    prompt: str,
+    models: list[str],
+    system: str | None = None,
+    temperature: float | None = None,
+    max_tokens: int | None = None,
+) -> list[ModelResult]:
+    tasks = [
+        _run_one(model, prompt, system, temperature, max_tokens)
+        for model in models
+    ]
+    return list(await asyncio.gather(*tasks))

assayer/scorer.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from assayer.models import ModelResult
+
+_model = None
+
+
+def _get_model():
+    global _model
+    if _model is None:
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError:
+            raise ImportError(
+                "sentence-transformers is required for --score. "
+                "Install it with: pip install 'assayer[score]'"
+            )
+        _model = SentenceTransformer("all-MiniLM-L6-v2")
+    return _model
+
+
+def compute_similarity(results: list[ModelResult]) -> dict[tuple[str, str], float]:
+    valid = [r for r in results if not r.error and r.output]
+    if len(valid) < 2:
+        return {}
+
+    import numpy as np
+
+    embeddings = _get_model().encode([r.output for r in valid], convert_to_numpy=True)
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+    normalized = embeddings / np.where(norms == 0, 1, norms)
+    similarity: dict[tuple[str, str], float] = {}
+
+    for i in range(len(valid)):
+        for j in range(i + 1, len(valid)):
+            score = float(np.dot(normalized[i], normalized[j]))
+            similarity[(valid[i].model, valid[j].model)] = score
+
+    return similarity
+
+
+def readability_stats(text: str) -> dict[str, float]:
+    sentences = [
+        s
+        for s in text.replace("!", ".").replace("?", ".").split(".")
+        if s.strip()
+    ]
+    words = text.split()
+    word_count = len(words)
+    sentence_count = len(sentences) or 1
+    return {
+        "word_count": float(word_count),
+        "sentence_count": float(sentence_count),
+        "avg_sentence_length": word_count / sentence_count,
+    }

assayer-0.1.0.dist-info/METADATA

@@ -0,0 +1,186 @@
+Metadata-Version: 2.4
+Name: assayer
+Version: 0.1.0
+Summary: Run a prompt across multiple LLMs and compare outputs side by side in the terminal.
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.1
+Requires-Dist: rich>=13.0
+Requires-Dist: litellm>=1.40
+Requires-Dist: httpx>=0.27
+Provides-Extra: score
+Requires-Dist: sentence-transformers>=3.0; extra == "score"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: sentence-transformers>=3.0; extra == "dev"
+Dynamic: license-file
+
+# assayer
+
+Send a prompt to multiple language models in parallel and compare their outputs in the terminal. Useful for evaluating which model handles a given task better, measuring semantic similarity between responses, or running an LLM-as-judge evaluation — without leaving the shell.
+
+## Installation
+
+```bash
+pip install assayer
+```
+
+Similarity scoring requires the optional `score` extra:
+
+```bash
+pip install "assayer[score]"
+```
+
+Python 3.11 or newer is required.
+
+> **Contributing?** See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, code style, and PR guidelines.
+
+## Supported Providers
+
+- **OpenAI**: All GPT models.
+- **Anthropic**: Claude 4.5 models (Opus, Sonnet, Haiku).
+- **Google Gemini**: 1.5 Pro and Flash models.
+- **Ollama**: Local models running on your machine.
+
+## Configuration
+
+Assayer looks for API keys in environment variables or a configuration file at `~/.assayer/config.json`.
+
+### Environment Variables
+
+```bash
+export OPENAI_API_KEY="your-key"
+export ANTHROPIC_API_KEY="your-key"
+export GEMINI_API_KEY="your-key"
+```
+
+### Configuration File
+
+```json
+{
+  "OPENAI_API_KEY": "sk-...",
+  "ANTHROPIC_API_KEY": "sk-ant-...",
+  "GEMINI_API_KEY": "..."
+}
+```
+
+Use `assayer models check` to verify your configuration.
+
+## Quickstart
+
+```bash
+assayer run "Explain recursion in one sentence." --models gpt-4o,claude-haiku-4-5-20251001
+```
+
+## Commands
+
+### run
+
+```bash
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5
+assayer run --prompt-file prompt.txt --models gpt-4o,ollama/llama3
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --score
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --judge gpt-4o --judge-criteria "clarity,brevity"
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --output results.json
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --output results.csv
+assayer run "prompt with {var}" --models gpt-4o --var key=value
+```
+
+| Flag | Description |
+|---|---|
+| `--models` | Comma-separated model identifiers (required) |
+| `--prompt-file` | Path to a `.txt` file instead of an inline prompt |
+| `--var` | `KEY=VALUE` template variable, repeatable |
+| `--system` | System prompt applied to all models |
+| `--temperature` | Sampling temperature |
+| `--max-tokens` | Maximum output tokens |
+| `--score` | Show pairwise similarity matrix |
+| `--judge` | Model to use as judge |
+| `--judge-criteria` | Comma-separated criteria for the judge |
+| `--output` | Save results to `.json` or `.csv` |
+
+### models
+
+```bash
+assayer models list               # list all supported model identifiers
+assayer models check              # check which API keys are configured
+assayer models check ollama       # check if Ollama is running and list local models
+```
+
+### config
+
+```bash
+assayer config set OPENAI_API_KEY sk-...
+assayer config show
+```
+
+Keys are saved to `~/.assayer/config.json`. Environment variables take precedence.
+
+## Providers
+
+### OpenAI
+
+```bash
+export OPENAI_API_KEY=sk-...
+```
+
+Supported models: `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo`, `o1-mini`
+
+### Anthropic
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+Supported models: `claude-opus-4-5`, `claude-sonnet-4-5`, `claude-haiku-4-5-20251001`
+
+### Google Gemini
+
+```bash
+export GEMINI_API_KEY=...
+```
+
+Supported models: `gemini-1.5-pro`, `gemini-1.5-flash`
+
+### Ollama (local)
+
+No API key needed. Start Ollama and use the `ollama/` prefix:
+
+```bash
+ollama serve
+assayer run "prompt" --models ollama/llama3,ollama/mistral,ollama/phi3
+```
+
+## Scoring
+
+`--score` embeds all outputs using `all-MiniLM-L6-v2` (runs locally, no API call) and displays a pairwise cosine similarity matrix. Values range from 0 (unrelated) to 1 (identical meaning).
+
+## LLM-as-judge
+
+`--judge <model>` sends all outputs to the specified model and asks it to pick a winner. Use `--judge-criteria` to focus the evaluation:
+
+```bash
+assayer run "Write a sorting algorithm." \
+  --models gpt-4o,claude-sonnet-4-5 \
+  --judge gpt-4o \
+  --judge-criteria "correctness,readability"
+```
+
+If the judge call fails, a warning is printed to stderr and the run continues normally.
+
+## Export
+
+`--output results.json` saves full results as JSON. `--output results.csv` saves as CSV. The file format is determined by the extension.

assayer-0.1.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+assayer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+assayer/config.py,sha256=cirsxcjt4JR75yL5-ZMKiaGhD_PAD93mUlBzBA9dqXg,837
+assayer/exporter.py,sha256=vGL4FzpRzYABWDcZqQuBjSnZtFv3rtm-dahiIzrX3wI,961
+assayer/judge.py,sha256=XYRmp4awujtdPd_6ba0MWJFkGzZKq8ihuBWgwtlC0I4,2573
+assayer/models.py,sha256=9Bag_3NEmjP1hn5P5qwXJK56cGZOaCnXWcSlxS_TY6k,218
+assayer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+assayer/renderer.py,sha256=uC-EMOUjIrfP4BUjWZQ3dbi2YXyu2oNDYgfT5CSmipE,3281
+assayer/runner.py,sha256=zaETgCvt_hZYk6H71uSH_55WZvVh1dLdw6WxvTg8CpE,1794
+assayer/scorer.py,sha256=OCNrWSuArV3Hn4dW4IvI2ed6mVLyUzrJulCXhtBxMrE,1707
+assayer/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+assayer/cli/main.py,sha256=Y7Q208JYJCyaHvDpgKqespcNld0iVnZFz_dA-OzCvUw,5802
+assayer/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+assayer/providers/anthropic.py,sha256=iREHmTARU49-yELK8xgwxde4c9p0rwdyPFOWRoAHq1I,2284
+assayer/providers/base.py,sha256=R3Yq4U3faK-eKxTrgt2vU6G_M-akFH8cAeIkkACwMS4,446
+assayer/providers/gemini.py,sha256=zIR-tMhh7wcXs11h8Ua_pHs7z1g4jBtVadr0Fxblikc,2272
+assayer/providers/ollama.py,sha256=0OatUWJrrHxbStldOjuxb7aTfQg6oT4Hfc0l2GtwFbs,2615
+assayer/providers/openai.py,sha256=4rC00QQyLrrURv-LneBo1kDXSuAxlr_scIp0BUmM0SA,2209
+assayer-0.1.0.dist-info/licenses/LICENSE,sha256=-mYCe4ncMaROc8WQ2awjMSHkEL9IyAhMYCJzxvW1aSI,1092
+assayer-0.1.0.dist-info/METADATA,sha256=RAdeXKtUoDJ3yDat5Cj6f5jxgSgBM3Z0gzFHpDry3CI,5524
+assayer-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+assayer-0.1.0.dist-info/entry_points.txt,sha256=sUmkqtoP0kEWrlY3XFM8ANePDEXiLJejT_z_h1BsHGo,49
+assayer-0.1.0.dist-info/top_level.txt,sha256=qQsw6NFoN_TjLkm9VW8IGeyAfronLQy4Yj4E6vP-6R8,8
+assayer-0.1.0.dist-info/RECORD,,

assayer-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

assayer-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+assayer = assayer.cli.main:cli

assayer-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Practical Mind
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

assayer-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+assayer