assayer 0.1.1__tar.gz

assayer-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Practical Mind
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

assayer-0.1.1/PKG-INFO

@@ -0,0 +1,186 @@
+Metadata-Version: 2.4
+Name: assayer
+Version: 0.1.1
+Summary: Run a prompt across multiple LLMs and compare outputs side by side in the terminal.
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.1
+Requires-Dist: rich>=13.0
+Requires-Dist: litellm>=1.40
+Requires-Dist: httpx>=0.27
+Provides-Extra: score
+Requires-Dist: sentence-transformers>=3.0; extra == "score"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: sentence-transformers>=3.0; extra == "dev"
+Dynamic: license-file
+
+# assayer
+
+Send a prompt to multiple language models in parallel and compare their outputs in the terminal. Useful for evaluating which model handles a given task better, measuring semantic similarity between responses, or running an LLM-as-judge evaluation — without leaving the shell.
+
+## Installation
+
+```bash
+pip install assayer
+```
+
+Similarity scoring requires the optional `score` extra:
+
+```bash
+pip install "assayer[score]"
+```
+
+Python 3.11 or newer is required.
+
+> **Contributing?** See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, code style, and PR guidelines.
+
+## Supported Providers
+
+- **OpenAI**: All GPT models.
+- **Anthropic**: Claude 4.5 models (Opus, Sonnet, Haiku).
+- **Google Gemini**: 1.5 Pro and Flash models.
+- **Ollama**: Local models running on your machine.
+
+## Configuration
+
+Assayer looks for API keys in environment variables or a configuration file at `~/.assayer/config.json`.
+
+### Environment Variables
+
+```bash
+export OPENAI_API_KEY="your-key"
+export ANTHROPIC_API_KEY="your-key"
+export GEMINI_API_KEY="your-key"
+```
+
+### Configuration File
+
+```json
+{
+  "OPENAI_API_KEY": "sk-...",
+  "ANTHROPIC_API_KEY": "sk-ant-...",
+  "GEMINI_API_KEY": "..."
+}
+```
+
+Use `assayer models check` to verify your configuration.
+
+## Quickstart
+
+```bash
+assayer run "Explain recursion in one sentence." --models gpt-4o,claude-haiku-4-5-20251001
+```
+
+## Commands
+
+### run
+
+```bash
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5
+assayer run --prompt-file prompt.txt --models gpt-4o,ollama/llama3
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --score
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --judge gpt-4o --judge-criteria "clarity,brevity"
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --output results.json
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --output results.csv
+assayer run "prompt with {var}" --models gpt-4o --var key=value
+```
+
+| Flag | Description |
+|---|---|
+| `--models` | Comma-separated model identifiers (required) |
+| `--prompt-file` | Path to a `.txt` file instead of an inline prompt |
+| `--var` | `KEY=VALUE` template variable, repeatable |
+| `--system` | System prompt applied to all models |
+| `--temperature` | Sampling temperature |
+| `--max-tokens` | Maximum output tokens |
+| `--score` | Show pairwise similarity matrix |
+| `--judge` | Model to use as judge |
+| `--judge-criteria` | Comma-separated criteria for the judge |
+| `--output` | Save results to `.json` or `.csv` |
+
+### models
+
+```bash
+assayer models list               # list all supported model identifiers
+assayer models check              # check which API keys are configured
+assayer models check ollama       # check if Ollama is running and list local models
+```
+
+### config
+
+```bash
+assayer config set OPENAI_API_KEY sk-...
+assayer config show
+```
+
+Keys are saved to `~/.assayer/config.json`. Environment variables take precedence.
+
+## Providers
+
+### OpenAI
+
+```bash
+export OPENAI_API_KEY=sk-...
+```
+
+Supported models: `gpt-5.5`, `gpt-5.5-pro`, `gpt-5.4`, `gpt-5.4-pro`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.2`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `gpt-4o`, `gpt-4o-mini`, `o3`, `o3-mini`, `o4-mini`
+
+### Anthropic
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+Supported models: `claude-opus-4-7`, `claude-sonnet-4-6`, `claude-haiku-4-5-20251001`, `claude-opus-4-6`, `claude-sonnet-4-5`, `claude-opus-4-5`
+
+### Google Gemini
+
+```bash
+export GEMINI_API_KEY=...
+```
+
+Supported models: `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite`, `gemini-3-flash-preview`, `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.0-flash`, `gemini-2.0-flash-lite`
+
+### Ollama (local)
+
+No API key needed. Start Ollama and use the `ollama/` prefix:
+
+```bash
+ollama serve
+assayer run "prompt" --models ollama/llama4-scout,ollama/llama3.2,ollama/qwen3
+```
+
+## Scoring
+
+`--score` embeds all outputs using `all-MiniLM-L6-v2` (runs locally, no API call) and displays a pairwise cosine similarity matrix. Values range from 0 (unrelated) to 1 (identical meaning).
+
+## LLM-as-judge
+
+`--judge <model>` sends all outputs to the specified model and asks it to pick a winner. Use `--judge-criteria` to focus the evaluation:
+
+```bash
+assayer run "Write a sorting algorithm." \
+  --models gpt-4o,claude-sonnet-4-5 \
+  --judge gpt-4o \
+  --judge-criteria "correctness,readability"
+```
+
+If the judge call fails, a warning is printed to stderr and the run continues normally.
+
+## Export
+
+`--output results.json` saves full results as JSON. `--output results.csv` saves as CSV. The file format is determined by the extension.

assayer-0.1.1/README.md

@@ -0,0 +1,156 @@
+# assayer
+
+Send a prompt to multiple language models in parallel and compare their outputs in the terminal. Useful for evaluating which model handles a given task better, measuring semantic similarity between responses, or running an LLM-as-judge evaluation — without leaving the shell.
+
+## Installation
+
+```bash
+pip install assayer
+```
+
+Similarity scoring requires the optional `score` extra:
+
+```bash
+pip install "assayer[score]"
+```
+
+Python 3.11 or newer is required.
+
+> **Contributing?** See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, code style, and PR guidelines.
+
+## Supported Providers
+
+- **OpenAI**: All GPT models.
+- **Anthropic**: Claude 4.5 models (Opus, Sonnet, Haiku).
+- **Google Gemini**: 1.5 Pro and Flash models.
+- **Ollama**: Local models running on your machine.
+
+## Configuration
+
+Assayer looks for API keys in environment variables or a configuration file at `~/.assayer/config.json`.
+
+### Environment Variables
+
+```bash
+export OPENAI_API_KEY="your-key"
+export ANTHROPIC_API_KEY="your-key"
+export GEMINI_API_KEY="your-key"
+```
+
+### Configuration File
+
+```json
+{
+  "OPENAI_API_KEY": "sk-...",
+  "ANTHROPIC_API_KEY": "sk-ant-...",
+  "GEMINI_API_KEY": "..."
+}
+```
+
+Use `assayer models check` to verify your configuration.
+
+## Quickstart
+
+```bash
+assayer run "Explain recursion in one sentence." --models gpt-4o,claude-haiku-4-5-20251001
+```
+
+## Commands
+
+### run
+
+```bash
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5
+assayer run --prompt-file prompt.txt --models gpt-4o,ollama/llama3
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --score
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --judge gpt-4o --judge-criteria "clarity,brevity"
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --output results.json
+assayer run "prompt" --models gpt-4o,claude-sonnet-4-5 --output results.csv
+assayer run "prompt with {var}" --models gpt-4o --var key=value
+```
+
+| Flag | Description |
+|---|---|
+| `--models` | Comma-separated model identifiers (required) |
+| `--prompt-file` | Path to a `.txt` file instead of an inline prompt |
+| `--var` | `KEY=VALUE` template variable, repeatable |
+| `--system` | System prompt applied to all models |
+| `--temperature` | Sampling temperature |
+| `--max-tokens` | Maximum output tokens |
+| `--score` | Show pairwise similarity matrix |
+| `--judge` | Model to use as judge |
+| `--judge-criteria` | Comma-separated criteria for the judge |
+| `--output` | Save results to `.json` or `.csv` |
+
+### models
+
+```bash
+assayer models list               # list all supported model identifiers
+assayer models check              # check which API keys are configured
+assayer models check ollama       # check if Ollama is running and list local models
+```
+
+### config
+
+```bash
+assayer config set OPENAI_API_KEY sk-...
+assayer config show
+```
+
+Keys are saved to `~/.assayer/config.json`. Environment variables take precedence.
+
+## Providers
+
+### OpenAI
+
+```bash
+export OPENAI_API_KEY=sk-...
+```
+
+Supported models: `gpt-5.5`, `gpt-5.5-pro`, `gpt-5.4`, `gpt-5.4-pro`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.2`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `gpt-4o`, `gpt-4o-mini`, `o3`, `o3-mini`, `o4-mini`
+
+### Anthropic
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+Supported models: `claude-opus-4-7`, `claude-sonnet-4-6`, `claude-haiku-4-5-20251001`, `claude-opus-4-6`, `claude-sonnet-4-5`, `claude-opus-4-5`
+
+### Google Gemini
+
+```bash
+export GEMINI_API_KEY=...
+```
+
+Supported models: `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite`, `gemini-3-flash-preview`, `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.0-flash`, `gemini-2.0-flash-lite`
+
+### Ollama (local)
+
+No API key needed. Start Ollama and use the `ollama/` prefix:
+
+```bash
+ollama serve
+assayer run "prompt" --models ollama/llama4-scout,ollama/llama3.2,ollama/qwen3
+```
+
+## Scoring
+
+`--score` embeds all outputs using `all-MiniLM-L6-v2` (runs locally, no API call) and displays a pairwise cosine similarity matrix. Values range from 0 (unrelated) to 1 (identical meaning).
+
+## LLM-as-judge
+
+`--judge <model>` sends all outputs to the specified model and asks it to pick a winner. Use `--judge-criteria` to focus the evaluation:
+
+```bash
+assayer run "Write a sorting algorithm." \
+  --models gpt-4o,claude-sonnet-4-5 \
+  --judge gpt-4o \
+  --judge-criteria "correctness,readability"
+```
+
+If the judge call fails, a warning is printed to stderr and the run continues normally.
+
+## Export
+
+`--output results.json` saves full results as JSON. `--output results.csv` saves as CSV. The file format is determined by the extension.

assayer-0.1.1/assayer/cli/main.py

@@ -0,0 +1,209 @@
+import asyncio
+import logging
+import sys
+
+import click
+import httpx
+
+from assayer.config import get_api_key, set_api_key, show_config
+
+logging.getLogger("LiteLLM").addFilter(
+    type("_F", (logging.Filter,), {"filter": lambda _, r: r.levelno >= logging.ERROR})()
+)
+
+_KNOWN_MODELS: dict[str, list[str]] = {
+    "openai": [
+        "gpt-5.5", "gpt-5.5-pro",
+        "gpt-5.4", "gpt-5.4-pro", "gpt-5.4-mini", "gpt-5.4-nano",
+        "gpt-5.2", "gpt-5", "gpt-5-mini", "gpt-5-nano",
+        "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano",
+        "gpt-4o", "gpt-4o-mini",
+        "o3", "o3-mini", "o4-mini",
+    ],
+    "anthropic": [
+        "claude-opus-4-7", "claude-sonnet-4-6", "claude-haiku-4-5-20251001",
+        "claude-opus-4-6", "claude-sonnet-4-5", "claude-opus-4-5",
+    ],
+    "gemini": [
+        "gemini-3.1-pro-preview", "gemini-3.1-flash-lite", "gemini-3-flash-preview",
+        "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite",
+        "gemini-2.0-flash", "gemini-2.0-flash-lite",
+    ],
+    "ollama": ["ollama/llama4-scout", "ollama/llama3.2", "ollama/qwen3", "ollama/gemma4", "ollama/mistral", "ollama/deepseek-r1", "ollama/phi4"],
+}
+
+
+@click.group()
+def cli() -> None:
+    pass
+
+
+@cli.command()
+@click.argument("prompt", required=False)
+@click.option("--models", required=True, help="Comma-separated model identifiers.")
+@click.option(
+    "--prompt-file",
+    type=click.Path(exists=True),
+    help="Path to a .txt prompt file.",
+)
+@click.option(
+    "--var",
+    multiple=True,
+    metavar="KEY=VALUE",
+    help="Template variables, repeatable.",
+)
+@click.option("--system", default=None, help="System prompt applied to all models.")
+@click.option("--temperature", type=float, default=None, help="Sampling temperature.")
+@click.option("--max-tokens", type=int, default=None, help="Max output tokens.")
+@click.option("--output", default=None, help="Save results to file (.json or .csv).")
+@click.option("--score", is_flag=True, default=False, help="Show similarity matrix.")
+@click.option("--judge", default=None, help="Model to use as judge.")
+@click.option(
+    "--judge-criteria", default=None, help="Comma-separated evaluation criteria."
+)
+def run(
+    prompt: str | None,
+    models: str,
+    prompt_file: str | None,
+    var: tuple[str, ...],
+    system: str | None,
+    temperature: float | None,
+    max_tokens: int | None,
+    output: str | None,
+    score: bool,
+    judge: str | None,
+    judge_criteria: str | None,
+) -> None:
+    if prompt_file:
+        with open(prompt_file) as f:
+            prompt_text = f.read().strip()
+    elif prompt:
+        prompt_text = prompt
+    else:
+        click.echo("Provide a prompt or --prompt-file.", err=True)
+        sys.exit(1)
+
+    if var:
+        variables: dict[str, str] = {}
+        for item in var:
+            if "=" not in item:
+                click.echo(
+                    f"Invalid --var format: {item!r}. Expected KEY=VALUE.", err=True
+                )
+                sys.exit(1)
+            key, _, value = item.partition("=")
+            variables[key.strip()] = value
+        try:
+            prompt_text = prompt_text.format_map(variables)
+        except KeyError as exc:
+            click.echo(f"Missing template variable: {exc}", err=True)
+            sys.exit(1)
+
+    from assayer.exporter import export
+    from assayer.judge import run_judge
+    from assayer.renderer import render_run
+    from assayer.runner import run_all
+    from assayer.scorer import compute_similarity
+
+    model_list = [m.strip() for m in models.split(",") if m.strip()]
+    results = asyncio.run(
+        run_all(
+            prompt_text,
+            model_list,
+            system=system,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+    )
+    similarity = compute_similarity(results) if score else None
+
+    criteria = (
+        [c.strip() for c in judge_criteria.split(",")] if judge_criteria else None
+    )
+    judge_result = (
+        asyncio.run(run_judge(prompt_text, results, judge, criteria)) if judge else None
+    )
+
+    render_run(prompt_text, results, similarity=similarity, judge_result=judge_result)
+
+    if output:
+        export(results, output)
+        click.echo(f"Results saved to {output}")
+
+
+@cli.group()
+def models_cmd() -> None:
+    pass
+
+
+cli.add_command(models_cmd, name="models")
+
+
+@models_cmd.command(name="list")
+def models_list() -> None:
+    for provider, names in _KNOWN_MODELS.items():
+        click.echo(f"\n{provider}")
+        for name in names:
+            click.echo(f"  {name}")
+
+
+@models_cmd.command(name="check")
+@click.argument("provider", required=False)
+def models_check(provider: str | None) -> None:
+    if provider == "ollama":
+        _check_ollama()
+        return
+
+    keys = {
+        "openai": "OPENAI_API_KEY",
+        "anthropic": "ANTHROPIC_API_KEY",
+        "gemini": "GEMINI_API_KEY",
+    }
+    for name, env_var in keys.items():
+        value = get_api_key(env_var)
+        status = "set" if value else "not set"
+        symbol = "+" if value else "-"
+        click.echo(f"  [{symbol}] {name}: {env_var} {status}")
+
+
+def _check_ollama() -> None:
+    try:
+        response = httpx.get("http://localhost:11434/api/tags", timeout=3.0)
+        response.raise_for_status()
+        data = response.json()
+        local_models: list[str] = [m["name"] for m in data.get("models", [])]
+        click.echo("Ollama is running.")
+        if local_models:
+            click.echo("Local models:")
+            for m in local_models:
+                click.echo(f"  ollama/{m}")
+        else:
+            click.echo("No local models found.")
+    except httpx.ConnectError:
+        click.echo("Ollama is not running at localhost:11434.", err=True)
+    except Exception as exc:
+        click.echo(f"Ollama check failed: {exc}", err=True)
+
+
+@cli.group()
+def config() -> None:
+    pass
+
+
+@config.command(name="set")
+@click.argument("key")
+@click.argument("value")
+def config_set(key: str, value: str) -> None:
+    set_api_key(key, value)
+    click.echo(f"{key} saved.")
+
+
+@config.command(name="show")
+def config_show() -> None:
+    data = show_config()
+    for key, value in data.items():
+        if value:
+            masked = value[:8] + "..." if len(value) > 8 else value
+            click.echo(f"  {key}: {masked}")
+        else:
+            click.echo(f"  {key}: not set")

assayer-0.1.1/assayer/config.py

@@ -0,0 +1,30 @@
+import json
+import os
+from pathlib import Path
+
+_CONFIG_PATH = Path.home() / ".assayer" / "config.json"
+
+_KNOWN_KEYS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY")
+
+
+def get_api_key(name: str) -> str | None:
+    value = os.environ.get(name)
+    if value:
+        return value
+    if _CONFIG_PATH.exists():
+        data: dict[str, str] = json.loads(_CONFIG_PATH.read_text())
+        return data.get(name)
+    return None
+
+
+def set_api_key(name: str, value: str) -> None:
+    _CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    data: dict[str, str] = {}
+    if _CONFIG_PATH.exists():
+        data = json.loads(_CONFIG_PATH.read_text())
+    data[name] = value
+    _CONFIG_PATH.write_text(json.dumps(data, indent=2))
+
+
+def show_config() -> dict[str, str | None]:
+    return {key: get_api_key(key) for key in _KNOWN_KEYS}

assayer-0.1.1/assayer/exporter.py

@@ -0,0 +1,32 @@
+import csv
+import json
+from pathlib import Path
+
+from assayer.models import ModelResult
+
+
+def _to_dict(result: ModelResult) -> dict:
+    return {
+        "model": result.model,
+        "output": result.output,
+        "tokens_input": result.tokens_input,
+        "tokens_output": result.tokens_output,
+        "latency_seconds": result.latency_seconds,
+        "cost_usd": result.cost_usd,
+        "error": result.error,
+    }
+
+
+def export(results: list[ModelResult], path: str) -> None:
+    dest = Path(path)
+    records = [_to_dict(r) for r in results]
+
+    if dest.suffix.lower() == ".csv":
+        with dest.open("w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=list(records[0].keys()))
+            writer.writeheader()
+            writer.writerows(records)
+    else:
+        dest.write_text(
+            json.dumps(records, indent=2, ensure_ascii=False), encoding="utf-8"
+        )