promptlab-cli 0.1.0__py3-none-any.whl

promptlab/__init__.py

@@ -0,0 +1,3 @@
+"""promptlab: Automated testing for LLM prompts."""
+
+__version__ = "0.1.0"

promptlab/assertions.py

@@ -0,0 +1,182 @@
+"""Assertion types for evaluating LLM outputs."""
+
+import re
+
+from promptlab.providers import call_llm
+
+
+def check_assertion(assertion: dict, output: str, model: str) -> dict:
+    """Check a single assertion against an LLM output.
+
+    Returns:
+        dict with keys: passed (bool), type, expected, got, message
+    """
+    atype = assertion["type"]
+    value = assertion.get("value")
+
+    checkers = {
+        "contains": _check_contains,
+        "not_contains": _check_not_contains,
+        "starts_with": _check_starts_with,
+        "regex": _check_regex,
+        "equals": _check_equals,
+        "max_tokens": _check_max_tokens,
+        "min_length": _check_min_length,
+        "max_length": _check_max_length,
+        "llm_judge": _check_llm_judge,
+    }
+
+    checker = checkers.get(atype)
+    if checker is None:
+        return {
+            "passed": False,
+            "type": atype,
+            "expected": value,
+            "got": None,
+            "message": f"Unknown assertion type: {atype}",
+        }
+
+    if atype == "llm_judge":
+        return checker(output, value, model)
+    return checker(output, value)
+
+
+def _check_contains(output: str, value: str) -> dict:
+    passed = value.lower() in output.lower()
+    return {
+        "passed": passed,
+        "type": "contains",
+        "expected": f'contains "{value}"',
+        "got": output[:200] + "..." if len(output) > 200 else output,
+        "message": "" if passed else f'Output does not contain "{value}"',
+    }
+
+
+def _check_not_contains(output: str, value: str) -> dict:
+    passed = value.lower() not in output.lower()
+    return {
+        "passed": passed,
+        "type": "not_contains",
+        "expected": f'does not contain "{value}"',
+        "got": output[:200] + "..." if len(output) > 200 else output,
+        "message": "" if passed else f'Output contains "{value}" (should not)',
+    }
+
+
+def _check_starts_with(output: str, value: str) -> dict:
+    passed = output.strip().lower().startswith(value.lower())
+    return {
+        "passed": passed,
+        "type": "starts_with",
+        "expected": f'starts with "{value}"',
+        "got": output[:100],
+        "message": "" if passed else f'Output does not start with "{value}"',
+    }
+
+
+def _check_regex(output: str, value: str) -> dict:
+    try:
+        passed = bool(re.search(value, output))
+    except re.error as e:
+        return {
+            "passed": False,
+            "type": "regex",
+            "expected": f"matches /{value}/",
+            "got": None,
+            "message": f"Invalid regex: {e}",
+        }
+    return {
+        "passed": passed,
+        "type": "regex",
+        "expected": f"matches /{value}/",
+        "got": output[:200] + "..." if len(output) > 200 else output,
+        "message": "" if passed else f"Output does not match regex /{value}/",
+    }
+
+
+def _check_equals(output: str, value: str) -> dict:
+    passed = output.strip() == value.strip()
+    return {
+        "passed": passed,
+        "type": "equals",
+        "expected": value,
+        "got": output.strip(),
+        "message": "" if passed else "Output does not exactly match expected value",
+    }
+
+
+def _check_max_tokens(output: str, value: int) -> dict:
+    # Rough token estimate: ~4 chars per token
+    token_estimate = len(output.split())
+    passed = token_estimate <= value
+    return {
+        "passed": passed,
+        "type": "max_tokens",
+        "expected": f"<= {value} tokens",
+        "got": f"~{token_estimate} tokens ({len(output)} chars)",
+        "message": "" if passed else f"Output has ~{token_estimate} tokens, max is {value}",
+    }
+
+
+def _check_min_length(output: str, value: int) -> dict:
+    length = len(output.strip())
+    passed = length >= value
+    return {
+        "passed": passed,
+        "type": "min_length",
+        "expected": f">= {value} chars",
+        "got": f"{length} chars",
+        "message": "" if passed else f"Output is {length} chars, minimum is {value}",
+    }
+
+
+def _check_max_length(output: str, value: int) -> dict:
+    length = len(output.strip())
+    passed = length <= value
+    return {
+        "passed": passed,
+        "type": "max_length",
+        "expected": f"<= {value} chars",
+        "got": f"{length} chars",
+        "message": "" if passed else f"Output is {length} chars, maximum is {value}",
+    }
+
+
+def _check_llm_judge(output: str, criteria: str, model: str) -> dict:
+    """Use an LLM to judge the output quality."""
+    judge_prompt = f"""You are evaluating an LLM output. Answer only YES or NO.
+
+Criteria: {criteria}
+
+Output to evaluate:
+---
+{output}
+---
+
+Does the output meet the criteria? Answer only YES or NO."""
+
+    try:
+        response = call_llm(
+            prompt=judge_prompt,
+            model=model,
+            temperature=0,
+            max_tokens=10,
+        )
+        answer = response["text"].strip().upper()
+        passed = answer.startswith("YES")
+    except Exception as e:
+        return {
+            "passed": False,
+            "type": "llm_judge",
+            "expected": criteria,
+            "got": f"Judge error: {e}",
+            "message": f"LLM judge failed: {e}",
+        }
+
+    return {
+        "passed": passed,
+        "type": "llm_judge",
+        "expected": criteria,
+        "got": f"Judge answered: {answer}",
+        "message": "" if passed else f"LLM judge said NO to: {criteria}",
+    }

promptlab/cli.py

@@ -0,0 +1,81 @@
+"""CLI entry point for promptlab."""
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import click
+
+from promptlab.loader import load_test_files
+from promptlab.runner import run_all_tests
+from promptlab.reporter import print_results, print_summary
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def main():
+    """promptlab — Automated testing for LLM prompts."""
+    pass
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True))
+@click.option("--verbose", "-v", is_flag=True, help="Show full LLM responses")
+@click.option("--json-output", "--json", "json_out", is_flag=True, help="Output results as JSON")
+@click.option("--dry-run", is_flag=True, help="Show tests without calling APIs")
+def run(path: str, verbose: bool, json_out: bool, dry_run: bool):
+    """Run prompt tests from a file or directory."""
+    target = Path(path)
+
+    # Collect test files
+    if target.is_file():
+        files = [target]
+    elif target.is_dir():
+        files = sorted(target.glob("**/*.yaml")) + sorted(target.glob("**/*.yml"))
+    else:
+        click.echo(f"Error: {path} is not a file or directory", err=True)
+        sys.exit(1)
+
+    if not files:
+        click.echo(f"No .yaml or .yml files found in {path}", err=True)
+        sys.exit(1)
+
+    # Load test suites
+    suites = load_test_files(files)
+
+    if dry_run:
+        for suite in suites:
+            click.echo(f"\n📄 {suite['file']}")
+            click.echo(f"   Model: {suite['model']}")
+            click.echo(f"   Tests: {len(suite['tests'])}")
+            for test in suite["tests"]:
+                assertions = ", ".join(a["type"] for a in test["assert"])
+                click.echo(f"   - {test['name']} [{assertions}]")
+        return
+
+    # Run tests
+    start = time.time()
+    results = run_all_tests(suites, verbose=verbose)
+    elapsed = time.time() - start
+
+    if json_out:
+        output = {
+            "results": results,
+            "elapsed_seconds": round(elapsed, 2),
+            "total": len(results),
+            "passed": sum(1 for r in results if r["passed"]),
+            "failed": sum(1 for r in results if not r["passed"]),
+        }
+        click.echo(json.dumps(output, indent=2))
+    else:
+        print_results(results, verbose=verbose)
+        print_summary(results, elapsed)
+
+    # Exit with non-zero if any test failed
+    if any(not r["passed"] for r in results):
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

promptlab/loader.py

@@ -0,0 +1,89 @@
+"""Load and validate YAML test files."""
+
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def load_test_files(files: list[Path]) -> list[dict]:
+    """Load and validate test suites from YAML files."""
+    suites = []
+    for file in files:
+        suite = _load_single_file(file)
+        suites.append(suite)
+    return suites
+
+
+def _load_single_file(file: Path) -> dict:
+    """Load a single YAML test file."""
+    with open(file) as f:
+        raw = yaml.safe_load(f)
+
+    if not isinstance(raw, dict):
+        raise ValueError(f"{file}: Expected a YAML mapping at the top level")
+
+    # Required fields
+    if "prompt" not in raw:
+        raise ValueError(f"{file}: Missing required 'prompt' field")
+    if "tests" not in raw:
+        raise ValueError(f"{file}: Missing required 'tests' field")
+
+    prompt_template = raw["prompt"]
+    model = raw.get("model", "claude-sonnet-4-20250514")
+    system = raw.get("system", None)
+    temperature = raw.get("temperature", 0)
+    max_tokens = raw.get("max_tokens", 1024)
+
+    tests = []
+    for i, test_raw in enumerate(raw["tests"]):
+        test = _validate_test(test_raw, file, i)
+        tests.append(test)
+
+    return {
+        "file": str(file),
+        "prompt_template": prompt_template,
+        "model": model,
+        "system": system,
+        "temperature": temperature,
+        "max_tokens": max_tokens,
+        "tests": tests,
+    }
+
+
+def _validate_test(test: dict, file: Path, index: int) -> dict:
+    """Validate a single test case."""
+    if not isinstance(test, dict):
+        raise ValueError(f"{file}: Test {index} must be a mapping")
+
+    name = test.get("name", f"test_{index}")
+    variables = test.get("vars", {})
+    assertions = test.get("assert", [])
+
+    if not assertions:
+        raise ValueError(f"{file}: Test '{name}' has no assertions")
+
+    validated_assertions = []
+    for a in assertions:
+        if "type" not in a:
+            raise ValueError(f"{file}: Test '{name}' has assertion without 'type'")
+        validated_assertions.append({
+            "type": a["type"],
+            "value": a.get("value"),
+        })
+
+    return {
+        "name": name,
+        "vars": variables,
+        "assert": validated_assertions,
+    }
+
+
+def render_prompt(template: str, variables: dict) -> str:
+    """Render a prompt template with variables using {{ var }} syntax."""
+    result = template
+    for key, value in variables.items():
+        result = result.replace("{{ " + key + " }}", str(value))
+        result = result.replace("{{" + key + "}}", str(value))
+    return result

promptlab/providers.py

@@ -0,0 +1,137 @@
+"""LLM API providers: Claude and OpenAI."""
+
+import os
+
+import httpx
+
+
+ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages"
+OPENAI_API_URL = "https://api.openai.com/v1/chat/completions"
+
+
+def _is_anthropic_model(model: str) -> bool:
+    return model.startswith("claude")
+
+
+def _is_openai_model(model: str) -> bool:
+    return model.startswith("gpt") or model.startswith("o1") or model.startswith("o3")
+
+
+def call_llm(
+    prompt: str,
+    model: str,
+    system: str | None = None,
+    temperature: float = 0,
+    max_tokens: int = 1024,
+) -> dict:
+    """Call an LLM and return the response.
+
+    Returns:
+        dict with keys: text, model, input_tokens, output_tokens
+    """
+    if _is_anthropic_model(model):
+        return _call_anthropic(prompt, model, system, temperature, max_tokens)
+    elif _is_openai_model(model):
+        return _call_openai(prompt, model, system, temperature, max_tokens)
+    else:
+        raise ValueError(
+            f"Unknown model: {model}. "
+            "Supported prefixes: 'claude' (Anthropic), 'gpt'/'o1'/'o3' (OpenAI)"
+        )
+
+
+def _call_anthropic(
+    prompt: str,
+    model: str,
+    system: str | None,
+    temperature: float,
+    max_tokens: int,
+) -> dict:
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "ANTHROPIC_API_KEY environment variable is not set. "
+            "Get a key at https://console.anthropic.com/"
+        )
+
+    headers = {
+        "x-api-key": api_key,
+        "content-type": "application/json",
+        "anthropic-version": "2023-06-01",
+    }
+
+    body: dict = {
+        "model": model,
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+        "messages": [{"role": "user", "content": prompt}],
+    }
+    if system:
+        body["system"] = system
+
+    with httpx.Client(timeout=60) as client:
+        resp = client.post(ANTHROPIC_API_URL, headers=headers, json=body)
+
+    if resp.status_code != 200:
+        raise RuntimeError(f"Anthropic API error ({resp.status_code}): {resp.text}")
+
+    data = resp.json()
+    text = ""
+    for block in data.get("content", []):
+        if block.get("type") == "text":
+            text += block.get("text", "")
+
+    return {
+        "text": text,
+        "model": data.get("model", model),
+        "input_tokens": data.get("usage", {}).get("input_tokens", 0),
+        "output_tokens": data.get("usage", {}).get("output_tokens", 0),
+    }
+
+
+def _call_openai(
+    prompt: str,
+    model: str,
+    system: str | None,
+    temperature: float,
+    max_tokens: int,
+) -> dict:
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "OPENAI_API_KEY environment variable is not set. "
+            "Get a key at https://platform.openai.com/"
+        )
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+
+    messages = []
+    if system:
+        messages.append({"role": "system", "content": system})
+    messages.append({"role": "user", "content": prompt})
+
+    body = {
+        "model": model,
+        "messages": messages,
+        "temperature": temperature,
+        "max_tokens": max_tokens,
+    }
+
+    with httpx.Client(timeout=60) as client:
+        resp = client.post(OPENAI_API_URL, headers=headers, json=body)
+
+    if resp.status_code != 200:
+        raise RuntimeError(f"OpenAI API error ({resp.status_code}): {resp.text}")
+
+    data = resp.json()
+    text = data["choices"][0]["message"]["content"]
+
+    return {
+        "text": text,
+        "model": data.get("model", model),
+        "input_tokens": data.get("usage", {}).get("prompt_tokens", 0),
+        "output_tokens": data.get("usage", {}).get("completion_tokens", 0),
+    }

promptlab/reporter.py

@@ -0,0 +1,69 @@
+"""Reporter: pretty terminal output for test results."""
+
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+def print_results(results: list[dict], verbose: bool = False):
+    """Print individual test results."""
+    console.print()
+
+    for result in results:
+        status = "[bold green]✅ PASS[/]" if result["passed"] else "[bold red]❌ FAIL[/]"
+        name = f"{result['suite']} :: {result['name']}"
+        elapsed = f"({result['elapsed']}s)"
+
+        console.print(f"  {status}  {name}  [dim]{elapsed}[/]")
+
+        if result["error"]:
+            console.print(f"         [red]Error: {result['error']}[/]")
+
+        if not result["passed"]:
+            for assertion in result["assertions"]:
+                if not assertion["passed"]:
+                    console.print(f"         [dim]Expected:[/] {assertion['expected']}")
+                    if assertion.get("message"):
+                        console.print(f"         [dim]Reason:[/]   {assertion['message']}")
+
+        if verbose and result["output"]:
+            console.print(f"         [dim]Output:[/]")
+            # Indent and truncate output
+            output_lines = result["output"][:500].split("\n")
+            for line in output_lines:
+                console.print(f"           [dim]{line}[/]")
+            if len(result["output"]) > 500:
+                console.print(f"           [dim]... (truncated)[/]")
+            console.print()
+
+
+def print_summary(results: list[dict], elapsed: float):
+    """Print summary line."""
+    total = len(results)
+    passed = sum(1 for r in results if r["passed"])
+    failed = total - passed
+
+    total_input = sum(r.get("input_tokens", 0) for r in results)
+    total_output = sum(r.get("output_tokens", 0) for r in results)
+
+    console.print()
+    console.print("━" * 50)
+
+    if failed == 0:
+        console.print(
+            f"  [bold green]Results: {passed} passed[/], "
+            f"{total} total ({elapsed:.1f}s)"
+        )
+    else:
+        console.print(
+            f"  [bold red]Results: {passed} passed, {failed} failed[/], "
+            f"{total} total ({elapsed:.1f}s)"
+        )
+
+    if total_input > 0 or total_output > 0:
+        console.print(
+            f"  [dim]Tokens: {total_input:,} input, {total_output:,} output[/]"
+        )
+
+    console.print()

promptlab/runner.py

@@ -0,0 +1,85 @@
+"""Test runner: executes test suites and collects results."""
+
+import time
+
+from promptlab.assertions import check_assertion
+from promptlab.loader import render_prompt
+from promptlab.providers import call_llm
+
+
+def run_all_tests(suites: list[dict], verbose: bool = False) -> list[dict]:
+    """Run all test suites and return results."""
+    results = []
+
+    for suite in suites:
+        suite_results = _run_suite(suite, verbose=verbose)
+        results.extend(suite_results)
+
+    return results
+
+
+def _run_suite(suite: dict, verbose: bool = False) -> list[dict]:
+    """Run a single test suite."""
+    results = []
+    file_name = suite["file"]
+    # Use the filename without extension as the suite name
+    suite_name = file_name.rsplit("/", 1)[-1].rsplit(".", 1)[0]
+
+    for test in suite["tests"]:
+        result = _run_single_test(suite, test, suite_name, verbose=verbose)
+        results.append(result)
+
+    return results
+
+
+def _run_single_test(suite: dict, test: dict, suite_name: str, verbose: bool = False) -> dict:
+    """Run a single test case."""
+    test_name = test["name"]
+    start = time.time()
+
+    # Render the prompt with variables
+    prompt = render_prompt(suite["prompt_template"], test["vars"])
+
+    # Call the LLM
+    try:
+        response = call_llm(
+            prompt=prompt,
+            model=suite["model"],
+            system=suite.get("system"),
+            temperature=suite.get("temperature", 0),
+            max_tokens=suite.get("max_tokens", 1024),
+        )
+        output = response["text"]
+        error = None
+    except Exception as e:
+        output = ""
+        error = str(e)
+        response = {"input_tokens": 0, "output_tokens": 0, "model": suite["model"]}
+
+    elapsed = time.time() - start
+
+    # Check assertions
+    assertion_results = []
+    all_passed = error is None
+
+    if error is None:
+        for assertion in test["assert"]:
+            result = check_assertion(assertion, output, suite["model"])
+            assertion_results.append(result)
+            if not result["passed"]:
+                all_passed = False
+    else:
+        all_passed = False
+
+    return {
+        "suite": suite_name,
+        "name": test_name,
+        "passed": all_passed,
+        "elapsed": round(elapsed, 2),
+        "output": output,
+        "error": error,
+        "assertions": assertion_results,
+        "model": response.get("model", suite["model"]),
+        "input_tokens": response.get("input_tokens", 0),
+        "output_tokens": response.get("output_tokens", 0),
+    }

promptlab_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,235 @@
+Metadata-Version: 2.4
+Name: promptlab-cli
+Version: 0.1.0
+Summary: Automated testing for LLM prompts. Like pytest, but for prompts.
+Project-URL: Homepage, https://github.com/vigp17/promptlab
+Project-URL: Repository, https://github.com/vigp17/promptlab
+Project-URL: Issues, https://github.com/vigp17/promptlab/issues
+Author: Vignesh Pai
+License-Expression: MIT
+Keywords: ai,claude,evaluation,llm,openai,prompts,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# promptlab ⚡
+
+Automated testing for LLM prompts. Write test cases in YAML, run them against Claude or OpenAI, get pass/fail results in your terminal.
+
+**Like pytest, but for prompts.**
+
+```bash
+pip install promptlab-cli
+promptlab run tests/
+```
+
+```
+✅ summarize_article :: returns_short_summary        PASS (1.2s)
+✅ summarize_article :: mentions_key_points           PASS (1.1s)
+❌ translate_text :: preserves_tone                   FAIL (0.9s)
+   Expected: contains "formal"
+   Got: "Here is the translated text in a casual style..."
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Results: 2 passed, 1 failed, 3 total (3.2s)
+```
+
+## Why?
+
+You're building an app with Claude or GPT. Your prompt works today. Tomorrow you tweak it and something breaks. You don't notice until a user complains.
+
+**promptlab catches prompt regressions before they ship.** Define what good output looks like, run tests on every change, and know immediately if something broke.
+
+## Quickstart
+
+### Install
+
+```bash
+pip install promptlab-cli
+```
+
+### Set your API key
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+# or
+export OPENAI_API_KEY=sk-...
+```
+
+### Write a test file
+
+Create `tests/summarize.yaml`:
+
+```yaml
+prompt: |
+  Summarize this article in 2-3 sentences:
+  {{ article }}
+
+model: claude-sonnet-4-20250514
+
+tests:
+  - name: short_summary
+    vars:
+      article: |
+        The Federal Reserve held interest rates steady on Wednesday,
+        keeping the benchmark rate in the 5.25%-5.50% range. Chair
+        Jerome Powell said the committee needs more confidence that
+        inflation is moving toward the 2% target before cutting rates.
+    assert:
+      - type: max_tokens
+        value: 100
+      - type: contains
+        value: "Federal Reserve"
+      - type: contains
+        value: "interest rate"
+
+  - name: handles_empty_input
+    vars:
+      article: ""
+    assert:
+      - type: not_contains
+        value: "error"
+      - type: min_length
+        value: 10
+```
+
+### Run it
+
+```bash
+promptlab run tests/
+```
+
+## Test File Format
+
+Each `.yaml` file defines a prompt and its test cases:
+
+```yaml
+# The prompt template. Use {{ variable }} for inputs.
+prompt: |
+  You are a helpful assistant. {{ instruction }}
+
+# Which model to use
+model: claude-sonnet-4-20250514   # or gpt-4o, claude-haiku-4-5-20251001, etc.
+
+# Optional system prompt
+system: "You are a concise technical writer."
+
+# Optional model parameters
+temperature: 0
+max_tokens: 500
+
+# Test cases
+tests:
+  - name: test_name
+    vars:
+      instruction: "Explain what a CPU does in one sentence."
+    assert:
+      - type: contains
+        value: "processor"
+      - type: max_tokens
+        value: 50
+```
+
+## Assertion Types
+
+| Type | Description | Example |
+|---|---|---|
+| `contains` | Output must contain this string (case-insensitive) | `value: "machine learning"` |
+| `not_contains` | Output must NOT contain this string | `value: "I'm sorry"` |
+| `starts_with` | Output must start with this string | `value: "Sure"` |
+| `regex` | Output must match this regex pattern | `value: "\\d{4}"` |
+| `max_tokens` | Output must be at most N tokens | `value: 100` |
+| `min_length` | Output must be at least N characters | `value: 50` |
+| `max_length` | Output must be at most N characters | `value: 500` |
+| `equals` | Output must exactly equal this string | `value: "42"` |
+| `llm_judge` | Ask another LLM to evaluate the output | `value: "Is this response helpful and accurate?"` |
+
+## LLM-as-Judge
+
+The most powerful assertion type. Uses a second LLM call to evaluate output quality:
+
+```yaml
+tests:
+  - name: helpful_response
+    vars:
+      question: "How do I fix a memory leak in Python?"
+    assert:
+      - type: llm_judge
+        value: "Does this response provide specific, actionable debugging steps? Answer YES or NO."
+```
+
+## Supported Models
+
+**Anthropic (Claude):**
+- `claude-sonnet-4-20250514`
+- `claude-haiku-4-5-20251001`
+- Set `ANTHROPIC_API_KEY` environment variable
+
+**OpenAI:**
+- `gpt-4o`
+- `gpt-4o-mini`
+- Set `OPENAI_API_KEY` environment variable
+
+## CLI Commands
+
+```bash
+# Run all test files in a directory
+promptlab run tests/
+
+# Run a single test file
+promptlab run tests/summarize.yaml
+
+# Verbose output (show full LLM responses)
+promptlab run tests/ --verbose
+
+# Output results as JSON
+promptlab run tests/ --json
+
+# Dry run (show what would be tested without calling APIs)
+promptlab run tests/ --dry-run
+```
+
+## Use Cases
+
+- **Prompt regression testing** — Run tests in CI/CD to catch regressions
+- **Prompt comparison** — Test the same cases across different models
+- **Guard rails validation** — Verify your prompt rejects harmful inputs
+- **Output format checking** — Ensure structured output matches expectations
+
+## Development
+
+```bash
+git clone https://github.com/vigp17/promptlab.git
+cd promptlab
+pip install -e ".[dev]"
+pytest
+```
+
+## Roadmap
+
+- [x] YAML test definitions
+- [x] Claude and OpenAI support
+- [x] 9 assertion types including LLM-as-judge
+- [x] CLI with colored output
+- [ ] Cost tracking per test run
+- [ ] HTML report generation
+- [ ] Parallel test execution
+- [ ] GitHub Actions integration
+- [ ] Prompt versioning and diff
+- [ ] Custom scoring functions
+
+## License
+
+MIT

promptlab_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+promptlab/__init__.py,sha256=os5p2uKFDJzFm1RuGX_yAwXOc6Jktvbtd79J9voVwPE,75
+promptlab/assertions.py,sha256=AD9CWAhktSJkh_4xtdKUipm3_PrTbmldxJ_iABsYWZw,5348
+promptlab/cli.py,sha256=_c00XcyyzmKs-5t-c6XwB5MJKn13gCe6kjgnuR2ricA,2452
+promptlab/loader.py,sha256=sg0aU0HA5OSPpl-cGWvUKCgPBwM5kfsK1xJltDCRmWQ,2579
+promptlab/providers.py,sha256=C6aRNyieptpGNa4wylUtsJgGxBnwE5Fngnk_-WwZOfQ,3743
+promptlab/reporter.py,sha256=QXjhWJpX5CtzH9tKAcAoI716bMfV998OQQP5lvm8e0M,2329
+promptlab/runner.py,sha256=wogLZ4K2hyURl6vCG-SS3czfIMElWZmgePpYKeXtUMs,2520
+promptlab_cli-0.1.0.dist-info/METADATA,sha256=9FnTyN5w9e_-AESCS2-ogLzn_WwH72dn797lIYGlRoU,6303
+promptlab_cli-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+promptlab_cli-0.1.0.dist-info/entry_points.txt,sha256=-_chAvLXxNGzv9TkKKVTmZLMe9rBMgu7qPXxuU4Io1Y,49
+promptlab_cli-0.1.0.dist-info/RECORD,,

promptlab_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

promptlab_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+promptlab = promptlab.cli:main