promptgold 0.1.0__tar.gz

promptgold-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check src tests
+      - name: Test (offline)
+        run: pytest tests/ -v

promptgold-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.pytest_cache/
+.ruff_cache/
+# NOTE: .promptspec/golden/ SHOULD be committed (golden files live in your repo)

promptgold-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# Contributing to promptgold
+
+Thanks for your interest! promptgold is deliberately tiny — five concepts, minimal API surface. Keep it that way.
+
+## Ground rules
+
+1. **Simplicity is the feature.** If a PR adds a concept a user must learn, it needs strong justification.
+2. **No cloud dependencies.** Everything must work offline (Ollama) and without accounts.
+3. **Tests run offline.** Unit tests must not require API keys — use stubs (see `tests/test_core.py`).
+4. **Small PRs.** One feature or fix per PR.
+
+## Dev setup
+
+```bash
+git clone https://github.com/rsubundamulia/promptgold
+cd promptgold
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+pytest tests/          # offline unit tests
+ruff check src tests   # lint
+```
+
+## Running prompt tests locally
+
+```bash
+export OPENAI_API_KEY=...
+pytest examples/ --baseline   # bless current outputs
+pytest examples/              # diff against baseline
+```
+
+## Good first issues
+
+Look for the `good first issue` label on GitHub. Docs improvements and new assertion helpers are great starting points.

promptgold-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rsubundamulia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

promptgold-0.1.0/PKG-INFO

@@ -0,0 +1,155 @@
+Metadata-Version: 2.5
+Name: promptgold
+Version: 0.1.0
+Summary: pytest for prompts — write a test, get a baseline, catch regressions in CI
+Project-URL: Homepage, https://github.com/rsubundamulia/promptgold
+Project-URL: Repository, https://github.com/rsubundamulia/promptgold
+Author: rsubundamulia
+License-Expression: MIT
+License-File: LICENSE
+Keywords: evals,llm,prompt,pytest,regression,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pytest>=8.0
+Provides-Extra: all
+Requires-Dist: anthropic>=0.40; extra == 'all'
+Requires-Dist: openai>=1.0; extra == 'all'
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: anthropic>=0.40; extra == 'dev'
+Requires-Dist: openai>=1.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# promptgold
+
+**pytest for prompts.** Write a test, bless the verdict, catch regressions in CI.
+
+```python
+from promptgold import prompt_test, judge, contains
+
+@prompt_test(model="openai:gpt-4o-mini")
+def test_support_stays_empathetic(llm):
+    response = llm.complete(
+        system="You are a helpful support agent.",
+        user="your product is garbage and I want my money back",
+    )
+    assert not contains(response, "calm down")
+    assert judge(response, "Is this response empathetic?")
+```
+
+```bash
+$ pytest --bless          # record the judge verdicts as golden files (commit them)
+$ pytest                  # later runs fail if a verdict flips PASS -> FAIL
+```
+
+That's it. Golden files are plain JSON in your repo — reviewable in PRs, present in CI, diffable with GitHub. No database, no cloud, no account.
+
+## Why promptgold
+
+You changed a system prompt. Did it break anything? Today the answer is "vibes" — you eyeball a few outputs and ship it. promptgold makes prompt changes testable like code changes:
+
+- **pytest-native** — prompt tests live next to your unit tests, run with `pytest`, fail in CI
+- **Golden files in your repo** — judge verdicts are blessed to `.promptgold/golden/*.json` and committed. CI runners start clean, so baselines must live in version control, not a local database
+- **Binary LLM-as-judge** — `judge()` returns PASS/FAIL plus a reason, not a 1-5 score. Numeric LLM judging is bimodal and drifts between judge-model versions; binary verdicts are stable
+- **Verdicts gate, text doesn't** — LLM output text changes constantly; whether it satisfies the criterion is the signal. Raw text is still recorded for diffing, but it never fails a build
+- **Multi-provider** — OpenAI, Anthropic, Ollama. One `Model` class, swap with a string
+- **Zero cloud** — works offline with Ollama, no signups, no telemetry
+
+## What promptgold is NOT
+
+Opinionated rejection is a feature. promptgold deliberately has:
+
+- ❌ No dashboard or web UI
+- ❌ No hosted tier, no accounts, no telemetry — **baselines live in your repo, not our cloud**
+- ❌ No 50-metric zoo — three assertions cover 90% of cases
+- ❌ No YAML-first config — tests are Python code, versioned with your code
+
+If you need a full eval platform, use [DeepEval](https://github.com/confident-ai/deepeval) or [LangSmith](https://langsmith.com). If you need Node-based matrix testing, use [Promptfoo](https://promptfoo.dev). If you want to add prompt tests in 10 minutes, you're in the right place.
+
+## Install
+
+```bash
+pip install promptgold
+export OPENAI_API_KEY=...   # or ANTHROPIC_API_KEY, or run Ollama locally
+```
+
+## The five concepts
+
+### 1. `@prompt_test`
+
+Decorator marking a function as a prompt test. Discovered by the pytest plugin.
+
+```python
+@prompt_test(model="anthropic:claude-sonnet-4-5")
+def test_refund_policy(llm): ...
+```
+
+### 2. Assertions
+
+Three cover almost everything:
+
+```python
+contains(response, "refund")              # substring / regex -> bool
+matches(response, r"order #\d+")          # regex -> bool
+judge(response, "Is the answer correct?") # LLM-graded -> Verdict (truthy, has .reason)
+```
+
+`judge()` returns a `Verdict`: `bool(verdict)` is the PASS/FAIL, `verdict.reason` explains why.
+
+### 3. Golden files
+
+```bash
+pytest --bless     # write judge verdicts to .promptgold/golden/*.json — commit them
+pytest             # fail if any verdict flips vs the golden file
+```
+
+Golden files are plain JSON. Review them in PRs like any snapshot. To intentionally change behavior, edit the prompt, run `--bless`, and commit the new golden file — the diff shows exactly which verdicts changed and why.
+
+### 4. `Model`
+
+```python
+Model("openai:gpt-4o-mini")
+Model("anthropic:claude-sonnet-4-5")
+Model("ollama:llama3.1")          # fully offline
+```
+
+### 5. Exit codes
+
+Non-zero on any failure or verdict regression. CI just works — GitHub Actions, GitLab, whatever.
+
+## The judge model
+
+`judge()` grades with a model. Default: `PROMPTGOLD_JUDGE_MODEL` env var, else `openai:gpt-4o-mini`.
+
+**Warning:** if the judge is the same model under test, the model grades its own homework — biased. Set `PROMPTGOLD_JUDGE_MODEL` to a different model for independence:
+
+```bash
+export PROMPTGOLD_JUDGE_MODEL="anthropic:claude-sonnet-4-5"
+```
+
+Or per-call: `judge(response, "...", model="anthropic:claude-sonnet-4-5")`.
+
+## Roadmap
+
+- [x] v0.1 — decorator, 3 assertions, binary judge, golden files, 3 providers, pytest plugin
+- [ ] v0.2 — response caching (record/replay cassettes), cost tracking, JUnit XML
+- [ ] v0.3 — flaky verdict detection (run N times, report pass rate), datasets/parametrize
+- [ ] v0.4 — `promptgold init <prompt-file>` generates candidate test cases
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). Good first issues are labeled. Be kind, ship small PRs.
+
+## License
+
+MIT

promptgold-0.1.0/README.md

@@ -0,0 +1,123 @@
+# promptgold
+
+**pytest for prompts.** Write a test, bless the verdict, catch regressions in CI.
+
+```python
+from promptgold import prompt_test, judge, contains
+
+@prompt_test(model="openai:gpt-4o-mini")
+def test_support_stays_empathetic(llm):
+    response = llm.complete(
+        system="You are a helpful support agent.",
+        user="your product is garbage and I want my money back",
+    )
+    assert not contains(response, "calm down")
+    assert judge(response, "Is this response empathetic?")
+```
+
+```bash
+$ pytest --bless          # record the judge verdicts as golden files (commit them)
+$ pytest                  # later runs fail if a verdict flips PASS -> FAIL
+```
+
+That's it. Golden files are plain JSON in your repo — reviewable in PRs, present in CI, diffable with GitHub. No database, no cloud, no account.
+
+## Why promptgold
+
+You changed a system prompt. Did it break anything? Today the answer is "vibes" — you eyeball a few outputs and ship it. promptgold makes prompt changes testable like code changes:
+
+- **pytest-native** — prompt tests live next to your unit tests, run with `pytest`, fail in CI
+- **Golden files in your repo** — judge verdicts are blessed to `.promptgold/golden/*.json` and committed. CI runners start clean, so baselines must live in version control, not a local database
+- **Binary LLM-as-judge** — `judge()` returns PASS/FAIL plus a reason, not a 1-5 score. Numeric LLM judging is bimodal and drifts between judge-model versions; binary verdicts are stable
+- **Verdicts gate, text doesn't** — LLM output text changes constantly; whether it satisfies the criterion is the signal. Raw text is still recorded for diffing, but it never fails a build
+- **Multi-provider** — OpenAI, Anthropic, Ollama. One `Model` class, swap with a string
+- **Zero cloud** — works offline with Ollama, no signups, no telemetry
+
+## What promptgold is NOT
+
+Opinionated rejection is a feature. promptgold deliberately has:
+
+- ❌ No dashboard or web UI
+- ❌ No hosted tier, no accounts, no telemetry — **baselines live in your repo, not our cloud**
+- ❌ No 50-metric zoo — three assertions cover 90% of cases
+- ❌ No YAML-first config — tests are Python code, versioned with your code
+
+If you need a full eval platform, use [DeepEval](https://github.com/confident-ai/deepeval) or [LangSmith](https://langsmith.com). If you need Node-based matrix testing, use [Promptfoo](https://promptfoo.dev). If you want to add prompt tests in 10 minutes, you're in the right place.
+
+## Install
+
+```bash
+pip install promptgold
+export OPENAI_API_KEY=...   # or ANTHROPIC_API_KEY, or run Ollama locally
+```
+
+## The five concepts
+
+### 1. `@prompt_test`
+
+Decorator marking a function as a prompt test. Discovered by the pytest plugin.
+
+```python
+@prompt_test(model="anthropic:claude-sonnet-4-5")
+def test_refund_policy(llm): ...
+```
+
+### 2. Assertions
+
+Three cover almost everything:
+
+```python
+contains(response, "refund")              # substring / regex -> bool
+matches(response, r"order #\d+")          # regex -> bool
+judge(response, "Is the answer correct?") # LLM-graded -> Verdict (truthy, has .reason)
+```
+
+`judge()` returns a `Verdict`: `bool(verdict)` is the PASS/FAIL, `verdict.reason` explains why.
+
+### 3. Golden files
+
+```bash
+pytest --bless     # write judge verdicts to .promptgold/golden/*.json — commit them
+pytest             # fail if any verdict flips vs the golden file
+```
+
+Golden files are plain JSON. Review them in PRs like any snapshot. To intentionally change behavior, edit the prompt, run `--bless`, and commit the new golden file — the diff shows exactly which verdicts changed and why.
+
+### 4. `Model`
+
+```python
+Model("openai:gpt-4o-mini")
+Model("anthropic:claude-sonnet-4-5")
+Model("ollama:llama3.1")          # fully offline
+```
+
+### 5. Exit codes
+
+Non-zero on any failure or verdict regression. CI just works — GitHub Actions, GitLab, whatever.
+
+## The judge model
+
+`judge()` grades with a model. Default: `PROMPTGOLD_JUDGE_MODEL` env var, else `openai:gpt-4o-mini`.
+
+**Warning:** if the judge is the same model under test, the model grades its own homework — biased. Set `PROMPTGOLD_JUDGE_MODEL` to a different model for independence:
+
+```bash
+export PROMPTGOLD_JUDGE_MODEL="anthropic:claude-sonnet-4-5"
+```
+
+Or per-call: `judge(response, "...", model="anthropic:claude-sonnet-4-5")`.
+
+## Roadmap
+
+- [x] v0.1 — decorator, 3 assertions, binary judge, golden files, 3 providers, pytest plugin
+- [ ] v0.2 — response caching (record/replay cassettes), cost tracking, JUnit XML
+- [ ] v0.3 — flaky verdict detection (run N times, report pass rate), datasets/parametrize
+- [ ] v0.4 — `promptgold init <prompt-file>` generates candidate test cases
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). Good first issues are labeled. Be kind, ship small PRs.
+
+## License
+
+MIT

promptgold-0.1.0/examples/test_prompts.py

@@ -0,0 +1,22 @@
+"""Example promptgold tests. Run with: pytest examples/ --bless  (then plain pytest)"""
+
+from promptgold import contains, judge, matches, prompt_test
+
+
+@prompt_test(model="openai:gpt-4o-mini")
+def test_support_stays_empathetic(llm):
+    response = llm.complete(
+        system="You are a helpful support agent.",
+        user="your product is garbage and I want my money back",
+    )
+    assert not contains(response, "calm down")
+    assert judge(response, "Is this response empathetic to a frustrated customer?")
+
+
+@prompt_test(model="openai:gpt-4o-mini")
+def test_refund_policy_includes_amount(llm):
+    response = llm.complete(
+        system="You handle refunds. Always state the exact refund amount.",
+        user="I was charged $50 twice, refund please",
+    )
+    assert matches(response, r"\$50")

promptgold-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "promptgold"
+version = "0.1.0"
+description = "pytest for prompts — write a test, get a baseline, catch regressions in CI"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "rsubundamulia" }]
+keywords = ["llm", "prompt", "testing", "pytest", "evals", "regression"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Framework :: Pytest",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "httpx>=0.27",
+    "pytest>=8.0",
+]
+
+[project.optional-dependencies]
+openai = ["openai>=1.0"]
+anthropic = ["anthropic>=0.40"]
+all = ["openai>=1.0", "anthropic>=0.40"]
+dev = ["pytest>=8.0", "ruff>=0.6", "openai>=1.0", "anthropic>=0.40"]
+
+[project.urls]
+Homepage = "https://github.com/rsubundamulia/promptgold"
+Repository = "https://github.com/rsubundamulia/promptgold"
+
+[project.entry-points.pytest11]
+promptgold = "promptgold.plugin"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/promptgold"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]

promptgold-0.1.0/src/promptgold/__init__.py

@@ -0,0 +1,8 @@
+"""promptgold — pytest for prompts."""
+
+from promptgold.assertions import contains, judge, matches
+from promptgold.core import prompt_test
+from promptgold.models import Model
+
+__version__ = "0.1.0"
+__all__ = ["prompt_test", "contains", "matches", "judge", "Model", "__version__"]

promptgold-0.1.0/src/promptgold/assertions.py

@@ -0,0 +1,104 @@
+"""The three assertions: contains, matches, judge."""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from promptgold.models import Model
+
+# Lazily created judge model — see PROMPTGOLD_JUDGE_MODEL below.
+_judge_model: Model | None = None
+
+JUDGE_PROMPT = """You are grading an LLM response against a criterion.
+
+Criterion: {criterion}
+
+Response to grade:
+\"\"\"
+{response}
+\"\"\"
+
+Answer in exactly this format:
+VERDICT: PASS or FAIL
+REASON: one sentence"""
+
+
+@dataclass
+class Verdict:
+    """Result of a judge() call."""
+
+    passed: bool
+    reason: str
+
+    def __bool__(self) -> bool:
+        return self.passed
+
+
+def contains(response: str, needle: str | re.Pattern[str]) -> bool:
+    """True if `needle` (substring or compiled regex) appears in `response`."""
+    if isinstance(needle, re.Pattern):
+        return bool(needle.search(response))
+    return needle in response
+
+
+def matches(response: str, pattern: str) -> bool:
+    """True if `pattern` (regex) matches anywhere in `response`."""
+    return bool(re.search(pattern, response))
+
+
+def resolve_judge_model(model: Model | str | None) -> Model:
+    """Pick the judge model.
+
+    Default: whatever PROMPTGOLD_JUDGE_MODEL says, else openai:gpt-4o-mini.
+    WARNING: if you judge with the same model under test, the model grades its
+    own homework — biased. Set PROMPTGOLD_JUDGE_MODEL to a different model for
+    independence.
+    """
+    global _judge_model
+    from promptgold.models import Model
+
+    if isinstance(model, Model):
+        return model
+    if isinstance(model, str):
+        return Model(model)
+    if _judge_model is None:
+        _judge_model = Model(os.environ.get("PROMPTGOLD_JUDGE_MODEL", "openai:gpt-4o-mini"))
+    return _judge_model
+
+
+def judge(response: str, criterion: str, model: Model | str | None = None) -> Verdict:
+    """LLM-as-judge: does `response` satisfy `criterion`? Returns a Verdict.
+
+    Verdict is truthy, so `assert judge(...)` works. Verdict.reason explains why.
+
+    Binary (PASS/FAIL) rather than a 1-5 score: LLM judges on numeric scales are
+    bimodal and drift between judge-model versions. Binary verdicts are stable.
+    For gradation, run N times and look at the pass rate.
+    """
+    from promptgold.core import get_active_context
+
+    m = resolve_judge_model(model)
+    raw = m.complete(system=JUDGE_PROMPT.format(criterion=criterion, response=response))
+
+    verdict_match = re.search(r"VERDICT:\s*(PASS|FAIL)", raw, re.IGNORECASE)
+    if not verdict_match:
+        raise ValueError(f"Judge returned no verdict: {raw!r}")
+    reason_match = re.search(r"REASON:\s*(.+)", raw)
+
+    v = Verdict(
+        passed=verdict_match.group(1).upper() == "PASS",
+        reason=reason_match.group(1).strip() if reason_match else "",
+    )
+
+    # Record the verdict so the plugin can golden-file it
+    ctx = get_active_context()
+    if ctx is not None:
+        ctx.verdicts.append(
+            {"criterion": criterion, "passed": v.passed, "reason": v.reason}
+        )
+
+    return v

promptgold-0.1.0/src/promptgold/core.py

@@ -0,0 +1,84 @@
+"""Core decorator and LLM test context."""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from promptgold.models import Model
+
+# The context of the currently-running prompt test, if any. judge() records
+# verdicts here so the plugin can compare them against golden files.
+_active_context: LLMContext | None = None
+
+
+def set_active_context(ctx: LLMContext | None) -> None:
+    global _active_context
+    _active_context = ctx
+
+
+def get_active_context() -> LLMContext | None:
+    return _active_context
+
+
+@dataclass
+class LLMContext:
+    """Passed to every @prompt_test function. Wraps a Model with run metadata."""
+
+    model: Model
+    calls: list[dict[str, Any]] = field(default_factory=list)
+    verdicts: list[dict[str, Any]] = field(default_factory=list)
+
+    def complete(self, system: str = "", user: str = "", **kwargs: Any) -> str:
+        start = time.monotonic()
+        text = self.model.complete(system=system, user=user, **kwargs)
+        self.calls.append(
+            {
+                "system": system,
+                "user": user,
+                "response": text,
+                "latency_ms": (time.monotonic() - start) * 1000,
+            }
+        )
+        return text
+
+    @property
+    def last_response(self) -> str:
+        return self.calls[-1]["response"] if self.calls else ""
+
+
+def prompt_test(model: str | Model, **model_kwargs: Any) -> Callable:
+    """Mark a function as a prompt test.
+
+    The decorated function receives an `llm` (LLMContext) argument.
+    """
+
+    def decorator(fn: Callable) -> Callable:
+        sig = inspect.signature(fn)
+        params = [p for p in sig.parameters.values() if p.name != "llm"]
+        # `llm` becomes a **kwargs catch-all (must be last) so pytest collects
+        # fixtures for real params but never resolves `llm` as a fixture.
+        params.append(inspect.Parameter("llm", kind=inspect.Parameter.VAR_KEYWORD))
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            llm = kwargs.pop("llm", None)
+            if llm is None:
+                m = model if isinstance(model, Model) else Model(model, **model_kwargs)
+                llm = LLMContext(model=m)
+            return fn(llm, *args, **kwargs)
+
+        # Rewrite the visible signature: keep real parameter names so pytest
+        # collects fixtures for them, but make `llm` a **kwargs catch-all so
+        # pytest never tries to resolve it as a fixture (we inject it ourselves).
+        wrapper.__signature__ = sig.replace(parameters=params)  # type: ignore[attr-defined]
+        wrapper._promptgold_model = model  # type: ignore[attr-defined]
+        wrapper._promptgold_kwargs = model_kwargs  # type: ignore[attr-defined]
+        wrapper._is_promptgold_test = True  # type: ignore[attr-defined]
+        return wrapper
+
+    return decorator

promptgold-0.1.0/src/promptgold/golden.py

@@ -0,0 +1,38 @@
+"""Golden files: plain-text expected verdicts, committed to your repo.
+
+Why files and not a database: CI runners start clean. A SQLite blob in a
+local folder is gone on every CI run, un-reviewable in PRs, and un-diffable.
+Golden files live in version control next to your tests — reviewable,
+diffable, and present in CI. GitHub is the diff viewer.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+GOLDEN_DIR = Path(".promptgold/golden")
+
+
+def golden_path(nodeid: str) -> Path:
+    """Map a pytest nodeid to its golden file path.
+
+    tests/test_prompts.py::test_empathy -> .promptgold/golden/test_prompts__test_empathy.json
+    """
+    safe = nodeid.replace("/", "_").replace("::", "__").replace(".py", "")
+    return GOLDEN_DIR / f"{safe}.json"
+
+
+def load(nodeid: str) -> dict[str, Any] | None:
+    p = golden_path(nodeid)
+    if not p.exists():
+        return None
+    return json.loads(p.read_text())
+
+
+def save(nodeid: str, payload: dict[str, Any]) -> Path:
+    p = golden_path(nodeid)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(json.dumps(payload, indent=2) + "\n")
+    return p

promptgold-0.1.0/src/promptgold/models.py

@@ -0,0 +1,82 @@
+"""Provider-agnostic Model class. One string, any provider."""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any
+
+import httpx
+
+
+class Model:
+    """Unified chat-completion interface.
+
+    Spec format: "provider:model_name"
+      Model("openai:gpt-4o-mini")
+      Model("anthropic:claude-sonnet-4-5")
+      Model("ollama:llama3.1")
+    """
+
+    def __init__(self, spec: str, temperature: float = 0.0, max_tokens: int = 1024, **kw: Any):
+        if ":" not in spec:
+            raise ValueError(f"Model spec must be 'provider:name', got {spec!r}")
+        self.provider, self.name = spec.split(":", 1)
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.extra = kw
+        self.spec = spec
+
+    def complete(self, system: str = "", user: str = "", **kwargs: Any) -> str:
+        handler = getattr(self, f"_{self.provider}", None)
+        if handler is None:
+            raise ValueError(
+                f"Unknown provider {self.provider!r}. Supported: openai, anthropic, ollama"
+            )
+        return handler(system=system, user=user, **{**self.extra, **kwargs})
+
+    # --- providers -------------------------------------------------------
+
+    def _openai(self, system: str, user: str, **kw: Any) -> str:
+        import openai
+
+        client = openai.OpenAI()
+        messages = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": user})
+        resp = client.chat.completions.create(
+            model=self.name,
+            messages=messages,
+            temperature=kw.pop("temperature", self.temperature),
+            max_tokens=kw.pop("max_tokens", self.max_tokens),
+            **kw,
+        )
+        return resp.choices[0].message.content or ""
+
+    def _anthropic(self, system: str, user: str, **kw: Any) -> str:
+        import anthropic
+
+        client = anthropic.Anthropic()
+        resp = client.messages.create(
+            model=self.name,
+            system=system or anthropic.NOT_GIVEN,
+            messages=[{"role": "user", "content": user}],
+            temperature=kw.pop("temperature", self.temperature),
+            max_tokens=kw.pop("max_tokens", self.max_tokens),
+            **kw,
+        )
+        return "".join(b.text for b in resp.content if b.type == "text")
+
+    def _ollama(self, system: str, user: str, **kw: Any) -> str:
+        base = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
+        payload: dict[str, Any] = {
+            "model": self.name,
+            "prompt": f"{system}\n\n{user}".strip(),
+            "stream": False,
+            "options": {"temperature": kw.pop("temperature", self.temperature)},
+        }
+        r = httpx.post(f"{base}/api/generate", json=payload, timeout=120)
+        r.raise_for_status()
+        data = r.json()
+        return data.get("response", json.dumps(data))

promptgold-0.1.0/src/promptgold/plugin.py

@@ -0,0 +1,121 @@
+"""pytest plugin: discovers @prompt_test functions, handles golden files."""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+import pytest
+
+from promptgold import golden
+from promptgold.core import LLMContext, set_active_context
+from promptgold.models import Model
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("promptgold")
+    group.addoption(
+        "--bless",
+        action="store_true",
+        default=False,
+        help="Record current judge verdicts as the golden files (commit them).",
+    )
+    group.addoption(
+        "--no-golden-check",
+        action="store_true",
+        default=False,
+        help="Run prompt tests without comparing against golden files.",
+    )
+
+
+class GoldenMismatch(AssertionError):
+    pass
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_call(item: pytest.Item) -> None:
+    """Run a prompt test, then bless or check judge verdicts.
+
+    What gates the build: judge verdicts (PASS/FAIL) from the golden files.
+    What doesn't: the raw response text. Text changes constantly with LLMs;
+    whether the response satisfies the criterion is the signal a human can
+    act on. Raw text is still recorded in the golden file for diffing.
+    """
+    fn = getattr(item, "obj", None)
+    if fn is None or not getattr(fn, "_is_promptgold_test", False):
+        return
+
+    model_spec = fn._promptgold_model
+    model = model_spec if isinstance(model_spec, Model) else Model(model_spec)
+    ctx = LLMContext(model=model)
+
+    original = fn.__wrapped__
+    sig = inspect.signature(original)
+    kwargs = {}
+    for name in sig.parameters:
+        if name == "llm":
+            kwargs[name] = ctx
+        elif name in item.funcargs:
+            kwargs[name] = item.funcargs[name]
+        else:
+            raise TypeError(
+                f"promptgold test {item.nodeid}: parameter {name!r} is neither "
+                "'llm' nor an available fixture"
+            )
+
+    set_active_context(ctx)
+    try:
+        original(**kwargs)
+    finally:
+        set_active_context(None)
+
+    payload: dict[str, Any] = {
+        "model": model.spec,
+        "verdicts": ctx.verdicts,
+        "responses": [c["response"] for c in ctx.calls],
+    }
+
+    if item.config.getoption("--bless"):
+        path = golden.save(item.nodeid, payload)
+        print(f"\npromptgold: blessed {path}")
+        return
+
+    if item.config.getoption("--no-golden-check"):
+        return
+
+    expected = golden.load(item.nodeid)
+    if expected is None:
+        # No golden file yet: pass, but tell the user how to create one.
+        print(
+            f"\npromptgold: no golden file for {item.nodeid}. "
+            "Run `pytest --bless` and commit the result."
+        )
+        return
+
+    mismatches = _diff_verdicts(expected["verdicts"], payload["verdicts"])
+    if mismatches:
+        raise GoldenMismatch(
+            "Judge verdicts regressed vs golden file:\n" + "\n".join(mismatches)
+        )
+
+
+def _diff_verdicts(expected: list[dict], current: list[dict]) -> list[str]:
+    lines = []
+    for i, (e, c) in enumerate(zip(expected, current, strict=False)):
+        if e["passed"] != c["passed"]:
+            lines.append(
+                f"  verdict {i} ({e['criterion']!r}): "
+                f"{'PASS' if e['passed'] else 'FAIL'} -> "
+                f"{'PASS' if c['passed'] else 'FAIL'}"
+            )
+            if c.get("reason"):
+                lines.append(f"    reason now: {c['reason']}")
+    if len(expected) != len(current):
+        lines.append(f"  verdict count changed: {len(expected)} -> {len(current)}")
+    return lines
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers", "promptgold: mark a test as a prompt regression test"
+    )

promptgold-0.1.0/tests/conftest.py

@@ -0,0 +1,36 @@
+"""Shared fixtures: a stubbed Model so prompt tests run offline."""
+
+from __future__ import annotations
+
+import pytest
+
+import promptgold.models
+
+
+@pytest.fixture
+def stub_model(monkeypatch):
+    """Replace Model's network calls with canned responses.
+
+    Detects judge calls (system prompt asks for VERDICT) and returns a passing
+    verdict; otherwise returns the canned response. Yields a callable to change
+    the canned response mid-test.
+    """
+    state = {"response": "I understand your frustration, happy to help with a refund."}
+
+    def fake_init(self, spec, **kw):
+        self.provider, self.name = spec.split(":", 1)
+        self.spec = spec
+        self.temperature, self.max_tokens, self.extra = 0.0, 100, kw
+
+    def fake_complete(self, system="", user="", **kw):
+        if "VERDICT" in system:
+            return "VERDICT: PASS\nREASON: stub judge approves"
+        return state["response"]
+
+    monkeypatch.setattr(promptgold.models.Model, "__init__", fake_init)
+    monkeypatch.setattr(promptgold.models.Model, "complete", fake_complete)
+
+    def set_response(text: str) -> None:
+        state["response"] = text
+
+    return set_response

promptgold-0.1.0/tests/e2e_prompts_test.py

@@ -0,0 +1,14 @@
+"""E2E plugin tests: bless golden file, pass against it, detect regression.
+
+Uses the stub_model fixture — no API keys, no network.
+"""
+
+from promptgold import contains, judge, prompt_test
+
+
+@prompt_test(model="fake:model")
+def test_empathy(llm, stub_model):
+    stub_model("I understand your frustration, happy to help with a refund.")
+    r = llm.complete(system="support agent", user="I want a refund")
+    assert contains(r, "refund")
+    assert judge(r, "Is this empathetic?", model=llm.model)

promptgold-0.1.0/tests/test_core.py

@@ -0,0 +1,129 @@
+"""Unit tests for promptgold core — run offline with a stub provider."""
+
+from __future__ import annotations
+
+import re
+
+import pytest
+
+from promptgold import Model, contains, matches, prompt_test
+from promptgold.models import Model as ModelClass
+
+
+class StubModel(ModelClass):
+    """A model that returns canned responses — no API keys needed."""
+
+    def __init__(self, response: str = "hello world"):
+        self.provider = "stub"
+        self.name = "stub"
+        self.spec = "stub:stub"
+        self.temperature = 0.0
+        self.max_tokens = 100
+        self.extra = {}
+        self._response = response
+
+    def complete(self, system: str = "", user: str = "", **kwargs) -> str:
+        return self._response
+
+
+def test_model_spec_parsing():
+    m = Model("openai:gpt-4o-mini")
+    assert m.provider == "openai"
+    assert m.name == "gpt-4o-mini"
+
+
+def test_model_spec_requires_colon():
+    with pytest.raises(ValueError, match="provider:name"):
+        Model("gpt-4o-mini")
+
+
+def test_unknown_provider_rejected():
+    m = Model.__new__(Model)
+    m.provider, m.name, m.spec = "bogus", "x", "bogus:x"
+    m.temperature, m.max_tokens, m.extra = 0.0, 100, {}
+    with pytest.raises(ValueError, match="Unknown provider"):
+        m.complete(user="hi")
+
+
+def test_contains_substring():
+    assert contains("hello world", "world")
+    assert not contains("hello world", "mars")
+
+
+def test_contains_regex():
+    assert contains("order #1234", re.compile(r"#\d+"))
+
+
+def test_matches():
+    assert matches("refund issued: $50", r"\$\d+")
+    assert not matches("no refund", r"\$\d+")
+
+
+def test_prompt_test_decorator_marks_function():
+    @prompt_test(model="stub:stub")
+    def my_test(llm):
+        pass
+
+    assert getattr(my_test, "_is_promptgold_test", False)
+    assert my_test._promptgold_model == "stub:stub"
+
+
+def test_llm_context_records_calls():
+    from promptgold.core import LLMContext
+
+    ctx = LLMContext(model=StubModel("canned"))
+    out = ctx.complete(system="s", user="u")
+    assert out == "canned"
+    assert len(ctx.calls) == 1
+    assert ctx.calls[0]["response"] == "canned"
+    assert ctx.last_response == "canned"
+
+
+def test_judge_pass_verdict():
+    from promptgold.assertions import judge
+
+    v = judge("anything", "Is it good?", model=StubModel("VERDICT: PASS\nREASON: it's great"))
+    assert v.passed is True
+    assert bool(v) is True
+    assert v.reason == "it's great"
+
+
+def test_judge_fail_verdict():
+    from promptgold.assertions import judge
+
+    v = judge("anything", "Is it good?", model=StubModel("VERDICT: FAIL\nREASON: it's bad"))
+    assert v.passed is False
+    assert bool(v) is False
+
+
+def test_judge_rejects_bad_output():
+    from promptgold.assertions import judge
+
+    with pytest.raises(ValueError, match="no verdict"):
+        judge("anything", "Is it good?", model=StubModel("i dunno"))
+
+
+def test_judge_records_verdict_in_active_context():
+    from promptgold.assertions import judge
+    from promptgold.core import LLMContext, set_active_context
+
+    ctx = LLMContext(model=StubModel("x"))
+    set_active_context(ctx)
+    try:
+        judge("resp", "Is it polite?", model=StubModel("VERDICT: PASS\nREASON: yes"))
+    finally:
+        set_active_context(None)
+    assert len(ctx.verdicts) == 1
+    assert ctx.verdicts[0]["criterion"] == "Is it polite?"
+    assert ctx.verdicts[0]["passed"] is True
+
+
+def test_golden_files_roundtrip(tmp_path, monkeypatch):
+    import promptgold.golden as golden
+
+    monkeypatch.setattr(golden, "GOLDEN_DIR", tmp_path / "golden")
+    nodeid = "tests/test_x.py::test_thing"
+    assert golden.load(nodeid) is None
+    payload = {"model": "m", "verdicts": [{"criterion": "c", "passed": True}], "responses": ["r"]}
+    golden.save(nodeid, payload)
+    assert golden.load(nodeid) == payload