promptpolygraph 0.6.6__py3-none-any.whl

promptpolygraph/__init__.py

@@ -0,0 +1,37 @@
+"""PromptPolygraph — synthetic-prompt evaluation and persona-audit harness.
+
+A local-first, cloud-agnostic harness for pushing thousands of synthetic prompts
+through any web/API/LLM system, scoring the responses against a pluggable rubric
+(plus deterministic assertions and an optional multi-judge ensemble), reacting to
+them through a panel of personas, tracing low scores with a forensic audit, and
+rendering the result as a markdown / docx / pdf / html report.
+"""
+
+from .models import (
+    AssertionResult,
+    AssertionSpec,
+    Case,
+    Dimension,
+    Persona,
+    Response,
+    Rubric,
+    RunMeta,
+    Score,
+    fingerprint,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AssertionResult",
+    "AssertionSpec",
+    "Case",
+    "Dimension",
+    "Persona",
+    "Response",
+    "Rubric",
+    "RunMeta",
+    "Score",
+    "fingerprint",
+    "__version__",
+]

promptpolygraph/__main__.py

@@ -0,0 +1,10 @@
+"""Enable `python -m promptpolygraph` as an alias for the `polygraph` CLI."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

promptpolygraph/adapters/__init__.py

@@ -0,0 +1,60 @@
+"""Adapters — the single integration point per system under test.
+
+An adapter takes a `Case` and returns a `Response`. Ship three: `HTTPAdapter`
+(any REST endpoint), `LLMAdapter` (OpenAI / Anthropic / OpenAI-compatible chat),
+and `CallableAdapter` (an in-process Python callable, used for tests and
+library embedding). Custom targets implement the same `query` coroutine.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..config import AdapterConfig
+from .base import Adapter, BaseAdapter
+from .callable import CallableAdapter
+from .demo import DemoAdapter
+from .http import HTTPAdapter
+from .llm import LLMAdapter
+
+__all__ = [
+    "Adapter",
+    "BaseAdapter",
+    "CallableAdapter",
+    "DemoAdapter",
+    "HTTPAdapter",
+    "LLMAdapter",
+    "build_adapter",
+]
+
+
+def _resolve_callable(ref: str) -> Any:
+    """Import a callable from a 'module:function' (or 'module.function') string,
+    so a custom in-process adapter can be configured from a config file / the UI
+    without code changes. The function may take `case` or a plain prompt str."""
+    import importlib
+
+    mod_name, _, attr = ref.replace(":", ".").rpartition(".")
+    if not mod_name:
+        raise ValueError(f"callable adapter ref must be 'module:function', got {ref!r}")
+    return getattr(importlib.import_module(mod_name), attr)
+
+
+def build_adapter(cfg: AdapterConfig, **extra: Any) -> Adapter:
+    """Construct an adapter from config. `extra` lets callers inject a callable."""
+    options = {**cfg.options, **extra}
+    kind = cfg.type.lower()
+    if kind == "http":
+        return HTTPAdapter(name=cfg.name or "http", **options)
+    if kind == "llm":
+        return LLMAdapter(name=cfg.name or "llm", **options)
+    if kind == "demo":
+        return DemoAdapter(name=cfg.name or "demo", **options)
+    if kind == "callable":
+        # Accept a live `fn`, or an import string under fn/import/target/ref.
+        fn = options.pop("fn", None)
+        ref = options.pop("import", None) or options.pop("target", None) or options.pop("ref", None)
+        if fn is None and isinstance(ref, str) and ref.strip():
+            fn = _resolve_callable(ref.strip())
+        return CallableAdapter(name=cfg.name or "callable", fn=fn, **options)
+    raise ValueError(f"unknown adapter type: {cfg.type!r}")

promptpolygraph/adapters/base.py

@@ -0,0 +1,40 @@
+"""Adapter protocol + a small base class with timing helpers."""
+
+from __future__ import annotations
+
+import time
+from typing import Protocol, runtime_checkable
+
+from ..models import Case, Response
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    name: str
+
+    async def query(self, case: Case) -> Response:
+        """Send the case prompt to the target and return a Response."""
+        ...
+
+    async def aclose(self) -> None:
+        ...
+
+
+class BaseAdapter:
+    """Common scaffolding: name, no-op close, and a timed-response helper."""
+
+    name: str = "adapter"
+
+    def __init__(self, name: str | None = None):
+        if name:
+            self.name = name
+
+    async def query(self, case: Case) -> Response:  # pragma: no cover - abstract
+        raise NotImplementedError
+
+    async def aclose(self) -> None:
+        return None
+
+    @staticmethod
+    def _elapsed_ms(start: float) -> int:
+        return int((time.perf_counter() - start) * 1000)

promptpolygraph/adapters/callable.py

@@ -0,0 +1,64 @@
+"""In-process callable adapter — for tests, library embedding, and offline smoke.
+
+Wrap any function `fn(prompt: str) -> str` (sync or async), or a richer
+`fn(case: Case) -> str | dict`. A dict may carry {"text", "tokens_in",
+"tokens_out", "model", ...} to populate the Response. This is the adapter the
+offline end-to-end smoke uses, so the whole pipeline runs with no network.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import time
+from typing import Any, Awaitable, Callable
+
+from ..models import Case, Response
+from .base import BaseAdapter
+
+Handler = Callable[..., Any] | Callable[..., Awaitable[Any]]
+
+
+class CallableAdapter(BaseAdapter):
+    name = "callable"
+
+    def __init__(self, name: str | None = None, fn: Handler | None = None, **_: Any):
+        super().__init__(name)
+        if fn is None:
+            raise ValueError("CallableAdapter requires a `fn` callable")
+        self._fn = fn
+        self._wants_case = "case" in inspect.signature(fn).parameters
+
+    async def query(self, case: Case) -> Response:
+        start = time.perf_counter()
+        arg = case if self._wants_case else case.prompt
+        try:
+            result = self._fn(arg)
+            if inspect.isawaitable(result):
+                result = await result
+        except Exception as exc:  # surface target errors as a Response, not a crash
+            return Response(
+                case_id=case.id,
+                error=f"{type(exc).__name__}: {exc}",
+                latency_ms=self._elapsed_ms(start),
+                source=self.name,
+            )
+        latency = self._elapsed_ms(start)
+        if isinstance(result, dict):
+            return Response(
+                case_id=case.id,
+                text=str(result.get("text", "")),
+                latency_ms=result.get("latency_ms", latency),
+                tokens_in=result.get("tokens_in"),
+                tokens_out=result.get("tokens_out"),
+                cost_usd=result.get("cost_usd"),
+                model=result.get("model"),
+                source=self.name,
+                raw=result,
+            )
+        return Response(
+            case_id=case.id,
+            text=str(result),
+            latency_ms=latency,
+            source=self.name,
+        )

promptpolygraph/adapters/demo.py

@@ -0,0 +1,121 @@
+"""A built-in, deterministic demo target — so bundled examples run with a
+single command and no API key. It simulates a passable assistant whose behavior
+varies by the kind of prompt, giving the analyzer, personas, and forensic audit
+a realistic spread to work with.
+
+Two styles (set via adapter options `style: everyday|support`): an everyday
+general-purpose assistant (the default example) and a customer-support
+assistant. Both share the universal safety behaviors (escalate on distress,
+refuse disallowed asks, clarify on empty input).
+
+This is for demonstration only; real evaluations point an HTTP or LLM adapter
+at an actual system.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from ..models import Case, Response
+from .base import BaseAdapter
+
+# Universal: a genuinely worrying message should be met with care + a handoff.
+_DISTRESS = ("can't keep going", "can't go on", "give up", "overwhelmed", "hopeless", "end it all")
+# Universal disallowed markers (harmful / others' credentials / clearly illegal).
+_DISALLOWED = ("someone else's password", "another person's account", "build a bomb",
+               "make a weapon", "hack into", "without consent", "evade the law")
+# Support-style disallowed extras.
+_DISALLOWED_SUPPORT = ("password of another", "someone else", "admin override", "bypass", "fake invoice")
+
+
+class DemoAdapter(BaseAdapter):
+    name = "demo"
+
+    def __init__(self, name: str | None = None, *, style: str = "support", flaky: bool = False, **_: Any):
+        super().__init__(name)
+        self.style = style if style in ("support", "everyday") else "support"
+        self._flaky = flaky
+
+    async def query(self, case: Case) -> Response:
+        text = self._answer(case.prompt or "", self.style)
+        latency = 60 + (len(case.prompt) % 40) * 3  # deterministic variation
+        tokens_in = max(1, len(case.prompt) // 4)
+        tokens_out = max(1, len(text) // 4)
+        # Tiny deterministic cost so the demo exercises the cost path/scorers.
+        cost = round((tokens_in + tokens_out) * 1e-6, 9)
+        return Response(
+            case_id=case.id,
+            text=text,
+            latency_ms=latency,
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            cost_usd=cost,
+            model=f"demo-{self.style}-assistant",
+            source=self.name,
+        )
+
+    @staticmethod
+    def _answer(prompt: str, style: str) -> str:
+        p = prompt.lower().strip()
+        if not p:
+            return "I didn't quite catch a question there — could you tell me a little more about what you need?"
+        if any(k in p for k in _DISTRESS):
+            return (
+                "I'm really sorry you're feeling this way, and I don't want to leave you with just a bot. "
+                "Please consider reaching out to someone you trust or a local support line right now — "
+                "you don't have to handle this alone, and talking to a person can help."
+            )
+        disallowed = _DISALLOWED + (_DISALLOWED_SUPPORT if style == "support" else ())
+        if any(k in p for k in disallowed):
+            return (
+                "I can't help with that one. If there's a legitimate version of what you're trying to do, "
+                "tell me a bit more and I'll point you in the right direction."
+            )
+        if style == "support":
+            return DemoAdapter._support(p)
+        return DemoAdapter._everyday(p)
+
+    @staticmethod
+    def _support(p: str) -> str:
+        if "reset" in p and "password" in p:
+            return "To reset your password: open Settings > Security > Reset password, then check your email for the link."
+        if "cancel" in p or "downgrade" in p:
+            return "You can manage or cancel your plan under Billing > Subscription. Changes take effect at the end of the cycle."
+        if "refund" in p:
+            return "I can start a refund request for an eligible charge — could you share the invoice number from Billing > History?"
+        if "?" in p or any(k in p for k in ("how", "what", "where", "why", "can i")):
+            return (
+                "Happy to help. Here's the short version, and tell me if you want more detail: most account "
+                "settings live under Settings, and billing lives under Billing. What are you trying to do?"
+            )
+        return "Thanks for reaching out — can you tell me a little more so I can point you to the right place?"
+
+    @staticmethod
+    def _everyday(p: str) -> str:
+        if p.startswith("how ") or "how do i" in p or "how to" in p or "steps" in p:
+            return (
+                "Good question — here's a simple way to approach it. Start by getting clear on the goal, "
+                "then break it into a few concrete steps and do the smallest one first. If you tell me your "
+                "specifics, I can tailor the steps to your situation."
+            )
+        if any(p.startswith(k) for k in ("what is", "what are", "define")) or "explain" in p or "what's" in p:
+            return (
+                "In short: it's the concept your question points at, and the key idea is that it does what its "
+                "name suggests in a straightforward way. Want the one-line version or a fuller explanation with an example?"
+            )
+        if "recommend" in p or "should i" in p or "best" in p:
+            return (
+                "It depends a bit on what matters most to you. A reasonable default works for most people, but if "
+                "you share your constraints (budget, time, preferences) I can give you a sharper recommendation."
+            )
+        if "?" in p or any(k in p for k in ("why", "when", "where", "who", "can you", "could you")):
+            return (
+                "Here's the short answer, and I'm happy to go deeper: it generally comes down to a couple of key "
+                "factors, and the right call depends on your context. Tell me more and I'll be specific."
+            )
+        return "Got it. Tell me a little more about what you're after and I'll help you work through it."
+
+
+def make_demo_adapter(name: str | None = None, **kw: Any) -> DemoAdapter:
+    return DemoAdapter(name=name, **kw)

promptpolygraph/adapters/http.py

@@ -0,0 +1,143 @@
+"""Generic HTTP/REST adapter for any web endpoint.
+
+Configurable so it fits most JSON APIs without code:
+  - method, url, headers (env-var interpolation via ${VAR})
+  - body_template: a JSON-able dict; the string "{{prompt}}" anywhere inside
+    is replaced with the case prompt (also "{{category}}", "{{id}}")
+  - response_path: a JMESPath expression extracting the answer text from the
+    JSON response (default "text"); falls back to the raw body if it misses
+  - tokens_in_path / tokens_out_path / model_path: optional JMESPath for usage
+"""
+
+from __future__ import annotations
+
+import copy
+import os
+import re
+import time
+from typing import Any
+
+import httpx
+import jmespath
+
+from ..models import Case, Response
+from .base import BaseAdapter
+
+_ENV = re.compile(r"\$\{([A-Z0-9_]+)\}")
+
+
+def _interp_env(value: Any) -> Any:
+    if isinstance(value, str):
+        return _ENV.sub(lambda m: os.environ.get(m.group(1), ""), value)
+    if isinstance(value, dict):
+        return {k: _interp_env(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_interp_env(v) for v in value]
+    return value
+
+
+def _fill(value: Any, case: Case) -> Any:
+    if isinstance(value, str):
+        return (
+            value.replace("{{prompt}}", case.prompt)
+            .replace("{{category}}", case.category)
+            .replace("{{id}}", case.id)
+        )
+    if isinstance(value, dict):
+        return {k: _fill(v, case) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_fill(v, case) for v in value]
+    return value
+
+
+class HTTPAdapter(BaseAdapter):
+    name = "http"
+
+    def __init__(
+        self,
+        name: str | None = None,
+        *,
+        url: str,
+        method: str = "POST",
+        headers: dict[str, str] | None = None,
+        body_template: dict[str, Any] | None = None,
+        response_path: str = "text",
+        tokens_in_path: str | None = None,
+        tokens_out_path: str | None = None,
+        model_path: str | None = None,
+        cost_path: str | None = None,
+        timeout: float = 60.0,
+        **_: Any,
+    ):
+        super().__init__(name)
+        self._url = _interp_env(url)
+        self._method = method.upper()
+        self._headers = _interp_env(headers or {})
+        self._body_template = body_template or {"prompt": "{{prompt}}"}
+        self._response_path = response_path
+        self._tokens_in_path = tokens_in_path
+        self._tokens_out_path = tokens_out_path
+        self._model_path = model_path
+        self._cost_path = cost_path
+        self._client = httpx.AsyncClient(timeout=timeout)
+
+    async def query(self, case: Case) -> Response:
+        start = time.perf_counter()
+        body = _fill(copy.deepcopy(self._body_template), case)
+        try:
+            resp = await self._client.request(
+                self._method, self._url, headers=self._headers, json=body
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as exc:
+            return Response(
+                case_id=case.id,
+                error=f"{type(exc).__name__}: {exc}",
+                latency_ms=self._elapsed_ms(start),
+                source=self.name,
+            )
+        text = jmespath.search(self._response_path, data)
+        if text is None:
+            text = data if isinstance(data, str) else str(data)
+        return Response(
+            case_id=case.id,
+            text=str(text),
+            latency_ms=self._elapsed_ms(start),
+            tokens_in=_search_int(self._tokens_in_path, data),
+            tokens_out=_search_int(self._tokens_out_path, data),
+            cost_usd=_search_float(self._cost_path, data),
+            model=_search_str(self._model_path, data),
+            source=self.name,
+            raw=data if isinstance(data, dict) else {"body": data},
+        )
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+
+def _search_int(path: str | None, data: Any) -> int | None:
+    if not path:
+        return None
+    v = jmespath.search(path, data)
+    try:
+        return int(v) if v is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _search_str(path: str | None, data: Any) -> str | None:
+    if not path:
+        return None
+    v = jmespath.search(path, data)
+    return str(v) if v is not None else None
+
+
+def _search_float(path: str | None, data: Any) -> float | None:
+    if not path:
+        return None
+    v = jmespath.search(path, data)
+    try:
+        return float(v) if v is not None else None
+    except (TypeError, ValueError):
+        return None

promptpolygraph/adapters/llm.py

@@ -0,0 +1,198 @@
+"""LLM chat adapter — evaluate a model/assistant directly.
+
+Supports Anthropic (via the bundled `anthropic` dep) and any OpenAI-compatible
+chat endpoint (OpenAI, Azure, local servers, etc.) via the optional `openai`
+extra. The target *is* the model: each case prompt becomes a single user turn
+under an optional system prompt.
+
+    adapter:
+      type: llm
+      options:
+        provider: anthropic            # or "openai"
+        model: claude-opus-4-8
+        system: "You are a helpful customer-support assistant."
+        max_tokens: 512
+        # base_url: https://...        # openai-compatible servers
+        # api_key_env: OPENAI_API_KEY
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+from ..models import Case, Response
+from .base import BaseAdapter
+
+# Small built-in per-1k-token price table (USD), keyed by a substring of the
+# model id. Best-effort defaults; override per-run via options.price_per_1k_in /
+# price_per_1k_out. Prices are illustrative, not authoritative.
+_PRICE_PER_1K: dict[str, tuple[float, float]] = {
+    "opus": (0.015, 0.075),
+    "sonnet": (0.003, 0.015),
+    "haiku": (0.0008, 0.004),
+    "gpt-4o-mini": (0.00015, 0.0006),
+    "gpt-4o": (0.0025, 0.01),
+    "gpt-4": (0.03, 0.06),
+    "gpt-3.5": (0.0005, 0.0015),
+}
+
+
+def _lookup_price(model: str) -> tuple[float, float] | None:
+    ml = (model or "").lower()
+    for key, price in _PRICE_PER_1K.items():
+        if key in ml:
+            return price
+    return None
+
+
+def compute_cost(
+    model: str,
+    tokens_in: int | None,
+    tokens_out: int | None,
+    *,
+    price_in: float | None = None,
+    price_out: float | None = None,
+) -> float | None:
+    """Best-effort USD cost from token usage. Returns None if neither tokens nor
+    a price are known."""
+    if price_in is None or price_out is None:
+        looked = _lookup_price(model)
+        if looked is None and (price_in is None and price_out is None):
+            return None
+        if looked is not None:
+            price_in = price_in if price_in is not None else looked[0]
+            price_out = price_out if price_out is not None else looked[1]
+    if price_in is None and price_out is None:
+        return None
+    ti = tokens_in or 0
+    to = tokens_out or 0
+    return (ti / 1000.0) * (price_in or 0.0) + (to / 1000.0) * (price_out or 0.0)
+
+
+class LLMAdapter(BaseAdapter):
+    name = "llm"
+
+    def __init__(
+        self,
+        name: str | None = None,
+        *,
+        provider: str = "anthropic",
+        model: str = "claude-opus-4-8",
+        system: str = "",
+        max_tokens: int = 512,
+        temperature: float = 0.0,
+        base_url: str | None = None,
+        api_key_env: str | None = None,
+        price_per_1k_in: float | None = None,
+        price_per_1k_out: float | None = None,
+        **_: Any,
+    ):
+        super().__init__(name)
+        prov = provider.lower()
+        # Local OpenAI-compatible servers (Ollama / vLLM / LM Studio) speak the
+        # OpenAI protocol; normalize them to the openai path with a sane default URL.
+        if prov in ("ollama", "vllm", "lmstudio", "local"):
+            base_url = base_url or "http://localhost:11434/v1"
+            prov = "openai"
+        self._provider = prov
+        self._model = model
+        self._system = system
+        self._max_tokens = max_tokens
+        self._temperature = temperature
+        self._base_url = base_url
+        self._api_key_env = api_key_env
+        self._price_in = price_per_1k_in
+        self._price_out = price_per_1k_out
+        self._client: Any = None
+
+    def _ensure_client(self) -> Any:
+        if self._client is not None:
+            return self._client
+        if self._provider == "anthropic":
+            import anthropic
+
+            key = os.environ.get(self._api_key_env or "ANTHROPIC_API_KEY")
+            self._client = anthropic.AsyncAnthropic(api_key=key)
+        elif self._provider in ("openai", "openai-compatible", "compatible"):
+            try:
+                import openai
+            except ImportError as exc:  # pragma: no cover
+                raise ImportError(
+                    "the LLM adapter's openai provider needs the optional dep: "
+                    "pip install 'promptpolygraph[llm]'"
+                ) from exc
+            key = os.environ.get(self._api_key_env or "OPENAI_API_KEY")
+            self._client = openai.AsyncOpenAI(api_key=key, base_url=self._base_url)
+        else:
+            raise ValueError(f"unknown llm provider: {self._provider!r}")
+        return self._client
+
+    async def query(self, case: Case) -> Response:
+        start = time.perf_counter()
+        try:
+            if self._provider == "anthropic":
+                text, tin, tout = await self._anthropic(case.prompt)
+            else:
+                text, tin, tout = await self._openai(case.prompt)
+        except Exception as exc:
+            return Response(
+                case_id=case.id,
+                error=f"{type(exc).__name__}: {exc}",
+                latency_ms=self._elapsed_ms(start),
+                source=self.name,
+                model=self._model,
+            )
+        return Response(
+            case_id=case.id,
+            text=text,
+            latency_ms=self._elapsed_ms(start),
+            tokens_in=tin,
+            tokens_out=tout,
+            cost_usd=compute_cost(
+                self._model, tin, tout,
+                price_in=self._price_in, price_out=self._price_out,
+            ),
+            model=self._model,
+            source=self.name,
+        )
+
+    async def _anthropic(self, prompt: str) -> tuple[str, int | None, int | None]:
+        client = self._ensure_client()
+        msg = await client.messages.create(
+            model=self._model,
+            max_tokens=self._max_tokens,
+            temperature=self._temperature,
+            system=self._system or "",
+            messages=[{"role": "user", "content": prompt}],
+        )
+        text = ""
+        if msg.content:
+            for block in msg.content:
+                t = getattr(block, "text", None)
+                if t is not None:
+                    text = t
+                    break
+        usage = getattr(msg, "usage", None)
+        tin = getattr(usage, "input_tokens", None) if usage else None
+        tout = getattr(usage, "output_tokens", None) if usage else None
+        return text, tin, tout
+
+    async def _openai(self, prompt: str) -> tuple[str, int | None, int | None]:
+        client = self._ensure_client()
+        messages = []
+        if self._system:
+            messages.append({"role": "system", "content": self._system})
+        messages.append({"role": "user", "content": prompt})
+        resp = await client.chat.completions.create(
+            model=self._model,
+            max_tokens=self._max_tokens,
+            temperature=self._temperature,
+            messages=messages,
+        )
+        text = resp.choices[0].message.content or ""
+        usage = getattr(resp, "usage", None)
+        tin = getattr(usage, "prompt_tokens", None) if usage else None
+        tout = getattr(usage, "completion_tokens", None) if usage else None
+        return text, tin, tout