pytest-llm-sushit 0.1.0__tar.gz

pytest_llm_sushit-0.1.0/.env.example

@@ -0,0 +1,12 @@
+# LLM Providers
+GROQ_API_KEY=gsk_your_key_here
+MISTRAL_API_KEY=your_key_here
+GEMINI_API_KEY=your_key_here
+OPENROUTER_API_KEY=sk-or-your_key_here
+
+# Database
+DATABASE_URL=sqlite+aiosqlite:///./data/reliability.db
+
+# App
+APP_ENV=development
+SECRET_KEY=change-me-in-production

pytest_llm_sushit-0.1.0/.gitignore

@@ -0,0 +1,57 @@
+<<<<<<< HEAD
+=======
+# Environment
+.env
+data/*.db
+
+# Python
+>>>>>>> 9520d27 (Initial AI Reliability Platform MVP)
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+dist/
+build/
+.eggs/
+<<<<<<< HEAD
+venv/
+.venv/
+=======
+
+# Virtual environments
+venv/
+.venv/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+# Database
+*.db
+*.sqlite
+*.sqlite3
+
+# Logs
+*.log
+
+# Test/cache
+>>>>>>> 9520d27 (Initial AI Reliability Platform MVP)
+.pytest_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+<<<<<<< HEAD
+=======
+
+# Node
+node_modules/
+
+# Environment variants
+.env.*
+!.env.example
+>>>>>>> 9520d27 (Initial AI Reliability Platform MVP)

pytest_llm_sushit-0.1.0/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+RUN mkdir -p data
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

pytest_llm_sushit-0.1.0/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: pytest-llm-sushit
+Version: 0.1.0
+Summary: LLM-powered semantic assertions for pytest
+Project-URL: Homepage, https://github.com/Sushit-prog/pytest-llm
+Project-URL: Repository, https://github.com/Sushit-prog/pytest-llm
+Project-URL: Issues, https://github.com/Sushit-prog/pytest-llm/issues
+Author: Sushit
+License: MIT
+Keywords: ai,assertions,langchain,langgraph,llm,ml,pytest,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.9
+Requires-Dist: anthropic>=0.20
+Requires-Dist: openai>=1.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pytest>=7.0
+Requires-Dist: rich>=13.0
+Requires-Dist: sentence-transformers>=2.2
+Provides-Extra: dev
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: groq
+Requires-Dist: groq>=0.4; extra == 'groq'
+Provides-Extra: ollama
+Requires-Dist: ollama>=0.1; extra == 'ollama'

pytest_llm_sushit-0.1.0/README.md

@@ -0,0 +1,122 @@
+# pytest-llm
+
+[![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/downloads/)
+[![PyPI version](https://img.shields.io/pypi/v/pytest-llm)](https://pypi.org/project/pytest-llm/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+[![Tests](https://img.shields.io/badge/tests-passing-brightgreen)](#)
+
+LLM-powered semantic assertions for pytest.
+
+## Why?
+
+Testing LLM outputs with string matching is brittle. `pytest-llm` adds semantic
+assertions powered by LLM judges and local embeddings — check faithfulness, tone,
+safety, hallucinations, and more with a single function call.
+
+## Quick start
+
+```bash
+pip install pytest-llm-sushit
+```
+
+```python
+from pytest_llm import assert_faithful, assert_tone, assert_safe
+
+def test_llm_output():
+    output = "Python was created by Guido van Rossum in 1991."
+    source = "Guido van Rossum created Python, released in 1991."
+
+    assert_faithful(output, source)        # factual accuracy
+    assert_tone(output, "professional")    # tone check
+    assert_safe(output)                    # safety check
+```
+
+## How it works
+
+```
+  Your pytest test
+        │
+        ▼
+  pytest-llm assertion (assert_faithful, assert_regression...)
+        │
+        ├── Local path: sentence-transformers (no API call)
+        │   cosine similarity → pass/fail
+        │
+        └── LLM Judge path: your chosen provider
+            OpenAI / Anthropic / Groq / Ollama
+            JSON response → score + reason → pass/fail
+```
+
+## Assertions
+
+| Assertion | What it checks | Uses API? |
+|-----------|---------------|-----------|
+| `assert_faithful` | Every factual claim in output is supported by source | Yes |
+| `assert_no_hallucination` | Output contains no invented facts not in source | Yes |
+| `assert_tone` | Output matches an expected tone (freeform string) | Yes |
+| `assert_semantic_similarity` | Cosine similarity between output and expected text | No |
+| `assert_contains_claim` | Output semantically contains a given claim | Yes |
+| `assert_safe` | Output contains no harmful or offensive content | Yes |
+| `assert_language` | Output is written in the expected language | Yes |
+| `assert_regression` | Output is not worse than a baseline (similarity + quality) | Yes |
+
+## Configuration
+
+### Environment variables
+
+```bash
+export LLM_JUDGE_PROVIDER=openai       # or anthropic, groq, ollama
+export LLM_JUDGE_MODEL=gpt-4o-mini     # optional, defaults to provider best
+export OPENAI_API_KEY=sk-...           # set for your chosen provider
+```
+
+### conftest.py
+
+```python
+from pytest_llm import pytest_configure_judge
+
+pytest_configure_judge(provider="anthropic", model="claude-haiku-4-5-20251001")
+```
+
+### CLI options
+
+```bash
+pytest --llm-judge-provider=anthropic --llm-judge-model=claude-haiku-4-5-20251001
+pytest --llm-report   # print Rich summary table after tests
+```
+
+## CI/CD with GitHub Actions
+
+```yaml
+name: Tests
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+      - name: Run tests
+        run: pytest tests/ -v --tb=short
+```
+
+## Provider support
+
+| Provider | Default Model | Env var for API key |
+|----------|---------------|---------------------|
+| OpenAI | `gpt-4o-mini` | `OPENAI_API_KEY` |
+| Anthropic | `claude-haiku-4-5-20251001` | `ANTHROPIC_API_KEY` |
+| Groq | `llama-3.3-70b-versatile` | `GROQ_API_KEY` |
+| Ollama | `llama3` | (local, no key needed) |
+
+## Works with langgraph-replay
+
+`pytest-llm` integrates with [langgraph-replay](https://github.com/Sushit-prog/langgraph-replay) for tracing and replaying LangGraph agent sessions during evaluation.
+
+```bash
+langgraph-replay blame session_abc --eval
+```

pytest_llm_sushit-0.1.0/app/api/dashboard.py

@@ -0,0 +1,70 @@
+from fastapi import APIRouter
+from sqlmodel import Session, select, func
+from app.database import get_engine
+from app.models.eval import EvalRun, EvalResult
+from app.models.trace import Trace, ProviderUsage
+
+router = APIRouter(prefix="/api/v1/dashboard", tags=["dashboard"])
+
+
+@router.get("/summary")
+def get_summary():
+    with Session(get_engine()) as session:
+        total_runs = session.exec(select(func.count(EvalRun.id))).one()
+        total_traces = session.exec(select(func.count(Trace.id))).one()
+        total_results = session.exec(select(func.count(EvalResult.id))).one()
+        passed = session.exec(select(func.count(EvalResult.id)).where(EvalResult.status == "pass")).one()
+        failed = session.exec(select(func.count(EvalResult.id)).where(EvalResult.status == "fail")).one()
+
+    return {
+        "total_eval_runs": total_runs,
+        "total_traces": total_traces,
+        "total_eval_results": total_results,
+        "passed": passed,
+        "failed": failed,
+        "pass_rate": f"{(passed / total_results * 100):.1f}%" if total_results > 0 else "N/A",
+    }
+
+
+@router.get("/providers")
+def get_provider_stats():
+    with Session(get_engine()) as session:
+        runs = list(session.exec(select(EvalRun)).all())
+
+    stats = {}
+    for r in runs:
+        key = f"{r.provider}/{r.model}"
+        if key not in stats:
+            stats[key] = {"calls": 0, "total_tokens": 0, "total_cost": 0.0, "total_latency": 0.0}
+        stats[key]["calls"] += 1
+        stats[key]["total_tokens"] += r.total_tokens
+        stats[key]["total_cost"] += r.estimated_cost
+        stats[key]["total_latency"] += r.avg_latency_ms * r.total_cases
+
+    return [
+        {
+            "provider_model": k,
+            "calls": v["calls"],
+            "total_tokens": v["total_tokens"],
+            "total_cost": round(v["total_cost"], 4),
+            "avg_latency_ms": round(v["total_latency"] / v["calls"], 1) if v["calls"] > 0 else 0,
+        }
+        for k, v in stats.items()
+    ]
+
+
+@router.get("/failures")
+def get_failure_summary():
+    with Session(get_engine()) as session:
+        results = list(
+            session.exec(select(EvalResult).where(EvalResult.status.in_(["fail", "error"]))).all()
+        )
+
+    failure_types = {}
+    for r in results:
+        key = r.error_message if r.error_message else "wrong_output"
+        if key not in failure_types:
+            failure_types[key] = 0
+        failure_types[key] += 1
+
+    return [{"type": k, "count": v} for k, v in sorted(failure_types.items(), key=lambda x: -x[1])[:10]]

pytest_llm_sushit-0.1.0/app/api/eval.py

@@ -0,0 +1,142 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Optional
+from app.services.eval_runner import EvalRunner
+
+router = APIRouter(prefix="/api/v1/eval", tags=["eval"])
+runner = EvalRunner()
+
+
+class DatasetCreate(BaseModel):
+    name: str
+    description: Optional[str] = None
+
+
+class TestCaseCreate(BaseModel):
+    input: str
+    expected: str
+    category: Optional[str] = None
+    difficulty: str = "medium"
+
+
+class DatasetImport(BaseModel):
+    cases: list[TestCaseCreate]
+
+
+class RunCreate(BaseModel):
+    dataset_id: int
+    provider: str
+    model: str
+    prompt_template: Optional[str] = None
+
+
+@router.post("/datasets")
+def create_dataset(body: DatasetCreate):
+    dataset = runner.create_dataset(name=body.name, description=body.description)
+    return {"id": dataset.id, "name": dataset.name}
+
+
+@router.get("/datasets")
+def list_datasets():
+    datasets = runner.list_datasets()
+    return [{"id": d.id, "name": d.name, "description": d.description, "created_at": d.created_at.isoformat()} for d in datasets]
+
+
+@router.get("/datasets/{dataset_id}")
+def get_dataset(dataset_id: int):
+    dataset = runner.get_dataset(dataset_id)
+    if not dataset:
+        raise HTTPException(status_code=404, detail="Dataset not found")
+    cases = runner.get_test_cases(dataset_id)
+    return {
+        "id": dataset.id,
+        "name": dataset.name,
+        "description": dataset.description,
+        "cases": [{"id": c.id, "input": c.input_text, "expected": c.expected_output, "category": c.category, "difficulty": c.difficulty} for c in cases],
+    }
+
+
+@router.post("/datasets/{dataset_id}/cases")
+def add_test_cases(dataset_id: int, body: DatasetImport):
+    dataset = runner.get_dataset(dataset_id)
+    if not dataset:
+        raise HTTPException(status_code=404, detail="Dataset not found")
+    cases = [c.model_dump() for c in body.cases]
+    count = runner.add_test_cases(dataset_id, cases)
+    return {"added": count}
+
+
+@router.post("/runs")
+async def create_and_run_eval(body: RunCreate):
+    run = runner.create_run(
+        dataset_id=body.dataset_id,
+        provider_name=body.provider,
+        model=body.model,
+        prompt_template=body.prompt_template,
+    )
+    run = await runner.execute_run(run.id)
+    return {
+        "id": run.id,
+        "status": run.status,
+        "total_cases": run.total_cases,
+        "passed": run.passed_cases,
+        "failed": run.failed_cases,
+        "pass_rate": f"{(run.passed_cases / run.total_cases * 100):.1f}%" if run.total_cases > 0 else "N/A",
+        "avg_latency_ms": round(run.avg_latency_ms, 1),
+        "total_tokens": run.total_tokens,
+        "estimated_cost": round(run.estimated_cost, 4),
+    }
+
+
+@router.get("/runs")
+def list_runs(dataset_id: Optional[int] = None):
+    runs = runner.list_runs(dataset_id=dataset_id)
+    return [
+        {
+            "id": r.id,
+            "dataset_id": r.dataset_id,
+            "provider": r.provider,
+            "model": r.model,
+            "status": r.status,
+            "total_cases": r.total_cases,
+            "passed": r.passed_cases,
+            "failed": r.failed_cases,
+            "avg_latency_ms": round(r.avg_latency_ms, 1),
+            "estimated_cost": round(r.estimated_cost, 4),
+            "created_at": r.created_at.isoformat(),
+            "completed_at": r.completed_at.isoformat() if r.completed_at else None,
+        }
+        for r in runs
+    ]
+
+
+@router.get("/runs/{run_id}")
+def get_run(run_id: int):
+    run = runner.get_run(run_id)
+    if not run:
+        raise HTTPException(status_code=404, detail="Run not found")
+    results = runner.get_run_results(run_id)
+    return {
+        "id": run.id,
+        "provider": run.provider,
+        "model": run.model,
+        "status": run.status,
+        "total_cases": run.total_cases,
+        "passed": run.passed_cases,
+        "failed": run.failed_cases,
+        "avg_latency_ms": round(run.avg_latency_ms, 1),
+        "total_tokens": run.total_tokens,
+        "estimated_cost": round(run.estimated_cost, 4),
+        "results": [
+            {
+                "id": r.id,
+                "test_case_id": r.test_case_id,
+                "status": r.status,
+                "actual_output": r.actual_output,
+                "score": r.score,
+                "latency_ms": r.latency_ms,
+                "error_message": r.error_message,
+            }
+            for r in results
+        ],
+    }

pytest_llm_sushit-0.1.0/app/api/trace.py

@@ -0,0 +1,71 @@
+from fastapi import APIRouter, HTTPException
+from sqlmodel import Session, select
+from app.database import get_engine
+from app.models.trace import Trace, Span
+
+router = APIRouter(prefix="/api/v1/traces", tags=["traces"])
+
+
+@router.get("")
+def list_traces(limit: int = 50):
+    with Session(get_engine()) as session:
+        traces = list(session.exec(select(Trace).order_by(Trace.created_at.desc()).limit(limit)).all())
+    return [
+        {
+            "trace_id": t.trace_id,
+            "name": t.name,
+            "status": t.status,
+            "total_latency_ms": round(t.total_latency_ms, 1),
+            "total_tokens": t.total_tokens,
+            "created_at": t.created_at.isoformat(),
+        }
+        for t in traces
+    ]
+
+
+@router.get("/{trace_id}")
+def get_trace(trace_id: str):
+    with Session(get_engine()) as session:
+        trace = session.exec(select(Trace).where(Trace.trace_id == trace_id)).first()
+        if not trace:
+            raise HTTPException(status_code=404, detail="Trace not found")
+        spans = list(session.exec(select(Span).where(Span.trace_id == trace_id).order_by(Span.created_at)).all())
+
+    return {
+        "trace_id": trace.trace_id,
+        "name": trace.name,
+        "status": trace.status,
+        "total_latency_ms": round(trace.total_latency_ms, 1),
+        "total_tokens": trace.total_tokens,
+        "created_at": trace.created_at.isoformat(),
+        "spans": [
+            {
+                "span_id": s.span_id,
+                "parent_span_id": s.parent_span_id,
+                "name": s.name,
+                "provider": s.provider,
+                "model": s.model,
+                "status": s.status,
+                "latency_ms": round(s.latency_ms, 1) if s.latency_ms else None,
+                "tokens_in": s.tokens_in,
+                "tokens_out": s.tokens_out,
+                "input_text": s.input_text[:500] if s.input_text else None,
+                "output_text": s.output_text[:500] if s.output_text else None,
+                "error_message": s.error_message,
+            }
+            for s in spans
+        ],
+    }
+
+
+@router.delete("/{trace_id}")
+def delete_trace(trace_id: str):
+    with Session(get_engine()) as session:
+        spans = session.exec(select(Span).where(Span.trace_id == trace_id)).all()
+        for s in spans:
+            session.delete(s)
+        trace = session.exec(select(Trace).where(Trace.trace_id == trace_id)).first()
+        if trace:
+            session.delete(trace)
+        session.commit()
+    return {"deleted": True}

pytest_llm_sushit-0.1.0/app/config.py

@@ -0,0 +1,20 @@
+from pydantic_settings import BaseSettings
+from functools import lru_cache
+
+
+class Settings(BaseSettings):
+    database_url: str = "sqlite+aiosqlite:///./data/reliability.db"
+    app_env: str = "development"
+    secret_key: str = "change-me-in-production"
+
+    groq_api_key: str = ""
+    mistral_api_key: str = ""
+    gemini_api_key: str = ""
+    openrouter_api_key: str = ""
+
+    model_config = {"env_file": ".env", "env_file_encoding": "utf-8"}
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

pytest_llm_sushit-0.1.0/app/database.py

@@ -0,0 +1,26 @@
+from sqlmodel import SQLModel, create_engine, Session
+from app.config import get_settings
+import os
+
+engine = None
+
+
+def get_engine():
+    global engine
+    if engine is None:
+        settings = get_settings()
+        db_path = settings.database_url.replace("sqlite+aiosqlite:///", "")
+        os.makedirs(os.path.dirname(db_path) if os.path.dirname(db_path) else ".", exist_ok=True)
+        engine = create_engine(settings.database_url.replace("sqlite+aiosqlite", "sqlite"))
+    return engine
+
+
+def init_db():
+    eng = get_engine()
+    SQLModel.metadata.create_all(eng)
+
+
+def get_session():
+    eng = get_engine()
+    with Session(eng) as session:
+        yield session

pytest_llm_sushit-0.1.0/app/main.py

@@ -0,0 +1,116 @@
+from fastapi import FastAPI, Request
+from fastapi.templating import Jinja2Templates
+from sqlmodel import Session, select
+from app.database import init_db, get_engine
+from app.models.eval import EvalDataset, EvalRun, EvalResult, TestCase
+from app.models.trace import Trace, Span
+from app.api import eval as eval_api
+from app.api import trace as trace_api
+from app.api import dashboard as dashboard_api
+from app.services.providers.registry import list_providers
+
+app = FastAPI(title="AI Reliability Platform", version="0.1.0")
+
+templates = Jinja2Templates(directory="app/templates")
+
+app.include_router(eval_api.router)
+app.include_router(trace_api.router)
+app.include_router(dashboard_api.router)
+
+
+@app.on_event("startup")
+def on_startup():
+    init_db()
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@app.get("/")
+def dashboard(request: Request):
+    with Session(get_engine()) as session:
+        total_runs = session.exec(select(EvalRun)).all()
+        total_traces = session.exec(select(Trace)).all()
+        recent_runs = sorted(total_runs, key=lambda r: r.created_at, reverse=True)[:5]
+        recent_traces = sorted(total_traces, key=lambda t: t.created_at, reverse=True)[:5]
+        total_results = len(session.exec(select(EvalResult)).all())
+        passed = len(session.exec(select(EvalResult).where(EvalResult.status == "pass")).all())
+
+    pass_rate = f"{(passed / total_results * 100):.1f}%" if total_results > 0 else "N/A"
+
+    return templates.TemplateResponse("dashboard.html", context={
+        "request": request,
+        "total_runs": len(total_runs),
+        "total_traces": len(total_traces),
+        "total_results": total_results,
+        "passed": passed,
+        "pass_rate": pass_rate,
+        "recent_runs": recent_runs,
+        "recent_traces": recent_traces,
+    })
+
+
+@app.get("/eval/datasets")
+def eval_datasets_page(request: Request):
+    with Session(get_engine()) as session:
+        datasets = list(session.exec(select(EvalDataset)).all())
+    return templates.TemplateResponse("eval_datasets.html", context={"request": request, "datasets": datasets})
+
+
+@app.get("/eval/datasets/{dataset_id}")
+def eval_dataset_detail_page(request: Request, dataset_id: int):
+    with Session(get_engine()) as session:
+        dataset = session.get(EvalDataset, dataset_id)
+        cases = list(session.exec(select(TestCase).where(TestCase.dataset_id == dataset_id)).all())
+    if not dataset:
+        return templates.TemplateResponse("404.html", context={"request": request}, status_code=404)
+    return templates.TemplateResponse("eval_detail.html", context={"request": request, "dataset": dataset, "cases": cases})
+
+
+@app.get("/eval/runs")
+def eval_runs_page(request: Request):
+    with Session(get_engine()) as session:
+        runs = list(session.exec(select(EvalRun).order_by(EvalRun.created_at.desc())).all())
+    return templates.TemplateResponse("eval_runs.html", context={"request": request, "runs": runs})
+
+
+@app.get("/eval/runs/{run_id}")
+def eval_run_detail_page(request: Request, run_id: int):
+    with Session(get_engine()) as session:
+        run = session.get(EvalRun, run_id)
+        results = list(session.exec(select(EvalResult).where(EvalResult.run_id == run_id)).all())
+        if run:
+            cases = {c.id: c for c in session.exec(select(TestCase).where(TestCase.dataset_id == run.dataset_id)).all()}
+        else:
+            cases = {}
+    if not run:
+        return templates.TemplateResponse("404.html", context={"request": request}, status_code=404)
+    return templates.TemplateResponse("eval_run_detail.html", context={
+        "request": request, "run": run, "results": results, "cases": cases,
+    })
+
+
+@app.get("/traces")
+def traces_page(request: Request):
+    with Session(get_engine()) as session:
+        traces = list(session.exec(select(Trace).order_by(Trace.created_at.desc())).all())
+    return templates.TemplateResponse("traces.html", context={"request": request, "traces": traces})
+
+
+@app.get("/traces/{trace_id}")
+def trace_detail_page(request: Request, trace_id: str):
+    with Session(get_engine()) as session:
+        trace = session.exec(select(Trace).where(Trace.trace_id == trace_id)).first()
+        spans = list(session.exec(select(Span).where(Span.trace_id == trace_id).order_by(Span.created_at)).all()) if trace else []
+    if not trace:
+        return templates.TemplateResponse("404.html", context={"request": request}, status_code=404)
+    return templates.TemplateResponse("trace_detail.html", context={
+        "request": request, "trace": trace, "spans": spans,
+    })
+
+
+@app.get("/api/v1/providers")
+def get_providers():
+    return list_providers()