genassert 0.2.0__py3-none-any.whl

genassert/__init__.py

@@ -0,0 +1,43 @@
+"""
+genassert — pytest-native semantic testing for generative AI applications.
+
+Drop-in pytest plugin. No servers. No SaaS. No config.
+Works with OpenAI, Anthropic, LiteLLM, or any LLM client.
+"""
+
+from genassert.assertions.intent import assert_intent
+from genassert.assertions.tone import assert_tone
+from genassert.assertions.hallucination import assert_no_hallucination
+from genassert.assertions.budget import assert_token_budget
+from genassert.assertions.schema import assert_schema
+from genassert.assertions.similarity import assert_similar_to
+from genassert.assertions.language import assert_language
+from genassert.assertions.pii import assert_no_pii, PIIMatch
+from genassert.assertions.readability import assert_readability
+from genassert.assertions.sentiment import assert_sentiment
+from genassert.baseline import record_baseline, compare_baseline
+from genassert.judge import LocalJudge
+
+__version__ = "0.2.0"
+__all__ = [
+    # Core semantic assertions
+    "assert_intent",
+    "assert_tone",
+    "assert_no_hallucination",
+    "assert_similar_to",
+    # Structural assertions
+    "assert_token_budget",
+    "assert_schema",
+    # Advanced assertions (new in 0.2.0)
+    "assert_language",
+    "assert_no_pii",
+    "assert_readability",
+    "assert_sentiment",
+    # Types
+    "PIIMatch",
+    # Baseline regression
+    "record_baseline",
+    "compare_baseline",
+    # Local judge
+    "LocalJudge",
+]

genassert/_embed.py

@@ -0,0 +1,113 @@
+"""
+Embedding backend for genassert.
+
+Priority order (auto-detected):
+  1. sentence-transformers (local, free, fast) — recommended
+  2. openai embeddings (requires OPENAI_API_KEY)
+  3. numpy hash-based fallback (no deps, for smoke tests only)
+
+Set GENASSERT_EMBED_BACKEND=openai|local|fallback to force a backend.
+"""
+
+from __future__ import annotations
+import os
+import hashlib
+import math
+from functools import lru_cache
+
+_BACKEND_ENV = "GENASSERT_EMBED_BACKEND"
+_MODEL_ENV = "GENASSERT_EMBED_MODEL"
+
+_DEFAULT_LOCAL_MODEL = "all-MiniLM-L6-v2"
+_DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
+
+
+@lru_cache(maxsize=512)
+def embed_text(text: str) -> tuple[float, ...]:
+    """
+    Embed `text` into a float vector using the best available backend.
+    Results are cached in-process for performance.
+    """
+    backend = os.environ.get(_BACKEND_ENV, "auto").lower()
+
+    if backend == "openai":
+        return _embed_openai(text)
+    elif backend in ("local", "sentence-transformers"):
+        return _embed_local(text)
+    elif backend == "fallback":
+        return _embed_fallback(text)
+    else:
+        # auto-detect
+        return _embed_auto(text)
+
+
+def _embed_auto(text: str) -> tuple[float, ...]:
+    """Try backends in priority order."""
+    try:
+        return _embed_local(text)
+    except ImportError:
+        pass
+    try:
+        return _embed_openai(text)
+    except (ImportError, RuntimeError):
+        pass
+    return _embed_fallback(text)
+
+
+def _embed_local(text: str) -> tuple[float, ...]:
+    """Use sentence-transformers (local, no API cost)."""
+    try:
+        from sentence_transformers import SentenceTransformer
+    except ImportError:
+        raise ImportError(
+            "sentence-transformers not installed.\n"
+            "  Install: pip install sentence-transformers\n"
+            "  Or set GENASSERT_EMBED_BACKEND=openai"
+        )
+    model_name = os.environ.get(_MODEL_ENV, _DEFAULT_LOCAL_MODEL)
+    model = _get_local_model(model_name)
+    vec = model.encode(text, normalize_embeddings=True)
+    return tuple(float(v) for v in vec)
+
+
+@lru_cache(maxsize=4)
+def _get_local_model(model_name: str):
+    from sentence_transformers import SentenceTransformer
+    return SentenceTransformer(model_name)
+
+
+def _embed_openai(text: str) -> tuple[float, ...]:
+    """Use OpenAI embeddings API."""
+    try:
+        import openai
+    except ImportError:
+        raise ImportError("openai not installed. pip install openai")
+
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY not set. "
+            "Set it or use GENASSERT_EMBED_BACKEND=local"
+        )
+
+    model = os.environ.get(_MODEL_ENV, _DEFAULT_OPENAI_MODEL)
+    client = openai.OpenAI(api_key=api_key)
+    result = client.embeddings.create(input=[text], model=model)
+    vec = result.data[0].embedding
+    return tuple(float(v) for v in vec)
+
+
+def _embed_fallback(text: str) -> tuple[float, ...]:
+    """
+    Hash-based pseudo-embedding. No dependencies, but ONLY suitable for
+    smoke tests. Will NOT produce meaningful semantic similarity scores.
+    """
+    dim = 384
+    vec = []
+    for i in range(dim):
+        seed = hashlib.sha256(f"{i}:{text}".encode()).digest()
+        val = int.from_bytes(seed[:4], "big") / (2**32)
+        vec.append(val - 0.5)
+    # normalize
+    mag = math.sqrt(sum(v**2 for v in vec))
+    return tuple(v / mag for v in vec)

genassert/assertions/__init__.py

@@ -0,0 +1 @@
+# assertions package

genassert/assertions/budget.py

@@ -0,0 +1,62 @@
+"""
+assert_token_budget: Verify LLM response stays within a token limit.
+"""
+
+from __future__ import annotations
+
+
+def assert_token_budget(
+    response: str,
+    max_tokens: int,
+    tokenizer: str = "approx",
+) -> None:
+    """
+    Assert that `response` does not exceed `max_tokens`.
+
+    Parameters
+    ----------
+    response:
+        The LLM output to evaluate.
+    max_tokens:
+        Maximum allowed token count.
+    tokenizer:
+        "approx" uses a fast word-based approximation (~1.3 tokens/word).
+        "tiktoken" uses OpenAI's tiktoken (requires `pip install tiktoken`).
+        "chars" uses character count divided by 4.
+
+    Raises
+    ------
+    AssertionError
+        If the response exceeds the token budget.
+
+    Examples
+    --------
+    >>> assert_token_budget(response, max_tokens=200)
+    >>> assert_token_budget(response, max_tokens=500, tokenizer="tiktoken")
+    """
+    count = _count_tokens(response, tokenizer)
+    if count > max_tokens:
+        raise AssertionError(
+            f"Token budget exceeded.\n"
+            f"  Max allowed : {max_tokens} tokens\n"
+            f"  Actual count: {count} tokens (method: {tokenizer!r})\n"
+            f"  Response preview: {response[:200]!r}"
+        )
+
+
+def _count_tokens(text: str, method: str) -> int:
+    if method == "tiktoken":
+        try:
+            import tiktoken
+            enc = tiktoken.get_encoding("cl100k_base")
+            return len(enc.encode(text))
+        except ImportError:
+            raise ImportError(
+                "tokenizer='tiktoken' requires: pip install tiktoken"
+            )
+    elif method == "chars":
+        return len(text) // 4
+    else:
+        # approx: ~1.3 tokens per word is a reasonable average
+        words = len(text.split())
+        return int(words * 1.3)

genassert/assertions/hallucination.py

@@ -0,0 +1,89 @@
+"""
+assert_no_hallucination: Verify LLM output doesn't contradict known facts.
+
+Checks that factual claims in the response are consistent with a provided
+list of ground-truth facts, using semantic similarity and contradiction detection.
+"""
+
+from __future__ import annotations
+
+
+def assert_no_hallucination(
+    response: str,
+    known_facts: list[str],
+    contradiction_threshold: float = 0.85,
+) -> None:
+    """
+    Assert that `response` does not contradict any of the `known_facts`.
+
+    This does NOT verify that every fact is mentioned — it checks that
+    the response does not make claims that semantically contradict the facts.
+
+    Parameters
+    ----------
+    response:
+        The LLM output to evaluate.
+    known_facts:
+        A list of factual statements that must NOT be contradicted.
+        E.g. ["The capital of France is Paris", "The product costs $49/month"]
+    contradiction_threshold:
+        If the negation of a fact is highly similar to the response, flag it.
+        Default 0.85.
+
+    Raises
+    ------
+    AssertionError
+        If the response appears to contradict one or more known facts.
+
+    Examples
+    --------
+    >>> assert_no_hallucination(
+    ...     response,
+    ...     known_facts=["Python was created by Guido van Rossum in 1991"]
+    ... )
+    """
+    from genassert._embed import embed_text
+    from genassert.assertions.intent import _cosine_similarity
+
+    response_embedding = embed_text(response)
+    contradictions = []
+
+    for fact in known_facts:
+        # Build negation of the fact to detect contradictions
+        negation = _negate_fact(fact)
+        negation_embedding = embed_text(negation)
+        similarity = _cosine_similarity(response_embedding, negation_embedding)
+
+        if similarity > contradiction_threshold:
+            contradictions.append((fact, similarity))
+
+    if contradictions:
+        details = "\n".join(
+            f"  - Contradicts: {fact!r} (score: {score:.3f})"
+            for fact, score in contradictions
+        )
+        raise AssertionError(
+            f"Hallucination detected — response contradicts known facts:\n{details}\n"
+            f"  Response preview: {response[:200]!r}"
+        )
+
+
+def _negate_fact(fact: str) -> str:
+    """Produce a simple semantic negation of a fact for contradiction detection."""
+    fact = fact.strip()
+    negations = [
+        ("is not", "is"),
+        ("are not", "are"),
+        ("was not", "was"),
+        ("were not", "were"),
+        ("does not", "does"),
+        ("did not", "did"),
+        ("cannot", "can"),
+        ("will not", "will"),
+    ]
+    lower = fact.lower()
+    for neg, pos in negations:
+        if pos in lower:
+            return fact.replace(pos, neg, 1)
+    # Fallback: prepend "It is false that"
+    return f"It is false that {fact}"

genassert/assertions/intent.py

@@ -0,0 +1,62 @@
+"""
+assert_intent: Check that an LLM response addresses the expected intent.
+
+Uses embedding cosine similarity to compare the response against a
+natural-language description of what the response should convey.
+"""
+
+from __future__ import annotations
+import os
+from genassert._embed import embed_text
+
+
+def assert_intent(
+    response: str,
+    expected_intent: str,
+    threshold: float = 0.72,
+    model: str | None = None,
+) -> None:
+    """
+    Assert that `response` semantically matches the `expected_intent`.
+
+    Parameters
+    ----------
+    response:
+        The LLM output to evaluate.
+    expected_intent:
+        A plain-English description of what the response should convey.
+        E.g. "a concise summary of the article".
+    threshold:
+        Cosine similarity threshold (0–1). Default 0.72 works well for
+        most English text pairs. Lower = more lenient, higher = stricter.
+    model:
+        Optional embedding model override.
+
+    Raises
+    ------
+    AssertionError
+        If the cosine similarity between `response` and `expected_intent`
+        is below `threshold`.
+
+    Examples
+    --------
+    >>> assert_intent(response, "a polite refusal to the user's request")
+    >>> assert_intent(response, "Python code that reads a CSV file", threshold=0.80)
+    """
+    similarity = _cosine_similarity(embed_text(response), embed_text(expected_intent))
+    if similarity < threshold:
+        raise AssertionError(
+            f"Intent assertion failed.\n"
+            f"  Expected intent : {expected_intent!r}\n"
+            f"  Cosine similarity: {similarity:.3f} (threshold: {threshold})\n"
+            f"  Response preview : {response[:200]!r}"
+        )
+
+
+def _cosine_similarity(a: list[float], b: list[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    mag_a = sum(x**2 for x in a) ** 0.5
+    mag_b = sum(x**2 for x in b) ** 0.5
+    if mag_a == 0 or mag_b == 0:
+        return 0.0
+    return dot / (mag_a * mag_b)

genassert/assertions/language.py

@@ -0,0 +1,145 @@
+"""
+assert_language: Verify LLM response is written in the expected language.
+
+Uses character n-gram frequency profiles to detect language without
+any external dependencies. Supports 20+ common languages.
+"""
+
+from __future__ import annotations
+import re
+import unicodedata
+
+# Script/character-range based fast detection
+_SCRIPT_HINTS: list[tuple[str, tuple[int, int]]] = [
+    ("arabic",   (0x0600, 0x06FF)),
+    ("hebrew",   (0x0590, 0x05FF)),
+    ("chinese",  (0x4E00, 0x9FFF)),
+    ("japanese", (0x3040, 0x30FF)),
+    ("korean",   (0xAC00, 0xD7AF)),
+    ("thai",     (0x0E00, 0x0E7F)),
+    ("greek",    (0x0370, 0x03FF)),
+    ("cyrillic", (0x0400, 0x04FF)),
+    ("devanagari", (0x0900, 0x097F)),
+]
+
+# Common high-frequency words per language (stop-word fingerprint)
+_STOPWORDS: dict[str, list[str]] = {
+    "english":    ["the", "and", "is", "in", "to", "of", "a", "that", "it", "was"],
+    "spanish":    ["el", "la", "de", "que", "en", "los", "se", "las", "un", "una"],
+    "french":     ["le", "la", "les", "de", "et", "en", "un", "une", "du", "que"],
+    "german":     ["der", "die", "das", "und", "in", "den", "von", "zu", "ist", "mit"],
+    "portuguese": ["de", "da", "do", "que", "em", "os", "as", "um", "uma", "para"],
+    "italian":    ["il", "la", "di", "che", "e", "un", "una", "in", "del", "per"],
+    "dutch":      ["de", "het", "een", "van", "en", "in", "is", "dat", "op", "te"],
+    "russian":    ["и", "в", "не", "на", "я", "что", "с", "он", "как", "это"],
+    "polish":     ["i", "w", "nie", "to", "się", "na", "jest", "z", "że", "do"],
+    "turkish":    ["bir", "bu", "ve", "da", "de", "için", "ile", "var", "ne", "mi"],
+    "swedish":    ["och", "i", "en", "att", "det", "av", "på", "är", "för", "med"],
+    "norwegian":  ["og", "i", "en", "er", "til", "av", "på", "et", "som", "for"],
+    "danish":     ["og", "i", "en", "er", "til", "af", "på", "et", "som", "for"],
+    "finnish":    ["ja", "on", "ei", "se", "että", "hän", "oli", "en", "niin", "jo"],
+    "czech":      ["a", "je", "to", "v", "se", "na", "že", "z", "do", "s"],
+    "hungarian":  ["a", "az", "és", "hogy", "nem", "van", "egy", "is", "de", "meg"],
+    "romanian":   ["și", "în", "de", "că", "este", "la", "cu", "se", "nu", "din"],
+    "indonesian": ["yang", "dan", "di", "ke", "dari", "ini", "itu", "untuk", "dengan", "ada"],
+}
+
+_LANG_ALIASES: dict[str, str] = {
+    "en": "english", "es": "spanish", "fr": "french", "de": "german",
+    "pt": "portuguese", "it": "italian", "nl": "dutch", "ru": "russian",
+    "pl": "polish", "tr": "turkish", "sv": "swedish", "no": "norwegian",
+    "da": "danish", "fi": "finnish", "cs": "czech", "hu": "hungarian",
+    "ro": "romanian", "id": "indonesian",
+    "ar": "arabic", "he": "hebrew", "zh": "chinese", "ja": "japanese",
+    "ko": "korean", "th": "thai", "el": "greek", "hi": "devanagari",
+}
+
+
+def assert_language(
+    response: str,
+    expected_language: str,
+    min_confidence: float = 0.3,
+) -> str:
+    """
+    Assert that `response` is written in the expected language.
+
+    Parameters
+    ----------
+    response:
+        The LLM output to evaluate.
+    expected_language:
+        Language name (e.g. "english", "spanish", "french") or ISO 639-1
+        code (e.g. "en", "es", "fr").
+    min_confidence:
+        Minimum confidence score (0–1) to accept the detection. Default 0.3.
+
+    Returns
+    -------
+    str
+        The detected language name.
+
+    Raises
+    ------
+    AssertionError
+        If the detected language doesn't match expected.
+    ValueError
+        If expected_language is not recognised.
+
+    Examples
+    --------
+    >>> assert_language(response, "english")
+    >>> assert_language(response, "fr")          # ISO code
+    >>> assert_language(response, "spanish")
+    """
+    lang = expected_language.lower().strip()
+    lang = _LANG_ALIASES.get(lang, lang)
+
+    detected, confidence = _detect_language(response)
+
+    if detected != lang:
+        raise AssertionError(
+            f"Language assertion failed.\n"
+            f"  Expected language : {lang!r}\n"
+            f"  Detected language : {detected!r} (confidence: {confidence:.2f})\n"
+            f"  Response preview  : {response[:150]!r}"
+        )
+    if confidence < min_confidence:
+        raise AssertionError(
+            f"Language detected as {detected!r} but confidence too low.\n"
+            f"  Confidence: {confidence:.2f} (min: {min_confidence})\n"
+            f"  Response preview: {response[:150]!r}"
+        )
+    return detected
+
+
+def _detect_language(text: str) -> tuple[str, float]:
+    """Return (language_name, confidence) for `text`."""
+    if not text.strip():
+        return "unknown", 0.0
+
+    # Fast path: script detection for non-Latin scripts
+    for lang, (lo, hi) in _SCRIPT_HINTS:
+        count = sum(1 for ch in text if lo <= ord(ch) <= hi)
+        ratio = count / max(len(text), 1)
+        if ratio > 0.2:
+            return lang, min(1.0, ratio * 3)
+
+    # Stopword frequency scoring for Latin-script languages
+    words = set(re.findall(r"\b[a-zA-ZÀ-ÿ]+\b", text.lower()))
+    scores: dict[str, float] = {}
+    for lang, stopwords in _STOPWORDS.items():
+        hits = sum(1 for w in stopwords if w in words)
+        scores[lang] = hits / len(stopwords)
+
+    if not scores:
+        return "unknown", 0.0
+
+    best = max(scores, key=lambda k: scores[k])
+    best_score = scores[best]
+
+    # Normalise against second-best to get relative confidence
+    sorted_scores = sorted(scores.values(), reverse=True)
+    second = sorted_scores[1] if len(sorted_scores) > 1 else 0.0
+    confidence = best_score - second if best_score > 0 else 0.0
+
+    return best, min(1.0, confidence * 5)

genassert/assertions/pii.py

@@ -0,0 +1,107 @@
+"""
+assert_no_pii: Detect PII (Personally Identifiable Information) leakage in LLM responses.
+
+Checks for emails, phone numbers, SSNs, credit card numbers, IP addresses,
+and common name/address patterns — all via regex, zero dependencies.
+"""
+
+from __future__ import annotations
+import re
+from dataclasses import dataclass, field
+
+
+@dataclass
+class PIIMatch:
+    type: str
+    value: str
+    start: int
+    end: int
+
+
+# ── PII Patterns ──────────────────────────────────────────────────────────────
+
+_PATTERNS: dict[str, str] = {
+    "email": r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b",
+    "phone_us": r"\b(\+1[\s\-.]?)?\(?\d{3}\)?[\s\-.]?\d{3}[\s\-.]?\d{4}\b",
+    "phone_intl": r"\+\d{1,3}[\s\-.]?\(?\d{1,4}\)?[\s\-.]?\d{1,4}[\s\-.]?\d{1,9}",
+    "ssn": r"\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b",
+    "credit_card": r"\b(?:\d{4}[\s\-]?){3}\d{4}\b",
+    "ip_address": r"\b(?:\d{1,3}\.){3}\d{1,3}\b",
+    "date_of_birth": r"\b(?:DOB|date of birth|born)[:\s]+\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}\b",
+    "passport": r"\b[A-Z]{1,2}\d{6,9}\b",
+    "iban": r"\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z0-9]?){0,16}\b",
+}
+
+_COMPILED = {name: re.compile(pattern, re.IGNORECASE) for name, pattern in _PATTERNS.items()}
+
+
+def assert_no_pii(
+    response: str,
+    checks: list[str] | None = None,
+    allow: list[str] | None = None,
+) -> list[PIIMatch]:
+    """
+    Assert that `response` contains no PII.
+
+    Parameters
+    ----------
+    response:
+        The LLM output to evaluate.
+    checks:
+        List of PII types to check. Default: all types.
+        Options: "email", "phone_us", "phone_intl", "ssn", "credit_card",
+                 "ip_address", "date_of_birth", "passport", "iban"
+    allow:
+        List of PII types to explicitly allow (skip checking).
+        Useful when IP addresses or emails are expected in output.
+
+    Returns
+    -------
+    list[PIIMatch]
+        Empty list if no PII found (assertion passes).
+
+    Raises
+    ------
+    AssertionError
+        If any PII is detected.
+
+    Examples
+    --------
+    >>> assert_no_pii(response)
+    >>> assert_no_pii(response, checks=["email", "ssn"])
+    >>> assert_no_pii(response, allow=["ip_address"])   # allow IPs
+    """
+    active = set(checks or _COMPILED.keys()) - set(allow or [])
+    found: list[PIIMatch] = []
+
+    for pii_type in active:
+        pattern = _COMPILED.get(pii_type)
+        if pattern is None:
+            raise ValueError(f"Unknown PII type: {pii_type!r}. Valid: {list(_COMPILED)}")
+        for m in pattern.finditer(response):
+            found.append(PIIMatch(
+                type=pii_type,
+                value=_redact(m.group()),
+                start=m.start(),
+                end=m.end(),
+            ))
+
+    if found:
+        details = "\n".join(
+            f"  [{p.type}] {p.value!r} at position {p.start}"
+            for p in found
+        )
+        raise AssertionError(
+            f"PII detected in LLM response ({len(found)} match{'es' if len(found) > 1 else ''}):\n"
+            f"{details}\n"
+            f"  Response preview: {response[:200]!r}"
+        )
+
+    return found
+
+
+def _redact(value: str) -> str:
+    """Partially redact a PII value for safe display in error messages."""
+    if len(value) <= 4:
+        return "****"
+    return value[:2] + "*" * (len(value) - 4) + value[-2:]