fixtureforge 2.1.0__tar.gz → 2.2.0__tar.gz

{fixtureforge-2.1.0 → fixtureforge-2.2.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: fixtureforge
-Version: 2.1.0
+Version: 2.2.0
 Summary: Agentic Test Data Harness: memory, multi-agent swarms, permission gates, coverage analysis. Provider-agnostic (Gemini, OpenAI, Anthropic, Ollama).
 License: MIT
 License-File: LICENSE
-Keywords: testing,fixtures,test-data,qa,automation,synthetic-data,llm,pytest
+Keywords: pytest,fixtures,test-data,pydantic,faker,factory,ai,testing,mock-data,qa,automation,synthetic-data,llm,test-fixtures,fixture-generator
 Author: Yaniv Metuku
 Requires-Python: >=3.11,<4.0
 Classifier: License :: OSI Approved :: MIT License
@@ -32,6 +32,10 @@ Project-URL: Homepage, https://fixtureforge.dev
 Project-URL: Repository, https://github.com/Yaniv2809/fixtureforge
 Description-Content-Type: text/markdown
 
+<p align="center">
+  <img src="fixtureforge-logo.png" alt="FixtureForge Logo" width="400"/>
+</p>
+
 # FixtureForge
 
 **Agentic Test Data Harness for Python.**  
@@ -421,7 +425,9 @@ MIT — see [LICENSE](LICENSE).
 
 ## Links
 
+- **Docs**: https://yaniv2809.github.io/fixtureforge/
 - **PyPI**: https://pypi.org/project/fixtureforge/
 - **Repository**: https://github.com/Yaniv2809/fixtureforge
 - **Issues**: https://github.com/Yaniv2809/fixtureforge/issues
 
+💬 [Join the discussion](https://github.com/Yaniv2809/fixtureforge/discussions/1)

{fixtureforge-2.1.0 → fixtureforge-2.2.0}/README.md

@@ -1,3 +1,7 @@
+<p align="center">
+  <img src="fixtureforge-logo.png" alt="FixtureForge Logo" width="400"/>
+</p>
+
 # FixtureForge
 
 **Agentic Test Data Harness for Python.**  
@@ -387,6 +391,9 @@ MIT — see [LICENSE](LICENSE).
 
 ## Links
 
+- **Docs**: https://yaniv2809.github.io/fixtureforge/
 - **PyPI**: https://pypi.org/project/fixtureforge/
 - **Repository**: https://github.com/Yaniv2809/fixtureforge
 - **Issues**: https://github.com/Yaniv2809/fixtureforge/issues
+
+💬 [Join the discussion](https://github.com/Yaniv2809/fixtureforge/discussions/1)

{fixtureforge-2.1.0 → fixtureforge-2.2.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "fixtureforge"
-version = "2.1.0"
+version = "2.2.0"
 description = "Agentic Test Data Harness: memory, multi-agent swarms, permission gates, coverage analysis. Provider-agnostic (Gemini, OpenAI, Anthropic, Ollama)."
 authors = ["Yaniv Metuku"]
 license = "MIT"
 readme = "README.md"
 homepage = "https://fixtureforge.dev"
 repository = "https://github.com/Yaniv2809/fixtureforge"
-keywords = ["testing", "fixtures", "test-data", "qa", "automation", "synthetic-data", "llm", "pytest"]
+keywords = ["pytest", "fixtures", "test-data", "pydantic", "faker", "factory", "ai", "testing", "mock-data", "qa", "automation", "synthetic-data", "llm", "test-fixtures", "fixture-generator"]
 
 # ---------------------------------------------------------------------------
 # Core dependencies — always installed, no AI required

fixtureforge-2.2.0/src/fixtureforge/ai/__init__.py

@@ -0,0 +1,25 @@
+"""
+AI-powered generation and validation for FixtureForge.
+
+Core (always available):
+    from fixtureforge.ai import AIEngine, ResponseCache
+
+v2.2.0 — Opt-in AI features (require GROQ_API_KEY for LLM mode):
+    from fixtureforge.ai import assert_semantic_match, SemanticResult
+    from fixtureforge.ai import SmartFailureAnalyzer
+"""
+from .engine import AIEngine
+from .cache import ResponseCache
+from .semantic_assert import assert_semantic_match, SemanticResult, SemanticAssertionError
+from .failure_analyzer import SmartFailureAnalyzer
+
+__all__ = [
+    # Core
+    "AIEngine",
+    "ResponseCache",
+    # v2.2.0 features
+    "assert_semantic_match",
+    "SemanticResult",
+    "SemanticAssertionError",
+    "SmartFailureAnalyzer",
+]

fixtureforge-2.2.0/src/fixtureforge/ai/_audit_log.py

@@ -0,0 +1,44 @@
+"""
+Audit log for FixtureForge AI features.
+
+Every LLM call made by assert_semantic_match or SmartFailureAnalyzer
+is appended here as a JSONL record. Zero API calls are made during import.
+
+Log location: .fixtureforge_ai_log.jsonl  (project root / cwd)
+Format:       one JSON object per line, UTF-8, append-only.
+"""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+LOG_FILENAME = ".fixtureforge_ai_log.jsonl"
+
+
+def _log_path() -> Path:
+    """Return the audit log path relative to the current working directory."""
+    return Path(os.getcwd()) / LOG_FILENAME
+
+
+def log_event(event_type: str, data: dict[str, Any]) -> None:
+    """
+    Append one event record to the audit log.
+
+    Args:
+        event_type: Short label — e.g. "semantic_assert_llm", "failure_analysis".
+        data:       Dict of arbitrary key-value pairs to record.
+    """
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "event": event_type,
+        **data,
+    }
+    try:
+        with open(_log_path(), "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except OSError:
+        # Never crash a test run because of a logging failure.
+        pass

fixtureforge-2.2.0/src/fixtureforge/ai/_groq_client.py

@@ -0,0 +1,109 @@
+"""
+Lightweight Groq REST client for FixtureForge AI features.
+
+Uses only `requests` (already a core dependency) — no extra pip install needed.
+Model: llama3-8b-8192 (fast, free tier, sufficient for analysis tasks).
+
+Usage:
+    from fixtureforge.ai._groq_client import groq_complete, LLMClientError
+    response = groq_complete(system="You are...", user="Analyze this...")
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Optional
+
+import requests
+
+GROQ_API_URL = "https://api.groq.com/openai/v1/chat/completions"
+DEFAULT_MODEL = "llama3-8b-8192"
+DEFAULT_TIMEOUT = 10
+DEFAULT_RETRIES = 2
+
+
+class LLMClientError(RuntimeError):
+    """Raised when the Groq API call fails after all retries."""
+
+
+def groq_complete(
+    system: str,
+    user: str,
+    model: str = DEFAULT_MODEL,
+    timeout: int = DEFAULT_TIMEOUT,
+    retries: int = DEFAULT_RETRIES,
+    api_key: Optional[str] = None,
+) -> str:
+    """
+    Send a chat completion request to Groq and return the response text.
+
+    Args:
+        system:  System prompt (role definition / instructions).
+        user:    User message (the actual task / input).
+        model:   Groq model identifier. Default: llama3-8b-8192.
+        timeout: Request timeout in seconds.
+        retries: Number of retry attempts on transient errors.
+        api_key: Override GROQ_API_KEY env var (optional).
+
+    Returns:
+        The model's response as a plain string.
+
+    Raises:
+        LLMClientError: If GROQ_API_KEY is not set or all retries fail.
+    """
+    key = api_key or os.environ.get("GROQ_API_KEY", "")
+    if not key:
+        raise LLMClientError(
+            "GROQ_API_KEY is not set. "
+            "Export it with: export GROQ_API_KEY=gsk_... "
+            "or set it in your .env file."
+        )
+
+    headers = {
+        "Authorization": f"Bearer {key}",
+        "Content-Type": "application/json",
+    }
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        "temperature": 0.2,
+        "max_tokens": 512,
+    }
+
+    last_error: Optional[Exception] = None
+    for attempt in range(retries + 1):
+        try:
+            resp = requests.post(
+                GROQ_API_URL,
+                headers=headers,
+                json=payload,
+                timeout=timeout,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+            return data["choices"][0]["message"]["content"].strip()
+        except requests.exceptions.Timeout as exc:
+            last_error = exc
+            if attempt < retries:
+                time.sleep(1.0 * (attempt + 1))
+        except requests.exceptions.HTTPError as exc:
+            # 429 rate limit — back off and retry
+            if resp.status_code == 429 and attempt < retries:
+                time.sleep(2.0 * (attempt + 1))
+                last_error = exc
+            else:
+                raise LLMClientError(
+                    f"Groq API returned HTTP {resp.status_code}: {resp.text[:200]}"
+                ) from exc
+        except (KeyError, ValueError, json.JSONDecodeError) as exc:
+            raise LLMClientError(
+                f"Unexpected Groq response format: {exc}"
+            ) from exc
+
+    raise LLMClientError(
+        f"Groq request failed after {retries + 1} attempts: {last_error}"
+    )

fixtureforge-2.2.0/src/fixtureforge/ai/failure_analyzer.py

@@ -0,0 +1,253 @@
+"""
+SmartFailureAnalyzer — AI-powered pytest failure diagnostics.
+
+Hooks into pytest's report pipeline and analyzes FAILED tests using Groq.
+Prints a structured diagnosis to the console after each failure.
+
+IMPORTANT:
+  - Never modifies test results or retry logic
+  - Never changes the failure status of any test
+  - Only activated on explicit opt-in (add to conftest.py)
+  - Requires GROQ_API_KEY — silently skips analysis if not set
+
+Setup (conftest.py):
+    from fixtureforge.ai import SmartFailureAnalyzer
+    pytest_plugins = [SmartFailureAnalyzer()]   # pass an instance
+
+Or register via pytest plugin list:
+    # conftest.py
+    from fixtureforge.ai.failure_analyzer import SmartFailureAnalyzer
+    def pytest_configure(config):
+        config.pluginmanager.register(SmartFailureAnalyzer(), "fixtureforge-analyzer")
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any, Optional
+
+from ._audit_log import log_event
+from ._groq_client import LLMClientError, groq_complete
+
+# ---------------------------------------------------------------------------
+# Terminal encoding safety
+# Detect if the terminal supports Unicode box-drawing characters.
+# Windows cmd / PowerShell with cp1255 / cp1252 encoding cannot render them.
+# ---------------------------------------------------------------------------
+
+def _supports_unicode() -> bool:
+    enc = getattr(sys.stdout, "encoding", None) or ""
+    return enc.lower().replace("-", "") in ("utf8", "utf8sig")
+
+
+_USE_UNICODE = _supports_unicode()
+
+
+def _box(title: str, lines: list[str]) -> str:
+    """Render a labeled box. ASCII-safe on non-UTF-8 terminals."""
+    if _USE_UNICODE:
+        width = max(len(title) + 4, max((len(l) for l in lines), default=0) + 4, 50)
+        top    = f"\u2554{'=' * (width - 2)}\u2557"
+        header = f"\u2551  {title:<{width - 4}}  \u2551"
+        sep    = f"\u2560{'-' * (width - 2)}\u2563"
+        bottom = f"\u255a{'=' * (width - 2)}\u255d"
+        body   = "\n".join(f"\u2551  {l:<{width - 4}}  \u2551" for l in lines)
+        return "\n".join([top, header, sep, body, bottom])
+    else:
+        width = max(len(title) + 4, max((len(l) for l in lines), default=0) + 4, 50)
+        border = "+" + "=" * (width - 2) + "+"
+        divider = "+" + "-" * (width - 2) + "+"
+        header = f"|  {title:<{width - 4}}  |"
+        body   = "\n".join(f"|  {l:<{width - 4}}  |" for l in lines)
+        return "\n".join([border, header, divider, body, border])
+
+
+# ---------------------------------------------------------------------------
+# Groq prompt
+# ---------------------------------------------------------------------------
+
+_SYSTEM_PROMPT = (
+    "You are a senior QA engineer and Python expert. "
+    "Analyze test failures and provide concise, actionable diagnostics. "
+    "Return ONLY valid JSON — no markdown, no explanation outside the JSON. "
+    "Format: {"
+    '"explanation": "one sentence describing what went wrong", '
+    '"likely_cause": "root cause in plain English", '
+    '"suggested_fix": "specific code-level action to resolve it"'
+    "}"
+)
+
+_TAIL_LINES = 20  # number of traceback lines to send
+
+
+# ---------------------------------------------------------------------------
+# SmartFailureAnalyzer plugin
+# ---------------------------------------------------------------------------
+
+class SmartFailureAnalyzer:
+    """
+    pytest plugin that analyzes FAILED tests with Groq AI.
+
+    Behavior:
+    - Triggers only when a test's call phase fails
+    - Sends: test ID + exception type + exception message + last 20 traceback lines
+    - Prints a structured analysis block to stdout
+    - Logs every interaction to .fixtureforge_ai_log.jsonl
+    - NEVER modifies report.passed or any test state
+
+    Registration (conftest.py):
+        from fixtureforge.ai import SmartFailureAnalyzer
+        def pytest_configure(config):
+            config.pluginmanager.register(SmartFailureAnalyzer(), "forge-analyzer")
+    """
+
+    def __init__(self, api_key: Optional[str] = None) -> None:
+        self._api_key = api_key
+        self._analyzed: int = 0
+
+    # ------------------------------------------------------------------
+    # pytest hook
+    # ------------------------------------------------------------------
+
+    def pytest_runtest_logreport(self, report: Any) -> None:
+        """Called by pytest after each test phase (setup / call / teardown)."""
+        if report.when != "call" or not report.failed:
+            return
+
+        # Skip if GROQ_API_KEY is not available — never block the test run
+        key = self._api_key or os.environ.get("GROQ_API_KEY", "")
+        if not key:
+            return
+
+        self._analyze(report, api_key=key)
+
+    # ------------------------------------------------------------------
+    # Analysis
+    # ------------------------------------------------------------------
+
+    def _extract_failure_info(self, report: Any) -> dict[str, str]:
+        """Pull structured info out of a pytest report object."""
+        nodeid = getattr(report, "nodeid", "unknown_test")
+        longrepr = report.longrepr
+
+        # longrepr can be a string, tuple, or ExceptionInfo object
+        if isinstance(longrepr, str):
+            full_tb = longrepr
+            exc_type = "UnknownError"
+            exc_msg  = longrepr[:300]
+        elif isinstance(longrepr, tuple):
+            # (path, lineno, message) format from some pytest versions
+            full_tb = "\n".join(str(p) for p in longrepr)
+            exc_type = "Error"
+            exc_msg  = str(longrepr[-1])[:300]
+        else:
+            # ExceptionInfo / ReprExceptionInfo
+            full_tb  = str(longrepr)
+            lines    = full_tb.splitlines()
+            exc_type = "UnknownError"
+            exc_msg  = ""
+            # find the last "E  " line — that's the exception message
+            for line in reversed(lines):
+                stripped = line.strip()
+                if stripped.startswith("E "):
+                    exc_msg = stripped[2:].strip()
+                    break
+            # find exception class name
+            for line in reversed(lines):
+                for keyword in ("Error", "Exception", "Failure", "assert"):
+                    if keyword in line:
+                        exc_type = line.strip()[:80]
+                        break
+
+        # last N lines of traceback (avoid huge prompts)
+        tail_lines = full_tb.splitlines()[-_TAIL_LINES:]
+        tail = "\n".join(tail_lines)
+
+        return {
+            "nodeid": nodeid,
+            "exc_type": exc_type,
+            "exc_msg": exc_msg,
+            "tail": tail,
+        }
+
+    def _call_groq(self, info: dict[str, str], api_key: str) -> dict[str, str]:
+        """Send failure info to Groq and parse structured JSON response."""
+        user_msg = (
+            f"Test: {info['nodeid']}\n"
+            f"Error type: {info['exc_type']}\n"
+            f"Error message: {info['exc_msg']}\n"
+            f"Traceback (last {_TAIL_LINES} lines):\n{info['tail']}"
+        )
+        raw = groq_complete(
+            system=_SYSTEM_PROMPT,
+            user=user_msg,
+            api_key=api_key,
+        )
+        # strip markdown fences if model ignored instructions
+        clean = raw.strip().lstrip("```json").lstrip("```").rstrip("```").strip()
+        parsed = json.loads(clean)
+        return {
+            "explanation":   str(parsed.get("explanation", "")),
+            "likely_cause":  str(parsed.get("likely_cause", "")),
+            "suggested_fix": str(parsed.get("suggested_fix", "")),
+        }
+
+    def _print_analysis(self, info: dict[str, str], analysis: dict[str, str]) -> None:
+        """Print the analysis block to stdout — encoding-safe."""
+        label = "FixtureForge AI Analysis"
+        lines = [
+            f"Test   : {info['nodeid']}",
+            "",
+            f"Cause  : {analysis['likely_cause']}",
+            "",
+            f"Detail : {analysis['explanation']}",
+            "",
+            f"Fix    : {analysis['suggested_fix']}",
+        ]
+        box = _box(label, lines)
+        # Write via sys.stdout with explicit encoding fallback
+        try:
+            print("\n" + box + "\n", flush=True)
+        except UnicodeEncodeError:
+            safe = (box
+                    .encode(sys.stdout.encoding or "ascii", errors="replace")
+                    .decode(sys.stdout.encoding or "ascii"))
+            print("\n" + safe + "\n", flush=True)
+
+    def _analyze(self, report: Any, api_key: str) -> None:
+        """Full analysis pipeline for one failed test."""
+        info = self._extract_failure_info(report)
+
+        try:
+            analysis = self._call_groq(info, api_key=api_key)
+        except (LLMClientError, json.JSONDecodeError, KeyError) as exc:
+            log_event("failure_analysis_error", {
+                "nodeid": info["nodeid"],
+                "error": str(exc),
+            })
+            return  # silently skip — never interrupt the test session
+
+        self._print_analysis(info, analysis)
+        self._analyzed += 1
+
+        log_event("failure_analysis", {
+            "nodeid":       info["nodeid"],
+            "exc_type":     info["exc_type"],
+            "explanation":  analysis["explanation"],
+            "likely_cause": analysis["likely_cause"],
+            "suggested_fix": analysis["suggested_fix"],
+        })
+
+    # ------------------------------------------------------------------
+    # Summary (optional hook)
+    # ------------------------------------------------------------------
+
+    def pytest_terminal_summary(self, terminalreporter: Any, exitstatus: int) -> None:
+        """Append a one-line summary at the end of the pytest session."""
+        if self._analyzed > 0:
+            terminalreporter.write_sep(
+                "-",
+                f"FixtureForge AI: analyzed {self._analyzed} failure(s) -- "
+                f"see .fixtureforge_ai_log.jsonl for full audit trail",
+            )

fixtureforge-2.2.0/src/fixtureforge/ai/semantic_assert.py

@@ -0,0 +1,252 @@
+"""
+assert_semantic_match — semantic validation for AI-generated or natural-language outputs.
+
+Two modes:
+  Local (default, use_llm=False):
+      Cosine similarity on TF-IDF vectors built from stdlib only.
+      Zero API calls. Zero cost. Always available offline.
+
+  LLM (use_llm=True):
+      Sends actual + expected_intent to Groq for a structured judgment.
+      Requires GROQ_API_KEY. Every call is logged to .fixtureforge_ai_log.jsonl.
+
+Usage:
+    from fixtureforge.ai import assert_semantic_match
+
+    # Local (no API key needed)
+    result = assert_semantic_match(
+        actual="The product was delivered on time and in perfect condition.",
+        expected_intent="positive delivery experience",
+    )
+
+    # LLM-powered (explicit opt-in)
+    result = assert_semantic_match(
+        actual=chatbot_response,
+        expected_intent="politely decline the refund request",
+        use_llm=True,
+        threshold=0.8,
+    )
+    # Raises SemanticAssertionError if result.passed is False
+"""
+from __future__ import annotations
+
+import json
+import math
+from collections import Counter
+from dataclasses import dataclass
+from typing import Optional
+
+from ._audit_log import log_event
+from ._groq_client import LLMClientError, groq_complete
+
+# ---------------------------------------------------------------------------
+# Public types
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SemanticResult:
+    """Return value of assert_semantic_match."""
+    passed: bool
+    score: float        # 0.0 – 1.0
+    reason: str         # human-readable explanation
+    method: str         # "local" or "llm"
+
+    def __str__(self) -> str:
+        status = "PASS" if self.passed else "FAIL"
+        return (
+            f"SemanticResult({status} | score={self.score:.2f} "
+            f"| method={self.method} | reason={self.reason!r})"
+        )
+
+
+class SemanticAssertionError(AssertionError):
+    """Raised when assert_semantic_match fails."""
+
+    def __init__(self, result: SemanticResult, actual: str, expected_intent: str) -> None:
+        self.result = result
+        self.actual = actual
+        self.expected_intent = expected_intent
+        super().__init__(
+            f"\n\nSemantic assertion failed.\n"
+            f"  Expected intent : {expected_intent!r}\n"
+            f"  Actual output   : {actual!r}\n"
+            f"  Score           : {result.score:.2f} (threshold not met)\n"
+            f"  Method          : {result.method}\n"
+            f"  Reason          : {result.reason}\n"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Local cosine similarity (stdlib only — no external dependencies)
+# ---------------------------------------------------------------------------
+
+def _tokenize(text: str) -> Counter:
+    """Lowercase, split on whitespace, return token frequency counter."""
+    tokens = text.lower().split()
+    # strip punctuation from token edges
+    cleaned = [t.strip(".,!?;:\"'()-[]{}") for t in tokens]
+    return Counter(t for t in cleaned if t)
+
+
+def _cosine_similarity(text_a: str, text_b: str) -> float:
+    """
+    TF-IDF cosine similarity between two strings using stdlib only.
+    Returns a value in [0.0, 1.0].
+    """
+    vec_a = _tokenize(text_a)
+    vec_b = _tokenize(text_b)
+
+    if not vec_a or not vec_b:
+        return 0.0
+
+    # dot product over shared tokens
+    shared = set(vec_a) & set(vec_b)
+    dot = sum(vec_a[w] * vec_b[w] for w in shared)
+
+    # magnitudes
+    mag_a = math.sqrt(sum(v * v for v in vec_a.values()))
+    mag_b = math.sqrt(sum(v * v for v in vec_b.values()))
+
+    if mag_a == 0.0 or mag_b == 0.0:
+        return 0.0
+
+    return dot / (mag_a * mag_b)
+
+
+def _local_assert(actual: str, expected_intent: str, threshold: float) -> SemanticResult:
+    score = _cosine_similarity(actual, expected_intent)
+    passed = score >= threshold
+    reason = (
+        f"Cosine similarity {score:.2f} >= threshold {threshold:.2f}"
+        if passed
+        else f"Cosine similarity {score:.2f} < threshold {threshold:.2f}"
+    )
+    return SemanticResult(passed=passed, score=score, reason=reason, method="local")
+
+
+# ---------------------------------------------------------------------------
+# LLM-powered assertion via Groq
+# ---------------------------------------------------------------------------
+
+_LLM_SYSTEM = (
+    "You are a test validation judge. "
+    "Given an actual output and an expected intent, "
+    "return ONLY valid JSON with no markdown, no explanation. "
+    'Format: {"passed": bool, "score": float, "reason": str}'
+)
+
+
+def _llm_assert(
+    actual: str,
+    expected_intent: str,
+    threshold: float,
+    api_key: Optional[str],
+) -> SemanticResult:
+    user_msg = (
+        f"Expected intent: {expected_intent}\n"
+        f"Actual output: {actual}\n"
+        f"Score threshold: {threshold}\n"
+        "Does the actual output satisfy the expected intent? "
+        "Return JSON only."
+    )
+
+    try:
+        raw = groq_complete(system=_LLM_SYSTEM, user=user_msg, api_key=api_key)
+        # strip markdown fences if model ignored instructions
+        raw = raw.strip().lstrip("```json").lstrip("```").rstrip("```").strip()
+        parsed = json.loads(raw)
+
+        score = float(parsed.get("score", 0.0))
+        passed = bool(parsed.get("passed", score >= threshold))
+        reason = str(parsed.get("reason", ""))
+
+        log_event("semantic_assert_llm", {
+            "actual": actual[:200],
+            "expected_intent": expected_intent[:200],
+            "threshold": threshold,
+            "score": score,
+            "passed": passed,
+            "reason": reason,
+        })
+
+        return SemanticResult(passed=passed, score=score, reason=reason, method="llm")
+
+    except (json.JSONDecodeError, KeyError, ValueError) as exc:
+        # Graceful degradation: fall back to local on parse error
+        log_event("semantic_assert_llm_parse_error", {"error": str(exc), "raw": raw[:300]})
+        result = _local_assert(actual, expected_intent, threshold)
+        result.reason = f"[LLM response unparseable, fell back to local] {result.reason}"
+        result.method = "local_fallback"
+        return result
+
+    except LLMClientError as exc:
+        # No API key or network error — fall back to local silently
+        log_event("semantic_assert_llm_error", {"error": str(exc)})
+        result = _local_assert(actual, expected_intent, threshold)
+        result.reason = f"[LLM unavailable, fell back to local] {result.reason}"
+        result.method = "local_fallback"
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def assert_semantic_match(
+    actual: str,
+    expected_intent: str,
+    use_llm: bool = False,
+    threshold: float = 0.75,
+    api_key: Optional[str] = None,
+    raise_on_failure: bool = True,
+) -> SemanticResult:
+    """
+    Assert that `actual` semantically matches `expected_intent`.
+
+    Args:
+        actual:           The string to validate (e.g. chatbot response, generated text).
+        expected_intent:  Natural language description of what the output should convey.
+        use_llm:          If True, use Groq for a more nuanced judgment.
+                          Requires GROQ_API_KEY. Default: False (local cosine similarity).
+        threshold:        Minimum score to pass. Range: [0.0, 1.0]. Default: 0.75.
+        api_key:          Override GROQ_API_KEY env var (optional).
+        raise_on_failure: If True (default), raise SemanticAssertionError on failure.
+
+    Returns:
+        SemanticResult(passed, score, reason, method)
+
+    Raises:
+        SemanticAssertionError: If the assertion fails and raise_on_failure=True.
+
+    Examples:
+        # Local — zero cost, always available
+        assert_semantic_match(
+            actual="Great experience, delivered next day!",
+            expected_intent="positive customer review",
+        )
+
+        # LLM — richer judgment, requires API key
+        result = assert_semantic_match(
+            actual=model_output,
+            expected_intent="professional refusal of the request",
+            use_llm=True,
+            threshold=0.8,
+        )
+        print(result.score, result.reason)
+    """
+    if not isinstance(actual, str) or not isinstance(expected_intent, str):
+        raise TypeError(
+            f"assert_semantic_match expects str arguments, "
+            f"got actual={type(actual).__name__}, "
+            f"expected_intent={type(expected_intent).__name__}"
+        )
+
+    if use_llm:
+        result = _llm_assert(actual, expected_intent, threshold, api_key)
+    else:
+        result = _local_assert(actual, expected_intent, threshold)
+
+    if not result.passed and raise_on_failure:
+        raise SemanticAssertionError(result, actual, expected_intent)
+
+    return result

fixtureforge-2.1.0/src/fixtureforge/ai/__init__.py

@@ -1,10 +0,0 @@
-"""
-AI-powered generation
-"""
-from .engine import AIEngine
-from .cache import ResponseCache
-
-__all__ = [
-    "AIEngine",
-    "ResponseCache",
-]