fixtureforge 2.0.2__tar.gz → 2.2.0__tar.gz

{fixtureforge-2.0.2 → fixtureforge-2.2.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: fixtureforge
-Version: 2.0.2
+Version: 2.2.0
 Summary: Agentic Test Data Harness: memory, multi-agent swarms, permission gates, coverage analysis. Provider-agnostic (Gemini, OpenAI, Anthropic, Ollama).
 License: MIT
 License-File: LICENSE
-Keywords: testing,fixtures,test-data,qa,automation,synthetic-data,llm
+Keywords: pytest,fixtures,test-data,pydantic,faker,factory,ai,testing,mock-data,qa,automation,synthetic-data,llm,test-fixtures,fixture-generator
 Author: Yaniv Metuku
 Requires-Python: >=3.11,<4.0
 Classifier: License :: OSI Approved :: MIT License
@@ -32,6 +32,10 @@ Project-URL: Homepage, https://fixtureforge.dev
 Project-URL: Repository, https://github.com/Yaniv2809/fixtureforge
 Description-Content-Type: text/markdown
 
+<p align="center">
+  <img src="fixtureforge-logo.png" alt="FixtureForge Logo" width="400"/>
+</p>
+
 # FixtureForge
 
 **Agentic Test Data Harness for Python.**  
@@ -421,7 +425,9 @@ MIT — see [LICENSE](LICENSE).
 
 ## Links
 
+- **Docs**: https://yaniv2809.github.io/fixtureforge/
 - **PyPI**: https://pypi.org/project/fixtureforge/
 - **Repository**: https://github.com/Yaniv2809/fixtureforge
 - **Issues**: https://github.com/Yaniv2809/fixtureforge/issues
 
+💬 [Join the discussion](https://github.com/Yaniv2809/fixtureforge/discussions/1)

{fixtureforge-2.0.2 → fixtureforge-2.2.0}/README.md

@@ -1,3 +1,7 @@
+<p align="center">
+  <img src="fixtureforge-logo.png" alt="FixtureForge Logo" width="400"/>
+</p>
+
 # FixtureForge
 
 **Agentic Test Data Harness for Python.**  
@@ -387,6 +391,9 @@ MIT — see [LICENSE](LICENSE).
 
 ## Links
 
+- **Docs**: https://yaniv2809.github.io/fixtureforge/
 - **PyPI**: https://pypi.org/project/fixtureforge/
 - **Repository**: https://github.com/Yaniv2809/fixtureforge
 - **Issues**: https://github.com/Yaniv2809/fixtureforge/issues
+
+💬 [Join the discussion](https://github.com/Yaniv2809/fixtureforge/discussions/1)

{fixtureforge-2.0.2 → fixtureforge-2.2.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "fixtureforge"
-version = "2.0.2"
+version = "2.2.0"
 description = "Agentic Test Data Harness: memory, multi-agent swarms, permission gates, coverage analysis. Provider-agnostic (Gemini, OpenAI, Anthropic, Ollama)."
 authors = ["Yaniv Metuku"]
 license = "MIT"
 readme = "README.md"
 homepage = "https://fixtureforge.dev"
 repository = "https://github.com/Yaniv2809/fixtureforge"
-keywords = ["testing", "fixtures", "test-data", "qa", "automation", "synthetic-data", "llm"]
+keywords = ["pytest", "fixtures", "test-data", "pydantic", "faker", "factory", "ai", "testing", "mock-data", "qa", "automation", "synthetic-data", "llm", "test-fixtures", "fixture-generator"]
 
 # ---------------------------------------------------------------------------
 # Core dependencies — always installed, no AI required
@@ -62,6 +62,13 @@ mypy            = "^1.8.0"
 [tool.poetry.scripts]
 forge = "fixtureforge.cli.commands:cli"
 
+# ---------------------------------------------------------------------------
+# pytest plugin — auto-registered when fixtureforge is installed
+# Makes the ``forge`` fixture available in every test without any conftest.py
+# ---------------------------------------------------------------------------
+[tool.poetry.plugins."pytest11"]
+fixtureforge = "fixtureforge.pytest_plugin"
+
 [build-system]
 requires      = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

{fixtureforge-2.0.2 → fixtureforge-2.2.0}/src/fixtureforge/__init__.py

@@ -59,7 +59,7 @@ from .security.permissions import (
     ForgeCoordinator,
 )
 
-__version__ = "2.0.2"
+__version__ = "2.1.0"
 
 T = TypeVar("T", bound=BaseModel)
 

fixtureforge-2.2.0/src/fixtureforge/ai/__init__.py

@@ -0,0 +1,25 @@
+"""
+AI-powered generation and validation for FixtureForge.
+
+Core (always available):
+    from fixtureforge.ai import AIEngine, ResponseCache
+
+v2.2.0 — Opt-in AI features (require GROQ_API_KEY for LLM mode):
+    from fixtureforge.ai import assert_semantic_match, SemanticResult
+    from fixtureforge.ai import SmartFailureAnalyzer
+"""
+from .engine import AIEngine
+from .cache import ResponseCache
+from .semantic_assert import assert_semantic_match, SemanticResult, SemanticAssertionError
+from .failure_analyzer import SmartFailureAnalyzer
+
+__all__ = [
+    # Core
+    "AIEngine",
+    "ResponseCache",
+    # v2.2.0 features
+    "assert_semantic_match",
+    "SemanticResult",
+    "SemanticAssertionError",
+    "SmartFailureAnalyzer",
+]

fixtureforge-2.2.0/src/fixtureforge/ai/_audit_log.py

@@ -0,0 +1,44 @@
+"""
+Audit log for FixtureForge AI features.
+
+Every LLM call made by assert_semantic_match or SmartFailureAnalyzer
+is appended here as a JSONL record. Zero API calls are made during import.
+
+Log location: .fixtureforge_ai_log.jsonl  (project root / cwd)
+Format:       one JSON object per line, UTF-8, append-only.
+"""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+LOG_FILENAME = ".fixtureforge_ai_log.jsonl"
+
+
+def _log_path() -> Path:
+    """Return the audit log path relative to the current working directory."""
+    return Path(os.getcwd()) / LOG_FILENAME
+
+
+def log_event(event_type: str, data: dict[str, Any]) -> None:
+    """
+    Append one event record to the audit log.
+
+    Args:
+        event_type: Short label — e.g. "semantic_assert_llm", "failure_analysis".
+        data:       Dict of arbitrary key-value pairs to record.
+    """
+    record = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "event": event_type,
+        **data,
+    }
+    try:
+        with open(_log_path(), "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except OSError:
+        # Never crash a test run because of a logging failure.
+        pass

fixtureforge-2.2.0/src/fixtureforge/ai/_groq_client.py

@@ -0,0 +1,109 @@
+"""
+Lightweight Groq REST client for FixtureForge AI features.
+
+Uses only `requests` (already a core dependency) — no extra pip install needed.
+Model: llama3-8b-8192 (fast, free tier, sufficient for analysis tasks).
+
+Usage:
+    from fixtureforge.ai._groq_client import groq_complete, LLMClientError
+    response = groq_complete(system="You are...", user="Analyze this...")
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Optional
+
+import requests
+
+GROQ_API_URL = "https://api.groq.com/openai/v1/chat/completions"
+DEFAULT_MODEL = "llama3-8b-8192"
+DEFAULT_TIMEOUT = 10
+DEFAULT_RETRIES = 2
+
+
+class LLMClientError(RuntimeError):
+    """Raised when the Groq API call fails after all retries."""
+
+
+def groq_complete(
+    system: str,
+    user: str,
+    model: str = DEFAULT_MODEL,
+    timeout: int = DEFAULT_TIMEOUT,
+    retries: int = DEFAULT_RETRIES,
+    api_key: Optional[str] = None,
+) -> str:
+    """
+    Send a chat completion request to Groq and return the response text.
+
+    Args:
+        system:  System prompt (role definition / instructions).
+        user:    User message (the actual task / input).
+        model:   Groq model identifier. Default: llama3-8b-8192.
+        timeout: Request timeout in seconds.
+        retries: Number of retry attempts on transient errors.
+        api_key: Override GROQ_API_KEY env var (optional).
+
+    Returns:
+        The model's response as a plain string.
+
+    Raises:
+        LLMClientError: If GROQ_API_KEY is not set or all retries fail.
+    """
+    key = api_key or os.environ.get("GROQ_API_KEY", "")
+    if not key:
+        raise LLMClientError(
+            "GROQ_API_KEY is not set. "
+            "Export it with: export GROQ_API_KEY=gsk_... "
+            "or set it in your .env file."
+        )
+
+    headers = {
+        "Authorization": f"Bearer {key}",
+        "Content-Type": "application/json",
+    }
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ],
+        "temperature": 0.2,
+        "max_tokens": 512,
+    }
+
+    last_error: Optional[Exception] = None
+    for attempt in range(retries + 1):
+        try:
+            resp = requests.post(
+                GROQ_API_URL,
+                headers=headers,
+                json=payload,
+                timeout=timeout,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+            return data["choices"][0]["message"]["content"].strip()
+        except requests.exceptions.Timeout as exc:
+            last_error = exc
+            if attempt < retries:
+                time.sleep(1.0 * (attempt + 1))
+        except requests.exceptions.HTTPError as exc:
+            # 429 rate limit — back off and retry
+            if resp.status_code == 429 and attempt < retries:
+                time.sleep(2.0 * (attempt + 1))
+                last_error = exc
+            else:
+                raise LLMClientError(
+                    f"Groq API returned HTTP {resp.status_code}: {resp.text[:200]}"
+                ) from exc
+        except (KeyError, ValueError, json.JSONDecodeError) as exc:
+            raise LLMClientError(
+                f"Unexpected Groq response format: {exc}"
+            ) from exc
+
+    raise LLMClientError(
+        f"Groq request failed after {retries + 1} attempts: {last_error}"
+    )

fixtureforge-2.2.0/src/fixtureforge/ai/failure_analyzer.py

@@ -0,0 +1,253 @@
+"""
+SmartFailureAnalyzer — AI-powered pytest failure diagnostics.
+
+Hooks into pytest's report pipeline and analyzes FAILED tests using Groq.
+Prints a structured diagnosis to the console after each failure.
+
+IMPORTANT:
+  - Never modifies test results or retry logic
+  - Never changes the failure status of any test
+  - Only activated on explicit opt-in (add to conftest.py)
+  - Requires GROQ_API_KEY — silently skips analysis if not set
+
+Setup (conftest.py):
+    from fixtureforge.ai import SmartFailureAnalyzer
+    pytest_plugins = [SmartFailureAnalyzer()]   # pass an instance
+
+Or register via pytest plugin list:
+    # conftest.py
+    from fixtureforge.ai.failure_analyzer import SmartFailureAnalyzer
+    def pytest_configure(config):
+        config.pluginmanager.register(SmartFailureAnalyzer(), "fixtureforge-analyzer")
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from typing import Any, Optional
+
+from ._audit_log import log_event
+from ._groq_client import LLMClientError, groq_complete
+
+# ---------------------------------------------------------------------------
+# Terminal encoding safety
+# Detect if the terminal supports Unicode box-drawing characters.
+# Windows cmd / PowerShell with cp1255 / cp1252 encoding cannot render them.
+# ---------------------------------------------------------------------------
+
+def _supports_unicode() -> bool:
+    enc = getattr(sys.stdout, "encoding", None) or ""
+    return enc.lower().replace("-", "") in ("utf8", "utf8sig")
+
+
+_USE_UNICODE = _supports_unicode()
+
+
+def _box(title: str, lines: list[str]) -> str:
+    """Render a labeled box. ASCII-safe on non-UTF-8 terminals."""
+    if _USE_UNICODE:
+        width = max(len(title) + 4, max((len(l) for l in lines), default=0) + 4, 50)
+        top    = f"\u2554{'=' * (width - 2)}\u2557"
+        header = f"\u2551  {title:<{width - 4}}  \u2551"
+        sep    = f"\u2560{'-' * (width - 2)}\u2563"
+        bottom = f"\u255a{'=' * (width - 2)}\u255d"
+        body   = "\n".join(f"\u2551  {l:<{width - 4}}  \u2551" for l in lines)
+        return "\n".join([top, header, sep, body, bottom])
+    else:
+        width = max(len(title) + 4, max((len(l) for l in lines), default=0) + 4, 50)
+        border = "+" + "=" * (width - 2) + "+"
+        divider = "+" + "-" * (width - 2) + "+"
+        header = f"|  {title:<{width - 4}}  |"
+        body   = "\n".join(f"|  {l:<{width - 4}}  |" for l in lines)
+        return "\n".join([border, header, divider, body, border])
+
+
+# ---------------------------------------------------------------------------
+# Groq prompt
+# ---------------------------------------------------------------------------
+
+_SYSTEM_PROMPT = (
+    "You are a senior QA engineer and Python expert. "
+    "Analyze test failures and provide concise, actionable diagnostics. "
+    "Return ONLY valid JSON — no markdown, no explanation outside the JSON. "
+    "Format: {"
+    '"explanation": "one sentence describing what went wrong", '
+    '"likely_cause": "root cause in plain English", '
+    '"suggested_fix": "specific code-level action to resolve it"'
+    "}"
+)
+
+_TAIL_LINES = 20  # number of traceback lines to send
+
+
+# ---------------------------------------------------------------------------
+# SmartFailureAnalyzer plugin
+# ---------------------------------------------------------------------------
+
+class SmartFailureAnalyzer:
+    """
+    pytest plugin that analyzes FAILED tests with Groq AI.
+
+    Behavior:
+    - Triggers only when a test's call phase fails
+    - Sends: test ID + exception type + exception message + last 20 traceback lines
+    - Prints a structured analysis block to stdout
+    - Logs every interaction to .fixtureforge_ai_log.jsonl
+    - NEVER modifies report.passed or any test state
+
+    Registration (conftest.py):
+        from fixtureforge.ai import SmartFailureAnalyzer
+        def pytest_configure(config):
+            config.pluginmanager.register(SmartFailureAnalyzer(), "forge-analyzer")
+    """
+
+    def __init__(self, api_key: Optional[str] = None) -> None:
+        self._api_key = api_key
+        self._analyzed: int = 0
+
+    # ------------------------------------------------------------------
+    # pytest hook
+    # ------------------------------------------------------------------
+
+    def pytest_runtest_logreport(self, report: Any) -> None:
+        """Called by pytest after each test phase (setup / call / teardown)."""
+        if report.when != "call" or not report.failed:
+            return
+
+        # Skip if GROQ_API_KEY is not available — never block the test run
+        key = self._api_key or os.environ.get("GROQ_API_KEY", "")
+        if not key:
+            return
+
+        self._analyze(report, api_key=key)
+
+    # ------------------------------------------------------------------
+    # Analysis
+    # ------------------------------------------------------------------
+
+    def _extract_failure_info(self, report: Any) -> dict[str, str]:
+        """Pull structured info out of a pytest report object."""
+        nodeid = getattr(report, "nodeid", "unknown_test")
+        longrepr = report.longrepr
+
+        # longrepr can be a string, tuple, or ExceptionInfo object
+        if isinstance(longrepr, str):
+            full_tb = longrepr
+            exc_type = "UnknownError"
+            exc_msg  = longrepr[:300]
+        elif isinstance(longrepr, tuple):
+            # (path, lineno, message) format from some pytest versions
+            full_tb = "\n".join(str(p) for p in longrepr)
+            exc_type = "Error"
+            exc_msg  = str(longrepr[-1])[:300]
+        else:
+            # ExceptionInfo / ReprExceptionInfo
+            full_tb  = str(longrepr)
+            lines    = full_tb.splitlines()
+            exc_type = "UnknownError"
+            exc_msg  = ""
+            # find the last "E  " line — that's the exception message
+            for line in reversed(lines):
+                stripped = line.strip()
+                if stripped.startswith("E "):
+                    exc_msg = stripped[2:].strip()
+                    break
+            # find exception class name
+            for line in reversed(lines):
+                for keyword in ("Error", "Exception", "Failure", "assert"):
+                    if keyword in line:
+                        exc_type = line.strip()[:80]
+                        break
+
+        # last N lines of traceback (avoid huge prompts)
+        tail_lines = full_tb.splitlines()[-_TAIL_LINES:]
+        tail = "\n".join(tail_lines)
+
+        return {
+            "nodeid": nodeid,
+            "exc_type": exc_type,
+            "exc_msg": exc_msg,
+            "tail": tail,
+        }
+
+    def _call_groq(self, info: dict[str, str], api_key: str) -> dict[str, str]:
+        """Send failure info to Groq and parse structured JSON response."""
+        user_msg = (
+            f"Test: {info['nodeid']}\n"
+            f"Error type: {info['exc_type']}\n"
+            f"Error message: {info['exc_msg']}\n"
+            f"Traceback (last {_TAIL_LINES} lines):\n{info['tail']}"
+        )
+        raw = groq_complete(
+            system=_SYSTEM_PROMPT,
+            user=user_msg,
+            api_key=api_key,
+        )
+        # strip markdown fences if model ignored instructions
+        clean = raw.strip().lstrip("```json").lstrip("```").rstrip("```").strip()
+        parsed = json.loads(clean)
+        return {
+            "explanation":   str(parsed.get("explanation", "")),
+            "likely_cause":  str(parsed.get("likely_cause", "")),
+            "suggested_fix": str(parsed.get("suggested_fix", "")),
+        }
+
+    def _print_analysis(self, info: dict[str, str], analysis: dict[str, str]) -> None:
+        """Print the analysis block to stdout — encoding-safe."""
+        label = "FixtureForge AI Analysis"
+        lines = [
+            f"Test   : {info['nodeid']}",
+            "",
+            f"Cause  : {analysis['likely_cause']}",
+            "",
+            f"Detail : {analysis['explanation']}",
+            "",
+            f"Fix    : {analysis['suggested_fix']}",
+        ]
+        box = _box(label, lines)
+        # Write via sys.stdout with explicit encoding fallback
+        try:
+            print("\n" + box + "\n", flush=True)
+        except UnicodeEncodeError:
+            safe = (box
+                    .encode(sys.stdout.encoding or "ascii", errors="replace")
+                    .decode(sys.stdout.encoding or "ascii"))
+            print("\n" + safe + "\n", flush=True)
+
+    def _analyze(self, report: Any, api_key: str) -> None:
+        """Full analysis pipeline for one failed test."""
+        info = self._extract_failure_info(report)
+
+        try:
+            analysis = self._call_groq(info, api_key=api_key)
+        except (LLMClientError, json.JSONDecodeError, KeyError) as exc:
+            log_event("failure_analysis_error", {
+                "nodeid": info["nodeid"],
+                "error": str(exc),
+            })
+            return  # silently skip — never interrupt the test session
+
+        self._print_analysis(info, analysis)
+        self._analyzed += 1
+
+        log_event("failure_analysis", {
+            "nodeid":       info["nodeid"],
+            "exc_type":     info["exc_type"],
+            "explanation":  analysis["explanation"],
+            "likely_cause": analysis["likely_cause"],
+            "suggested_fix": analysis["suggested_fix"],
+        })
+
+    # ------------------------------------------------------------------
+    # Summary (optional hook)
+    # ------------------------------------------------------------------
+
+    def pytest_terminal_summary(self, terminalreporter: Any, exitstatus: int) -> None:
+        """Append a one-line summary at the end of the pytest session."""
+        if self._analyzed > 0:
+            terminalreporter.write_sep(
+                "-",
+                f"FixtureForge AI: analyzed {self._analyzed} failure(s) -- "
+                f"see .fixtureforge_ai_log.jsonl for full audit trail",
+            )

fixtureforge-2.2.0/src/fixtureforge/ai/semantic_assert.py

@@ -0,0 +1,252 @@
+"""
+assert_semantic_match — semantic validation for AI-generated or natural-language outputs.
+
+Two modes:
+  Local (default, use_llm=False):
+      Cosine similarity on TF-IDF vectors built from stdlib only.
+      Zero API calls. Zero cost. Always available offline.
+
+  LLM (use_llm=True):
+      Sends actual + expected_intent to Groq for a structured judgment.
+      Requires GROQ_API_KEY. Every call is logged to .fixtureforge_ai_log.jsonl.
+
+Usage:
+    from fixtureforge.ai import assert_semantic_match
+
+    # Local (no API key needed)
+    result = assert_semantic_match(
+        actual="The product was delivered on time and in perfect condition.",
+        expected_intent="positive delivery experience",
+    )
+
+    # LLM-powered (explicit opt-in)
+    result = assert_semantic_match(
+        actual=chatbot_response,
+        expected_intent="politely decline the refund request",
+        use_llm=True,
+        threshold=0.8,
+    )
+    # Raises SemanticAssertionError if result.passed is False
+"""
+from __future__ import annotations
+
+import json
+import math
+from collections import Counter
+from dataclasses import dataclass
+from typing import Optional
+
+from ._audit_log import log_event
+from ._groq_client import LLMClientError, groq_complete
+
+# ---------------------------------------------------------------------------
+# Public types
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SemanticResult:
+    """Return value of assert_semantic_match."""
+    passed: bool
+    score: float        # 0.0 – 1.0
+    reason: str         # human-readable explanation
+    method: str         # "local" or "llm"
+
+    def __str__(self) -> str:
+        status = "PASS" if self.passed else "FAIL"
+        return (
+            f"SemanticResult({status} | score={self.score:.2f} "
+            f"| method={self.method} | reason={self.reason!r})"
+        )
+
+
+class SemanticAssertionError(AssertionError):
+    """Raised when assert_semantic_match fails."""
+
+    def __init__(self, result: SemanticResult, actual: str, expected_intent: str) -> None:
+        self.result = result
+        self.actual = actual
+        self.expected_intent = expected_intent
+        super().__init__(
+            f"\n\nSemantic assertion failed.\n"
+            f"  Expected intent : {expected_intent!r}\n"
+            f"  Actual output   : {actual!r}\n"
+            f"  Score           : {result.score:.2f} (threshold not met)\n"
+            f"  Method          : {result.method}\n"
+            f"  Reason          : {result.reason}\n"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Local cosine similarity (stdlib only — no external dependencies)
+# ---------------------------------------------------------------------------
+
+def _tokenize(text: str) -> Counter:
+    """Lowercase, split on whitespace, return token frequency counter."""
+    tokens = text.lower().split()
+    # strip punctuation from token edges
+    cleaned = [t.strip(".,!?;:\"'()-[]{}") for t in tokens]
+    return Counter(t for t in cleaned if t)
+
+
+def _cosine_similarity(text_a: str, text_b: str) -> float:
+    """
+    TF-IDF cosine similarity between two strings using stdlib only.
+    Returns a value in [0.0, 1.0].
+    """
+    vec_a = _tokenize(text_a)
+    vec_b = _tokenize(text_b)
+
+    if not vec_a or not vec_b:
+        return 0.0
+
+    # dot product over shared tokens
+    shared = set(vec_a) & set(vec_b)
+    dot = sum(vec_a[w] * vec_b[w] for w in shared)
+
+    # magnitudes
+    mag_a = math.sqrt(sum(v * v for v in vec_a.values()))
+    mag_b = math.sqrt(sum(v * v for v in vec_b.values()))
+
+    if mag_a == 0.0 or mag_b == 0.0:
+        return 0.0
+
+    return dot / (mag_a * mag_b)
+
+
+def _local_assert(actual: str, expected_intent: str, threshold: float) -> SemanticResult:
+    score = _cosine_similarity(actual, expected_intent)
+    passed = score >= threshold
+    reason = (
+        f"Cosine similarity {score:.2f} >= threshold {threshold:.2f}"
+        if passed
+        else f"Cosine similarity {score:.2f} < threshold {threshold:.2f}"
+    )
+    return SemanticResult(passed=passed, score=score, reason=reason, method="local")
+
+
+# ---------------------------------------------------------------------------
+# LLM-powered assertion via Groq
+# ---------------------------------------------------------------------------
+
+_LLM_SYSTEM = (
+    "You are a test validation judge. "
+    "Given an actual output and an expected intent, "
+    "return ONLY valid JSON with no markdown, no explanation. "
+    'Format: {"passed": bool, "score": float, "reason": str}'
+)
+
+
+def _llm_assert(
+    actual: str,
+    expected_intent: str,
+    threshold: float,
+    api_key: Optional[str],
+) -> SemanticResult:
+    user_msg = (
+        f"Expected intent: {expected_intent}\n"
+        f"Actual output: {actual}\n"
+        f"Score threshold: {threshold}\n"
+        "Does the actual output satisfy the expected intent? "
+        "Return JSON only."
+    )
+
+    try:
+        raw = groq_complete(system=_LLM_SYSTEM, user=user_msg, api_key=api_key)
+        # strip markdown fences if model ignored instructions
+        raw = raw.strip().lstrip("```json").lstrip("```").rstrip("```").strip()
+        parsed = json.loads(raw)
+
+        score = float(parsed.get("score", 0.0))
+        passed = bool(parsed.get("passed", score >= threshold))
+        reason = str(parsed.get("reason", ""))
+
+        log_event("semantic_assert_llm", {
+            "actual": actual[:200],
+            "expected_intent": expected_intent[:200],
+            "threshold": threshold,
+            "score": score,
+            "passed": passed,
+            "reason": reason,
+        })
+
+        return SemanticResult(passed=passed, score=score, reason=reason, method="llm")
+
+    except (json.JSONDecodeError, KeyError, ValueError) as exc:
+        # Graceful degradation: fall back to local on parse error
+        log_event("semantic_assert_llm_parse_error", {"error": str(exc), "raw": raw[:300]})
+        result = _local_assert(actual, expected_intent, threshold)
+        result.reason = f"[LLM response unparseable, fell back to local] {result.reason}"
+        result.method = "local_fallback"
+        return result
+
+    except LLMClientError as exc:
+        # No API key or network error — fall back to local silently
+        log_event("semantic_assert_llm_error", {"error": str(exc)})
+        result = _local_assert(actual, expected_intent, threshold)
+        result.reason = f"[LLM unavailable, fell back to local] {result.reason}"
+        result.method = "local_fallback"
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def assert_semantic_match(
+    actual: str,
+    expected_intent: str,
+    use_llm: bool = False,
+    threshold: float = 0.75,
+    api_key: Optional[str] = None,
+    raise_on_failure: bool = True,
+) -> SemanticResult:
+    """
+    Assert that `actual` semantically matches `expected_intent`.
+
+    Args:
+        actual:           The string to validate (e.g. chatbot response, generated text).
+        expected_intent:  Natural language description of what the output should convey.
+        use_llm:          If True, use Groq for a more nuanced judgment.
+                          Requires GROQ_API_KEY. Default: False (local cosine similarity).
+        threshold:        Minimum score to pass. Range: [0.0, 1.0]. Default: 0.75.
+        api_key:          Override GROQ_API_KEY env var (optional).
+        raise_on_failure: If True (default), raise SemanticAssertionError on failure.
+
+    Returns:
+        SemanticResult(passed, score, reason, method)
+
+    Raises:
+        SemanticAssertionError: If the assertion fails and raise_on_failure=True.
+
+    Examples:
+        # Local — zero cost, always available
+        assert_semantic_match(
+            actual="Great experience, delivered next day!",
+            expected_intent="positive customer review",
+        )
+
+        # LLM — richer judgment, requires API key
+        result = assert_semantic_match(
+            actual=model_output,
+            expected_intent="professional refusal of the request",
+            use_llm=True,
+            threshold=0.8,
+        )
+        print(result.score, result.reason)
+    """
+    if not isinstance(actual, str) or not isinstance(expected_intent, str):
+        raise TypeError(
+            f"assert_semantic_match expects str arguments, "
+            f"got actual={type(actual).__name__}, "
+            f"expected_intent={type(expected_intent).__name__}"
+        )
+
+    if use_llm:
+        result = _llm_assert(actual, expected_intent, threshold, api_key)
+    else:
+        result = _local_assert(actual, expected_intent, threshold)
+
+    if not result.passed and raise_on_failure:
+        raise SemanticAssertionError(result, actual, expected_intent)
+
+    return result

{fixtureforge-2.0.2 → fixtureforge-2.2.0}/src/fixtureforge/core/generator.py

@@ -38,8 +38,14 @@ class BasicGenerator:
         self.verbose = verbose
 
         if seed is not None:
-            Faker.seed(seed)
-            random.seed(seed)
+            # seed_instance() seeds THIS faker object's private RNG only —
+            # two Forge(seed=42) instances are fully independent.
+            self.faker.seed_instance(seed)
+            # Use a dedicated Random instance (not global random module)
+            # so two Forge objects with the same seed don't interfere.
+            self._rng = random.Random(seed)
+        else:
+            self._rng = random
 
         if ai_engine is not None:
             self.ai_engine = ai_engine
@@ -112,7 +118,7 @@ class BasicGenerator:
             target_model = name[: -len("_id")]
             records = self.registry.get(target_model, [])
             if records:
-                return getattr(random.choice(records), "id", None)
+                return getattr(self._rng.choice(records), "id", None)
 
         # Integer PK / sequential ID
         if "int" in field.type_name.lower():
@@ -181,19 +187,19 @@ class BasicGenerator:
         if "time" in name_parts:
             return self.faker.time()
         if "gender" in name_parts:
-            return random.choice(["male", "female", "non-binary"])
+            return self._rng.choice(["male", "female", "non-binary"])
         if "age" in name_parts:
-            return random.randint(18, 80)
+            return self._rng.randint(18, 80)
         if "salary" in name or "amount" in name or "price" in name:
-            return round(random.uniform(10, 10_000), 2)
+            return round(self._rng.uniform(10, 10_000), 2)
         if "score" in name or "rating" in name:
-            return round(random.uniform(1, 5), 1)
+            return round(self._rng.uniform(1, 5), 1)
         if "quantity" in name or "qty" in name:
-            return random.randint(1, 100)
+            return self._rng.randint(1, 100)
         if "weight" in name_parts:
-            return round(random.uniform(0.1, 500), 2)
+            return round(self._rng.uniform(0.1, 500), 2)
         if "height" in name_parts:
-            return round(random.uniform(1.0, 2.5), 2)
+            return round(self._rng.uniform(1.0, 2.5), 2)
 
         # Type fallbacks
         if "int" in type_name:
@@ -201,7 +207,7 @@ class BasicGenerator:
         if "bool" in type_name:
             return self.faker.boolean()
         if "float" in type_name:
-            return round(random.uniform(0, 1000), 2)
+            return round(self._rng.uniform(0, 1000), 2)
         if "list" in type_name:
             return []
         if "dict" in type_name:
@@ -215,7 +221,7 @@ class BasicGenerator:
         max_val = c.get("le", c.get("lt", 10001) - 1 if "lt" in c else 10000)
         if min_val > max_val:
             max_val = min_val + 100
-        return random.randint(min_val, max_val)
+        return self._rng.randint(min_val, max_val)
 
 
 # Sentinel object returned for COMPUTED fields so the generator knows to

fixtureforge-2.2.0/src/fixtureforge/pytest_plugin.py

@@ -0,0 +1,264 @@
+"""
+FixtureForge pytest plugin.
+
+Provides two integration points:
+
+1. ``forge`` fixture — auto-available in every test, zero config.
+   CI-safe by default (use_ai=False). Activated by FORGE_AI=1.
+
+2. ``forge_fixture()`` — declare model fixtures in conftest.py with one line.
+   Injects a named pytest fixture into the caller's namespace automatically.
+
+-------------------------------------------------------------------------------
+QUICK START
+-------------------------------------------------------------------------------
+
+# conftest.py
+from fixtureforge.pytest_plugin import forge_fixture
+from myapp.models import User, Order, Product
+
+# Declare fixtures — one line per model
+forge_fixture(User,    count=10,  seed=42)
+forge_fixture(Order,   count=50,  seed=42, context="e-commerce orders")
+forge_fixture(Product, count=1)            # singular — returns one instance
+
+# test_users.py
+def test_all_users_have_email(users):      # ← 10 User instances
+    assert all("@" in u.email for u in users)
+
+def test_order_links_customer(orders, users):  # ← FK resolved automatically
+    assert all(o.user_id in {u.id for u in users} for o in orders)
+
+def test_product_name(product):            # ← single Product instance
+    assert len(product.name) > 0
+
+-------------------------------------------------------------------------------
+ADVANCED USAGE
+-------------------------------------------------------------------------------
+
+# Explicit forge fixture for one-off generation
+def test_custom(forge):
+    vip = forge.create(User, context="VIP enterprise customer")
+    assert vip.name
+
+# Full AI mode in dev (override with env var)
+# FORGE_AI=1 pytest → uses real LLM
+# FORGE_SEED=42 pytest → reproducible seed
+
+# Scoped fixtures (shared across all tests in a module)
+forge_fixture(User, count=100, seed=42, scope="module")
+
+# Named override (when you want a different name than the model)
+forge_fixture(User, count=5, seed=0, name="admin_users", context="admin accounts")
+
+def test_admins(admin_users):
+    assert len(admin_users) == 5
+"""
+from __future__ import annotations
+
+import os
+import sys
+from typing import Any, Dict, List, Optional, Type
+
+import pytest
+from pydantic import BaseModel
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _resolve_forge_config() -> Dict[str, Any]:
+    """Read Forge config from environment variables."""
+    use_ai = os.environ.get("FORGE_AI", "0").strip().lower() in ("1", "true", "yes")
+    seed_env = os.environ.get("FORGE_SEED", "").strip()
+    seed = int(seed_env) if seed_env.isdigit() else None
+    verbose = os.environ.get("FORGE_VERBOSE", "0").strip().lower() in ("1", "true", "yes")
+    return {"use_ai": use_ai, "seed": seed, "verbose": verbose}
+
+
+def _fixture_name_for(model: Type[BaseModel], count: int, name: Optional[str]) -> str:
+    """Derive fixture name: explicit name > plural/singular from model name."""
+    if name:
+        return name
+    base = model.__name__.lower()
+    return base + "s" if count > 1 else base
+
+
+# ---------------------------------------------------------------------------
+# Auto-provided ``forge`` fixture (available in every test, zero config)
+# ---------------------------------------------------------------------------
+
+@pytest.fixture
+def forge():
+    """
+    Auto-provided Forge instance.
+
+    CI-safe by default (use_ai=False, seed from FORGE_SEED env var).
+    Set FORGE_AI=1 to enable real AI generation in dev/staging.
+
+    Example::
+
+        def test_something(forge):
+            users = forge.create_batch(User, count=10)
+            assert len(users) == 10
+    """
+    from fixtureforge import Forge  # lazy import — keeps pytest startup fast
+
+    cfg = _resolve_forge_config()
+    return Forge(
+        use_ai=cfg["use_ai"],
+        seed=cfg["seed"],
+        verbose=cfg["verbose"],
+    )
+
+
+# ---------------------------------------------------------------------------
+# forge_fixture() — one-line fixture declaration
+# ---------------------------------------------------------------------------
+
+def forge_fixture(
+    model: Type[BaseModel],
+    count: int = 1,
+    *,
+    context: Optional[str] = None,
+    seed: Optional[int] = None,
+    use_ai: Optional[bool] = None,
+    scope: str = "function",
+    name: Optional[str] = None,
+    **overrides: Any,
+) -> Any:
+    """
+    Declare a pytest fixture that generates model instances.
+
+    Call this at module level in ``conftest.py``.  The fixture is injected
+    directly into the caller's namespace under the derived or given name.
+
+    Parameters
+    ----------
+    model   : Pydantic BaseModel subclass to generate
+    count   : number of records (>1 → list, ==1 → single instance)
+    context : AI prompt context, e.g. "angry customers"
+    seed    : random seed for deterministic output (overrides FORGE_SEED)
+    use_ai  : override env-based detection (default: False / FORGE_AI env var)
+    scope   : pytest fixture scope — "function" | "class" | "module" | "session"
+    name    : explicit fixture name (default: model name in lower-case, pluralised)
+    **overrides : field overrides forwarded to forge.create / create_batch
+
+    Returns
+    -------
+    The pytest fixture callable (also injected into caller's namespace).
+
+    Example::
+
+        # conftest.py
+        forge_fixture(User,  count=50, seed=42)
+        forge_fixture(Order, count=200, seed=42, context="high-value orders")
+
+        # test_orders.py
+        def test_order_count(orders):
+            assert len(orders) == 200
+    """
+    fixture_name = _fixture_name_for(model, count, name)
+
+    # Resolve use_ai: explicit arg > env var
+    _use_ai: bool
+    if use_ai is not None:
+        _use_ai = use_ai
+    else:
+        _use_ai = os.environ.get("FORGE_AI", "0").strip().lower() in ("1", "true", "yes")
+
+    # Resolve seed: explicit arg > env var
+    _seed: Optional[int]
+    if seed is not None:
+        _seed = seed
+    else:
+        seed_env = os.environ.get("FORGE_SEED", "").strip()
+        _seed = int(seed_env) if seed_env.isdigit() else None
+
+    # Capture all params in closure (avoids late-binding issues in loops)
+    _model    = model
+    _count    = count
+    _context  = context
+    _overrides = overrides
+
+    def _fixture_fn():
+        from fixtureforge import Forge  # lazy import
+
+        f = Forge(use_ai=_use_ai, seed=_seed)
+
+        if _count == 1:
+            return f.create(_model, context=_context, **_overrides)
+        return f.create_batch(_model, count=_count, context=_context, **_overrides)
+
+    # Give the function the correct name so pytest identifies the fixture by it
+    _fixture_fn.__name__ = fixture_name
+    _fixture_fn.__qualname__ = fixture_name
+
+    # Apply the pytest.fixture decorator with the requested scope
+    marked = pytest.fixture(scope=scope, name=fixture_name)(_fixture_fn)
+
+    # Inject into caller's conftest.py namespace so pytest can discover it
+    frame = sys._getframe(1)
+    frame.f_locals[fixture_name] = marked
+
+    return marked
+
+
+# ---------------------------------------------------------------------------
+# forge_swarm_fixture() — parallel multi-model fixture in one call
+# ---------------------------------------------------------------------------
+
+def forge_swarm_fixture(
+    models: List[Type[BaseModel]],
+    counts: Optional[List[int]] = None,
+    contexts: Optional[List[Optional[str]]] = None,
+    *,
+    seed: Optional[int] = None,
+    use_ai: Optional[bool] = None,
+    scope: str = "session",
+) -> Any:
+    """
+    Declare a single fixture that generates multiple models in parallel
+    using DataSwarms.  Returns a dict mapping model_name → list.
+
+    Best used with ``scope="session"`` so the swarm runs once per test run.
+
+    Example::
+
+        # conftest.py
+        forge_swarm_fixture(
+            [User, Order, Product],
+            counts=[10, 50, 100],
+            seed=42,
+        )
+
+        # test_something.py
+        def test_data_volume(swarm_data):
+            assert len(swarm_data["User"]) == 10
+            assert len(swarm_data["Order"]) == 50
+    """
+    _use_ai = (
+        use_ai
+        if use_ai is not None
+        else os.environ.get("FORGE_AI", "0").strip().lower() in ("1", "true", "yes")
+    )
+    _seed = seed or (
+        int(s) if (s := os.environ.get("FORGE_SEED", "").strip()).isdigit() else None
+    )
+
+    def _swarm_fn():
+        from fixtureforge import Forge
+
+        f = Forge(use_ai=_use_ai, seed=_seed)
+        return f.swarm(models=models, counts=counts, contexts=contexts)
+
+    _swarm_fn.__name__ = "swarm_data"
+    _swarm_fn.__qualname__ = "swarm_data"
+
+    marked = pytest.fixture(scope=scope, name="swarm_data")(_swarm_fn)
+
+    frame = sys._getframe(1)
+    frame.f_locals["swarm_data"] = marked
+
+    return marked

fixtureforge-2.0.2/src/fixtureforge/ai/__init__.py

@@ -1,10 +0,0 @@
-"""
-AI-powered generation
-"""
-from .engine import AIEngine
-from .cache import ResponseCache
-
-__all__ = [
-    "AIEngine",
-    "ResponseCache",
-]