semanticheck 0.1.0__tar.gz

semanticheck-0.1.0/.gitignore

@@ -0,0 +1,20 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.eggs/
+.venv/
+venv/
+env/
+.env
+*.env
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.llmassert_baselines/
+htmlcov/
+.coverage
+coverage.xml
+*.log
+.pypirc

semanticheck-0.1.0/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## [0.1.0] — 2026-04-06
+
+### Added
+- `assert_intent` — semantic intent matching via cosine similarity
+- `assert_tone` — tone/style verification (professional, casual, friendly, etc.)
+- `assert_no_hallucination` — contradiction detection against known facts
+- `assert_similar_to` — semantic similarity between response and reference
+- `assert_token_budget` — token count enforcement (tiktoken or word-based)
+- `assert_schema` — JSON Schema and Pydantic model validation
+- `assert_language` — language detection via Unicode script analysis
+- `assert_no_pii` — PII detection (email, SSN, credit card, phone, IBAN, etc.)
+- `assert_readability` — Flesch-Kincaid readability scoring
+- `assert_sentiment` — sentiment polarity assertion
+- `record_baseline` / `compare_baseline` — golden baseline regression testing
+- `LocalJudge` — local LLM judge (transformers / llama.cpp / embeddings fallback)
+- pytest plugin with `--record-baselines`, `--llm-threshold`, `--skip-llm` flags
+- `llm_record` and `llm_threshold` pytest fixtures
+- Auto-detecting embedding backend: sentence-transformers → OpenAI → hash fallback

semanticheck-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 llmassert contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

semanticheck-0.1.0/PKG-INFO

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: semanticheck
+Version: 0.1.0
+Summary: pytest-native semantic assertions for LLM and generative AI applications. No servers. No SaaS. Works with OpenAI, Anthropic, LiteLLM and any LLM client.
+Project-URL: Homepage, https://github.com/semanticheck/semanticheck
+Project-URL: Documentation, https://github.com/semanticheck/semanticheck#readme
+Project-URL: Repository, https://github.com/semanticheck/semanticheck
+Project-URL: Bug Tracker, https://github.com/semanticheck/semanticheck/issues
+Project-URL: Changelog, https://github.com/semanticheck/semanticheck/blob/main/CHANGELOG.md
+Author: semanticheck contributors
+License: MIT
+License-File: LICENSE
+Keywords: agent testing,ai quality assurance,ai testing,anthropic,claude testing,genai testing,generative ai,golden baseline,gpt testing,hallucination detection,langchain,llm,llm assertions,llm evaluation,llm quality,llm testing,machine learning testing,openai,prompt testing,pytest,pytest plugin,rag testing,regression testing,semantic assertions,semantic testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: all
+Requires-Dist: jsonschema>=4.0.0; extra == 'all'
+Requires-Dist: openai>=1.0.0; extra == 'all'
+Requires-Dist: pydantic>=2.0.0; extra == 'all'
+Requires-Dist: sentence-transformers>=2.7.0; extra == 'all'
+Requires-Dist: tiktoken>=0.5.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: openai>=1.0.0; extra == 'dev'
+Requires-Dist: pydantic>=2.0.0; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: sentence-transformers>=2.7.0; extra == 'dev'
+Provides-Extra: jsonschema
+Requires-Dist: jsonschema>=4.0.0; extra == 'jsonschema'
+Provides-Extra: judge
+Requires-Dist: torch>=2.0.0; extra == 'judge'
+Requires-Dist: transformers>=4.40.0; extra == 'judge'
+Provides-Extra: local
+Requires-Dist: sentence-transformers>=2.7.0; extra == 'local'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
+Provides-Extra: pydantic
+Requires-Dist: pydantic>=2.0.0; extra == 'pydantic'
+Provides-Extra: tiktoken
+Requires-Dist: tiktoken>=0.5.0; extra == 'tiktoken'
+Description-Content-Type: text/markdown
+
+# semanticheck
+
+**pytest-native semantic assertions for LLM and generative AI applications.**
+
+No servers. No SaaS. No config. Works with OpenAI, Anthropic, LiteLLM, or any LLM client.
+
+[![PyPI](https://img.shields.io/pypi/v/semanticheck)](https://pypi.org/project/semanticheck/)
+[![Python](https://img.shields.io/pypi/pyversions/semanticheck)](https://pypi.org/project/semanticheck/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![CI](https://github.com/semanticheck/semanticheck/actions/workflows/ci.yml/badge.svg)](https://github.com/semanticheck/semanticheck/actions)
+
+---
+
+## Installation
+
+```bash
+pip install semanticheck
+```
+
+With local embeddings (recommended, no API cost):
+
+```bash
+pip install "semanticheck[local]"
+```
+
+With OpenAI embeddings:
+
+```bash
+pip install "semanticheck[openai]"
+```
+
+---
+
+## Quick Start
+
+```python
+from semanticheck import assert_intent, assert_tone, assert_no_hallucination, assert_no_pii
+
+def test_customer_support_reply():
+    response = my_llm("Help me reset my password")
+
+    assert_intent(response, "instructions for resetting a password")
+    assert_tone(response, "friendly")
+    assert_no_pii(response)
+    assert_no_hallucination(response, known_facts=["Password reset links expire after 24 hours"])
+```
+
+---
+
+## Assertions
+
+| Function | What it checks |
+|---|---|
+| `assert_intent(response, expected_intent)` | Semantic match to expected meaning |
+| `assert_tone(response, tone)` | Tone: professional, casual, friendly, formal, etc. |
+| `assert_no_hallucination(response, known_facts)` | No contradiction of known facts |
+| `assert_similar_to(response, reference)` | Cosine similarity to a reference text |
+| `assert_token_budget(response, max_tokens)` | Response stays within token limit |
+| `assert_schema(response, schema)` | Valid JSON Schema or Pydantic model |
+| `assert_language(response, language)` | Written in expected language |
+| `assert_no_pii(response)` | No emails, SSNs, credit cards, phones, etc. |
+| `assert_readability(response, min_score=60)` | Flesch Reading Ease score |
+| `assert_sentiment(response, "positive")` | Sentiment polarity |
+
+---
+
+## Baseline Regression Testing
+
+```python
+from semanticheck import record_baseline, compare_baseline
+
+def test_summarizer_regression(llm_record):
+    response = my_summarizer(article)
+    if llm_record:
+        record_baseline("summarizer_v1", response)
+    else:
+        compare_baseline("summarizer_v1", response, threshold=0.85)
+```
+
+Run with:
+
+```bash
+pytest --record-baselines   # record golden baselines
+pytest                      # compare against them
+```
+
+---
+
+## LocalJudge
+
+Use a local model as a judge — zero API cost in CI:
+
+```python
+from semanticheck import LocalJudge
+
+judge = LocalJudge()  # uses Qwen2.5-0.5B by default
+result = judge.evaluate(
+    response="Paris is the capital of France.",
+    criterion="Correctly answers a geography question about European capitals.",
+)
+assert result.passed
+print(result.score, result.reasoning)
+```
+
+---
+
+## pytest Plugin
+
+semanticheck auto-registers as a pytest plugin. Available flags:
+
+```bash
+pytest --skip-llm           # skip all @pytest.mark.llm tests
+pytest --llm-threshold 0.8  # override similarity threshold globally
+pytest --record-baselines   # record golden baselines
+```
+
+Available markers:
+
+```python
+@pytest.mark.llm        # LLM semantic test
+@pytest.mark.llm_slow   # slow test using local judge inference
+```
+
+---
+
+## Embedding Backends
+
+| Backend | How to activate | Cost |
+|---|---|---|
+| sentence-transformers | `pip install semanticheck[local]` | Free |
+| OpenAI | Set `OPENAI_API_KEY` | Per token |
+| Hash fallback | `LLMASSERT_EMBED_BACKEND=fallback` | Free (smoke tests only) |
+
+Override via env vars:
+
+```bash
+LLMASSERT_EMBED_BACKEND=openai
+LLMASSERT_EMBED_MODEL=text-embedding-3-large
+```
+
+---
+
+## License
+
+MIT

semanticheck-0.1.0/README.md

@@ -0,0 +1,145 @@
+# semanticheck
+
+**pytest-native semantic assertions for LLM and generative AI applications.**
+
+No servers. No SaaS. No config. Works with OpenAI, Anthropic, LiteLLM, or any LLM client.
+
+[![PyPI](https://img.shields.io/pypi/v/semanticheck)](https://pypi.org/project/semanticheck/)
+[![Python](https://img.shields.io/pypi/pyversions/semanticheck)](https://pypi.org/project/semanticheck/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![CI](https://github.com/semanticheck/semanticheck/actions/workflows/ci.yml/badge.svg)](https://github.com/semanticheck/semanticheck/actions)
+
+---
+
+## Installation
+
+```bash
+pip install semanticheck
+```
+
+With local embeddings (recommended, no API cost):
+
+```bash
+pip install "semanticheck[local]"
+```
+
+With OpenAI embeddings:
+
+```bash
+pip install "semanticheck[openai]"
+```
+
+---
+
+## Quick Start
+
+```python
+from semanticheck import assert_intent, assert_tone, assert_no_hallucination, assert_no_pii
+
+def test_customer_support_reply():
+    response = my_llm("Help me reset my password")
+
+    assert_intent(response, "instructions for resetting a password")
+    assert_tone(response, "friendly")
+    assert_no_pii(response)
+    assert_no_hallucination(response, known_facts=["Password reset links expire after 24 hours"])
+```
+
+---
+
+## Assertions
+
+| Function | What it checks |
+|---|---|
+| `assert_intent(response, expected_intent)` | Semantic match to expected meaning |
+| `assert_tone(response, tone)` | Tone: professional, casual, friendly, formal, etc. |
+| `assert_no_hallucination(response, known_facts)` | No contradiction of known facts |
+| `assert_similar_to(response, reference)` | Cosine similarity to a reference text |
+| `assert_token_budget(response, max_tokens)` | Response stays within token limit |
+| `assert_schema(response, schema)` | Valid JSON Schema or Pydantic model |
+| `assert_language(response, language)` | Written in expected language |
+| `assert_no_pii(response)` | No emails, SSNs, credit cards, phones, etc. |
+| `assert_readability(response, min_score=60)` | Flesch Reading Ease score |
+| `assert_sentiment(response, "positive")` | Sentiment polarity |
+
+---
+
+## Baseline Regression Testing
+
+```python
+from semanticheck import record_baseline, compare_baseline
+
+def test_summarizer_regression(llm_record):
+    response = my_summarizer(article)
+    if llm_record:
+        record_baseline("summarizer_v1", response)
+    else:
+        compare_baseline("summarizer_v1", response, threshold=0.85)
+```
+
+Run with:
+
+```bash
+pytest --record-baselines   # record golden baselines
+pytest                      # compare against them
+```
+
+---
+
+## LocalJudge
+
+Use a local model as a judge — zero API cost in CI:
+
+```python
+from semanticheck import LocalJudge
+
+judge = LocalJudge()  # uses Qwen2.5-0.5B by default
+result = judge.evaluate(
+    response="Paris is the capital of France.",
+    criterion="Correctly answers a geography question about European capitals.",
+)
+assert result.passed
+print(result.score, result.reasoning)
+```
+
+---
+
+## pytest Plugin
+
+semanticheck auto-registers as a pytest plugin. Available flags:
+
+```bash
+pytest --skip-llm           # skip all @pytest.mark.llm tests
+pytest --llm-threshold 0.8  # override similarity threshold globally
+pytest --record-baselines   # record golden baselines
+```
+
+Available markers:
+
+```python
+@pytest.mark.llm        # LLM semantic test
+@pytest.mark.llm_slow   # slow test using local judge inference
+```
+
+---
+
+## Embedding Backends
+
+| Backend | How to activate | Cost |
+|---|---|---|
+| sentence-transformers | `pip install semanticheck[local]` | Free |
+| OpenAI | Set `OPENAI_API_KEY` | Per token |
+| Hash fallback | `LLMASSERT_EMBED_BACKEND=fallback` | Free (smoke tests only) |
+
+Override via env vars:
+
+```bash
+LLMASSERT_EMBED_BACKEND=openai
+LLMASSERT_EMBED_MODEL=text-embedding-3-large
+```
+
+---
+
+## License
+
+MIT

semanticheck-0.1.0/pyproject.toml

@@ -0,0 +1,144 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "semanticheck"
+version = "0.1.0"
+description = "pytest-native semantic assertions for LLM and generative AI applications. No servers. No SaaS. Works with OpenAI, Anthropic, LiteLLM and any LLM client."
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.9"
+
+keywords = [
+  "llm",
+  "llm testing",
+  "llm assertions",
+  "llm evaluation",
+  "pytest",
+  "pytest plugin",
+  "ai testing",
+  "generative ai",
+  "semantic testing",
+  "semantic assertions",
+  "openai",
+  "anthropic",
+  "langchain",
+  "hallucination detection",
+  "prompt testing",
+  "rag testing",
+  "agent testing",
+  "ai quality assurance",
+  "machine learning testing",
+  "golden baseline",
+  "regression testing",
+  "gpt testing",
+  "claude testing",
+  "genai testing",
+  "llm quality",
+]
+
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Testing",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Framework :: Pytest",
+  "Typing :: Typed",
+]
+
+authors = [
+  { name = "semanticheck contributors" },
+]
+
+dependencies = []
+
+[project.optional-dependencies]
+local = [
+  "sentence-transformers>=2.7.0",
+]
+openai = [
+  "openai>=1.0.0",
+]
+tiktoken = [
+  "tiktoken>=0.5.0",
+]
+pydantic = [
+  "pydantic>=2.0.0",
+]
+jsonschema = [
+  "jsonschema>=4.0.0",
+]
+judge = [
+  "transformers>=4.40.0",
+  "torch>=2.0.0",
+]
+all = [
+  "sentence-transformers>=2.7.0",
+  "openai>=1.0.0",
+  "tiktoken>=0.5.0",
+  "pydantic>=2.0.0",
+  "jsonschema>=4.0.0",
+]
+dev = [
+  "pytest>=7.0.0",
+  "pytest-cov",
+  "sentence-transformers>=2.7.0",
+  "pydantic>=2.0.0",
+  "openai>=1.0.0",
+  "ruff",
+  "mypy",
+]
+
+[project.urls]
+Homepage      = "https://github.com/semanticheck/semanticheck"
+Documentation = "https://github.com/semanticheck/semanticheck#readme"
+Repository    = "https://github.com/semanticheck/semanticheck"
+"Bug Tracker" = "https://github.com/semanticheck/semanticheck/issues"
+Changelog     = "https://github.com/semanticheck/semanticheck/blob/main/CHANGELOG.md"
+
+[project.entry-points."pytest11"]
+semanticheck = "semanticheck.plugin"
+
+[tool.hatch.build.targets.wheel]
+packages = ["semanticheck"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "semanticheck/",
+  "tests/",
+  "README.md",
+  "LICENSE",
+  "CHANGELOG.md",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py39"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+ignore = ["E501"]
+
+[tool.mypy]
+python_version = "3.9"
+strict = false
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+markers = [
+  "llm: LLM semantic tests (may call embedding APIs)",
+  "llm_slow: slow tests using local judge model inference",
+]

semanticheck-0.1.0/semanticheck/__init__.py

@@ -0,0 +1,37 @@
+"""
+semanticheck — pytest-native semantic testing for LLM and generative AI applications.
+
+Drop-in pytest plugin. No servers. No SaaS. No config.
+Works with OpenAI, Anthropic, LiteLLM, or any LLM client.
+"""
+
+from semanticheck.assertions.intent import assert_intent
+from semanticheck.assertions.tone import assert_tone
+from semanticheck.assertions.hallucination import assert_no_hallucination
+from semanticheck.assertions.budget import assert_token_budget
+from semanticheck.assertions.schema import assert_schema
+from semanticheck.assertions.similarity import assert_similar_to
+from semanticheck.assertions.language import assert_language
+from semanticheck.assertions.pii import assert_no_pii, PIIMatch
+from semanticheck.assertions.readability import assert_readability
+from semanticheck.assertions.sentiment import assert_sentiment
+from semanticheck.baseline import record_baseline, compare_baseline
+from semanticheck.judge import LocalJudge
+
+__version__ = "0.1.0"
+__all__ = [
+    "assert_intent",
+    "assert_tone",
+    "assert_no_hallucination",
+    "assert_similar_to",
+    "assert_token_budget",
+    "assert_schema",
+    "assert_language",
+    "assert_no_pii",
+    "assert_readability",
+    "assert_sentiment",
+    "PIIMatch",
+    "record_baseline",
+    "compare_baseline",
+    "LocalJudge",
+]

semanticheck-0.1.0/semanticheck/_embed.py

@@ -0,0 +1,93 @@
+"""
+Embedding backend for semanticheck.
+
+Priority order (auto-detected):
+  1. sentence-transformers (local, free, fast) — recommended
+  2. openai embeddings (requires OPENAI_API_KEY)
+  3. hash-based fallback (no deps, smoke tests only)
+
+Set LLMASSERT_EMBED_BACKEND=openai|local|fallback to force a backend.
+"""
+
+from __future__ import annotations
+import os
+import hashlib
+import math
+from functools import lru_cache
+
+_BACKEND_ENV = "LLMASSERT_EMBED_BACKEND"
+_MODEL_ENV = "LLMASSERT_EMBED_MODEL"
+_DEFAULT_LOCAL_MODEL = "all-MiniLM-L6-v2"
+_DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
+
+
+@lru_cache(maxsize=512)
+def embed_text(text: str) -> tuple[float, ...]:
+    """Embed `text` into a float vector. Results are cached in-process."""
+    backend = os.environ.get(_BACKEND_ENV, "auto").lower()
+    if backend == "openai":
+        return _embed_openai(text)
+    elif backend in ("local", "sentence-transformers"):
+        return _embed_local(text)
+    elif backend == "fallback":
+        return _embed_fallback(text)
+    return _embed_auto(text)
+
+
+def _embed_auto(text: str) -> tuple[float, ...]:
+    try:
+        return _embed_local(text)
+    except ImportError:
+        pass
+    try:
+        return _embed_openai(text)
+    except (ImportError, RuntimeError):
+        pass
+    return _embed_fallback(text)
+
+
+def _embed_local(text: str) -> tuple[float, ...]:
+    try:
+        from sentence_transformers import SentenceTransformer  # noqa: F401
+    except ImportError:
+        raise ImportError(
+            "sentence-transformers not installed.\n"
+            "  pip install sentence-transformers\n"
+            "  Or set LLMASSERT_EMBED_BACKEND=openai"
+        )
+    model_name = os.environ.get(_MODEL_ENV, _DEFAULT_LOCAL_MODEL)
+    model = _get_local_model(model_name)
+    vec = model.encode(text, normalize_embeddings=True)
+    return tuple(float(v) for v in vec)
+
+
+@lru_cache(maxsize=4)
+def _get_local_model(model_name: str):
+    from sentence_transformers import SentenceTransformer
+    return SentenceTransformer(model_name)
+
+
+def _embed_openai(text: str) -> tuple[float, ...]:
+    try:
+        import openai
+    except ImportError:
+        raise ImportError("openai not installed. pip install openai")
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError("OPENAI_API_KEY not set. Set it or use LLMASSERT_EMBED_BACKEND=local")
+    model = os.environ.get(_MODEL_ENV, _DEFAULT_OPENAI_MODEL)
+    client = openai.OpenAI(api_key=api_key)
+    result = client.embeddings.create(input=[text], model=model)
+    return tuple(float(v) for v in result.data[0].embedding)
+
+
+def _embed_fallback(text: str) -> tuple[float, ...]:
+    """Hash-based pseudo-embedding. Smoke tests only — no real semantic similarity."""
+    dim = 384
+    vec = []
+    for i in range(dim):
+        seed = hashlib.sha256(f"{i}:{text}".encode()).digest()
+        val = int.from_bytes(seed[:4], "big") / (2**32)
+        vec.append(val - 0.5)
+    mag = math.sqrt(sum(v**2 for v in vec))
+    return tuple(v / mag for v in vec)

semanticheck-0.1.0/semanticheck/assertions/__init__.py

@@ -0,0 +1 @@
+# assertions package