evidentia-eval 0.10.5__tar.gz

evidentia_eval-0.10.5/.gitignore

@@ -0,0 +1,148 @@
+# Secrets & credential stores (belt-and-suspenders safety net; the
+# user's working directory MUST keep secrets in C:\Users\<user>\.secrets\
+# and never inside the repo, but these patterns block accidental adds).
+# .env.example IS tracked intentionally — it documents the expected
+# variable names for new contributors.
+.env
+.env.*
+!.env.example
+!.env.template
+*.pem
+*.key
+*.crt
+*.p12
+*.pfx
+secrets/
+credentials.json
+
+# v0.4.0 — frontend build output lands in the Python package's static
+# directory at wheel-assembly time via the hatchling build hook. The
+# .gitkeep file in static/ is tracked; everything else is regenerated.
+packages/evidentia-api/src/evidentia_api/static/assets/
+packages/evidentia-api/src/evidentia_api/static/index.html
+packages/evidentia-api/src/evidentia_api/static/*.js
+packages/evidentia-api/src/evidentia_api/static/*.css
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+# NB: `lib/` and `lib64/` above would otherwise also match
+# packages/evidentia-ui/src/lib/ (TypeScript utils). Scope to top-level
+# only — there's no real Python-venv lib/ we'd fail to ignore because
+# .venv/ and venv/ below cover that case.
+!packages/evidentia-ui/src/lib/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# uv
+# NOTE: uv.lock is committed for reproducible builds.
+# https://docs.astral.sh/uv/concepts/projects/sync/#locking-dependencies
+
+# Testing
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.cache
+coverage.xml
+*.cover
+.hypothesis/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Ruff
+.ruff_cache/
+
+# Supply-chain scan artifact — the CycloneDX SBOM is regenerated by
+# scripts/run_osv_scan.py and by release.yml; it is never committed.
+evidentia-sbom.cdx.json
+# Step 7 post-tag verification artifacts: SBOMs downloaded from the
+# GitHub Release attached assets for osv-scan re-verification. The
+# canonical SBOM lives on the published release, not in the repo.
+published-sbom*.cdx.json
+
+# IDE
+# .vscode/ is ignored by default but the canonical shared workspace
+# config files are version-controlled (see docs/ide-setup.md). Per-developer
+# overrides (.vscode/*.local.json, scratch files, etc.) stay ignored.
+.vscode/*
+!.vscode/settings.json
+!.vscode/launch.json
+!.vscode/tasks.json
+!.vscode/extensions.json
+.idea/
+
+# .cursor/ is the per-developer private Cursor workspace directory
+# (project rules under .cursor/rules/*.mdc, MCP server configs, and any
+# other Cursor IDE state). The public-facing Cursor conventions live at
+# the repo-root .cursorrules file (already version-controlled). The
+# .cursor/ directory is for per-developer extensions that should not be
+# committed.
+.cursor/
+
+# Local-only / per-developer scratch directory for working notes,
+# drafts, and anything not ready to share. The convention follows the
+# .vscode/ split: ignore the whole directory by default; un-ignore
+# specific files only if they're meant to be shared across the team.
+.local/
+
+# Private competitive-strategy / market-research working docs. The repo
+# is public (Polycentric-Labs/evidentia); strategy material naming
+# specific competitor-feature adoption decisions or commercial
+# positioning stays out of the public tree. Neutral landscape analysis
+# lives in docs/ (e.g. docs/positioning-and-value.md) and IS tracked.
+/private/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Claude Code local state
+.claude/
+
+# Evidentia runtime — user project state (NOT bundled examples).
+# `.controlbridge/` and `/controlbridge.yaml` remain ignored as a courtesy
+# for legacy project workspaces (v0.1.0 – v0.5.0) so files generated by
+# pre-rename code don't leak into git when those projects are migrated.
+.evidentia/
+.controlbridge/
+/evidentia.yaml
+/controlbridge.yaml
+*.local.yaml
+evidence/
+reports/
+risks/
+
+# Generated reports from examples (keep source files, ignore generated ones)
+examples/**/report.json
+examples/**/report.csv
+examples/**/report.md
+examples/**/report.oscal.json
+examples/**/risks.json

evidentia_eval-0.10.5/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: evidentia-eval
+Version: 0.10.5
+Summary: DFAH (Decision-Faithfulness Assessment Harness) determinism + faithfulness eval harness for Evidentia — dev-time AI-output quality gates
+Project-URL: Homepage, https://github.com/polycentric-labs/evidentia
+Project-URL: Repository, https://github.com/polycentric-labs/evidentia
+Project-URL: Issues, https://github.com/polycentric-labs/evidentia/issues
+Project-URL: Changelog, https://github.com/polycentric-labs/evidentia/blob/main/CHANGELOG.md
+Author-email: Allen Byrd <allen@allenfbyrd.com>
+License-Expression: Apache-2.0
+Keywords: ai-quality,compliance,determinism,dfah,faithfulness,grc,llm-eval
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: evidentia-core<0.11.0,>=0.10.5
+Provides-Extra: faithfulness-semantic
+Requires-Dist: numpy>=1.26; extra == 'faithfulness-semantic'
+Requires-Dist: sentence-transformers>=3.0; extra == 'faithfulness-semantic'
+Description-Content-Type: text/markdown
+
+# evidentia-eval
+
+Dev-time AI-output quality eval harness for Evidentia.
+
+Hosts the **DFAH (Decision-Faithfulness Assessment Harness)** —
+the auditor-defensible numerical proof layer that validates
+LLM-driven artifact production is deterministic, replay-
+equivalent, and faithful to its source policy clauses.
+
+## Why this package exists (v0.10.5 P9 extraction)
+
+The DFAH harness was originally bundled into `evidentia-ai` (the
+risk-statement generator + control explainer package). That
+conflated two very different deployment surfaces:
+
+- **`evidentia-ai`** — PRODUCTION runtime. Needed in air-gap
+  installs to actually generate risk statements.
+- **`evidentia-eval`** — DEVELOPMENT-time evaluation. NOT needed
+  in air-gap installs; only fires when a CI pipeline runs a
+  determinism / faithfulness gate before tagging a release.
+
+Extracting the eval harness lets air-gap installs of
+`evidentia-ai` skip the optional sentence-transformers stack
+entirely (it now lives behind `evidentia-eval[faithfulness-semantic]`
+instead of `evidentia-ai[eval-faithfulness]`).
+
+## Quick start
+
+```bash
+# Stdlib Jaccard baseline (no extra needed; <10 MB install)
+pip install evidentia-eval
+
+# Optional semantic-similarity faithfulness (~250 MB extra
+# for sentence-transformers + numpy + model cache on first use)
+pip install 'evidentia-eval[faithfulness-semantic]'
+```
+
+CLI verbs:
+
+```bash
+# Smoke test against a deterministic stub generator (no LLM
+# tokens burned)
+evidentia eval stub-smoke
+
+# Real-LLM determinism gate against the risk-statement generator
+evidentia eval risk-determinism --gap-report gaps.json \
+    --system-context ctx.yaml \
+    --fail-on-determinism-rate-below 0.95
+
+# Verify a previously-signed eval bundle
+evidentia eval verify path/to/eval-output.json
+```
+
+The CLI verbs live in `evidentia.cli.eval` (the meta-package);
+this package contributes the underlying library.
+
+## Public API
+
+| Symbol | Purpose |
+|---|---|
+| `DFAHarness` | Owns the run loop + audit emit |
+| `EvalResult` | Top-level harness output (JSON-serializable, Sigstore-signable) |
+| `EvalSample` | One prompt's inputs (immutable; audit-trail-stable) |
+| `DeterminismResult` | Per-prompt determinism outcome |
+| `ReplayResult` | Per-prompt replay-equivalence outcome |
+| `FaithfulnessResult` | Per-claim faithfulness outcome |
+| `PromptFaithfulnessResult` | Aggregated per-prompt faithfulness |
+| `faithfulness_score` | Stdlib Jaccard token-overlap baseline |
+| `faithfulness_score_semantic` | Sentence-transformers path (optional extra) |
+| `determinism_score` | Computes the modal-output pass rate |
+| `replay_equivalent` | Binary replay-equivalence check |
+| `extract_claims` | Atomic-claim extraction from generated artifacts |
+| `normalize_for_determinism` | Canonical whitespace + punctuation normalization |
+| `hash_output` | SHA-256 hex of normalized output |
+| `sign_eval_result` | Sigstore-sign an `EvalResult` JSON |
+| `verify_eval_result` | Verify a previously-signed eval bundle |
+
+## Backward-compat shim
+
+For external scripts that still import `from evidentia_ai.eval
+import ...`, `evidentia-ai` ships a deprecation shim that
+re-exports from `evidentia_eval`. The shim warns once at import
+time and is scheduled for removal in **v0.12.0**.
+
+## License
+
+Apache-2.0. See the workspace root LICENSE file.

evidentia_eval-0.10.5/README.md

@@ -0,0 +1,87 @@
+# evidentia-eval
+
+Dev-time AI-output quality eval harness for Evidentia.
+
+Hosts the **DFAH (Decision-Faithfulness Assessment Harness)** —
+the auditor-defensible numerical proof layer that validates
+LLM-driven artifact production is deterministic, replay-
+equivalent, and faithful to its source policy clauses.
+
+## Why this package exists (v0.10.5 P9 extraction)
+
+The DFAH harness was originally bundled into `evidentia-ai` (the
+risk-statement generator + control explainer package). That
+conflated two very different deployment surfaces:
+
+- **`evidentia-ai`** — PRODUCTION runtime. Needed in air-gap
+  installs to actually generate risk statements.
+- **`evidentia-eval`** — DEVELOPMENT-time evaluation. NOT needed
+  in air-gap installs; only fires when a CI pipeline runs a
+  determinism / faithfulness gate before tagging a release.
+
+Extracting the eval harness lets air-gap installs of
+`evidentia-ai` skip the optional sentence-transformers stack
+entirely (it now lives behind `evidentia-eval[faithfulness-semantic]`
+instead of `evidentia-ai[eval-faithfulness]`).
+
+## Quick start
+
+```bash
+# Stdlib Jaccard baseline (no extra needed; <10 MB install)
+pip install evidentia-eval
+
+# Optional semantic-similarity faithfulness (~250 MB extra
+# for sentence-transformers + numpy + model cache on first use)
+pip install 'evidentia-eval[faithfulness-semantic]'
+```
+
+CLI verbs:
+
+```bash
+# Smoke test against a deterministic stub generator (no LLM
+# tokens burned)
+evidentia eval stub-smoke
+
+# Real-LLM determinism gate against the risk-statement generator
+evidentia eval risk-determinism --gap-report gaps.json \
+    --system-context ctx.yaml \
+    --fail-on-determinism-rate-below 0.95
+
+# Verify a previously-signed eval bundle
+evidentia eval verify path/to/eval-output.json
+```
+
+The CLI verbs live in `evidentia.cli.eval` (the meta-package);
+this package contributes the underlying library.
+
+## Public API
+
+| Symbol | Purpose |
+|---|---|
+| `DFAHarness` | Owns the run loop + audit emit |
+| `EvalResult` | Top-level harness output (JSON-serializable, Sigstore-signable) |
+| `EvalSample` | One prompt's inputs (immutable; audit-trail-stable) |
+| `DeterminismResult` | Per-prompt determinism outcome |
+| `ReplayResult` | Per-prompt replay-equivalence outcome |
+| `FaithfulnessResult` | Per-claim faithfulness outcome |
+| `PromptFaithfulnessResult` | Aggregated per-prompt faithfulness |
+| `faithfulness_score` | Stdlib Jaccard token-overlap baseline |
+| `faithfulness_score_semantic` | Sentence-transformers path (optional extra) |
+| `determinism_score` | Computes the modal-output pass rate |
+| `replay_equivalent` | Binary replay-equivalence check |
+| `extract_claims` | Atomic-claim extraction from generated artifacts |
+| `normalize_for_determinism` | Canonical whitespace + punctuation normalization |
+| `hash_output` | SHA-256 hex of normalized output |
+| `sign_eval_result` | Sigstore-sign an `EvalResult` JSON |
+| `verify_eval_result` | Verify a previously-signed eval bundle |
+
+## Backward-compat shim
+
+For external scripts that still import `from evidentia_ai.eval
+import ...`, `evidentia-ai` ships a deprecation shim that
+re-exports from `evidentia_eval`. The shim warns once at import
+time and is scheduled for removal in **v0.12.0**.
+
+## License
+
+Apache-2.0. See the workspace root LICENSE file.

evidentia_eval-0.10.5/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "evidentia-eval"
+version = "0.10.5"
+description = "DFAH (Decision-Faithfulness Assessment Harness) determinism + faithfulness eval harness for Evidentia — dev-time AI-output quality gates"
+readme = "README.md"
+authors = [{name = "Allen Byrd", email = "allen@allenfbyrd.com"}]
+license = "Apache-2.0"
+requires-python = ">=3.12"
+keywords = ["grc", "compliance", "llm-eval", "dfah", "determinism", "faithfulness", "ai-quality"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Information Technology",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Typing :: Typed",
+]
+dependencies = [
+    "evidentia-core>=0.10.5,<0.11.0",
+]
+
+[project.optional-dependencies]
+# v0.10.5 P9 extraction: the v0.8.3 P1.1 sentence-transformers
+# faithfulness path lives on evidentia-eval now. Same opt-in shape:
+# ~90 MB model download (default all-MiniLM-L6-v2) on first use;
+# cached at ~/.cache/huggingface/. Operators relying on the stdlib
+# Jaccard baseline are unaffected (no extra needed). Heavyweight;
+# intentionally extra-gated.
+faithfulness-semantic = [
+    "sentence-transformers>=3.0",
+    "numpy>=1.26",
+]
+
+[project.urls]
+Homepage = "https://github.com/polycentric-labs/evidentia"
+Repository = "https://github.com/polycentric-labs/evidentia"
+Issues = "https://github.com/polycentric-labs/evidentia/issues"
+Changelog = "https://github.com/polycentric-labs/evidentia/blob/main/CHANGELOG.md"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evidentia_eval"]
+
+[tool.uv.sources]
+evidentia-core = { workspace = true }

evidentia_eval-0.10.5/src/evidentia_eval/__init__.py

@@ -0,0 +1,131 @@
+"""evidentia-eval — DFAH determinism + faithfulness harness for Evidentia.
+
+Decision-Faithfulness Assessment Harness per arXiv 2601.15322.
+Validates that risk-statement generation (or any AI-driven
+artifact production) is auditor-defensibly stable: same input +
+same model + same temperature produces the same output, and a
+re-run with pinned ``(input, model, temperature, prompt_hash,
+run_id)`` is byte-equivalent to the original.
+
+Three metrics ship:
+
+- **Decision determinism** — same prompt produces the same
+  normalized output across N samples. The pass rate is the
+  fraction of samples that match the modal output (modulo
+  whitespace + punctuation normalization). Reported as a 0..1
+  score; CI-gateable via
+  ``evidentia eval --fail-on-determinism-rate-below 0.95``.
+- **Replay equivalence** — re-running with a pinned context
+  (``GenerationContext`` instance) produces an output whose
+  SHA-256 hash matches the original. Either the run is replay-
+  equivalent or it isn't — there is no graceful degradation.
+- **Faithfulness** — do the atomic claims in a generated artifact
+  trace back to source policy clauses? Stdlib Jaccard baseline
+  (always available) + optional sentence-transformers semantic
+  path (``[faithfulness-semantic]`` extra).
+
+Public API:
+
+- :class:`DFAHarness` — owns the run loop + audit emit.
+- :class:`DeterminismResult` — Pydantic model summarizing one
+  prompt's determinism outcome (modal output + pass rate +
+  per-sample hashes).
+- :class:`ReplayResult` — Pydantic model summarizing replay-
+  equivalence for a single ``GenerationContext`` re-run.
+- :class:`EvalResult` — top-level harness output covering all
+  prompts in one ``run_id``.
+- :class:`FaithfulnessResult` — per-claim faithfulness outcome.
+- :class:`PromptFaithfulnessResult` — aggregated per-prompt
+  faithfulness outcome.
+- :func:`faithfulness_score` — stdlib Jaccard token-overlap
+  baseline.
+- :func:`faithfulness_score_semantic` — sentence-transformers
+  semantic-similarity path (opt-in extra).
+- :func:`extract_claims` — atomic-claim extraction from generated
+  artifacts.
+- :func:`normalize_for_determinism` — canonical normalization
+  (whitespace + punctuation) used by the determinism check.
+- :func:`hash_output` — SHA-256 hex of normalized output.
+- :func:`sign_eval_result` / :func:`verify_eval_result` —
+  Sigstore-sign + verify the eval output.
+
+The harness is generator-agnostic: it accepts any callable
+``(prompt: str, context: GenerationContext) -> str`` so the
+same machinery validates risk statements, control
+explanations, future PRT-traced outputs, and any third-party
+plugin's AI-generated artifacts. Unit tests use a deterministic
+fake generator; live operator runs wire in
+``evidentia_ai.risk_statements.RiskStatementGenerator.generate``.
+
+v0.10.5 P9 extraction: this package was carved out of
+``evidentia_ai.eval.*`` to keep air-gap installs of the
+risk-statement runtime from pulling sentence-transformers /
+numpy / instructor heavy-dep stacks. The dev-time eval harness
+now installs separately (or via ``pip install
+evidentia-eval[faithfulness-semantic]`` for the optional
+semantic path).
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
+from evidentia_eval.claim_extraction import (
+    CLAIM_EXTRACTION_PROMPT,
+    extract_claims,
+)
+from evidentia_eval.faithfulness import (
+    DEFAULT_FAITHFULNESS_THRESHOLD,
+    FaithfulnessResult,
+    PromptFaithfulnessResult,
+    faithfulness_score,
+)
+from evidentia_eval.faithfulness_semantic import (
+    DEFAULT_SEMANTIC_MODEL,
+    DEFAULT_SEMANTIC_THRESHOLD,
+    SemanticFaithfulnessNotAvailableError,
+    faithfulness_score_semantic,
+)
+from evidentia_eval.harness import DFAHarness, EvalResult, EvalSample
+from evidentia_eval.metrics import (
+    DeterminismResult,
+    ReplayResult,
+    determinism_score,
+    replay_equivalent,
+)
+from evidentia_eval.seeds import hash_output, normalize_for_determinism
+from evidentia_eval.signing import (
+    sign_eval_result,
+    verify_eval_result,
+)
+
+try:
+    __version__ = _pkg_version("evidentia-eval")
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "CLAIM_EXTRACTION_PROMPT",
+    "DEFAULT_FAITHFULNESS_THRESHOLD",
+    "DEFAULT_SEMANTIC_MODEL",
+    "DEFAULT_SEMANTIC_THRESHOLD",
+    "DFAHarness",
+    "DeterminismResult",
+    "EvalResult",
+    "EvalSample",
+    "FaithfulnessResult",
+    "PromptFaithfulnessResult",
+    "ReplayResult",
+    "SemanticFaithfulnessNotAvailableError",
+    "__version__",
+    "determinism_score",
+    "extract_claims",
+    "faithfulness_score",
+    "faithfulness_score_semantic",
+    "hash_output",
+    "normalize_for_determinism",
+    "replay_equivalent",
+    "sign_eval_result",
+    "verify_eval_result",
+]