aicert 0.1.0__tar.gz

aicert-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Quinto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aicert-0.1.0/PKG-INFO

@@ -0,0 +1,306 @@
+Metadata-Version: 2.4
+Name: aicert
+Version: 0.1.0
+Summary: CI for LLM JSON outputs - Validate, test, and measure LLM outputs
+Author: aicert contributors
+License: MIT
+Keywords: llm,cli,validation,testing,json
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.21.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: jsonschema>=4.20.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: rich>=14.0.0
+Requires-Dist: cryptography>=41.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Dynamic: license-file
+
+# aicert
+
+**Reliability tooling for structured LLM outputs.**
+
+Measure, validate, and enforce JSON stability in CI.
+
+---
+
+## Why aicert?
+
+LLMs are probabilistic systems.
+
+If your application depends on structured JSON generated by an LLM, that output becomes an implicit contract within your system. Model updates, prompt changes, or configuration tweaks can cause that contract to drift — often without immediate visibility.
+
+Traditional tests typically validate a single run. They do not measure variability across repeated executions.
+
+Aicert treats LLM output like any other production dependency:
+
+* Validate against a JSON Schema
+* Measure stability across repeated runs
+* Track latency and variability
+* Enforce reliability thresholds in CI
+
+Instead of assuming structured outputs remain stable, you can verify that they do.
+
+---
+
+## What It Enables
+
+Aicert helps you:
+
+* Detect schema breakage before deployment
+* Quantify output variability across runs
+* Catch regressions caused by model or prompt changes
+* Enforce reliability standards automatically in CI
+* Treat prompts and structured outputs as testable infrastructure
+
+It converts non-deterministic behavior into measurable signals.
+
+---
+
+## Who It’s For
+
+Aicert is designed for teams shipping structured LLM outputs into production systems:
+
+* Backend APIs powered by LLMs
+* Extraction and classification pipelines
+* Decision-support systems
+* Any workflow where JSON output drives downstream logic
+
+If structured LLM output is part of your system contract, Aicert provides guardrails.
+
+---
+
+## Installation
+
+```bash
+pip install aicert
+```
+
+For development:
+
+```bash
+pip install -e .
+```
+
+---
+
+## What It Measures
+
+* **Compliance** — % of outputs matching JSON Schema
+* **Stability** — % of identical outputs across repeated runs
+* **Latency** — P50 / P95 response times
+* **Similarity** — Structural or semantic similarity
+* **CI Gating** — Threshold-based pass/fail enforcement
+
+---
+
+## Basic Workflow
+
+### 1. Create `aicert.yaml`
+
+```yaml
+project: my-project
+
+providers:
+  - id: openai-gpt4
+    provider: openai
+    model: gpt-4
+    temperature: 0.1
+
+prompt_file: prompt.txt
+cases_file: cases.jsonl
+schema_file: schema.json
+
+runs: 50
+concurrency: 10
+timeout_s: 30
+
+validation:
+  extract_json: true
+  allow_extra_keys: false
+
+thresholds:
+  min_stability: 85
+  min_compliance: 95
+  p95_latency_ms: 5000
+
+ci:
+  runs: 10
+  save_on_fail: true
+```
+
+---
+
+### 2. Run Stability Tests
+
+```bash
+aicert stability aicert.yaml
+```
+
+---
+
+### 3. Enforce in CI
+
+```bash
+aicert ci aicert.yaml
+```
+
+Non-zero exit codes indicate threshold failures.
+
+---
+
+## Commands
+
+```bash
+aicert init
+aicert doctor aicert.yaml
+aicert run aicert.yaml
+aicert stability aicert.yaml
+aicert compare aicert.yaml
+aicert ci aicert.yaml
+aicert diff <run_a> <run_b>
+aicert report <run_dir>
+```
+
+---
+
+# Aicert Pro
+
+Aicert Pro turns measurement into enforcement.
+
+The core package tells you how your LLM behaves.
+Pro defines what behavior is acceptable — and blocks regressions automatically.
+
+When structured outputs drive production systems, “informational metrics” are not enough.
+You need a reliability boundary.
+
+Aicert Pro adds:
+
+* Baseline locking — capture a known-good state
+* Regression enforcement — fail CI when stability or compliance degrades
+* Schema and prompt drift detection — detect structural changes immediately
+* Cost regression limits — prevent silent cost creep
+* Policy-backed CI gating — enforce standards across commits
+
+Pro is designed for teams that treat LLM output as production infrastructure.
+
+---
+
+## What Changes With Pro?
+
+Without Pro:
+
+You measure stability.
+You review results manually.
+Regressions can slip through if thresholds are ignored.
+
+With Pro:
+
+Known-good behavior is locked.
+Deviations are compared automatically.
+CI fails when output contracts drift.
+Reliability becomes enforceable policy.
+Pro moves you from observation to control.
+
+## Installing Aicert Pro
+
+After purchasing a subscription:
+
+```bash
+pip install aicert-pro
+```
+
+Set your license key:
+
+```bash
+export AICERT_LICENSE="your_signed_license_key"
+```
+
+Verify:
+
+```bash
+aicert-pro license verify
+```
+
+Use baseline enforcement:
+
+```bash
+aicert-pro baseline save aicert.yaml
+aicert-pro baseline check aicert.yaml
+```
+
+---
+
+## Licensing
+
+Aicert Pro uses signed offline license keys.
+
+* No SaaS dependency
+* No account login
+* CI-friendly
+* Time-limited per subscription
+
+---
+
+## Pricing
+
+* **$29 / month**
+* **$290 / year (2 months free)**
+
+Purchase:
+[https://mfifth.github.io/aicert/](https://mfifth.github.io/aicert/)
+
+Questions:
+[mfifth@gmail.com](mailto:mfifth@gmail.com)
+
+---
+
+## GitHub Actions Example
+
+```yaml
+- run: pip install aicert
+- run: aicert ci aicert.yaml
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+```
+
+Pro enforcement in CI:
+
+```yaml
+- run: pip install aicert-pro
+- run: aicert-pro baseline check aicert.yaml
+  env:
+    AICERT_LICENSE: ${{ secrets.AICERT_LICENSE }}
+```
+
+---
+
+## Exit Codes
+
+| Code | Meaning                  |
+| ---- | ------------------------ |
+| 0    | Success                  |
+| 2    | Threshold failure        |
+| 3    | Config/schema error      |
+| 4    | Provider/auth error      |
+| 5    | License error (Pro only) |
+
+---
+
+## License
+
+Core package (`aicert`) is MIT licensed.
+Aicert Pro is commercial software.

aicert-0.1.0/README.md

@@ -0,0 +1,275 @@
+# aicert
+
+**Reliability tooling for structured LLM outputs.**
+
+Measure, validate, and enforce JSON stability in CI.
+
+---
+
+## Why aicert?
+
+LLMs are probabilistic systems.
+
+If your application depends on structured JSON generated by an LLM, that output becomes an implicit contract within your system. Model updates, prompt changes, or configuration tweaks can cause that contract to drift — often without immediate visibility.
+
+Traditional tests typically validate a single run. They do not measure variability across repeated executions.
+
+Aicert treats LLM output like any other production dependency:
+
+* Validate against a JSON Schema
+* Measure stability across repeated runs
+* Track latency and variability
+* Enforce reliability thresholds in CI
+
+Instead of assuming structured outputs remain stable, you can verify that they do.
+
+---
+
+## What It Enables
+
+Aicert helps you:
+
+* Detect schema breakage before deployment
+* Quantify output variability across runs
+* Catch regressions caused by model or prompt changes
+* Enforce reliability standards automatically in CI
+* Treat prompts and structured outputs as testable infrastructure
+
+It converts non-deterministic behavior into measurable signals.
+
+---
+
+## Who It’s For
+
+Aicert is designed for teams shipping structured LLM outputs into production systems:
+
+* Backend APIs powered by LLMs
+* Extraction and classification pipelines
+* Decision-support systems
+* Any workflow where JSON output drives downstream logic
+
+If structured LLM output is part of your system contract, Aicert provides guardrails.
+
+---
+
+## Installation
+
+```bash
+pip install aicert
+```
+
+For development:
+
+```bash
+pip install -e .
+```
+
+---
+
+## What It Measures
+
+* **Compliance** — % of outputs matching JSON Schema
+* **Stability** — % of identical outputs across repeated runs
+* **Latency** — P50 / P95 response times
+* **Similarity** — Structural or semantic similarity
+* **CI Gating** — Threshold-based pass/fail enforcement
+
+---
+
+## Basic Workflow
+
+### 1. Create `aicert.yaml`
+
+```yaml
+project: my-project
+
+providers:
+  - id: openai-gpt4
+    provider: openai
+    model: gpt-4
+    temperature: 0.1
+
+prompt_file: prompt.txt
+cases_file: cases.jsonl
+schema_file: schema.json
+
+runs: 50
+concurrency: 10
+timeout_s: 30
+
+validation:
+  extract_json: true
+  allow_extra_keys: false
+
+thresholds:
+  min_stability: 85
+  min_compliance: 95
+  p95_latency_ms: 5000
+
+ci:
+  runs: 10
+  save_on_fail: true
+```
+
+---
+
+### 2. Run Stability Tests
+
+```bash
+aicert stability aicert.yaml
+```
+
+---
+
+### 3. Enforce in CI
+
+```bash
+aicert ci aicert.yaml
+```
+
+Non-zero exit codes indicate threshold failures.
+
+---
+
+## Commands
+
+```bash
+aicert init
+aicert doctor aicert.yaml
+aicert run aicert.yaml
+aicert stability aicert.yaml
+aicert compare aicert.yaml
+aicert ci aicert.yaml
+aicert diff <run_a> <run_b>
+aicert report <run_dir>
+```
+
+---
+
+# Aicert Pro
+
+Aicert Pro turns measurement into enforcement.
+
+The core package tells you how your LLM behaves.
+Pro defines what behavior is acceptable — and blocks regressions automatically.
+
+When structured outputs drive production systems, “informational metrics” are not enough.
+You need a reliability boundary.
+
+Aicert Pro adds:
+
+* Baseline locking — capture a known-good state
+* Regression enforcement — fail CI when stability or compliance degrades
+* Schema and prompt drift detection — detect structural changes immediately
+* Cost regression limits — prevent silent cost creep
+* Policy-backed CI gating — enforce standards across commits
+
+Pro is designed for teams that treat LLM output as production infrastructure.
+
+---
+
+## What Changes With Pro?
+
+Without Pro:
+
+You measure stability.
+You review results manually.
+Regressions can slip through if thresholds are ignored.
+
+With Pro:
+
+Known-good behavior is locked.
+Deviations are compared automatically.
+CI fails when output contracts drift.
+Reliability becomes enforceable policy.
+Pro moves you from observation to control.
+
+## Installing Aicert Pro
+
+After purchasing a subscription:
+
+```bash
+pip install aicert-pro
+```
+
+Set your license key:
+
+```bash
+export AICERT_LICENSE="your_signed_license_key"
+```
+
+Verify:
+
+```bash
+aicert-pro license verify
+```
+
+Use baseline enforcement:
+
+```bash
+aicert-pro baseline save aicert.yaml
+aicert-pro baseline check aicert.yaml
+```
+
+---
+
+## Licensing
+
+Aicert Pro uses signed offline license keys.
+
+* No SaaS dependency
+* No account login
+* CI-friendly
+* Time-limited per subscription
+
+---
+
+## Pricing
+
+* **$29 / month**
+* **$290 / year (2 months free)**
+
+Purchase:
+[https://mfifth.github.io/aicert/](https://mfifth.github.io/aicert/)
+
+Questions:
+[mfifth@gmail.com](mailto:mfifth@gmail.com)
+
+---
+
+## GitHub Actions Example
+
+```yaml
+- run: pip install aicert
+- run: aicert ci aicert.yaml
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+```
+
+Pro enforcement in CI:
+
+```yaml
+- run: pip install aicert-pro
+- run: aicert-pro baseline check aicert.yaml
+  env:
+    AICERT_LICENSE: ${{ secrets.AICERT_LICENSE }}
+```
+
+---
+
+## Exit Codes
+
+| Code | Meaning                  |
+| ---- | ------------------------ |
+| 0    | Success                  |
+| 2    | Threshold failure        |
+| 3    | Config/schema error      |
+| 4    | Provider/auth error      |
+| 5    | License error (Pro only) |
+
+---
+
+## License
+
+Core package (`aicert`) is MIT licensed.
+Aicert Pro is commercial software.

aicert-0.1.0/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "aicert"
+version = "0.1.0"
+description = "CI for LLM JSON outputs - Validate, test, and measure LLM outputs"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT"}
+authors = [
+    {name = "aicert contributors"}
+]
+keywords = ["llm", "cli", "validation", "testing", "json"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+dependencies = [
+    "typer>=0.21.0",
+    "pydantic>=2.5.0",
+    "pyyaml>=6.0.1",
+    "jsonschema>=4.20.0",
+    "httpx>=0.25.0",
+    "rich>=14.0.0",
+    "cryptography>=41.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4.0",
+    "pytest-asyncio>=0.21.0",
+    "black>=23.0.0",
+    "ruff>=0.1.0",
+]
+
+[project.scripts]
+aicert = "aicert.cli:app"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]
+
+[tool.black]
+line-length = 100
+target-version = ["py311"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

aicert-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

aicert-0.1.0/src/aicert/__init__.py

@@ -0,0 +1,3 @@
+"""aicert - CI for LLM JSON outputs."""
+
+__version__ = "0.1.0"

aicert-0.1.0/src/aicert/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for running aicert as a module."""
+
+from aicert.cli import app
+
+if __name__ == "__main__":
+    app()