aicert 0.1.0__py3-none-any.whl

aicert/validation.py

@@ -0,0 +1,322 @@
+"""Validation utilities for aicert."""
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+from jsonschema import ValidationError, validate
+
+
+@dataclass
+class ValidationResult:
+    """Result of validating LLM output."""
+    ok_json: bool = False
+    ok_schema: bool = False
+    extra_keys: List[str] = field(default_factory=list)
+    error: Optional[str] = None
+    parsed: Optional[Any] = None
+
+
+def extract_json_from_block(text: str) -> Optional[str]:
+    """Extract JSON from a ```json ... ``` block."""
+    # Match ```json at start of line, followed by any content, ending with ```
+    pattern = r'```json\s*\n(.*?)\n```'
+    match = re.search(pattern, text, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+    return None
+
+
+def extract_json_basic(text: str) -> Optional[str]:
+    """Basic extraction of first JSON object/array from text."""
+    # Find the first { or [ that starts a JSON object/array
+    text = text.strip()
+    
+    # Find first { or [ at the start or after whitespace
+    start_idx = -1
+    for i, char in enumerate(text):
+        if char == '{':
+            start_idx = i
+            break
+        elif char == '[':
+            start_idx = i
+            break
+    
+    if start_idx == -1:
+        return None
+    
+    # Try to parse from this position
+    # We'll try to find the matching closing bracket
+    end_idx = -1
+    stack = []
+    for i in range(start_idx, len(text)):
+        char = text[i]
+        if char == '{' or char == '[':
+            stack.append(char)
+        elif char == '}' or char == ']':
+            if not stack:
+                continue
+            opening = stack.pop()
+            if (opening == '{' and char == '}') or (opening == '[' and char == ']'):
+                if not stack:
+                    end_idx = i + 1
+                    break
+    
+    if end_idx > 0:
+        return text[start_idx:end_idx]
+    
+    # Fallback: try to find any closing bracket at the end
+    # This is a simple heuristic
+    if start_idx == 0:
+        # Try to find the last } or ]
+        for i in range(len(text) - 1, start_idx - 1, -1):
+            if text[i] in '}]':
+                return text[start_idx:i + 1]
+    
+    return None
+
+
+def parse_output(text: str, extract_json: bool) -> tuple[bool, Any | None, str | None]:
+    """
+    Parse output text to extract JSON.
+    
+    Args:
+        text: The raw output text from the LLM
+        extract_json: Whether to try extracting JSON from blocks or text
+        
+    Returns:
+        Tuple of (ok_json: bool, parsed: Any|None, error: str|None)
+    """
+    if extract_json:
+        # Try to extract from ```json block first
+        json_text = extract_json_from_block(text)
+        if json_text is None:
+            # Fall back to basic extraction
+            json_text = extract_json_basic(text)
+        
+        if json_text is None:
+            return False, None, "No JSON found in output"
+    else:
+        # Use entire text as JSON
+        json_text = text
+    
+    if json_text is None:
+        return False, None, "No JSON found in output"
+    
+    try:
+        parsed = json.loads(json_text)
+        return True, parsed, None
+    except json.JSONDecodeError as e:
+        return False, None, f"Invalid JSON: {str(e)}"
+
+
+def find_extra_keys(obj: Any, schema: Dict[str, Any], parent_path: str = "") -> List[str]:
+    """
+    Find extra keys in an object that are not in the schema.
+    
+    Args:
+        obj: The parsed JSON object
+        schema: The JSON schema
+        parent_path: Path for error reporting
+        
+    Returns:
+        List of extra key names
+    """
+    extra_keys: List[str] = []
+    
+    if not isinstance(obj, dict) or not isinstance(schema, dict):
+        return extra_keys
+    
+    # Check if this object has additionalProperties: false
+    additional_properties = schema.get("additionalProperties", None)
+    is_strict = additional_properties is False
+    
+    # Get defined properties
+    properties = schema.get("properties", {})
+    required_fields = schema.get("required", [])
+    
+    for key, value in obj.items():
+        key_path = f"{parent_path}.{key}" if parent_path else key
+        
+        if key in properties:
+            # Key is defined, check nested schema
+            prop_schema = properties[key]
+            if isinstance(prop_schema, dict):
+                # Recursively check nested objects
+                nested_extra = find_extra_keys(value, prop_schema, key_path)
+                extra_keys.extend(nested_extra)
+        elif is_strict:
+            # Key not defined and additionalProperties is false
+            extra_keys.append(key_path)
+    
+    return extra_keys
+
+
+def remove_additional_properties(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Remove additionalProperties from schema and nested schemas."""
+    if not isinstance(schema, dict):
+        return schema
+    
+    result = {}
+    for key, value in schema.items():
+        if key == "additionalProperties":
+            continue
+        elif key == "properties" and isinstance(value, dict):
+            result[key] = {k: remove_additional_properties(v) for k, v in value.items()}
+        elif isinstance(value, dict):
+            result[key] = remove_additional_properties(value)
+        else:
+            result[key] = value
+    return result
+
+
+def validate_schema(parsed: Any, schema: Dict[str, Any], allow_extra_keys: bool = True) -> tuple[bool, str | None, List[str]]:
+    """
+    Validate parsed JSON against a schema.
+    
+    Args:
+        parsed: The parsed JSON object
+        schema: The JSON schema
+        allow_extra_keys: If False, enforce that no extra keys exist
+        
+    Returns:
+        Tuple of (ok_schema: bool, error: str|None, extra_keys: List[str])
+    """
+    extra_keys: List[str] = []
+    
+    if not isinstance(schema, dict):
+        return True, None, []
+    
+    # Check for strict mode - extra key enforcement
+    if not allow_extra_keys:
+        extra_keys = find_extra_keys(parsed, schema)
+        if extra_keys:
+            return False, f"Extra keys not allowed: {', '.join(extra_keys)}", extra_keys
+    
+    # If allow_extra_keys=True, remove additionalProperties restriction
+    # before validation to allow extra keys
+    validation_schema = schema if allow_extra_keys else schema
+    if allow_extra_keys and schema.get("additionalProperties") is False:
+        validation_schema = remove_additional_properties(schema)
+    
+    try:
+        validate(instance=parsed, schema=validation_schema)
+        return True, None, extra_keys
+    except ValidationError as e:
+        return False, f"Schema validation error: {e.message}", extra_keys
+
+
+def validate_output(
+    text: str,
+    schema: Dict[str, Any],
+    extract_json: bool,
+    allow_extra_keys: bool = True
+) -> ValidationResult:
+    """
+    Validate LLM output against a schema.
+    
+    Args:
+        text: The raw output text from the LLM
+        schema: The JSON schema to validate against
+        extract_json: Whether to try extracting JSON from blocks
+        allow_extra_keys: If False, enforce that no extra keys exist
+        
+    Returns:
+        ValidationResult with ok_json, ok_schema, extra_keys, and error
+    """
+    # Step 1: Parse JSON from output
+    ok_json, parsed, error = parse_output(text, extract_json)
+    
+    if not ok_json:
+        return ValidationResult(
+            ok_json=False,
+            ok_schema=False,
+            error=error
+        )
+    
+    # Step 2: Validate against schema
+    ok_schema, schema_error, extra_keys = validate_schema(parsed, schema, allow_extra_keys)
+    
+    return ValidationResult(
+        ok_json=True,
+        ok_schema=ok_schema,
+        extra_keys=extra_keys,
+        error=schema_error,
+        parsed=parsed
+    )
+
+
+# Keep old functions for backward compatibility
+
+class SchemaLoadError(Exception):
+    """Error raised when schema loading fails."""
+
+    def __init__(self, message: str, schema_path: str = None, hint: str = None):
+        self.message = message
+        self.schema_path = schema_path
+        self.hint = hint
+        super().__init__(self._format_message())
+
+    def _format_message(self) -> str:
+        """Format the error message with context."""
+        parts = []
+        if self.schema_path:
+            parts.append(f"[bold red]Schema file: {self.schema_path}[/bold red]")
+        parts.append(f"[bold red]Error:[/bold red] {self.message}")
+        if self.hint:
+            parts.append(f"[bold yellow]Hint:[/bold yellow] {self.hint}")
+        return "\n".join(parts)
+
+
+def load_json_schema(schema_path: str) -> Dict[str, Any]:
+    """Load a JSON schema from a file with better error messages."""
+    import yaml
+    from pathlib import Path
+
+    try:
+        with open(schema_path, "r") as f:
+            schema = yaml.safe_load(f)
+    except FileNotFoundError:
+        raise SchemaLoadError(
+            message=f"Schema file not found: {schema_path}",
+            schema_path=schema_path,
+            hint="Make sure the path is correct and the file exists."
+        )
+    except yaml.YAMLError as e:
+        raise SchemaLoadError(
+            message=f"Invalid YAML in schema file: {e}",
+            schema_path=schema_path,
+            hint="Check for syntax errors like incorrect indentation."
+        )
+    except Exception as e:
+        raise SchemaLoadError(
+            message=f"Failed to load schema: {e}",
+            schema_path=schema_path,
+            hint="Make sure the file is valid JSON or YAML."
+        )
+
+    # Validate that it's a dict (could be JSON or YAML parsed)
+    if not isinstance(schema, dict):
+        raise SchemaLoadError(
+            message=f"Schema must be a dictionary/object, got {type(schema).__name__}",
+            schema_path=schema_path,
+            hint="The schema file should contain a JSON object with '$schema', 'type', 'properties', etc."
+        )
+
+    return schema
+
+
+def validate_json(data: Any, schema: Dict[str, Any]) -> bool:
+    """Validate JSON data against a schema."""
+    validate(instance=data, schema=schema)
+    return True
+
+
+def validate_json_file(data_path: str, schema_path: str) -> bool:
+    """Validate a JSON file against a schema."""
+    import yaml
+    with open(data_path, "r") as f:
+        data = yaml.safe_load(f)
+    schema = load_json_schema(schema_path)
+    return validate_json(data, schema)

aicert-0.1.0.dist-info/METADATA

@@ -0,0 +1,306 @@
+Metadata-Version: 2.4
+Name: aicert
+Version: 0.1.0
+Summary: CI for LLM JSON outputs - Validate, test, and measure LLM outputs
+Author: aicert contributors
+License: MIT
+Keywords: llm,cli,validation,testing,json
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.21.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: jsonschema>=4.20.0
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: rich>=14.0.0
+Requires-Dist: cryptography>=41.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Dynamic: license-file
+
+# aicert
+
+**Reliability tooling for structured LLM outputs.**
+
+Measure, validate, and enforce JSON stability in CI.
+
+---
+
+## Why aicert?
+
+LLMs are probabilistic systems.
+
+If your application depends on structured JSON generated by an LLM, that output becomes an implicit contract within your system. Model updates, prompt changes, or configuration tweaks can cause that contract to drift — often without immediate visibility.
+
+Traditional tests typically validate a single run. They do not measure variability across repeated executions.
+
+Aicert treats LLM output like any other production dependency:
+
+* Validate against a JSON Schema
+* Measure stability across repeated runs
+* Track latency and variability
+* Enforce reliability thresholds in CI
+
+Instead of assuming structured outputs remain stable, you can verify that they do.
+
+---
+
+## What It Enables
+
+Aicert helps you:
+
+* Detect schema breakage before deployment
+* Quantify output variability across runs
+* Catch regressions caused by model or prompt changes
+* Enforce reliability standards automatically in CI
+* Treat prompts and structured outputs as testable infrastructure
+
+It converts non-deterministic behavior into measurable signals.
+
+---
+
+## Who It’s For
+
+Aicert is designed for teams shipping structured LLM outputs into production systems:
+
+* Backend APIs powered by LLMs
+* Extraction and classification pipelines
+* Decision-support systems
+* Any workflow where JSON output drives downstream logic
+
+If structured LLM output is part of your system contract, Aicert provides guardrails.
+
+---
+
+## Installation
+
+```bash
+pip install aicert
+```
+
+For development:
+
+```bash
+pip install -e .
+```
+
+---
+
+## What It Measures
+
+* **Compliance** — % of outputs matching JSON Schema
+* **Stability** — % of identical outputs across repeated runs
+* **Latency** — P50 / P95 response times
+* **Similarity** — Structural or semantic similarity
+* **CI Gating** — Threshold-based pass/fail enforcement
+
+---
+
+## Basic Workflow
+
+### 1. Create `aicert.yaml`
+
+```yaml
+project: my-project
+
+providers:
+  - id: openai-gpt4
+    provider: openai
+    model: gpt-4
+    temperature: 0.1
+
+prompt_file: prompt.txt
+cases_file: cases.jsonl
+schema_file: schema.json
+
+runs: 50
+concurrency: 10
+timeout_s: 30
+
+validation:
+  extract_json: true
+  allow_extra_keys: false
+
+thresholds:
+  min_stability: 85
+  min_compliance: 95
+  p95_latency_ms: 5000
+
+ci:
+  runs: 10
+  save_on_fail: true
+```
+
+---
+
+### 2. Run Stability Tests
+
+```bash
+aicert stability aicert.yaml
+```
+
+---
+
+### 3. Enforce in CI
+
+```bash
+aicert ci aicert.yaml
+```
+
+Non-zero exit codes indicate threshold failures.
+
+---
+
+## Commands
+
+```bash
+aicert init
+aicert doctor aicert.yaml
+aicert run aicert.yaml
+aicert stability aicert.yaml
+aicert compare aicert.yaml
+aicert ci aicert.yaml
+aicert diff <run_a> <run_b>
+aicert report <run_dir>
+```
+
+---
+
+# Aicert Pro
+
+Aicert Pro turns measurement into enforcement.
+
+The core package tells you how your LLM behaves.
+Pro defines what behavior is acceptable — and blocks regressions automatically.
+
+When structured outputs drive production systems, “informational metrics” are not enough.
+You need a reliability boundary.
+
+Aicert Pro adds:
+
+* Baseline locking — capture a known-good state
+* Regression enforcement — fail CI when stability or compliance degrades
+* Schema and prompt drift detection — detect structural changes immediately
+* Cost regression limits — prevent silent cost creep
+* Policy-backed CI gating — enforce standards across commits
+
+Pro is designed for teams that treat LLM output as production infrastructure.
+
+---
+
+## What Changes With Pro?
+
+Without Pro:
+
+You measure stability.
+You review results manually.
+Regressions can slip through if thresholds are ignored.
+
+With Pro:
+
+Known-good behavior is locked.
+Deviations are compared automatically.
+CI fails when output contracts drift.
+Reliability becomes enforceable policy.
+Pro moves you from observation to control.
+
+## Installing Aicert Pro
+
+After purchasing a subscription:
+
+```bash
+pip install aicert-pro
+```
+
+Set your license key:
+
+```bash
+export AICERT_LICENSE="your_signed_license_key"
+```
+
+Verify:
+
+```bash
+aicert-pro license verify
+```
+
+Use baseline enforcement:
+
+```bash
+aicert-pro baseline save aicert.yaml
+aicert-pro baseline check aicert.yaml
+```
+
+---
+
+## Licensing
+
+Aicert Pro uses signed offline license keys.
+
+* No SaaS dependency
+* No account login
+* CI-friendly
+* Time-limited per subscription
+
+---
+
+## Pricing
+
+* **$29 / month**
+* **$290 / year (2 months free)**
+
+Purchase:
+[https://mfifth.github.io/aicert/](https://mfifth.github.io/aicert/)
+
+Questions:
+[mfifth@gmail.com](mailto:mfifth@gmail.com)
+
+---
+
+## GitHub Actions Example
+
+```yaml
+- run: pip install aicert
+- run: aicert ci aicert.yaml
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+```
+
+Pro enforcement in CI:
+
+```yaml
+- run: pip install aicert-pro
+- run: aicert-pro baseline check aicert.yaml
+  env:
+    AICERT_LICENSE: ${{ secrets.AICERT_LICENSE }}
+```
+
+---
+
+## Exit Codes
+
+| Code | Meaning                  |
+| ---- | ------------------------ |
+| 0    | Success                  |
+| 2    | Threshold failure        |
+| 3    | Config/schema error      |
+| 4    | Provider/auth error      |
+| 5    | License error (Pro only) |
+
+---
+
+## License
+
+Core package (`aicert`) is MIT licensed.
+Aicert Pro is commercial software.

aicert-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+aicert/__init__.py,sha256=Wr15C3CQxuxepOgzREU4WQl9TkSRSNTMHTslmrV73_Y,63
+aicert/__main__.py,sha256=4P50AkHMqohCf03NwoGtALlQFF0w0KZAu-braeWXxLY,116
+aicert/artifacts.py,sha256=lbSkeIYR3XUPLTQBI8VqY6O5iMBSJGpdpaYmA8NC88Q,3031
+aicert/cli.py,sha256=7nL9CCyb_PoAvdAS7kF5Mi0GbwK8_EtTqll9Wvq2iPY,56603
+aicert/config.py,sha256=EeBHZcH_aMKvAgm7aojJ-1DccfkHte5bQpGYLbMZihk,7188
+aicert/doctor.py,sha256=Ig1SwOrXbDV_pEFE2UpjzNnbyVfSLeWuGhjCv1QxoZU,12569
+aicert/hashing.py,sha256=uqQmg2qp7Lfb3N3EYqpfw6bBDvletks8ZoHHKmJHcUU,705
+aicert/metrics.py,sha256=sh-yMk7qrH_3QC_IZEr-PASnaYi43sHql3ueCNym8JA,11152
+aicert/runner.py,sha256=l3BHDpypQKTiHm2c3B_ClUO_P2kNEOCz0ncU3eVIDFI,20722
+aicert/templating.py,sha256=EXqjw69VjHKTNATp-_sXycs6HABraMZnq3bRrKiA0lY,2472
+aicert/validation.py,sha256=m2Qx7vyNxUiajzFXW2UDjAWl7qqihbvaPLifviV1fiY,10371
+aicert/providers/__init__.py,sha256=z4JPvhJkVsz7gW8gf4b4lUgvMg9cK6rj78p7WhA1v68,374
+aicert/providers/anthropic.py,sha256=FUlZZ6LP_o9GP5Qbk0sVXEiBtf3Iz8Fj0gvqnfXfelc,6381
+aicert/providers/base.py,sha256=pp3mGD6EY8AMRgOEOfsB1ZXXZHAGxLEufUWtfMX3-tE,966
+aicert/providers/openai.py,sha256=aQCqGUxEWjOUyUpu9ceCaqNUYig1dRWqWiF0Qu3z_iI,5272
+aicert/providers/openai_compatible.py,sha256=o9BWUcNQUmWJljWIqXx5LH0Q1ysl7tW52I9Wok66FQA,5373
+aicert-0.1.0.dist-info/licenses/LICENSE,sha256=m8Wu9ollES96-xdQn31baYdbK4xIDfHR8S0lCQt0Aow,1068
+aicert-0.1.0.dist-info/METADATA,sha256=uIgMchOoGJKD8shY0W1I-yAsafy9WtRd3GlGygvw6dc,6413
+aicert-0.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+aicert-0.1.0.dist-info/entry_points.txt,sha256=iCeFwGlD159K_db1wkq5ExrF6RTklQYVioQC7ZtnRoo,42
+aicert-0.1.0.dist-info/top_level.txt,sha256=RnxinJ8aabE4rxqgfQ68ZutBZKUC2NRDgMBVfciv4dQ,7
+aicert-0.1.0.dist-info/RECORD,,

aicert-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

aicert-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+aicert = aicert.cli:app

aicert-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Quinto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aicert-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+aicert