aicert-pro 0.1.0__tar.gz

aicert_pro-0.1.0/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.4
+Name: aicert-pro
+Version: 0.1.0
+Summary: Pro baseline regression enforcement for aicert CLI
+Author-email: aicert <dev@aicert.io>
+License: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: typer>=0.9.0
+Requires-Dist: cryptography>=41.0.0
+Requires-Dist: aicert>=0.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+
+# aicert-pro
+
+Pro baseline regression enforcement for aicert CLI.
+
+## Installation
+
+```bash
+pip install -e .
+```
+
+## Usage
+
+### License Verification
+
+```bash
+aicert-pro license verify
+```
+
+Requires `AICERT_PRO_LICENSE` environment variable to be set with a valid license.
+
+### Baseline Management
+
+Save baseline from a run:
+
+```bash
+aicert-pro baseline save --from-run /path/to/run/dir
+```
+
+Check current run against baseline:
+
+```bash
+aicert-pro baseline check --from-run /path/to/run/dir
+```
+
+Show baseline metrics:
+
+```bash
+aicert-pro baseline show <project>
+```
+
+## License
+
+MIT

aicert_pro-0.1.0/README.md

@@ -0,0 +1,43 @@
+# aicert-pro
+
+Pro baseline regression enforcement for aicert CLI.
+
+## Installation
+
+```bash
+pip install -e .
+```
+
+## Usage
+
+### License Verification
+
+```bash
+aicert-pro license verify
+```
+
+Requires `AICERT_PRO_LICENSE` environment variable to be set with a valid license.
+
+### Baseline Management
+
+Save baseline from a run:
+
+```bash
+aicert-pro baseline save --from-run /path/to/run/dir
+```
+
+Check current run against baseline:
+
+```bash
+aicert-pro baseline check --from-run /path/to/run/dir
+```
+
+Show baseline metrics:
+
+```bash
+aicert-pro baseline show <project>
+```
+
+## License
+
+MIT

aicert_pro-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "aicert-pro"
+version = "0.1.0"
+description = "Pro baseline regression enforcement for aicert CLI"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    {name = "aicert", email = "dev@aicert.io"}
+]
+dependencies = [
+    "typer>=0.9.0",
+    "cryptography>=41.0.0",
+    "aicert>=0.1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+]
+
+[project.scripts]
+aicert-pro = "aicert_pro.cli:app"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]

aicert_pro-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

aicert_pro-0.1.0/src/aicert_pro/__init__.py

@@ -0,0 +1,3 @@
+"""aicert_pro - Pro baseline regression enforcement for aicert CLI."""
+
+__version__ = "0.1.0"

aicert_pro-0.1.0/src/aicert_pro/baseline.py

@@ -0,0 +1,209 @@
+"""Baseline regression enforcement for aicert-pro."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+# Default thresholds for regression detection
+DEFAULT_MAX_STABILITY_DROP = 5
+DEFAULT_MAX_COMPLIANCE_DROP = 2
+DEFAULT_MAX_P95_LATENCY_REGRESSION_PCT = 25
+DEFAULT_MAX_COST_REGRESSION_PCT = 20
+
+
+def provider_key(provider: str, model: str, temperature: float | None) -> str:
+    """Generate unique provider key for baseline tracking.
+    
+    Args:
+        provider: Provider name (e.g., "anthropic", "openai").
+        model: Model name (e.g., "claude-3-5-sonnet-20241022").
+        temperature: Temperature setting or None.
+        
+    Returns:
+        Unique key string combining provider, model, and temperature.
+    """
+    temp_str = f"_{temperature}" if temperature is not None else ""
+    return f"{provider}/{model}{temp_str}"
+
+
+def load_baseline(path: Path) -> dict:
+    """Load baseline data from JSON file.
+    
+    Args:
+        path: Path to baseline JSON file.
+        
+    Returns:
+        Baseline data dictionary.
+    """
+    with open(path, "r") as f:
+        return json.load(f)
+
+
+def save_baseline(path: Path, data: dict) -> None:
+    """Save baseline data to JSON file.
+    
+    Args:
+        path: Path to save baseline JSON file.
+        data: Baseline data dictionary.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def baseline_path(project: str, baseline_dir: Path | None) -> Path:
+    """Get baseline file path for a project.
+    
+    Args:
+        project: Project name.
+        baseline_dir: Directory to store baselines, or None for default.
+        
+    Returns:
+        Path to baseline JSON file.
+    """
+    if baseline_dir is None:
+        baseline_dir = Path.home() / ".aicert" / "baselines"
+    baseline_dir.mkdir(parents=True, exist_ok=True)
+    return baseline_dir / f"{project}.json"
+
+
+def save_from_run(run_dir: Path, baseline_dir: Path | None = None) -> dict:
+    """Save baseline metrics from a run directory.
+    
+    Reads summary.json from the run directory and saves provider metrics
+    to the baseline store.
+    
+    Args:
+        run_dir: Path to run directory containing summary.json.
+        baseline_dir: Directory to store baselines, or None for default.
+        
+    Returns:
+        The saved baseline data dictionary.
+    """
+    summary_path = run_dir / "summary.json"
+    if not summary_path.exists():
+        raise FileNotFoundError(f"summary.json not found in {run_dir}")
+    
+    with open(summary_path, "r") as f:
+        summary = json.load(f)
+    
+    project = summary.get("project", "default")
+    
+    # Extract provider metrics from summary
+    metrics = summary.get("metrics", {})
+    provider = summary.get("provider", "")
+    model = summary.get("model", "")
+    temperature = summary.get("temperature")
+    
+    provider_key_str = provider_key(provider, model, temperature)
+    
+    baseline_data = {
+        "project": project,
+        "provider_key": provider_key_str,
+        "created_at": summary.get("timestamp", ""),
+        "metrics": {
+            "stability": metrics.get("stability", 0),
+            "compliance": metrics.get("compliance", 0),
+            "p95_latency_ms": metrics.get("p95_latency_ms", 0),
+            "mean_latency_ms": metrics.get("mean_latency_ms", 0),
+            "mean_cost_usd": metrics.get("mean_cost_usd"),
+            "prompt_hash": summary.get("prompt_hash", ""),
+            "schema_hash": summary.get("schema_hash", ""),
+        }
+    }
+    
+    # Remove None values
+    baseline_data["metrics"] = {k: v for k, v in baseline_data["metrics"].items() if v is not None}
+    
+    path = baseline_path(project, baseline_dir)
+    save_baseline(path, baseline_data)
+    
+    return baseline_data
+
+
+def check_from_run(
+    run_dir: Path,
+    baseline_dir: Path | None = None,
+    max_stability_drop: int = DEFAULT_MAX_STABILITY_DROP,
+    max_compliance_drop: int = DEFAULT_MAX_COMPLIANCE_DROP,
+    max_p95_latency_regression_pct: int = DEFAULT_MAX_P95_LATENCY_REGRESSION_PCT,
+    max_cost_regression_pct: int = DEFAULT_MAX_COST_REGRESSION_PCT,
+) -> bool:
+    """Check current run against baseline for regressions.
+    
+    Args:
+        run_dir: Path to run directory containing summary.json.
+        baseline_dir: Directory to store baselines, or None for default.
+        max_stability_drop: Maximum allowed stability drop (percentage points).
+        max_compliance_drop: Maximum allowed compliance drop (percentage points).
+        max_p95_latency_regression_pct: Maximum allowed P95 latency regression (%).
+        max_cost_regression_pct: Maximum allowed cost regression (%).
+        
+    Returns:
+        True if all metrics pass baseline thresholds, False otherwise.
+    """
+    summary_path = run_dir / "summary.json"
+    if not summary_path.exists():
+        raise FileNotFoundError(f"summary.json not found in {run_dir}")
+    
+    with open(summary_path, "r") as f:
+        summary = json.load(f)
+    
+    project = summary.get("project", "default")
+    baseline_file = baseline_path(project, baseline_dir)
+    
+    if not baseline_file.exists():
+        raise FileNotFoundError(f"Baseline not found for project '{project}'. Run 'baseline save' first.")
+    
+    baseline = load_baseline(baseline_file)
+    baseline_metrics = baseline.get("metrics", {})
+    current_metrics = summary.get("metrics", {})
+    
+    failures: list[str] = []
+    
+    # Check stability drop
+    baseline_stability = baseline_metrics.get("stability", 0)
+    current_stability = current_metrics.get("stability", 0)
+    stability_drop = baseline_stability - current_stability
+    if stability_drop > max_stability_drop:
+        failures.append(
+            f"Stability dropped by {stability_drop:.1f} (max: {max_stability_drop})"
+        )
+    
+    # Check compliance drop
+    baseline_compliance = baseline_metrics.get("compliance", 0)
+    current_compliance = current_metrics.get("compliance", 0)
+    compliance_drop = baseline_compliance - current_compliance
+    if compliance_drop > max_compliance_drop:
+        failures.append(
+            f"Compliance dropped by {compliance_drop:.1f} (max: {max_compliance_drop})"
+        )
+    
+    # Check P95 latency regression
+    baseline_p95 = baseline_metrics.get("p95_latency_ms", 0)
+    current_p95 = current_metrics.get("p95_latency_ms", 0)
+    if baseline_p95 > 0 and current_p95 > 0:
+        p95_regression_pct = ((current_p95 - baseline_p95) / baseline_p95) * 100
+        if p95_regression_pct > max_p95_latency_regression_pct:
+            failures.append(
+                f"P95 latency regressed by {p95_regression_pct:.1f}% (max: {max_p95_latency_regression_pct}%)"
+            )
+    
+    # Check cost regression
+    baseline_cost = baseline_metrics.get("mean_cost_usd")
+    current_cost = current_metrics.get("mean_cost_usd")
+    if baseline_cost is not None and current_cost is not None and baseline_cost > 0:
+        cost_regression_pct = ((current_cost - baseline_cost) / baseline_cost) * 100
+        if cost_regression_pct > max_cost_regression_pct:
+            failures.append(
+                f"Cost regressed by {cost_regression_pct:.1f}% (max: {max_cost_regression_pct}%)"
+            )
+    
+    if failures:
+        print("Baseline regression detected:")
+        for failure in failures:
+            print(f"  - {failure}")
+        return False
+    
+    print("All metrics pass baseline thresholds.")
+    return True

aicert_pro-0.1.0/src/aicert_pro/cli.py

@@ -0,0 +1,171 @@
+"""CLI for aicert-pro baseline regression enforcement."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from .baseline import (
+    baseline_path,
+    check_from_run,
+    load_baseline,
+    save_from_run,
+)
+from .license import LicenseError, require_pro
+
+app = typer.Typer(
+    name="aicert-pro",
+    help="Pro baseline regression enforcement for aicert CLI.",
+    add_completion=False,
+)
+
+
+def run_aicert(config_path: Path) -> Path:
+    """Run aicert stability evaluation and return the run directory.
+    
+    Args:
+        config_path: Path to aicert.yaml configuration file.
+        
+    Returns:
+        Path to the run directory containing summary.json.
+        
+    Exits:
+        With the same exit code if aicert fails.
+    """
+    cmd = [sys.executable, "-m", "aicert", "stability", str(config_path), "--ci"]
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    
+    if result.returncode != 0:
+        typer.echo(f"aicert failed:\n{result.stderr}", err=True)
+        raise typer.Exit(result.returncode)
+    
+    # Try to extract run directory from stdout
+    # aicert outputs something like "Run complete. Results in: /path/to/run/dir"
+    for line in result.stdout.strip().split("\n"):
+        if "Run complete" in line or "Results in" in line:
+            # Extract path from line
+            parts = line.split()
+            for part in parts:
+                if part.startswith("/") or (len(part) > 1 and part[1] == ":"):
+                    run_dir = Path(part)
+                    if run_dir.exists():
+                        return run_dir
+    
+    # Fallback: assume default artifact location
+    # Default is typically ~/.aicert/runs/<timestamp>
+    from datetime import datetime
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    run_dir = Path.home() / ".aicert" / "runs" / timestamp
+    
+    if not run_dir.exists():
+        raise typer.BadParameter(f"Could not determine run directory from aicert output. "
+                                  f"Please specify run directory directly.")
+    
+    return run_dir
+
+
+@app.command("license")
+def license_verify() -> None:
+    """Verify pro license."""
+    require_pro()
+    typer.echo("License is valid.")
+
+
+baseline_app = typer.Typer(help="Manage baselines for regression enforcement.")
+app.add_typer(baseline_app, name="baseline")
+
+
+@baseline_app.command("save")
+def baseline_save(
+    path: Path = typer.Argument(
+        ...,
+        help="Path to run directory or .yaml config file",
+    ),
+    baseline_dir: Path | None = typer.Option(
+        None,
+        "--baseline-dir",
+        "-b",
+        help="Directory to store baselines",
+    ),
+) -> None:
+    """Save baseline metrics from a run directory or run aicert first."""
+    try:
+        require_pro()
+    except LicenseError:
+        raise typer.Exit(5)
+    
+    # Determine if path is a config file or run directory
+    if path.suffix == ".yaml" or path.suffix == ".yml":
+        # Run aicert first
+        run_dir = run_aicert(path)
+        typer.echo(f"Evaluation complete. Run directory: {run_dir}")
+    else:
+        run_dir = path
+    
+    data = save_from_run(run_dir, baseline_dir)
+    result_path = baseline_path(data["project"], baseline_dir)
+    typer.echo(f"Baseline saved to {result_path}")
+
+
+@baseline_app.command("check")
+def baseline_check(
+    path: Path = typer.Argument(
+        ...,
+        help="Path to run directory or .yaml config file",
+    ),
+    baseline_dir: Path | None = typer.Option(
+        None,
+        "--baseline-dir",
+        "-b",
+        help="Directory to store baselines",
+    ),
+) -> None:
+    """Check current run against baseline for regressions."""
+    try:
+        require_pro()
+    except LicenseError:
+        raise typer.Exit(5)
+    
+    # Determine if path is a config file or run directory
+    if path.suffix == ".yaml" or path.suffix == ".yml":
+        # Run aicert first
+        run_dir = run_aicert(path)
+        typer.echo(f"Evaluation complete. Run directory: {run_dir}")
+    else:
+        run_dir = path
+    
+    passed = check_from_run(run_dir, baseline_dir)
+    if not passed:
+        raise typer.Exit(2)
+    typer.echo("Baseline check passed.")
+
+
+@baseline_app.command("show")
+def baseline_show(
+    project: str = typer.Argument(..., help="Project name"),
+    baseline_dir: Path | None = typer.Option(
+        None,
+        "--baseline-dir",
+        "-b",
+        help="Directory to store baselines",
+    ),
+) -> None:
+    """Show baseline metrics for a project."""
+    require_pro()
+    path = baseline_path(project, baseline_dir)
+    if not path.exists():
+        raise typer.BadParameter(f"Baseline not found for project '{project}'")
+    data = load_baseline(path)
+    typer.echo(json.dumps(data, indent=2))
+
+
+def main() -> None:
+    """Entry point for aicert-pro CLI."""
+    app()
+
+
+if __name__ == "__main__":
+    app()

aicert_pro-0.1.0/src/aicert_pro/license.py

@@ -0,0 +1,144 @@
+"""License verification for aicert-pro using Ed25519 signatures."""
+
+import base64
+import os
+import sys
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Optional
+
+from cryptography.hazmat.primitives.asymmetric import ed25519
+from cryptography.hazmat.primitives import serialization
+from cryptography.exceptions import InvalidSignature
+
+# Embedded Ed25519 public key for license verification (placeholder)
+# Replace with actual public key for production
+LICENSE_PUBLIC_KEY = ed25519.Ed25519PublicKey.from_public_bytes(
+    base64.b64decode(
+        "ewWxgkhpOJpLGcQKky3gMuEKaBlvlEZjy9MJ6YYCpuY="
+    )
+)
+
+LICENSE_ENV_VAR = "AICERT_PRO_LICENSE"
+
+
+class LicenseError(Exception):
+    """Base exception for license errors."""
+
+    exit_code = 5
+
+
+class LicenseMissingError(LicenseError):
+    """Raised when license is not found."""
+
+    def __init__(self) -> None:
+        super().__init__("License not found. Set AICERT_PRO_LICENSE environment variable.")
+
+
+class LicenseInvalidError(LicenseError):
+    """Raised when license signature is invalid."""
+
+    def __init__(self) -> None:
+        super().__init__("License signature is invalid.")
+
+
+class LicenseExpiredError(LicenseError):
+    """Raised when license has expired."""
+
+    def __init__(self, expired_at: str) -> None:
+        super().__init__(f"License expired at {expired_at}.")
+
+
+@dataclass
+class LicenseData:
+    """License data parsed from base64-encoded JSON."""
+
+    license_id: str
+    owner: str
+    issued_at: str
+    expires_at: Optional[str] = None
+    features: Optional[list[str]] = None
+    signature: Optional[str] = None
+
+    @property
+    def is_expired(self) -> bool:
+        """Check if license is expired."""
+        if self.expires_at is None:
+            return False
+        try:
+            expires = datetime.fromisoformat(self.expires_at.replace("Z", "+00:00"))
+            return datetime.now(timezone.utc) > expires
+        except ValueError:
+            return False
+
+
+def verify_license(license_data: LicenseData) -> bool:
+    """Verify Ed25519 signature on license data.
+    
+    Args:
+        license_data: Parsed license data including signature.
+        
+    Returns:
+        True if signature is valid.
+        
+    Raises:
+        LicenseInvalidError: If signature verification fails.
+        LicenseExpiredError: If license has expired.
+    """
+    if license_data.is_expired:
+        raise LicenseExpiredError(license_data.expires_at)
+    
+    if license_data.signature is None:
+        raise LicenseInvalidError()
+    
+    # Create signable payload (license data without signature)
+    payload = f"{license_data.license_id}:{license_data.owner}:{license_data.issued_at}"
+    if license_data.expires_at:
+        payload += f":{license_data.expires_at}"
+    if license_data.features:
+        payload += f":{','.join(license_data.features)}"
+    
+    try:
+        signature_bytes = base64.b64decode(license_data.signature)
+        LICENSE_PUBLIC_KEY.verify(signature_bytes, payload.encode("utf-8"))
+    except (InvalidSignature, ValueError) as e:
+        raise LicenseInvalidError()
+    
+    return True
+
+
+def read_license_from_env() -> str:
+    """Read license string from environment variable.
+    
+    Returns:
+        Base64-encoded license string.
+        
+    Raises:
+        LicenseMissingError: If environment variable is not set.
+    """
+    license_str = os.environ.get(LICENSE_ENV_VAR)
+    if not license_str:
+        raise LicenseMissingError()
+    return license_str
+
+
+def require_pro() -> LicenseData:
+    """Require and verify a valid pro license.
+    
+    Returns:
+        Parsed and verified license data.
+        
+    Exits:
+        Exit code 5 if license is missing or invalid.
+    """
+    try:
+        license_str = read_license_from_env()
+        import json
+        data = json.loads(base64.b64decode(license_str))
+        license_data = LicenseData(**data)
+        verify_license(license_data)
+        return license_data
+    except LicenseError:
+        raise
+    except Exception:
+        raise LicenseInvalidError()