aireliability 0.1.0__tar.gz

aireliability-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,114 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint & Format Check (Ruff)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ruff
+
+      - name: Run Ruff Lint
+        run: ruff check .
+
+      - name: Run Ruff Format Check
+        run: ruff format --check .
+
+  tests:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    needs: [lint]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install core package and dev dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Run test suite with JUnit report
+        run: |
+          pytest --junitxml=junit-results-${{ matrix.python-version }}.xml
+
+      - name: Upload test results artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-results-python-${{ matrix.python-version }}
+          path: junit-results-${{ matrix.python-version }}.xml
+          retention-days: 7
+
+  package:
+    name: Package Build & Clean Install Verification
+    runs-on: ubuntu-latest
+    needs: [tests]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install packaging tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+
+      - name: Build package distribution (sdist and wheel)
+        run: python -m build
+
+      - name: Validate distribution metadata with Twine
+        run: |
+          ls -la dist/
+          twine check dist/*
+
+      - name: Clean environment installation test
+        run: |
+          # Create an isolated temporary virtual environment outside repository tree
+          python -m venv /tmp/aireliability-clean
+          /tmp/aireliability-clean/bin/pip install --upgrade pip
+          /tmp/aireliability-clean/bin/pip install dist/*.whl
+          
+          # Verify Python import and CLI entrypoint
+          /tmp/aireliability-clean/bin/python -c "import aireliability; print('aireliability version:', aireliability.__version__)"
+          /tmp/aireliability-clean/bin/airel --help

aireliability-0.1.0/.github/workflows/integration.yml

@@ -0,0 +1,43 @@
+name: Integration
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  framework-matrix:
+    name: Optional Frameworks (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install package with optional framework dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev,langgraph,langchain,openai,opentelemetry,prometheus]"
+
+      - name: Run framework and provider integration test suite
+        run: |
+          pytest tests/unit/test_phase19_integrations.py tests/unit/test_phase21_provider_integration.py tests/unit/test_phase22_telemetry.py -v

aireliability-0.1.0/.github/workflows/reliability.yml

@@ -0,0 +1,81 @@
+name: Reliability
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  reliability:
+    name: Deterministic Reliability & Regression Suite
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Run Deterministic Reliability Benchmarks & Scenarios
+        run: |
+          pytest tests/unit/test_phase14_benchmarks.py \
+                 tests/unit/test_phase18_regression.py \
+                 tests/unit/test_phase20_ci_reliability.py \
+                 tests/unit/test_phase23_distributed.py \
+                 tests/unit/test_phase24_persistence.py \
+                 tests/unit/test_phase25_control_plane.py \
+                 tests/unit/test_phase26_resilience.py \
+                 -v
+
+      - name: Run Smoke Benchmark (100 Iterations)
+        run: |
+          python benchmarks/run_benchmarks.py --smoke
+
+      - name: Verify CLI PR Reliability Reporting & Step Summary
+        run: |
+          # 1. Initialize clean test project
+          mkdir -p /tmp/pr-reliability-eval
+          airel init --dir /tmp/pr-reliability-eval
+          
+          # 2. Establish main baseline with nominal agent
+          airel test --dir /tmp/pr-reliability-eval --save-baseline
+          
+          # 3. Evaluate PR with comparison and generate reports
+          airel compare default \
+            --dir /tmp/pr-reliability-eval \
+            --report-json /tmp/reliability-report.json \
+            --report-markdown /tmp/reliability-report.md
+
+          # 4. Append to GitHub Step Summary if running in GitHub Actions
+          if [ -n "$GITHUB_STEP_SUMMARY" ]; then
+            cat /tmp/reliability-report.md >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Upload Reliability Reports
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: reliability-reports
+          path: |
+            /tmp/reliability-report.json
+            /tmp/reliability-report.md
+            benchmarks/phase14_results.json
+          retention-days: 14

aireliability-0.1.0/.gitignore

@@ -0,0 +1,88 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# Testing and coverage
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+
+# Ruff
+.ruff_cache/
+
+# IDE / Editor
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+# Python
+__pycache__/
+*.py[cod]
+*.so
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Build artifacts
+build/
+dist/
+*.egg-info/
+
+# Test / coverage
+.pytest_cache/
+.coverage
+htmlcov/
+.coverage.*
+
+# Ruff
+.ruff_cache/
+
+# AI Reliability Engine runtime data
+.aireliability/
+*.db
+
+# macOS
+.DS_Store
+# Environment / secrets
+.env
+.env.*
+!.env.example

aireliability-0.1.0/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **Phase 1: Foundation & Setup**
+  - Package structure, `pyproject.toml` configuration with Hatchling build backend.
+  - Python 3.11+ requirement, Pydantic runtime dependency.
+  - Development tools (`pytest`, `ruff`).
+- **Phase 2: Core Data Models**
+  - Implemented immutable Pydantic models: `TestCase`, `TraceStep`, `ExecutionTrace`, `EvaluationResult`, `FailureReport`, `RegressionTest`, and `RunResult`.
+  - Added enums: `StepType`, `ExecutionStatus`, and `FailureSeverity`.
+- **Phase 3: Core Protocols & Interfaces**
+  - Added `runtime_checkable` protocols: `Traceable`, `ExecutionAdapter`, `Evaluator`, and `Expectation`.
+- **Phase 4: Deterministic Assertions**
+  - Implemented non-LLM, microsecond-level assertions: `ToolCalled`, `ToolNotCalled`, `ToolOrder`, `ToolArguments`, `OutputEquals`, `OutputContains`, `SchemaMatch`, `MaxLatency`, and `MaxCost`.
+- **Phase 5: Execution Engine**
+  - Implemented `ReliabilityRunner` orchestrating agent invocation, trace capture, evaluator execution, and failure compilation.
+- **Phase 6: Failure Taxonomy & Analysis**
+  - Implemented `FailureTaxonomy` across 6 categories (`TASK`, `TOOL`, `RETRIEVAL`, `OUTPUT`, `SAFETY`, `PERFORMANCE`).
+  - Implemented deterministic `FailureAnalyzer` mapping evaluation outcomes into structured `FailureReports`.
+- **Phase 7: Regression Generator & Baseline Management**
+  - Implemented `RegressionGenerator` creating reproducible `RegressionTest` cases preserving provenance (`source_failure_id`).
+  - Implemented `BaselineManager` with 5-state comparison logic (`PASSING`, `REGRESSION`, `KNOWN_FAILURE`, `FIXED`, `NEW`).
+- **Phase 8: SQLite Storage Subsystem**
+  - Implemented serverless, transactional `SQLiteStorage` implementing `StorageBackend`.
+  - Added index optimization and safe schema initialization.
+- **Phase 9: Command-Line Interface (`airel`)**
+  - Implemented CLI commands: `airel init`, `airel test`, `airel failures`, `airel regressions`, and `airel compare`.
+- **Phase 10: CI/CD Workflows & Packaging**
+  - Added GitHub Actions workflow (`.github/workflows/ci.yml`) testing Python 3.11, 3.12, 3.13, Ruff linting, and package builds.
+  - Implemented `--ci` strict exit code flag (`airel test --ci`).
+  - Added executable CI script `examples/ci_pipeline.sh`.
+- **Phase 11: Reproducible Benchmarks**
+  - Created reproducible benchmark runner `benchmarks/run_benchmarks.py` and scenario definitions `benchmarks/scenarios.py`.
+  - Empirical overhead verification (< 0.04 ms added per agent run; 500k–700k ops/sec assertion throughput).
+- **Phase 12 (Documentation & Community Readiness)**
+  - Comprehensive `README.md` covering architecture, deterministic vs semantic evaluation, CI/CD, and extension points.
+  - Added `docs/architecture.md`, `docs/evaluation_concepts.md`, and `docs/README.md`.
+  - Added practical runnable examples: `examples/end_to_end_workflow.py` and `examples/custom_evaluator_and_adapter.py`.

aireliability-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,149 @@
+# Contributing to AI Reliability Engine
+
+Thank you for your interest in contributing to `aireliability`! We welcome community contributions, bug reports, documentation improvements, and architectural suggestions.
+
+---
+
+## 1. Code of Conduct
+
+We are committed to providing a welcoming, inclusive, and harassment-free environment for all contributors. Please treat all community members with respect.
+
+---
+
+## 2. Technical Philosophy & Design Invariants
+
+Before contributing code, please keep our core architectural rules in mind:
+- **Zero Heavy External Dependencies**: The core package relies only on Python standard libraries and `pydantic`. External frameworks (e.g. LangChain, LangGraph) are strictly optional adapters.
+- **Provider & Framework Agnostic**: Never couple internal models or runners to a specific LLM vendor or agent framework. Use standard protocols.
+- **Deterministic Confidence**: Never report confidence `1.0` on failure analyses unless backed by verifiable, deterministic evidence.
+- **Zero API Key Requirement in CI**: All CI and local test suites run deterministically without API keys, GPUs, or network access.
+- **Microsecond Overhead**: Evaluators and tracing logic must remain lightweight (< 0.1 ms overhead).
+
+---
+
+## 3. Development Setup
+
+### Prerequisites
+- Python 3.11, 3.12, or 3.13
+- `git`
+
+### Installation
+1. **Fork and clone the repository:**
+   ```bash
+   git clone https://github.com/aireliability/aireliability.git
+   cd aireliability
+   ```
+
+2. **Set up a virtual environment:**
+   ```bash
+   python3 -m venv .venv
+   source .venv/bin/activate
+   ```
+
+3. **Install dependencies in editable mode:**
+   ```bash
+   pip install --upgrade pip
+   pip install -e ".[dev]"
+   pip install build twine
+   ```
+
+4. **Install optional frameworks (if testing integrations):**
+   ```bash
+   pip install -e ".[langgraph,langchain]"
+   ```
+
+---
+
+## 4. Running Checks Locally
+
+All contributions must pass local linting, formatting, tests, and packaging checks before opening a pull request.
+
+### 1. Running Unit & Integration Tests
+```bash
+# Run all unit tests
+pytest
+
+# Run tests with verbose output
+pytest -v
+
+# Run with JUnit XML generation (same as CI)
+pytest --junitxml=test-results.xml
+```
+
+### 2. Running Reliability Scenarios & Distributed Tests
+```bash
+# Run deterministic reliability benchmark scenarios
+pytest tests/unit/test_phase14_benchmarks.py \
+       tests/unit/test_phase18_regression.py \
+       tests/unit/test_phase20_ci_reliability.py
+
+# Run Phase 23-29 distributed, resilience, tenancy, security, & observability tests
+pytest tests/unit/test_phase23_distributed.py \
+       tests/unit/test_phase24_persistence.py \
+       tests/unit/test_phase25_control_plane.py \
+       tests/unit/test_phase26_resilience.py \
+       tests/unit/test_phase27_multitenancy.py \
+       tests/unit/test_phase28_security.py \
+       tests/unit/test_phase29_observability.py
+```
+
+### 3. Running Benchmarks
+```bash
+python benchmarks/run_benchmarks.py --smoke
+python benchmarks/run_phase28_benchmarks.py
+python benchmarks/run_phase29_benchmarks.py
+```
+
+### 4. Running Optional Framework Tests
+If you have `langgraph` and `langchain` installed:
+```bash
+pytest tests/unit/test_phase19_integrations.py
+```
+
+### 5. Running Ruff (Linter & Formatter)
+We enforce clean code via `ruff` with an 88-character line length:
+```bash
+# Check for lint violations
+ruff check .
+
+# Check code formatting
+ruff format --check .
+
+# Automatically apply safe lint fixes and formatting
+ruff check --fix .
+ruff format .
+```
+
+### 6. Validating Package Distribution
+Ensure the project builds cleanly and installs in an isolated clean environment:
+```bash
+# Build distribution
+python -m build
+twine check dist/*
+
+# Test clean wheel installation
+python -m venv /tmp/airel-clean
+source /tmp/airel-clean/bin/activate
+pip install dist/*.whl
+python -c "import aireliability; print(aireliability.__version__)"
+airel --help
+deactivate
+rm -rf /tmp/airel-clean
+```
+
+---
+
+## 5. Pull Request Expectations & Handling Reliability Regressions
+
+1. **Deterministic Test Execution**:
+   - Tests in CI do not use network access or external LLM API keys. Do not add tests that require external network access.
+2. **Baseline Comparison Policy**:
+   - The reliability CI job compares execution against established baselines using `BaselineManager`.
+   - `PASSING` (previously passing, still passing): Allowed.
+   - `FIXED` (previously failing, now passing): Allowed.
+   - `KNOWN_FAILURE` (previously failing, still failing): Allowed (does not fail CI).
+   - `REGRESSION` (previously passing, now failing): **Strict CI failure**.
+3. **Diagnosing a Reliability Regression**:
+   - If CI flags a regression, inspect the step summary and generated `reliability-report.json` / `reliability-report.md` artifacts.
+   - The report pinpoints the test ID, failure category (`tool`, `output`, `performance`), root cause diagnosis, and synthesized regression test.
+   - Run the scenario locally using `airel test` or `airel compare` to reproduce and resolve the regression before requesting review.