correctover-test 0.3.0__tar.gz

correctover_test-0.3.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build tools
+        run: pip install build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}

correctover_test-0.3.0/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+.env
+.venv/
+venv/
+.DS_Store

correctover_test-0.3.0/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: correctover-test
+Version: 0.3.0
+Summary: 🔒 Correctover Agent Security Audit — test your AI agents for guardrail gaps in 5 seconds. Supports CrewAI, Smolagents, LlamaIndex.
+Project-URL: Homepage, https://correctover.com
+Project-URL: Documentation, https://correctover.com/docs
+Project-URL: Repository, https://github.com/Correctover/correctover-test
+Project-URL: Issues, https://github.com/Correctover/correctover-test/issues
+Author-email: NeuralBridge <wangguigui@correctover.com>
+License: Apache-2.0
+Keywords: agent,ai,audit,correctover,guardrail,llm,security,self-healing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: click>=8.0
+Requires-Dist: correctover>=2.2.0
+Requires-Dist: pydantic>=2.0
+Provides-Extra: all
+Requires-Dist: crewai; extra == 'all'
+Requires-Dist: llama-index-core; extra == 'all'
+Requires-Dist: smolagents; extra == 'all'
+Provides-Extra: crewai
+Requires-Dist: crewai; extra == 'crewai'
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core; extra == 'llamaindex'
+Provides-Extra: smolagents
+Requires-Dist: smolagents; extra == 'smolagents'
+Description-Content-Type: text/markdown
+
+# 🔒 correctover-test
+
+**AI Agent Security Audit — 5 seconds to know if your agents are protected.**
+
+```bash
+pip install correctover-test
+correctover-test --quick
+```
+
+[Fix issues → correctover.com](https://correctover.com)

correctover_test-0.3.0/README.md

@@ -0,0 +1,10 @@
+# 🔒 correctover-test
+
+**AI Agent Security Audit — 5 seconds to know if your agents are protected.**
+
+```bash
+pip install correctover-test
+correctover-test --quick
+```
+
+[Fix issues → correctover.com](https://correctover.com)

correctover_test-0.3.0/correctover_test/__init__.py

@@ -0,0 +1,2 @@
+"""Correctover Self-Healing Test Agent v0.1.0"""
+__version__ = "0.1.0"

correctover_test-0.3.0/correctover_test/cli.py

@@ -0,0 +1,222 @@
+"""
+correctover-test CLI — self-healing test agent entry point.
+
+Usage:
+    correctover-test --quick              # 5-second smoke test
+    correctover-test --framework crewai   # Single framework
+    correctover-test --all --html         # Full audit, HTML report
+    correctover-test --quick --cloud      # Upload results to Correctover Cloud
+"""
+
+import json
+import sys
+import urllib.request
+from typing import Optional
+
+import click
+
+from .core.engine import TestEngine
+from .core.reporter import Reporter
+from .core.license import get_validator, LicenseExceededError
+from .core.adapters import CrewAIAdapter, SmolagentsAdapter, LlamaIndexAdapter
+from .core.scenarios import MCPSelfHealScenario, EnvProtectionScenario, ContractBreakScenario
+
+
+ADAPTER_REGISTRY = {
+    "crewai": CrewAIAdapter,
+    "smolagents": SmolagentsAdapter,
+    "llamaindex": LlamaIndexAdapter,
+}
+
+SCENARIO_REGISTRY = {
+    "mcp_self_heal": MCPSelfHealScenario,
+    "env_protection": EnvProtectionScenario,
+    "contract_break": ContractBreakScenario,
+}
+
+VERSION = "0.2.0"
+
+
+@click.command()
+@click.option(
+    "--framework", "-f",
+    default="all",
+    help="Target framework: crewai, smolagents, llamaindex, or all",
+)
+@click.option(
+    "--scenario", "-s",
+    default="all",
+    help="Test scenario: mcp_self_heal, env_protection, contract_break, or all",
+)
+@click.option(
+    "--output", "-o",
+    default=None,
+    help="Output file path (.json / .md / .html based on extension)",
+)
+@click.option(
+    "--quick", "-q",
+    is_flag=True,
+    help="5-second smoke test: env_protection on all frameworks",
+)
+@click.option(
+    "--all", "run_all",
+    is_flag=True,
+    help="Full audit: all scenarios on all frameworks",
+)
+@click.option(
+    "--format", "output_format",
+    type=click.Choice(["terminal", "json", "markdown", "html"]),
+    default="terminal",
+    help="Output format (default: terminal)",
+)
+@click.option(
+    "--cloud",
+    default=None,
+    help="Correctover Cloud endpoint URL (e.g. https://cloud.correctover.com/api/v1/audit)",
+)
+@click.option(
+    "--no-cta",
+    is_flag=True,
+    help="Suppress the 'Fix this' CTA in output",
+)
+@click.version_option(version=VERSION, prog_name="correctover-test")
+def main(
+    framework: str,
+    scenario: str,
+    output: Optional[str],
+    quick: bool,
+    run_all: bool,
+    output_format: str,
+    cloud: Optional[str],
+    no_cta: bool,
+):
+    """🔒 Correctover Agent Security Audit
+
+    Tests your AI agents for guardrail gaps, command injection,
+    environment variable leaks, and contract violations.
+
+    \b
+    Quick start:
+      correctover-test --quick          # 5-second audit
+      correctover-test --all --html     # Full report
+
+    \b
+    Correctover Cloud:
+      correctover-test --quick --cloud https://cloud.correctover.com/api/v1/audit
+    """
+
+    # Resolve frameworks
+    if quick:
+        framework = "all"
+        scenario = "env_protection"
+    elif run_all:
+        framework = "all"
+        scenario = "all"
+
+    if framework == "all":
+        adapters = [cls() for cls in ADAPTER_REGISTRY.values()]
+    else:
+        fws = [f.strip() for f in framework.split(",")]
+        adapters = []
+        for fw in fws:
+            if fw not in ADAPTER_REGISTRY:
+                click.echo(
+                    f"❌ Unknown framework: {fw}. Available: {list(ADAPTER_REGISTRY.keys())}",
+                    err=True,
+                )
+                sys.exit(1)
+            adapters.append(ADAPTER_REGISTRY[fw]())
+
+    # Resolve scenarios
+    if scenario == "all":
+        scenarios = [cls() for cls in SCENARIO_REGISTRY.values()]
+    else:
+        scs = [s.strip() for s in scenario.split(",")]
+        scenarios = []
+        for sc in scs:
+            if sc not in SCENARIO_REGISTRY:
+                click.echo(
+                    f"❌ Unknown scenario: {sc}. Available: {list(SCENARIO_REGISTRY.keys())}",
+                    err=True,
+                )
+                sys.exit(1)
+            scenarios.append(SCENARIO_REGISTRY[sc]())
+
+    # License check — download-to-pay enforcement
+    try:
+        from .core.license import LicenseValidator
+        license_key = LicenseValidator.get_license_from_env()
+        validator = get_validator()
+        if license_key:
+            validator.set_license_key(license_key)
+        status = validator.record_call()
+        if status["tier"] == "free":
+            click.echo(
+                f"📊 Free tier: {status['calls_remaining']} audits remaining today "
+                f"({status['calls_today']}/{status['limit']} used)",
+                err=True,
+            )
+    except LicenseExceededError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+    # Run
+    click.echo(
+        f"🔍 Auditing {len(adapters)} framework(s) × {len(scenarios)} scenario(s)...",
+        err=True,
+    )
+    engine = TestEngine(adapters)
+    results = engine.run_all(scenarios=scenarios)
+
+    # Upload to cloud
+    if cloud:
+        try:
+            payload = json.dumps({
+                "source": "correctover-test",
+                "version": VERSION,
+                "results": [r.to_dict() for r in results],
+            }).encode("utf-8")
+            req = urllib.request.Request(
+                cloud,
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                cloud_resp = json.loads(resp.read())
+                click.echo(
+                    f"☁️  Uploaded to Correctover Cloud: {cloud_resp.get('dashboard_url', cloud)}",
+                    err=True,
+                )
+        except Exception as e:
+            click.echo(f"⚠️  Cloud upload failed: {e}", err=True)
+
+    # Output
+    if output:
+        if output.endswith(".md"):
+            content = Reporter.to_markdown(results)
+        elif output.endswith(".html"):
+            content = Reporter.to_html(results)
+        else:
+            content = Reporter.to_json(results)
+        with open(output, "w") as f:
+            f.write(content)
+        click.echo(f"📄 Report written to {output}", err=True)
+    else:
+        if output_format == "json":
+            click.echo(Reporter.to_json(results))
+        elif output_format == "markdown":
+            click.echo(Reporter.to_markdown(results))
+        elif output_format == "html":
+            click.echo(Reporter.to_html(results))
+        else:
+            Reporter.to_terminal(results, show_cta=not no_cta)
+
+    # Exit code
+    total = len(results)
+    failed = sum(1 for r in results if r.verdict.value == "FAIL")
+    sys.exit(0 if failed == 0 else 1)
+
+
+if __name__ == "__main__":
+    main()

correctover_test-0.3.0/correctover_test/core/adapters/__init__.py

@@ -0,0 +1,6 @@
+from .base import FrameworkAdapter
+from .crewai_adapter import CrewAIAdapter
+from .smolagents_adapter import SmolagentsAdapter
+from .llamaindex_adapter import LlamaIndexAdapter
+
+__all__ = ["FrameworkAdapter", "CrewAIAdapter", "SmolagentsAdapter", "LlamaIndexAdapter"]

correctover_test-0.3.0/correctover_test/core/adapters/base.py

@@ -0,0 +1,206 @@
+"""
+FrameworkAdapter ABC and shared data models.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+import time
+
+
+class FaultType(str, Enum):
+    COMMAND_INJECTION = "command_injection"
+    ENV_EXPOSURE = "env_exposure"
+    CONTRACT_BREAK = "contract_break"
+
+
+class Verdict(str, Enum):
+    PASS = "PASS"
+    FAIL = "FAIL"
+    SKIP = "SKIP"
+
+
+@dataclass
+class MetricSnapshot:
+    """6-dimensional verification metrics per test run."""
+    dimension: str          # integrity | authorization | auditability | latency | coverage | recovery
+    value: float
+    threshold: float        # pass threshold
+    passed: bool
+    detail: str = ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "dimension": self.dimension,
+            "value": self.value,
+            "threshold": self.threshold,
+            "passed": self.passed,
+            "detail": self.detail,
+        }
+
+
+@dataclass
+class TestResult:
+    """Single scenario run result."""
+    framework: str
+    scenario: str
+    verdict: Verdict
+    metrics: List[MetricSnapshot] = field(default_factory=list)
+    decisions_count: int = 0
+    envelopes_count: int = 0
+    integrity_verified: bool = False
+    duration_ms: float = 0.0
+    errors: List[str] = field(default_factory=list)
+    details: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def self_heal_rate(self) -> float:
+        """Fraction of metrics that passed."""
+        if not self.metrics:
+            return 0.0
+        return sum(1 for m in self.metrics if m.passed) / len(self.metrics)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "framework": self.framework,
+            "scenario": self.scenario,
+            "verdict": self.verdict.value,
+            "self_heal_rate": round(self.self_heal_rate, 4),
+            "metrics": [m.to_dict() for m in self.metrics],
+            "decisions_count": self.decisions_count,
+            "envelopes_count": self.envelopes_count,
+            "integrity_verified": self.integrity_verified,
+            "duration_ms": round(self.duration_ms, 2),
+            "errors": self.errors,
+            "details": self.details,
+        }
+
+
+class FrameworkAdapter(ABC):
+    """Abstract adapter for hooking correctover into an Agent framework."""
+
+    name: str
+
+    @abstractmethod
+    def create_agent(self, tools: List[Any], config: Optional[Dict] = None) -> Any:
+        """Create a native agent instance for this framework."""
+        ...
+
+    @abstractmethod
+    def register_guardrail(self, agent: Any, provider: Any) -> Any:
+        """
+        Inject a correctover GuardrailProvider into the agent.
+        Returns the modified agent or a wrapper.
+        """
+        ...
+
+    @abstractmethod
+    def run_task(self, agent: Any, task: str) -> Dict[str, Any]:
+        """
+        Execute a task through the agent.
+        Returns {"output": str, "duration_ms": float, "error": Optional[str]}.
+        """
+        ...
+
+    @abstractmethod
+    def detect_missing_guardrail(self, agent: Any) -> List[Dict]:
+        """
+        Scan the agent for missing guardrail hooks.
+        Returns list of findings, empty list = no issues.
+        """
+        ...
+
+    @abstractmethod
+    def inject_fault(self, agent: Any, fault_type: FaultType) -> Any:
+        """
+        Inject a fault into the agent's tool call pipeline.
+        Returns the modified agent/context.
+        """
+        ...
+
+
+class TestScenario(ABC):
+    """Abstract test scenario that runs against an adapter+agent pair."""
+
+    name: str
+    description: str
+    fault_type: FaultType
+
+    @abstractmethod
+    def setup(self, adapter: FrameworkAdapter, agent: Any) -> Any:
+        """Prepare the agent for this scenario. Return modified agent/context."""
+        ...
+
+    @abstractmethod
+    def execute(self, adapter: FrameworkAdapter, agent: Any) -> Dict[str, Any]:
+        """Run the scenario. Return raw results dict."""
+        ...
+
+    @abstractmethod
+    def verify(
+        self,
+        adapter: FrameworkAdapter,
+        agent: Any,
+        raw_result: Dict[str, Any],
+        trail: Any,  # correctover.AuditTrail
+    ) -> List[MetricSnapshot]:
+        """Verify the 6-dimension metrics from AuditTrail."""
+        ...
+
+    def run(
+        self,
+        adapter: FrameworkAdapter,
+        agent: Any,
+        trail: Any,
+    ) -> TestResult:
+        """Orchestrate full scenario lifecycle."""
+        t0 = time.time()
+        errors = []
+
+        try:
+            agent = self.setup(adapter, agent)
+        except Exception as e:
+            errors.append(f"setup: {e}")
+            return TestResult(
+                framework=adapter.name,
+                scenario=self.name,
+                verdict=Verdict.FAIL,
+                errors=errors,
+                details={"phase": "setup"},
+            )
+
+        try:
+            raw = self.execute(adapter, agent)
+        except Exception as e:
+            errors.append(f"execute: {e}")
+            return TestResult(
+                framework=adapter.name,
+                scenario=self.name,
+                verdict=Verdict.FAIL,
+                errors=errors,
+                details={"phase": "execute"},
+            )
+
+        try:
+            metrics = self.verify(adapter, agent, raw, trail)
+        except Exception as e:
+            errors.append(f"verify: {e}")
+            metrics = []
+
+        duration_ms = (time.time() - t0) * 1000
+        all_pass = all(m.passed for m in metrics) if metrics else False
+        integrity_ok = trail.verify_all() if trail else False
+
+        return TestResult(
+            framework=adapter.name,
+            scenario=self.name,
+            verdict=Verdict.PASS if all_pass and not errors else Verdict.FAIL,
+            metrics=metrics,
+            decisions_count=trail.decision_count if trail else 0,
+            envelopes_count=trail.envelope_count if trail else 0,
+            integrity_verified=integrity_ok,
+            duration_ms=duration_ms,
+            errors=errors,
+            details={"raw_output": raw.get("output", "")[:500]},
+        )