correctover-test 0.3.0__py3-none-any.whl

correctover_test/__init__.py

@@ -0,0 +1,2 @@
+"""Correctover Self-Healing Test Agent v0.1.0"""
+__version__ = "0.1.0"

correctover_test/cli.py

@@ -0,0 +1,222 @@
+"""
+correctover-test CLI — self-healing test agent entry point.
+
+Usage:
+    correctover-test --quick              # 5-second smoke test
+    correctover-test --framework crewai   # Single framework
+    correctover-test --all --html         # Full audit, HTML report
+    correctover-test --quick --cloud      # Upload results to Correctover Cloud
+"""
+
+import json
+import sys
+import urllib.request
+from typing import Optional
+
+import click
+
+from .core.engine import TestEngine
+from .core.reporter import Reporter
+from .core.license import get_validator, LicenseExceededError
+from .core.adapters import CrewAIAdapter, SmolagentsAdapter, LlamaIndexAdapter
+from .core.scenarios import MCPSelfHealScenario, EnvProtectionScenario, ContractBreakScenario
+
+
+ADAPTER_REGISTRY = {
+    "crewai": CrewAIAdapter,
+    "smolagents": SmolagentsAdapter,
+    "llamaindex": LlamaIndexAdapter,
+}
+
+SCENARIO_REGISTRY = {
+    "mcp_self_heal": MCPSelfHealScenario,
+    "env_protection": EnvProtectionScenario,
+    "contract_break": ContractBreakScenario,
+}
+
+VERSION = "0.2.0"
+
+
+@click.command()
+@click.option(
+    "--framework", "-f",
+    default="all",
+    help="Target framework: crewai, smolagents, llamaindex, or all",
+)
+@click.option(
+    "--scenario", "-s",
+    default="all",
+    help="Test scenario: mcp_self_heal, env_protection, contract_break, or all",
+)
+@click.option(
+    "--output", "-o",
+    default=None,
+    help="Output file path (.json / .md / .html based on extension)",
+)
+@click.option(
+    "--quick", "-q",
+    is_flag=True,
+    help="5-second smoke test: env_protection on all frameworks",
+)
+@click.option(
+    "--all", "run_all",
+    is_flag=True,
+    help="Full audit: all scenarios on all frameworks",
+)
+@click.option(
+    "--format", "output_format",
+    type=click.Choice(["terminal", "json", "markdown", "html"]),
+    default="terminal",
+    help="Output format (default: terminal)",
+)
+@click.option(
+    "--cloud",
+    default=None,
+    help="Correctover Cloud endpoint URL (e.g. https://cloud.correctover.com/api/v1/audit)",
+)
+@click.option(
+    "--no-cta",
+    is_flag=True,
+    help="Suppress the 'Fix this' CTA in output",
+)
+@click.version_option(version=VERSION, prog_name="correctover-test")
+def main(
+    framework: str,
+    scenario: str,
+    output: Optional[str],
+    quick: bool,
+    run_all: bool,
+    output_format: str,
+    cloud: Optional[str],
+    no_cta: bool,
+):
+    """🔒 Correctover Agent Security Audit
+
+    Tests your AI agents for guardrail gaps, command injection,
+    environment variable leaks, and contract violations.
+
+    \b
+    Quick start:
+      correctover-test --quick          # 5-second audit
+      correctover-test --all --html     # Full report
+
+    \b
+    Correctover Cloud:
+      correctover-test --quick --cloud https://cloud.correctover.com/api/v1/audit
+    """
+
+    # Resolve frameworks
+    if quick:
+        framework = "all"
+        scenario = "env_protection"
+    elif run_all:
+        framework = "all"
+        scenario = "all"
+
+    if framework == "all":
+        adapters = [cls() for cls in ADAPTER_REGISTRY.values()]
+    else:
+        fws = [f.strip() for f in framework.split(",")]
+        adapters = []
+        for fw in fws:
+            if fw not in ADAPTER_REGISTRY:
+                click.echo(
+                    f"❌ Unknown framework: {fw}. Available: {list(ADAPTER_REGISTRY.keys())}",
+                    err=True,
+                )
+                sys.exit(1)
+            adapters.append(ADAPTER_REGISTRY[fw]())
+
+    # Resolve scenarios
+    if scenario == "all":
+        scenarios = [cls() for cls in SCENARIO_REGISTRY.values()]
+    else:
+        scs = [s.strip() for s in scenario.split(",")]
+        scenarios = []
+        for sc in scs:
+            if sc not in SCENARIO_REGISTRY:
+                click.echo(
+                    f"❌ Unknown scenario: {sc}. Available: {list(SCENARIO_REGISTRY.keys())}",
+                    err=True,
+                )
+                sys.exit(1)
+            scenarios.append(SCENARIO_REGISTRY[sc]())
+
+    # License check — download-to-pay enforcement
+    try:
+        from .core.license import LicenseValidator
+        license_key = LicenseValidator.get_license_from_env()
+        validator = get_validator()
+        if license_key:
+            validator.set_license_key(license_key)
+        status = validator.record_call()
+        if status["tier"] == "free":
+            click.echo(
+                f"📊 Free tier: {status['calls_remaining']} audits remaining today "
+                f"({status['calls_today']}/{status['limit']} used)",
+                err=True,
+            )
+    except LicenseExceededError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+
+    # Run
+    click.echo(
+        f"🔍 Auditing {len(adapters)} framework(s) × {len(scenarios)} scenario(s)...",
+        err=True,
+    )
+    engine = TestEngine(adapters)
+    results = engine.run_all(scenarios=scenarios)
+
+    # Upload to cloud
+    if cloud:
+        try:
+            payload = json.dumps({
+                "source": "correctover-test",
+                "version": VERSION,
+                "results": [r.to_dict() for r in results],
+            }).encode("utf-8")
+            req = urllib.request.Request(
+                cloud,
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                cloud_resp = json.loads(resp.read())
+                click.echo(
+                    f"☁️  Uploaded to Correctover Cloud: {cloud_resp.get('dashboard_url', cloud)}",
+                    err=True,
+                )
+        except Exception as e:
+            click.echo(f"⚠️  Cloud upload failed: {e}", err=True)
+
+    # Output
+    if output:
+        if output.endswith(".md"):
+            content = Reporter.to_markdown(results)
+        elif output.endswith(".html"):
+            content = Reporter.to_html(results)
+        else:
+            content = Reporter.to_json(results)
+        with open(output, "w") as f:
+            f.write(content)
+        click.echo(f"📄 Report written to {output}", err=True)
+    else:
+        if output_format == "json":
+            click.echo(Reporter.to_json(results))
+        elif output_format == "markdown":
+            click.echo(Reporter.to_markdown(results))
+        elif output_format == "html":
+            click.echo(Reporter.to_html(results))
+        else:
+            Reporter.to_terminal(results, show_cta=not no_cta)
+
+    # Exit code
+    total = len(results)
+    failed = sum(1 for r in results if r.verdict.value == "FAIL")
+    sys.exit(0 if failed == 0 else 1)
+
+
+if __name__ == "__main__":
+    main()

correctover_test/core/adapters/__init__.py

@@ -0,0 +1,6 @@
+from .base import FrameworkAdapter
+from .crewai_adapter import CrewAIAdapter
+from .smolagents_adapter import SmolagentsAdapter
+from .llamaindex_adapter import LlamaIndexAdapter
+
+__all__ = ["FrameworkAdapter", "CrewAIAdapter", "SmolagentsAdapter", "LlamaIndexAdapter"]

correctover_test/core/adapters/base.py

@@ -0,0 +1,206 @@
+"""
+FrameworkAdapter ABC and shared data models.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+import time
+
+
+class FaultType(str, Enum):
+    COMMAND_INJECTION = "command_injection"
+    ENV_EXPOSURE = "env_exposure"
+    CONTRACT_BREAK = "contract_break"
+
+
+class Verdict(str, Enum):
+    PASS = "PASS"
+    FAIL = "FAIL"
+    SKIP = "SKIP"
+
+
+@dataclass
+class MetricSnapshot:
+    """6-dimensional verification metrics per test run."""
+    dimension: str          # integrity | authorization | auditability | latency | coverage | recovery
+    value: float
+    threshold: float        # pass threshold
+    passed: bool
+    detail: str = ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "dimension": self.dimension,
+            "value": self.value,
+            "threshold": self.threshold,
+            "passed": self.passed,
+            "detail": self.detail,
+        }
+
+
+@dataclass
+class TestResult:
+    """Single scenario run result."""
+    framework: str
+    scenario: str
+    verdict: Verdict
+    metrics: List[MetricSnapshot] = field(default_factory=list)
+    decisions_count: int = 0
+    envelopes_count: int = 0
+    integrity_verified: bool = False
+    duration_ms: float = 0.0
+    errors: List[str] = field(default_factory=list)
+    details: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def self_heal_rate(self) -> float:
+        """Fraction of metrics that passed."""
+        if not self.metrics:
+            return 0.0
+        return sum(1 for m in self.metrics if m.passed) / len(self.metrics)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "framework": self.framework,
+            "scenario": self.scenario,
+            "verdict": self.verdict.value,
+            "self_heal_rate": round(self.self_heal_rate, 4),
+            "metrics": [m.to_dict() for m in self.metrics],
+            "decisions_count": self.decisions_count,
+            "envelopes_count": self.envelopes_count,
+            "integrity_verified": self.integrity_verified,
+            "duration_ms": round(self.duration_ms, 2),
+            "errors": self.errors,
+            "details": self.details,
+        }
+
+
+class FrameworkAdapter(ABC):
+    """Abstract adapter for hooking correctover into an Agent framework."""
+
+    name: str
+
+    @abstractmethod
+    def create_agent(self, tools: List[Any], config: Optional[Dict] = None) -> Any:
+        """Create a native agent instance for this framework."""
+        ...
+
+    @abstractmethod
+    def register_guardrail(self, agent: Any, provider: Any) -> Any:
+        """
+        Inject a correctover GuardrailProvider into the agent.
+        Returns the modified agent or a wrapper.
+        """
+        ...
+
+    @abstractmethod
+    def run_task(self, agent: Any, task: str) -> Dict[str, Any]:
+        """
+        Execute a task through the agent.
+        Returns {"output": str, "duration_ms": float, "error": Optional[str]}.
+        """
+        ...
+
+    @abstractmethod
+    def detect_missing_guardrail(self, agent: Any) -> List[Dict]:
+        """
+        Scan the agent for missing guardrail hooks.
+        Returns list of findings, empty list = no issues.
+        """
+        ...
+
+    @abstractmethod
+    def inject_fault(self, agent: Any, fault_type: FaultType) -> Any:
+        """
+        Inject a fault into the agent's tool call pipeline.
+        Returns the modified agent/context.
+        """
+        ...
+
+
+class TestScenario(ABC):
+    """Abstract test scenario that runs against an adapter+agent pair."""
+
+    name: str
+    description: str
+    fault_type: FaultType
+
+    @abstractmethod
+    def setup(self, adapter: FrameworkAdapter, agent: Any) -> Any:
+        """Prepare the agent for this scenario. Return modified agent/context."""
+        ...
+
+    @abstractmethod
+    def execute(self, adapter: FrameworkAdapter, agent: Any) -> Dict[str, Any]:
+        """Run the scenario. Return raw results dict."""
+        ...
+
+    @abstractmethod
+    def verify(
+        self,
+        adapter: FrameworkAdapter,
+        agent: Any,
+        raw_result: Dict[str, Any],
+        trail: Any,  # correctover.AuditTrail
+    ) -> List[MetricSnapshot]:
+        """Verify the 6-dimension metrics from AuditTrail."""
+        ...
+
+    def run(
+        self,
+        adapter: FrameworkAdapter,
+        agent: Any,
+        trail: Any,
+    ) -> TestResult:
+        """Orchestrate full scenario lifecycle."""
+        t0 = time.time()
+        errors = []
+
+        try:
+            agent = self.setup(adapter, agent)
+        except Exception as e:
+            errors.append(f"setup: {e}")
+            return TestResult(
+                framework=adapter.name,
+                scenario=self.name,
+                verdict=Verdict.FAIL,
+                errors=errors,
+                details={"phase": "setup"},
+            )
+
+        try:
+            raw = self.execute(adapter, agent)
+        except Exception as e:
+            errors.append(f"execute: {e}")
+            return TestResult(
+                framework=adapter.name,
+                scenario=self.name,
+                verdict=Verdict.FAIL,
+                errors=errors,
+                details={"phase": "execute"},
+            )
+
+        try:
+            metrics = self.verify(adapter, agent, raw, trail)
+        except Exception as e:
+            errors.append(f"verify: {e}")
+            metrics = []
+
+        duration_ms = (time.time() - t0) * 1000
+        all_pass = all(m.passed for m in metrics) if metrics else False
+        integrity_ok = trail.verify_all() if trail else False
+
+        return TestResult(
+            framework=adapter.name,
+            scenario=self.name,
+            verdict=Verdict.PASS if all_pass and not errors else Verdict.FAIL,
+            metrics=metrics,
+            decisions_count=trail.decision_count if trail else 0,
+            envelopes_count=trail.envelope_count if trail else 0,
+            integrity_verified=integrity_ok,
+            duration_ms=duration_ms,
+            errors=errors,
+            details={"raw_output": raw.get("output", "")[:500]},
+        )

correctover_test/core/adapters/crewai_adapter.py

@@ -0,0 +1,204 @@
+"""
+CrewAI Framework Adapter.
+
+Hooks into CrewAI's global before_tool_call / after_tool_call decorator system
+and the agent.guardrail attribute for guardrail detection.
+
+For testing without an LLM, run_task simulates tool call guardrail checks
+directly against the registered provider, bypassing actual LLM calls.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from .base import FrameworkAdapter, FaultType
+
+
+class CrewAIAdapter(FrameworkAdapter):
+    name = "crewai"
+
+    def create_agent(self, tools: List[Any], config: Optional[Dict] = None) -> Any:
+        from crewai import Agent
+
+        return Agent(
+            role=config.get("role", "test-agent") if config else "test-agent",
+            goal=config.get("goal", "Execute tools safely with guardrail protection") if config else "Execute tools safely with guardrail protection",
+            backstory=config.get("backstory", "A test agent for correctover self-healing validation") if config else "A test agent for correctover self-healing validation",
+            tools=tools,
+            verbose=False,
+            allow_delegation=False,
+        )
+
+    def register_guardrail(self, agent: Any, provider: Any) -> Any:
+        """
+        Inject guardrail via CrewAI's global before_tool_call hook system.
+        Wraps the provider into a callable that matches CrewAI's hook signature.
+        """
+        from crewai.hooks.tool_hooks import register_before_tool_call_hook
+        from correctover import GuardrailContext, ToolCallContext, AuditTrail
+
+        trail = AuditTrail()
+        ctx = GuardrailContext(provider=provider, trail=trail)
+
+        def before_hook(*, tool=None, tool_input=None, agent_role=None, agent=None, **kwargs):
+            tool_name = getattr(tool, "name", str(tool))
+            tool_args = tool_input if isinstance(tool_input, dict) else {}
+            agent_id = agent_role or "unknown"
+
+            tc = ToolCallContext(
+                tool_name=tool_name,
+                tool_args=tool_args,
+                agent_id=agent_id,
+                metadata={"framework": "crewai"},
+            )
+            decision = ctx.authorize(tc)
+            if not decision.authorized:
+                raise PermissionError(
+                    f"Tool '{tool_name}' denied by correctover guardrail"
+                )
+            return tool_input
+
+        register_before_tool_call_hook(before_hook)
+
+        agent._correctover_trail = trail
+        agent._correctover_context = ctx
+
+        return agent
+
+    def run_task(self, agent: Any, task: str) -> Dict[str, Any]:
+        """
+        Simulate a tool call against the guardrail directly — no LLM required.
+        Extracts tool name from the task string and tests the guardrail provider.
+        """
+        import time
+        from correctover import ToolCallContext
+
+        t0 = time.time()
+        error = None
+        output = ""
+
+        trail = getattr(agent, "_correctover_trail", None)
+        ctx = getattr(agent, "_correctover_context", None)
+
+        # Parse tool name from task: "Use X to do Y" -> X
+        tool_name = "unknown_tool"
+        task_lower = task.lower()
+        for word in task.split():
+            if word in ["dangerous_exec", "read_env", "unreliable_output"]:
+                tool_name = word
+                break
+        # Fallback: find tool from agent.tools
+        if tool_name == "unknown_tool" and hasattr(agent, "tools"):
+            for t in agent.tools:
+                tname = getattr(t, "name", str(t))
+                if tname.lower() in task_lower:
+                    tool_name = tname
+                    break
+
+        # Extract command arg from task if relevant
+        tool_args = {}
+        if "rm -rf" in task:
+            tool_args["command"] = "rm -rf /"
+        elif "TEST_API_KEY" in task:
+            tool_args["key"] = "TEST_API_KEY"
+        else:
+            tool_args["query"] = task
+
+        # Direct guardrail check
+        if ctx and ctx.provider:
+            tc = ToolCallContext(
+                tool_name=tool_name,
+                tool_args=tool_args,
+                agent_id=getattr(agent, "role", "unknown"),
+                metadata={"framework": "crewai", "mode": "direct-check"},
+            )
+            try:
+                decision = ctx.authorize(tc)
+                ctx.after_tool_call(decision, {"simulated": True}, time.time())
+                if decision.authorized:
+                    output = f"Tool '{tool_name}' was authorized"
+                else:
+                    error = f"Tool '{tool_name}' denied by guardrail: {decision.claims.get('reason', 'unauthorized')}"
+                    output = f"[BLOCKED] {error}"
+            except Exception as e:
+                error = str(e)
+                output = f"[ERROR] {e}"
+        else:
+            # Fallback: try actual agent kickoff (needs LLM)
+            try:
+                result = agent.kickoff(messages=task)
+                output = str(result) if result else ""
+            except Exception as e:
+                error = str(e)
+                output = ""
+
+        return {
+            "output": output,
+            "duration_ms": (time.time() - t0) * 1000,
+            "error": error,
+        }
+
+    def detect_missing_guardrail(self, agent: Any) -> List[Dict]:
+        findings = []
+
+        if getattr(agent, "guardrail", None) is None:
+            findings.append({
+                "severity": "WARNING",
+                "pattern": "CREWAI-GUARDRAIL-MISS-001",
+                "agent": agent.role,
+                "message": f"Agent '{agent.role}' has no CrewAI guardrail set",
+                "remediation": "Set agent.guardrail or inject correctover via register_guardrail()",
+            })
+
+        if not hasattr(agent, "_correctover_trail"):
+            findings.append({
+                "severity": "CRITICAL",
+                "pattern": "AS-GUARDRAIL-MISS-001",
+                "agent": agent.role,
+                "message": f"Agent '{agent.role}' has no correctover guardrail injected",
+                "remediation": "Call adapter.register_guardrail(agent, provider)",
+            })
+
+        from crewai.hooks.tool_hooks import get_before_tool_call_hooks
+        if not get_before_tool_call_hooks():
+            findings.append({
+                "severity": "HIGH",
+                "pattern": "CREWAI-HOOK-MISS-001",
+                "agent": agent.role,
+                "message": "No global before_tool_call hooks registered",
+                "remediation": "Register a before_tool_call hook with correctover provider",
+            })
+
+        return findings
+
+    def inject_fault(self, agent: Any, fault_type: FaultType) -> Any:
+        from crewai.tools import tool
+
+        if fault_type == FaultType.COMMAND_INJECTION:
+
+            @tool("dangerous_exec")
+            def dangerous_exec(command: str) -> str:
+                """Execute a shell command. DANGEROUS — should be blocked by guardrail."""
+                return f"executed: {command}"
+
+            agent.tools.append(dangerous_exec)
+
+        elif fault_type == FaultType.ENV_EXPOSURE:
+
+            @tool("read_env")
+            def read_env(key: str) -> str:
+                """Read an environment variable. Should be blocked by guardrail."""
+                import os
+                return os.environ.get(key, "NOT_SET")
+
+            agent.tools.append(read_env)
+
+        elif fault_type == FaultType.CONTRACT_BREAK:
+
+            @tool("unreliable_output")
+            def unreliable_output(query: str) -> dict:
+                """Return incomplete output missing required fields — contract break test."""
+                return {"partial": query, "_incomplete": True}
+
+            agent.tools.append(unreliable_output)
+
+        return agent