correctover-test 0.3.0__py3-none-any.whl

correctover_test/core/adapters/llamaindex_adapter.py

@@ -0,0 +1,185 @@
+"""
+LlamaIndex Framework Adapter.
+
+LlamaIndex's workflow agents use event streams. For testing without an LLM,
+we run guardrail checks directly against the provider by iterating tools.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from .base import FrameworkAdapter, FaultType
+
+
+class LlamaIndexAdapter(FrameworkAdapter):
+    name = "llamaindex"
+
+    def create_agent(self, tools: List[Any], config: Optional[Dict] = None) -> Any:
+        from llama_index.core.agent.workflow import FunctionAgent
+        from llama_index.core.llms import MockLLM
+
+        cfg = config or {}
+        llm = cfg.get("llm")
+        if llm is None:
+            llm = MockLLM()
+
+        return FunctionAgent(
+            name=config.get("name", "test-agent") if config else "test-agent",
+            description=config.get("description", "Test agent for correctover validation") if config else "Test agent for correctover validation",
+            system_prompt=config.get("system_prompt", "You are a test agent. Use tools to complete tasks.") if config else "You are a test agent. Use tools to complete tasks.",
+            tools=tools,
+            llm=llm,
+            **(config or {}),
+        )
+
+    def register_guardrail(self, agent: Any, provider: Any) -> Any:
+        """
+        Store guardrail context on the agent for direct-check mode.
+        """
+        from correctover import GuardrailContext, AuditTrail
+
+        trail = AuditTrail()
+        ctx = GuardrailContext(provider=provider, trail=trail)
+
+        agent._correctover_trail = trail
+        agent._correctover_context = ctx
+
+        return agent
+
+    def run_task(self, agent: Any, task: str) -> Dict[str, Any]:
+        """
+        Direct guardrail check — no LLM required.
+        Iterates over agent tools and tests each against the guardrail provider.
+        """
+        import time
+        from correctover import ToolCallContext
+
+        t0 = time.time()
+        error = None
+        output_parts = []
+
+        trail = getattr(agent, "_correctover_trail", None)
+        ctx = getattr(agent, "_correctover_context", None)
+
+        if ctx and ctx.provider:
+            tools = getattr(agent, "tools", None)
+            if tools:
+                for tool_obj in tools:
+                    # Get tool name from various LlamaIndex tool structures
+                    tname = None
+                    if hasattr(tool_obj, "metadata"):
+                        meta = tool_obj.metadata
+                        tname = getattr(meta, "name", None) if hasattr(meta, "name") else str(meta)
+                    if tname is None:
+                        fn = getattr(tool_obj, "fn", None)
+                        tname = getattr(fn, "__name__", str(tool_obj)) if fn else str(tool_obj)
+
+                    tool_args = {}
+                    if "rm -rf" in task or "dangerous_exec" in str(tname):
+                        tool_args["command"] = "rm -rf /"
+                    elif "TEST_API_KEY" in task or "read_env" in str(tname):
+                        tool_args["key"] = "TEST_API_KEY"
+                    else:
+                        tool_args["query"] = task
+
+                    tc = ToolCallContext(
+                        tool_name=tname,
+                        tool_args=tool_args,
+                        agent_id=getattr(agent, "name", "llamaindex-agent"),
+                        metadata={"framework": "llamaindex", "mode": "direct-check"},
+                    )
+                    try:
+                        decision = ctx.authorize(tc)
+                        ctx.after_tool_call(decision, {"simulated": True}, time.time())
+                        if decision.authorized:
+                            output_parts.append(f"Tool '{tname}' authorized")
+                        else:
+                            msg = f"Tool '{tname}' DENIED: {decision.claims.get('reason', 'unauthorized')}"
+                            output_parts.append(msg)
+                            error = msg
+                    except Exception as e:
+                        output_parts.append(f"[ERROR] {e}")
+                        error = str(e)
+            else:
+                output_parts.append("No tools to test")
+        else:
+            output_parts.append("No guardrail context — skipping")
+
+        output = "\n".join(output_parts) if output_parts else str(task)
+
+        return {
+            "output": output,
+            "duration_ms": (time.time() - t0) * 1000,
+            "error": error,
+        }
+
+    def detect_missing_guardrail(self, agent: Any) -> List[Dict]:
+        findings = []
+
+        if not hasattr(agent, "_correctover_trail"):
+            findings.append({
+                "severity": "CRITICAL",
+                "pattern": "AS-GUARDRAIL-MISS-001",
+                "agent": agent.name,
+                "message": f"Agent '{agent.name}' has no correctover guardrail injected",
+                "remediation": "Call adapter.register_guardrail(agent, provider)",
+            })
+
+        tools = getattr(agent, "tools", None)
+        if tools is None or (isinstance(tools, list) and len(tools) == 0):
+            findings.append({
+                "severity": "INFO",
+                "pattern": "LLAMA-NO-TOOLS-001",
+                "agent": agent.name,
+                "message": "Agent has no tools configured",
+                "remediation": "Add tools to the agent for guardrail testing",
+            })
+
+        return findings
+
+    def inject_fault(self, agent: Any, fault_type: FaultType) -> Any:
+        from llama_index.core.tools import FunctionTool
+
+        if fault_type == FaultType.COMMAND_INJECTION:
+
+            def dangerous_exec(command: str) -> str:
+                return f"executed: {command}"
+
+            tool = FunctionTool.from_defaults(
+                fn=dangerous_exec,
+                name="dangerous_exec",
+                description="Execute a shell command (DANGEROUS)",
+            )
+            if agent.tools is None:
+                agent.tools = []
+            agent.tools.append(tool)
+
+        elif fault_type == FaultType.ENV_EXPOSURE:
+
+            def read_env(key: str) -> str:
+                import os
+                return os.environ.get(key, "NOT_SET")
+
+            tool = FunctionTool.from_defaults(
+                fn=read_env,
+                name="read_env",
+                description="Read an environment variable",
+            )
+            if agent.tools is None:
+                agent.tools = []
+            agent.tools.append(tool)
+
+        elif fault_type == FaultType.CONTRACT_BREAK:
+
+            def unreliable_output(query: str) -> dict:
+                return {"partial": query, "_incomplete": True}
+
+            tool = FunctionTool.from_defaults(
+                fn=unreliable_output,
+                name="unreliable_output",
+                description="Returns output that may break contract validation",
+            )
+            if agent.tools is None:
+                agent.tools = []
+            agent.tools.append(tool)
+
+        return agent

correctover_test/core/adapters/smolagents_adapter.py

@@ -0,0 +1,193 @@
+"""
+Smolagents Framework Adapter.
+
+Smolagents lacks before_tool_call hooks. For testing without an LLM,
+we run guardrail checks directly against the provider by simulating
+ToolCallContext entries for each tool in the agent.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from .base import FrameworkAdapter, FaultType
+
+
+class SmolagentsAdapter(FrameworkAdapter):
+    name = "smolagents"
+
+    def create_agent(self, tools: List[Any], config: Optional[Dict] = None) -> Any:
+        from smolagents import ToolCallingAgent
+
+        # Use a mock model to avoid real API calls during testing
+        try:
+            from smolagents.models import LiteLLMModel
+            model = LiteLLMModel(model_id="deepseek/deepseek-chat", api_key="sk-test")
+        except Exception:
+            model = None
+
+        return ToolCallingAgent(
+            tools=tools,
+            model=model,
+            **(config or {}),
+        )
+
+    def register_guardrail(self, agent: Any, provider: Any) -> Any:
+        """
+        Register guardrail context on the agent for direct-check mode.
+        Smolagents has no native before_tool_call hook, so we store the
+        provider on the agent for run_task to use directly.
+        """
+        from correctover import GuardrailContext, AuditTrail
+
+        trail = AuditTrail()
+        ctx = GuardrailContext(provider=provider, trail=trail)
+
+        agent._correctover_trail = trail
+        agent._correctover_context = ctx
+
+        return agent
+
+    def run_task(self, agent: Any, task: str) -> Dict[str, Any]:
+        """
+        Direct guardrail check — no LLM required.
+        Iterates over agent tools and tests each against the guardrail provider.
+        """
+        import time
+        from correctover import ToolCallContext
+
+        t0 = time.time()
+        error = None
+        output_parts = []
+
+        trail = getattr(agent, "_correctover_trail", None)
+        ctx = getattr(agent, "_correctover_context", None)
+
+        if ctx and ctx.provider:
+            tools = getattr(agent, "tools", {})
+            if tools:
+                # Test each tool against the guardrail
+                for tname, tool_obj in (tools.items() if isinstance(tools, dict) else [(getattr(t, "name", str(t)), t) for t in tools]):
+                    tool_args = {}
+                    if "rm -rf" in task or "dangerous_exec" in tname:
+                        tool_args["command"] = "rm -rf /"
+                    elif "TEST_API_KEY" in task or "read_env" in tname:
+                        tool_args["key"] = "TEST_API_KEY"
+                    else:
+                        tool_args["query"] = task
+
+                    tc = ToolCallContext(
+                        tool_name=tname,
+                        tool_args=tool_args,
+                        agent_id=getattr(agent, "name", "smolagents-agent"),
+                        metadata={"framework": "smolagents", "mode": "direct-check"},
+                    )
+                    try:
+                        decision = ctx.authorize(tc)
+                        ctx.after_tool_call(decision, {"simulated": True}, time.time())
+                        if decision.authorized:
+                            output_parts.append(f"Tool '{tname}' authorized")
+                        else:
+                            msg = f"Tool '{tname}' DENIED: {decision.claims.get('reason', 'unauthorized')}"
+                            output_parts.append(msg)
+                            error = msg
+                    except Exception as e:
+                        output_parts.append(f"[ERROR] {e}")
+                        error = str(e)
+            else:
+                output_parts.append("No tools to test")
+        else:
+            output_parts.append("No guardrail context — skipping")
+
+        output = "\n".join(output_parts) if output_parts else str(task)
+
+        return {
+            "output": output,
+            "duration_ms": (time.time() - t0) * 1000,
+            "error": error,
+        }
+
+    def detect_missing_guardrail(self, agent: Any) -> List[Dict]:
+        findings = []
+
+        if not hasattr(agent, "_correctover_trail"):
+            findings.append({
+                "severity": "CRITICAL",
+                "pattern": "AS-GUARDRAIL-MISS-001",
+                "agent": getattr(agent, "name", "unknown"),
+                "message": "Agent has no correctover guardrail injected",
+                "remediation": "Call adapter.register_guardrail(agent, provider)",
+            })
+
+        checks = getattr(agent, "final_answer_checks", None)
+        if checks is None or (isinstance(checks, list) and len(checks) == 0):
+            findings.append({
+                "severity": "WARNING",
+                "pattern": "SMOL-FINAL-CHECK-MISS-001",
+                "agent": getattr(agent, "name", "unknown"),
+                "message": "No final_answer_checks configured",
+                "remediation": "Set agent.final_answer_checks with validation functions",
+            })
+
+        if hasattr(agent, "authorized_imports"):
+            imports = getattr(agent, "authorized_imports", [])
+            if "os" in imports or "subprocess" in imports:
+                findings.append({
+                    "severity": "HIGH",
+                    "pattern": "SMOL-IMPORT-RISK-001",
+                    "agent": getattr(agent, "name", "unknown"),
+                    "message": "Dangerous imports allowed: os, subprocess",
+                    "remediation": "Remove os/subprocess from authorized_imports",
+                })
+
+        return findings
+
+    def inject_fault(self, agent: Any, fault_type: FaultType) -> Any:
+        from smolagents import tool
+
+        if fault_type == FaultType.COMMAND_INJECTION:
+
+            @tool
+            def dangerous_exec(command: str) -> str:
+                """Execute a shell command. DANGEROUS — should be blocked by guardrail.
+
+                Args:
+                    command: Shell command to execute.
+                """
+                import subprocess
+                return subprocess.getoutput(command)
+
+            if agent.tools is None:
+                agent.tools = {}
+            agent.tools["dangerous_exec"] = dangerous_exec
+
+        elif fault_type == FaultType.ENV_EXPOSURE:
+
+            @tool
+            def read_env(key: str) -> str:
+                """Read an environment variable. Should be blocked by guardrail.
+
+                Args:
+                    key: Environment variable name to read.
+                """
+                import os
+                return os.environ.get(key, "NOT_SET")
+
+            if agent.tools is None:
+                agent.tools = {}
+            agent.tools["read_env"] = read_env
+
+        elif fault_type == FaultType.CONTRACT_BREAK:
+
+            @tool
+            def unreliable_output(query: str) -> dict:
+                """Return incomplete output missing required fields — contract break test.
+
+                Args:
+                    query: Input query to process.
+                """
+                return {"partial": query, "_incomplete": True}
+
+            if agent.tools is None:
+                agent.tools = {}
+            agent.tools["unreliable_output"] = unreliable_output
+
+        return agent

correctover_test/core/engine.py

@@ -0,0 +1,104 @@
+"""
+TestEngine — orchestrates scenario runs across frameworks, collects AuditTrail data.
+"""
+
+from typing import Any, Dict, List, Optional
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from correctover import AuditTrail, make_guardrail_hook, EnvProtectionProvider
+
+from .adapters.base import FrameworkAdapter, TestResult, Verdict
+from .scenarios.base import TestScenario
+
+
+class TestEngine:
+    """Orchestrates self-healing test runs across framework adapters."""
+
+    def __init__(self, adapters: Optional[List[FrameworkAdapter]] = None):
+        self.adapters = adapters or []
+
+    def register_adapter(self, adapter: FrameworkAdapter) -> None:
+        self.adapters.append(adapter)
+
+    def run_scenario(
+        self,
+        adapter: FrameworkAdapter,
+        scenario: TestScenario,
+    ) -> TestResult:
+        """Run a single scenario on a single adapter."""
+        agent = adapter.create_agent([], {})
+
+        # Inject guardrail — use scenario-appropriate provider
+        from correctover import EnvProtectionProvider, DenyAllGuardrailProvider, CompositeGuardrailProvider, ToolListGuardrailProvider
+        from .adapters.base import FaultType
+
+        if scenario.fault_type == FaultType.COMMAND_INJECTION:
+            # MCP: deny dangerous_exec explicitly
+            provider = ToolListGuardrailProvider(denied_tools={"dangerous_exec"})
+        elif scenario.fault_type == FaultType.ENV_EXPOSURE:
+            provider = EnvProtectionProvider()
+        elif scenario.fault_type == FaultType.CONTRACT_BREAK:
+            # Contract break: allow all (post-execution validation via CCSValidator)
+            from correctover import AllowAllGuardrailProvider
+            provider = AllowAllGuardrailProvider()
+        else:
+            provider = EnvProtectionProvider()
+
+        agent = adapter.register_guardrail(agent, provider)
+
+        # Use the trail from agent (created by register_guardrail)
+        trail = getattr(agent, "_correctover_trail", AuditTrail())
+
+        # Run scenario
+        return scenario.run(adapter, agent, trail)
+
+    def run_all(
+        self,
+        scenarios: Optional[List[TestScenario]] = None,
+        frameworks: Optional[List[str]] = None,
+        max_workers: int = 4,
+    ) -> List[TestResult]:
+        """Run all scenarios on all (or selected) adapters."""
+        from .scenarios import (
+            MCPSelfHealScenario,
+            EnvProtectionScenario,
+            ContractBreakScenario,
+        )
+
+        if scenarios is None:
+            scenarios = [
+                MCPSelfHealScenario(),
+                EnvProtectionScenario(),
+                ContractBreakScenario(),
+            ]
+
+        targets = self.adapters
+        if frameworks:
+            targets = [a for a in self.adapters if a.name in frameworks]
+
+        results: List[TestResult] = []
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {
+                pool.submit(self.run_scenario, adapter, scenario): (adapter.name, scenario.name)
+                for adapter in targets
+                for scenario in scenarios
+            }
+            for future in as_completed(futures):
+                try:
+                    result = future.result()
+                    results.append(result)
+                except Exception as e:
+                    fw, sc = futures[future]
+                    results.append(TestResult(
+                        framework=fw,
+                        scenario=sc,
+                        verdict=Verdict.FAIL,
+                        errors=[str(e)],
+                    ))
+
+        return results
+
+    def quick_smoke_test(self) -> List[TestResult]:
+        """Run a minimal smoke test: EnvProtectionScenario only."""
+        from .scenarios import EnvProtectionScenario
+        return self.run_all(scenarios=[EnvProtectionScenario()])

correctover_test/core/license.py

@@ -0,0 +1,167 @@
+"""
+License validator — enforces free tier limits (50 audits/day).
+Embedded in all Correctover Agent products.
+"""
+
+import hashlib
+import json
+import os
+import time
+from pathlib import Path
+from typing import Dict, Optional
+
+
+class LicenseValidator:
+    """Enforces usage limits for Correctover Agent products."""
+
+    FREE_LIMIT_PER_DAY = 50
+    STATE_FILE = Path.home() / ".correctover" / "license.json"
+
+    def __init__(self, product: str):
+        self.product = product
+        self.state = self._load_state()
+
+    def _load_state(self) -> Dict:
+        if self.STATE_FILE.exists():
+            try:
+                return json.loads(self.STATE_FILE.read_text())
+            except (json.JSONDecodeError, OSError):
+                pass
+        return {"products": {}, "license_key": None, "installed_at": time.time()}
+
+    def _save_state(self) -> None:
+        self.STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
+        self.STATE_FILE.write_text(json.dumps(self.state, indent=2))
+
+    def _get_product_state(self) -> Dict:
+        return self.state["products"].setdefault(self.product, {
+            "calls_today": 0,
+            "date": time.strftime("%Y-%m-%d"),
+            "total_calls": 0,
+        })
+
+    def check_license(self) -> Dict:
+        """Check if the current usage is within limits. Returns status dict."""
+        today = time.strftime("%Y-%m-%d")
+        ps = self._get_product_state()
+
+        # Reset daily counter if it's a new day
+        if ps.get("date") != today:
+            ps["calls_today"] = 0
+            ps["date"] = today
+
+        license_key = self.state.get("license_key")
+        has_license = bool(license_key)
+
+        if has_license:
+            # Pro/Enterprise — no limits
+            return {
+                "authorized": True,
+                "tier": "pro" if self._verify_license_key(license_key) else "free",
+                "calls_remaining": float("inf"),
+                "calls_today": ps["calls_today"],
+                "limit": float("inf"),
+                "license_key": license_key[:8] + "..." if license_key else None,
+            }
+
+        # Free tier
+        remaining = max(0, self.FREE_LIMIT_PER_DAY - ps["calls_today"])
+        return {
+            "authorized": remaining > 0,
+            "tier": "free",
+            "calls_remaining": remaining,
+            "calls_today": ps["calls_today"],
+            "limit": self.FREE_LIMIT_PER_DAY,
+            "license_key": None,
+        }
+
+    def record_call(self) -> Dict:
+        """Record a single API call. Returns updated status."""
+        status = self.check_license()
+        if not status["authorized"]:
+            return status
+
+        ps = self._get_product_state()
+        ps["calls_today"] += 1
+        ps["total_calls"] = ps.get("total_calls", 0) + 1
+        self._save_state()
+
+        status["calls_remaining"] = max(0, status["limit"] - ps["calls_today"])
+        status["calls_today"] = ps["calls_today"]
+        return status
+
+    def set_license_key(self, key: str) -> bool:
+        """Set and validate a license key."""
+        if self._verify_license_key(key):
+            self.state["license_key"] = key
+            self._save_state()
+            return True
+        return False
+
+    def _verify_license_key(self, key: str) -> bool:
+        """Basic license key validation (HMAC-based)."""
+        if not key or len(key) < 16:
+            return False
+
+        # Format: COV-<product>-<hash>
+        parts = key.split("-")
+        if len(parts) < 3 or parts[0] != "COV":
+            return False
+
+        # Verify the HMAC signature
+        expected_prefix = self._compute_key_prefix(parts[1])
+        return parts[2].startswith(expected_prefix)
+
+    def _compute_key_prefix(self, product_code: str) -> str:
+        secret = f"correctover-{product_code}-2026"
+        return hashlib.sha256(secret.encode()).hexdigest()[:12]
+
+    def get_upgrade_message(self) -> str:
+        """Return the appropriate upgrade CTA."""
+        status = self.check_license()
+        if status["tier"] == "free":
+            remaining = status["calls_remaining"]
+            if remaining <= 0:
+                return (
+                    f"\n🚫 Free tier limit reached ({self.FREE_LIMIT_PER_DAY} audits/day).\n"
+                    f"   Upgrade to Pro for unlimited audits: https://correctover.com/pricing\n"
+                    f"   Or set your license key: export CORRECTOVER_LICENSE_KEY=<your-key>\n"
+                )
+            return (
+                f"\n📊 Free tier: {remaining} audits remaining today.\n"
+                f"   Upgrade to Pro: https://correctover.com/pricing\n"
+            )
+        return ""
+
+    @staticmethod
+    def get_license_from_env() -> Optional[str]:
+        return os.environ.get("CORRECTOVER_LICENSE_KEY")
+
+
+# Global singleton
+_validators: Dict[str, LicenseValidator] = {}
+
+
+def get_validator(product: str = "correctover-test") -> LicenseValidator:
+    if product not in _validators:
+        _validators[product] = LicenseValidator(product)
+    return _validators[product]
+
+
+def check_and_record(product: str = "correctover-test") -> Dict:
+    """Check license, record call, return status. Raise if over limit."""
+    v = get_validator(product)
+    status = v.check_license()
+
+    if not status["authorized"]:
+        msg = v.get_upgrade_message()
+        raise LicenseExceededError(
+            f"Free tier limit ({v.FREE_LIMIT_PER_DAY}/day) exceeded. {msg}"
+        )
+
+    return v.record_call()
+
+
+class LicenseExceededError(Exception):
+    """Raised when free tier limit is exceeded."""
+    pass