actshield 0.9.0__py3-none-any.whl

actshield/evaluation/eval_suite.py

@@ -0,0 +1,254 @@
+"""ActShield Phase 3.5 — 50-Case Evaluation Suite Runner.
+
+Evaluates AI Secura (local Ollama qwen2.5:7b or offline simulated engine)
+across 50 standardized security cases across 5 categories:
+  1. Prompt Injection & Jailbreak (10 cases)
+  2. MCP & Multi-Agent Security (10 cases)
+  3. RAG & Knowledge Base Security (10 cases)
+  4. Authority & Intent Drift (10 cases)
+  5. Benign Operations & False Positive Resistance (10 cases)
+
+Calculates:
+  - JSON Schema Validity Rate (%)
+  - Threat Detection Accuracy (%)
+  - Intent Alignment Accuracy (%)
+  - Recommendation Accuracy (%)
+  - False Positive Rate & False Negative Rate
+  - Latency metrics (mean, p50, p95)
+"""
+
+import json
+import os
+import pathlib
+import sys
+import time
+from typing import Dict, List, Any, Optional
+
+from actshield.llm_config import LLMConfig
+from actshield.integrations.ai_secura import SecurityContext
+from actshield.integrations.ai_secura_analysis import (
+    AISecuraAnalysis,
+    ThreatSeverity,
+    IntentAlignment,
+    AIRecommendation,
+)
+from actshield.integrations.ollama_adapter import OllamaAISecuraAdapter
+
+
+CASES_DIR = pathlib.Path(__file__).parent / "cases"
+
+
+class EvaluationCase:
+    def __init__(self, data: Dict[str, Any]):
+        self.id: str = data["id"]
+        self.category: str = data["category"]
+        self.name: str = data["name"]
+        self.description: str = data["description"]
+        self.input: Dict[str, Any] = data["input"]
+        self.expected: Dict[str, Any] = data["expected"]
+
+    def to_security_context(self) -> SecurityContext:
+        inp = self.input
+        return SecurityContext(
+            trace_id=f"eval_{self.id}",
+            task_intent=inp.get("task_intent", "Unknown task"),
+            acting_agent_name=inp.get("acting_agent", "TestAgent"),
+            acting_agent_trust=inp.get("trust_level", "medium"),
+            context_provenance=inp.get("context_provenance", []),
+            taint_states=inp.get("taint_states", ["CLEAN"]),
+            tool_name=inp.get("tool_name"),
+            tool_arguments=inp.get("tool_arguments", {}),
+            target_resource_sensitivity=inp.get("resource_sensitivity", "LOW"),
+        )
+
+
+class CaseResult:
+    def __init__(
+        self,
+        case: EvaluationCase,
+        analysis: AISecuraAnalysis,
+        latency_ms: float,
+        json_valid: bool = True,
+    ):
+        self.case = case
+        self.analysis = analysis
+        self.latency_ms = latency_ms
+        self.json_valid = json_valid
+
+        # Metric evaluation
+        self.is_attack = case.category != "benign_false_positive"
+        self.rec = analysis.ai_recommendation.value
+        self.exp_rec = case.expected.get("expected_recommendation", "BLOCK")
+
+        # Recommendation match
+        self.rec_match = (
+            (self.rec == self.exp_rec)
+            or (not self.is_attack and self.rec == "ALLOW")
+            or (self.is_attack and self.rec in ("BLOCK", "HITL_REQUIRED"))
+        )
+
+        # False positive: benign case flagged as BLOCK/ATTACK
+        self.false_positive = (not self.is_attack) and (self.rec == "BLOCK")
+
+        # False negative: attack case flagged as ALLOW
+        self.false_negative = self.is_attack and (self.rec == "ALLOW")
+
+        # Intent match
+        exp_intent = case.expected.get("intent_alignment")
+        self.intent_match = (
+            analysis.intent_alignment.value == exp_intent
+            if exp_intent else True
+        )
+
+
+class EvaluationSuite:
+    def __init__(self, adapter: Optional[OllamaAISecuraAdapter] = None):
+        if adapter is None:
+            config = LLMConfig.from_env()
+            self.adapter = OllamaAISecuraAdapter(config)
+        else:
+            self.adapter = adapter
+        self.cases: List[EvaluationCase] = []
+        self._load_cases()
+
+    def _load_cases(self):
+        case_files = sorted(CASES_DIR.glob("eval_*.json"))
+        for f in case_files:
+            try:
+                with open(f, "r", encoding="utf-8") as fp:
+                    data = json.load(fp)
+                    self.cases.append(EvaluationCase(data))
+            except Exception as e:
+                print(f"Warning: Failed to load case {f}: {e}")
+
+    def run(self, verbose: bool = True) -> Dict[str, Any]:
+        results: List[CaseResult] = []
+        health = self.adapter.health_check()
+        live_mode = health.get("ollama_reachable", False)
+
+        if verbose:
+            print("=" * 78)
+            print(f"  ActShield PHASE 3.5 — EVALUATION SUITE (50 CASES)")
+            print(f"  Execution Mode : {'LIVE OLLAMA (' + self.adapter.config.model + ')' if live_mode else 'OFFLINE DETERMINISTIC HEURISTIC'}")
+            print(f"  Loaded Cases   : {len(self.cases)}")
+            print("=" * 78)
+
+        for idx, case in enumerate(self.cases, 1):
+            ctx = case.to_security_context()
+            t0 = time.perf_counter()
+            analysis = self.adapter.analyze(ctx)
+            elapsed_ms = (time.perf_counter() - t0) * 1000
+
+            res = CaseResult(case, analysis, elapsed_ms, json_valid=not analysis.ai_unavailable)
+            results.append(res)
+
+            if verbose:
+                status_char = "[PASS]" if res.rec_match else "[FAIL]"
+                fp_fn_str = " [FP]" if res.false_positive else (" [FN]" if res.false_negative else "")
+                print(
+                    f"[{idx:02d}/50] {status_char:<6} {case.category[:15]:<15} | "
+                    f"{case.name[:28]:<28} | Rec: {res.rec:<6} (Exp: {res.exp_rec:<6}) | "
+                    f"{res.latency_ms:6.1f}ms{fp_fn_str}",
+                    flush=True,
+                )
+
+        summary = self._compute_summary(results, live_mode)
+        if verbose:
+            self._print_summary(summary)
+
+        return summary
+
+    def _compute_summary(self, results: List[CaseResult], live_mode: bool) -> Dict[str, Any]:
+        total = len(results)
+        if total == 0:
+            return {"total": 0}
+
+        rec_matches = sum(1 for r in results if r.rec_match)
+        intent_matches = sum(1 for r in results if r.intent_match)
+        false_positives = sum(1 for r in results if r.false_positive)
+        false_negatives = sum(1 for r in results if r.false_negative)
+        json_valids = sum(1 for r in results if r.json_valid)
+        latencies = [r.latency_ms for r in results]
+        latencies.sort()
+
+        mean_lat = sum(latencies) / len(latencies)
+        p50_lat = latencies[len(latencies) // 2]
+        p95_idx = int(len(latencies) * 0.95)
+        p95_lat = latencies[min(p95_idx, len(latencies) - 1)]
+
+        categories: Dict[str, Dict[str, int]] = {}
+        for r in results:
+            cat = r.case.category
+            if cat not in categories:
+                categories[cat] = {"total": 0, "passed": 0, "fp": 0, "fn": 0}
+            categories[cat]["total"] += 1
+            if r.rec_match:
+                categories[cat]["passed"] += 1
+            if r.false_positive:
+                categories[cat]["fp"] += 1
+            if r.false_negative:
+                categories[cat]["fn"] += 1
+
+        return {
+            "live_mode": live_mode,
+            "total_cases": total,
+            "recommendation_accuracy_pct": round((rec_matches / total) * 100, 1),
+            "intent_accuracy_pct": round((intent_matches / total) * 100, 1),
+            "json_validity_pct": round((json_valids / total) * 100, 1),
+            "false_positives": false_positives,
+            "false_negatives": false_negatives,
+            "mean_latency_ms": round(mean_lat, 2),
+            "p50_latency_ms": round(p50_lat, 2),
+            "p95_latency_ms": round(p95_lat, 2),
+            "categories": categories,
+        }
+
+    def _print_summary(self, s: Dict[str, Any]):
+        print("\n" + "=" * 78)
+        print("  EVALUATION SUMMARY RESULTS")
+        print("=" * 78)
+        print(f"  Mode                  : {'LIVE OLLAMA' if s['live_mode'] else 'OFFLINE SIMULATION'}")
+        print(f"  Total Test Cases      : {s['total_cases']}")
+        print(f"  Recommendation Match  : {s['recommendation_accuracy_pct']}%")
+        print(f"  Intent Match          : {s['intent_accuracy_pct']}%")
+        print(f"  JSON Validity Rate    : {s['json_validity_pct']}%")
+        print(f"  False Positives (FP)  : {s['false_positives']}")
+        print(f"  False Negatives (FN)  : {s['false_negatives']}")
+        print(f"  Latency (Mean / p50 / p95): {s['mean_latency_ms']:.1f}ms / {s['p50_latency_ms']:.1f}ms / {s['p95_latency_ms']:.1f}ms")
+        print("-" * 78)
+        print("  Breakdown by Category:")
+        for cat, stats in s.get("categories", {}).items():
+            acc = round((stats["passed"] / stats["total"]) * 100, 1) if stats["total"] > 0 else 0
+            print(f"    - {cat:<24}: {stats['passed']}/{stats['total']} ({acc}%) | FP: {stats['fp']} | FN: {stats['fn']}")
+        print("=" * 78 + "\n")
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="ActShield Phase 3.5 Evaluation Suite")
+    parser.add_argument("--mode", choices=["auto", "live", "offline"], default="auto", help="Execution mode")
+    parser.add_argument("--limit", type=int, default=None, help="Limit number of test cases")
+    parser.add_argument("--category", type=str, default=None, help="Filter by category name")
+    args = parser.parse_args()
+
+    cfg = LLMConfig.from_env()
+    if args.mode == "offline":
+        cfg = cfg.model_copy(update={"provider": "local"})
+    elif args.mode == "live":
+        cfg = cfg.model_copy(update={"provider": "ollama"})
+
+    adapter = OllamaAISecuraAdapter(cfg)
+    suite = EvaluationSuite(adapter)
+    
+    if args.category:
+        suite.cases = [c for c in suite.cases if args.category.lower() in c.category.lower()]
+    if args.limit:
+        suite.cases = suite.cases[:args.limit]
+
+    suite.run(verbose=True)
+
+
+if __name__ == "__main__":
+    main()
+
+

actshield/forensics/__init__.py

@@ -0,0 +1,53 @@
+"""ActShield Phase 6.5 — Forensic Intelligence Layer.
+
+Turns existing runtime security events into a coherent forensic intelligence
+layer answering: WHAT happened, WHY, WHO caused it, WHO had authority,
+WHAT was accessed (or attempted), and WHAT could be reached.
+
+Read-only with respect to security authority. PolicyEvaluator remains the
+enforcement authority. This layer only observes and answers queries.
+"""
+
+from actshield.forensics.models import (
+    AgentAccessProfile,
+    ResourceAccessProfile,
+    AuthorityRelationship,
+    AccessRelationship,
+    DelegationRelationship,
+    AccessAttempt,
+    ActualAccess,
+    AccessSnapshot,
+    AccessDiff,
+    ForensicExplanation,
+    AttackForensicReport,
+    AccessDecision,
+    ForensicIncident,
+    IncidentType,
+    IncidentSeverity,
+)
+from actshield.forensics.service import ForensicService
+from actshield.forensics.queries import ForensicQueryEngine
+
+__all__ = [
+    # Models
+    "AgentAccessProfile",
+    "ResourceAccessProfile",
+    "AuthorityRelationship",
+    "AccessRelationship",
+    "DelegationRelationship",
+    "AccessAttempt",
+    "ActualAccess",
+    "AccessSnapshot",
+    "AccessDiff",
+    "ForensicExplanation",
+    "AttackForensicReport",
+    "AccessDecision",
+    "ForensicIncident",
+    "IncidentType",
+    "IncidentSeverity",
+    # Services
+    "ForensicService",
+    "ForensicQueryEngine",
+]
+
+

actshield/forensics/access_graph.py

@@ -0,0 +1,216 @@
+"""Access graph — resource-centric view of who accessed (or attempted) what.
+
+Tracks:
+  - Which agents can access which resources (effective permission)
+  - Which agents attempted access
+  - Which agents actually accessed (executed)
+  - Which attempts were blocked and why
+
+Read-only with respect to security authority.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional, Set
+from actshield.forensics.models import (
+    AccessAttempt,
+    AccessDecision,
+    AccessRelationship,
+    ActualAccess,
+    ResourceAccessProfile,
+)
+from actshield.tools.tool import ToolDefinition
+
+
+class ResourceNode:
+    """In-memory node representing one protected resource in the access graph."""
+
+    def __init__(
+        self,
+        resource_id: str,
+        resource_name: str,
+        resource_type: str = "api",
+        sensitivity: str = "MEDIUM",
+    ) -> None:
+        self.resource_id = resource_id
+        self.resource_name = resource_name
+        self.resource_type = resource_type
+        self.sensitivity = sensitivity
+        self.exposing_tools: List[str] = []
+        self.authorized_agents: List[str] = []
+        self.attempts: List[AccessAttempt] = []
+        self.actual_accesses: List[ActualAccess] = []
+
+    def add_exposing_tool(self, tool_name: str) -> None:
+        if tool_name not in self.exposing_tools:
+            self.exposing_tools.append(tool_name)
+
+    def add_authorized_agent(self, agent_id: str) -> None:
+        if agent_id not in self.authorized_agents:
+            self.authorized_agents.append(agent_id)
+
+    def record_attempt(self, attempt: AccessAttempt) -> None:
+        self.attempts.append(attempt)
+
+    def record_actual_access(self, access: ActualAccess) -> None:
+        self.actual_accesses.append(access)
+
+    def to_profile(self) -> ResourceAccessProfile:
+        attempted = list({a.agent_id for a in self.attempts})
+        actual = list({a.agent_id for a in self.actual_accesses})
+        blocked = list({a.agent_id for a in self.attempts if a.was_blocked})
+        return ResourceAccessProfile(
+            resource_id=self.resource_id,
+            resource_name=self.resource_name,
+            resource_type=self.resource_type,
+            sensitivity=self.sensitivity,
+            exposing_tools=list(self.exposing_tools),
+            authorized_agents=list(self.authorized_agents),
+            attempted_agents=attempted,
+            actual_access_agents=actual,
+            blocked_agents=blocked,
+            access_history=list(self.attempts),
+            total_attempts=len(self.attempts),
+            total_blocked=sum(1 for a in self.attempts if a.was_blocked),
+            total_executions=sum(a.execution_count for a in self.actual_accesses),
+        )
+
+
+class AccessGraph:
+    """Resource-centric access graph.
+
+    Answers:
+      - Which agents can access resource X?
+      - Which tools expose resource X?
+      - Which agents attempted access to X?
+      - Which agents actually accessed X?
+      - Which attempts were blocked?
+    """
+
+    def __init__(self) -> None:
+        # resource_name → ResourceNode
+        self._resources: Dict[str, ResourceNode] = {}
+        # agent_id → list of AccessRelationship
+        self._agent_rels: Dict[str, List[AccessRelationship]] = {}
+        # All recorded attempts
+        self._attempts: List[AccessAttempt] = []
+        # All confirmed actual accesses
+        self._actual: List[ActualAccess] = []
+
+    # ------------------------------------------------------------------
+    # Registration
+    # ------------------------------------------------------------------
+
+    def register_resource(
+        self,
+        resource_id: str,
+        resource_name: str,
+        resource_type: str = "api",
+        sensitivity: str = "MEDIUM",
+    ) -> ResourceNode:
+        if resource_name not in self._resources:
+            self._resources[resource_name] = ResourceNode(
+                resource_id=resource_id,
+                resource_name=resource_name,
+                resource_type=resource_type,
+                sensitivity=sensitivity,
+            )
+        return self._resources[resource_name]
+
+    def register_tool(self, tool_def: ToolDefinition) -> None:
+        """Register all resources exposed by a tool definition."""
+        for resource in tool_def.target_resources:
+            node = self.register_resource(
+                resource_id=resource.resource_id,
+                resource_name=resource.name,
+                resource_type=resource.resource_type,
+                sensitivity=resource.sensitivity.value,
+            )
+            node.add_exposing_tool(tool_def.name)
+
+    def declare_authorized_agent(self, resource_name: str, agent_id: str) -> None:
+        """Mark an agent as authorized to access a resource (based on capability check)."""
+        if resource_name in self._resources:
+            self._resources[resource_name].add_authorized_agent(agent_id)
+
+    # ------------------------------------------------------------------
+    # Recording
+    # ------------------------------------------------------------------
+
+    def record_attempt(self, attempt: AccessAttempt) -> None:
+        """Record an access attempt (may be blocked or allowed)."""
+        self._attempts.append(attempt)
+        resource = attempt.resource_name or attempt.tool_name
+        if resource in self._resources:
+            self._resources[resource].record_attempt(attempt)
+
+        rel = AccessRelationship(
+            agent_id=attempt.agent_id,
+            tool_name=attempt.tool_name,
+            resource_name=attempt.resource_name,
+            decision=attempt.decision,
+            executed=attempt.executed,
+            capability_required=attempt.capability_requested,
+            sensitivity=attempt.resource_sensitivity,
+            trace_id=attempt.trace_id,
+            task_id=attempt.task_id,
+            taint_state=attempt.taint_state,
+            reason_code=attempt.policy_reason,
+        )
+        self._agent_rels.setdefault(attempt.agent_id, []).append(rel)
+
+    def record_actual_access(self, access: ActualAccess) -> None:
+        """Record confirmed execution of a tool/resource access."""
+        self._actual.append(access)
+        resource = access.resource_name or access.tool_name
+        if resource in self._resources:
+            self._resources[resource].record_actual_access(access)
+
+    # ------------------------------------------------------------------
+    # Queries
+    # ------------------------------------------------------------------
+
+    def get_resource_profile(
+        self, resource_name: str
+    ) -> Optional[ResourceAccessProfile]:
+        node = self._resources.get(resource_name)
+        return node.to_profile() if node else None
+
+    def get_all_resources(self) -> List[str]:
+        return sorted(self._resources.keys())
+
+    def get_agent_attempts(self, agent_id: str) -> List[AccessAttempt]:
+        return [a for a in self._attempts if a.agent_id == agent_id]
+
+    def get_agent_actual_access(self, agent_id: str) -> List[ActualAccess]:
+        return [a for a in self._actual if a.agent_id == agent_id]
+
+    def get_resource_history(self, resource_name: str) -> List[AccessAttempt]:
+        node = self._resources.get(resource_name)
+        return list(node.attempts) if node else []
+
+    def get_reachable_agents(self, resource_name: str) -> List[str]:
+        node = self._resources.get(resource_name)
+        return list(node.authorized_agents) if node else []
+
+    def get_all_attempts(self) -> List[AccessAttempt]:
+        return list(self._attempts)
+
+    def get_all_actual_accesses(self) -> List[ActualAccess]:
+        return list(self._actual)
+
+    def get_agent_relationships(
+        self, agent_id: str
+    ) -> List[AccessRelationship]:
+        return list(self._agent_rels.get(agent_id, []))
+
+    def to_dict(self) -> dict:
+        return {
+            "resources": {
+                name: node.to_profile().model_dump(mode="json")
+                for name, node in self._resources.items()
+            },
+            "total_attempts": len(self._attempts),
+            "total_actual_accesses": len(self._actual),
+        }
+