icdev 0.0.3__py3-none-any.whl

tools/audit_engine/engine.py

@@ -0,0 +1,614 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+# Controlled by: Department of Defense
+# CUI Category: CTI
+# Distribution: D
+# POC: ICDEV System Administrator
+"""Audit Engine — Core orchestrator for compliance report card generation.
+
+Runs the 5-stage audit pipeline: SCAN → EVALUATE → SCORE → ADVISE → REPORT.
+Produces per-regime scores with side-by-side gap analysis across all supported
+compliance frameworks.
+
+Usage:
+    # Audit ICDEV itself
+    python tools/audit_engine/engine.py --target . --project-id sparkpilot --json
+
+    # Audit a child app
+    python tools/audit_engine/engine.py --target /path/to/child --type child_app --json
+
+    # Audit BYOS directory
+    python tools/audit_engine/engine.py --target /path/to/code --type byos --json
+
+    # Audit with specific regimes
+    python tools/audit_engine/engine.py --target . --regimes nist_800_53 fedramp_moderate --json
+
+    # Compare two regimes side-by-side
+    python tools/audit_engine/engine.py --target . --regimes nist_800_171 nist_800_53 --compare --json
+
+Databases:
+    - data/icdev.db: audit_engine_results, audit_engine_scores
+"""
+
+import json
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+# Safe import of storage layer
+try:
+    from tools.db.storage import get_connection
+except ImportError:
+    get_connection = None
+
+from tools.audit_engine.regime_loader import (
+    load_regime, list_regimes, get_regime_controls, get_regime_metadata,
+)
+from tools.audit_engine.scanner import scan_target
+
+
+# NIST control-family to evidence-source mapping
+CONTROL_FAMILY_EVIDENCE_MAP = {
+    "AC": ["sast", "code_patterns", "stig"],       # Access Control
+    "AU": ["code_patterns", "stig"],                 # Audit and Accountability
+    "AT": ["code_patterns"],                         # Awareness and Training
+    "CA": ["code_patterns", "stig"],                 # Security Assessment
+    "CM": ["file_permissions", "stig", "sbom"],      # Configuration Management
+    "CP": ["code_patterns"],                         # Contingency Planning
+    "IA": ["secrets", "code_patterns", "stig"],      # Identification & Auth
+    "IR": ["code_patterns"],                         # Incident Response
+    "MA": ["dependencies"],                          # Maintenance
+    "MP": ["file_permissions"],                      # Media Protection
+    "PE": [],                                        # Physical (N/A for code)
+    "PL": ["code_patterns"],                         # Planning
+    "PS": [],                                        # Personnel (N/A for code)
+    "PM": ["code_patterns"],                         # Program Management
+    "RA": ["sast", "dependencies"],                  # Risk Assessment
+    "SA": ["sast", "dependencies", "sbom", "sbd"],   # System & Services Acq.
+    "SC": ["code_patterns", "secrets", "stig"],      # System & Comm Protection
+    "SI": ["sast", "code_patterns", "stig"],         # System & Info Integrity
+    "SR": ["dependencies", "sbom"],                  # Supply Chain Risk
+}
+
+# NIST 800-171 chapter → NIST 800-53 family mapping
+_NIST_171_FAMILY_MAP = {
+    "3.1": "AC",   # Access Control
+    "3.2": "AT",   # Awareness and Training
+    "3.3": "AU",   # Audit and Accountability
+    "3.4": "CM",   # Configuration Management
+    "3.5": "IA",   # Identification and Authentication
+    "3.6": "IR",   # Incident Response
+    "3.7": "MA",   # Maintenance
+    "3.8": "MP",   # Media Protection
+    "3.9": "PS",   # Personnel Security
+    "3.10": "PE",  # Physical Protection
+    "3.11": "RA",  # Risk Assessment
+    "3.12": "CA",  # Security Assessment
+    "3.13": "SC",  # System and Communications Protection
+    "3.14": "SI",  # System and Information Integrity
+}
+
+
+def run_audit(
+    target_path: str,
+    target_type: str = "auto",
+    project_id: Optional[str] = None,
+    regime_ids: Optional[List[str]] = None,
+    skip_tools: Optional[List[str]] = None,
+    store_results: bool = True,
+) -> Dict:
+    """Execute the full audit pipeline.
+
+    Args:
+        target_path: Path to scan.
+        target_type: Target type (icdev, module, child_app, byos, auto).
+        project_id: ICDEV project ID for tool integration.
+        regime_ids: List of regime IDs to evaluate. None = all available.
+        skip_tools: Scanner tools to skip.
+        store_results: Whether to persist results to DB.
+
+    Returns:
+        Complete audit result with per-regime scores and evidence.
+    """
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Stage 1: SCAN
+    scan_results = scan_target(target_path, target_type, project_id, skip_tools)
+
+    # Determine regimes to evaluate
+    if not regime_ids:
+        available = list_regimes()
+        regime_ids = [r["id"] for r in available]
+
+    if not regime_ids:
+        return {
+            "success": False,
+            "error": "No regime definitions found. Add YAML files to args/audit_regimes/",
+            "scan_results": scan_results,
+        }
+
+    # Stage 2 & 3: EVALUATE + SCORE per regime
+    regime_scores = {}
+    for regime_id in regime_ids:
+        regime = load_regime(regime_id)
+        if not regime:
+            regime_scores[regime_id] = {
+                "error": f"Regime '{regime_id}' not found",
+                "score": 0,
+            }
+            continue
+        regime_scores[regime_id] = _evaluate_regime(regime, scan_results)
+
+    # Build result
+    result = {
+        "success": True,
+        "target_path": scan_results["target_path"],
+        "target_type": scan_results["target_type"],
+        "project_id": project_id,
+        "timestamp": timestamp,
+        "scan_summary": scan_results["summary"],
+        "regime_scores": regime_scores,
+        "regimes_evaluated": len(regime_scores),
+    }
+
+    # Stage 5: REPORT (summary)
+    result["report_card"] = _build_report_card(regime_scores)
+
+    # Persist results
+    if store_results and get_connection:
+        try:
+            _store_results(result)
+        except Exception as e:
+            result["storage_warning"] = f"Failed to store results: {e}"
+
+    return result
+
+
+def _evaluate_regime(regime: Dict, scan_results: Dict) -> Dict:
+    """Evaluate scan evidence against a regime's controls.
+
+    Returns per-category scores and overall regime score.
+    """
+    meta = regime.get("regime", {})
+    categories = regime.get("categories", [])
+    evidence = scan_results.get("evidence", {})
+
+    category_scores = []
+    all_control_results = []
+
+    for category in categories:
+        cat_id = category.get("id", "")
+        cat_name = category.get("name", "")
+        controls = category.get("controls", [])
+
+        cat_passed = 0
+        cat_partial = 0
+        cat_failed = 0
+        cat_na = 0
+        cat_results = []
+
+        for ctrl in controls:
+            ctrl_id = ctrl.get("id", "")
+            ctrl_result = _evaluate_control(ctrl, cat_id, evidence)
+            cat_results.append(ctrl_result)
+
+            status = ctrl_result["status"]
+            if status == "pass":
+                cat_passed += 1
+            elif status == "partial":
+                cat_partial += 1
+            elif status == "not_applicable":
+                cat_na += 1
+            else:
+                cat_failed += 1
+
+        total_applicable = cat_passed + cat_partial + cat_failed
+        if total_applicable > 0:
+            cat_score = ((cat_passed + 0.5 * cat_partial) / total_applicable) * 100
+        else:
+            cat_score = 100.0  # All N/A = full score
+
+        category_scores.append({
+            "category_id": cat_id,
+            "category_name": cat_name,
+            "score": round(cat_score, 1),
+            "passed": cat_passed,
+            "partial": cat_partial,
+            "failed": cat_failed,
+            "not_applicable": cat_na,
+            "total": len(controls),
+            "controls": cat_results,
+        })
+        all_control_results.extend(cat_results)
+
+    # Overall regime score (weighted by applicable controls)
+    total_applicable = sum(c["passed"] + c["partial"] + c["failed"] for c in category_scores)
+    total_passed = sum(c["passed"] for c in category_scores)
+    total_partial = sum(c["partial"] for c in category_scores)
+
+    if total_applicable > 0:
+        overall_score = ((total_passed + 0.5 * total_partial) / total_applicable) * 100
+    else:
+        overall_score = 100.0
+
+    return {
+        "regime_id": meta.get("id", ""),
+        "regime_name": meta.get("name", ""),
+        "version": meta.get("version", ""),
+        "overall_score": round(overall_score, 1),
+        "grade": _score_to_grade(overall_score),
+        "categories": category_scores,
+        "summary": {
+            "total_controls": sum(c["total"] for c in category_scores),
+            "passed": total_passed,
+            "partial": total_partial,
+            "failed": sum(c["failed"] for c in category_scores),
+            "not_applicable": sum(c["not_applicable"] for c in category_scores),
+        },
+    }
+
+
+def _evaluate_control(ctrl: Dict, category_id: str, evidence: Dict) -> Dict:
+    """Evaluate a single control against collected evidence.
+
+    Maps control to evidence sources using three methods:
+    1. Explicit evidence_sources in the regime YAML (tool + code_pattern types)
+    2. NIST control-family heuristic mapping (enhanced for cross-framework IDs)
+    3. Crosswalk bridge — inherit evidence from mapped NIST 800-53 controls
+    """
+    ctrl_id = ctrl.get("id", "")
+    title = ctrl.get("title", "")
+    severity = ctrl.get("severity", "medium")
+
+    # Collect relevant evidence
+    relevant_findings = []
+
+    # Method 1: Explicit evidence sources from regime YAML
+    for source in ctrl.get("evidence_sources", []):
+        source_type = source.get("type", "")
+        if source_type == "tool":
+            tool_name = source.get("tool", "")
+            tool_evidence = evidence.get(tool_name, {})
+            for finding in tool_evidence.get("findings", []):
+                hint = finding.get("control_hint", "")
+                if hint == ctrl_id or hint.startswith(category_id):
+                    relevant_findings.append(finding)
+        elif source_type == "code_pattern":
+            # Search all evidence findings for regex pattern matches
+            pattern = source.get("pattern", "")
+            if pattern:
+                _collect_pattern_evidence(pattern, evidence, relevant_findings)
+
+    # Method 2: Control-family heuristic mapping (enhanced for cross-framework)
+    if not relevant_findings:
+        family = _resolve_nist_family(ctrl_id, category_id)
+        if family:
+            evidence_sources = CONTROL_FAMILY_EVIDENCE_MAP.get(family, [])
+            for source_name in evidence_sources:
+                source_evidence = evidence.get(source_name, {})
+                for finding in source_evidence.get("findings", []):
+                    hint = finding.get("control_hint", "")
+                    if hint == ctrl_id or hint.startswith(family):
+                        relevant_findings.append(finding)
+
+    # Method 3: Crosswalk bridge — inherit evidence from mapped NIST 800-53 controls
+    if not relevant_findings:
+        for xwalk in ctrl.get("crosswalk", []):
+            if xwalk.get("framework") == "nist_800_53":
+                nist_ctrl = xwalk.get("control", "")
+                # Extract family: "SA-11(1)" → "SA", "AC-2" → "AC"
+                nist_base = nist_ctrl.split("(")[0] if "(" in nist_ctrl else nist_ctrl
+                nist_family = nist_base.split("-")[0] if "-" in nist_base else ""
+                if nist_family and nist_family in CONTROL_FAMILY_EVIDENCE_MAP:
+                    for source_name in CONTROL_FAMILY_EVIDENCE_MAP[nist_family]:
+                        source_evidence = evidence.get(source_name, {})
+                        for finding in source_evidence.get("findings", []):
+                            hint = finding.get("control_hint", "")
+                            if hint.startswith(nist_family) and finding not in relevant_findings:
+                                relevant_findings.append(finding)
+                    if relevant_findings:
+                        break  # Found evidence via crosswalk, stop
+
+    # Determine control status from findings
+    if not relevant_findings:
+        status = "not_assessed"
+    else:
+        pass_count = sum(1 for f in relevant_findings if f.get("status") == "pass")
+        fail_count = sum(1 for f in relevant_findings if f.get("status") == "fail")
+        na_count = sum(1 for f in relevant_findings if f.get("status") == "not_applicable")
+
+        if na_count == len(relevant_findings):
+            status = "not_applicable"
+        elif fail_count == 0 and pass_count > 0:
+            status = "pass"
+        elif pass_count > 0 and fail_count > 0:
+            status = "partial"
+        else:
+            status = "fail"
+
+    return {
+        "control_id": ctrl_id,
+        "title": title,
+        "severity": severity,
+        "status": status,
+        "evidence_count": len(relevant_findings),
+        "findings": relevant_findings[:5],  # Cap at 5 for report size
+        "crosswalk": ctrl.get("crosswalk", []),
+    }
+
+
+def _resolve_nist_family(ctrl_id: str, category_id: str) -> Optional[str]:
+    """Resolve any control ID format to its NIST 800-53 family prefix.
+
+    Handles: NIST 800-53 (AC-2), CMMC (AC.L2-3.1.1), SBD (SBD-MS-01),
+    800-171 (3.1.1), CSSP (CSSP-S-01), IEEE (IVV-PL-01), DES (DES-MOD-01).
+    """
+    # NIST 800-53 direct: AC-2, AU-3, SI-10
+    m = re.match(r"^([A-Z]{2})-\d+", ctrl_id)
+    if m and m.group(1) in CONTROL_FAMILY_EVIDENCE_MAP:
+        return m.group(1)
+
+    # CMMC: AC.L2-3.1.1 → AC
+    m = re.match(r"^([A-Z]{2})\.L\d+", ctrl_id)
+    if m and m.group(1) in CONTROL_FAMILY_EVIDENCE_MAP:
+        return m.group(1)
+
+    # NIST 800-171: 3.1.1 → look up chapter
+    m = re.match(r"^(3\.\d+)\.\d+", ctrl_id)
+    if m:
+        return _NIST_171_FAMILY_MAP.get(m.group(1))
+
+    # Category-based: if category_id is a NIST family (e.g., "AC", "SI")
+    if category_id in CONTROL_FAMILY_EVIDENCE_MAP:
+        return category_id
+
+    # 800-171 category: "3.1" → AC, "3.5" → IA, etc.
+    if category_id in _NIST_171_FAMILY_MAP:
+        return _NIST_171_FAMILY_MAP[category_id]
+
+    # Fallback: original split behavior, only if result is a known family
+    family = ctrl_id.split("-")[0] if "-" in ctrl_id else category_id
+    return family if family in CONTROL_FAMILY_EVIDENCE_MAP else None
+
+
+def _collect_pattern_evidence(pattern: str, evidence: Dict, findings_out: list) -> None:
+    """Search all evidence findings for matches against a code_pattern regex.
+
+    Patterns from regime YAML (e.g., "auth|login|session") are searched against
+    finding titles, descriptions, and file paths across all scanner outputs.
+    """
+    try:
+        regex = re.compile(pattern, re.IGNORECASE)
+    except re.error:
+        return
+
+    for _tool_name, tool_ev in evidence.items():
+        if not isinstance(tool_ev, dict):
+            continue
+        for finding in tool_ev.get("findings", []):
+            search_text = "{} {} {}".format(
+                finding.get("title", ""),
+                finding.get("description", ""),
+                finding.get("file_path", ""),
+            )
+            if regex.search(search_text) and finding not in findings_out:
+                findings_out.append(finding)
+
+
+def _score_to_grade(score: float) -> str:
+    """Convert numeric score to letter grade."""
+    if score >= 95:
+        return "A+"
+    elif score >= 90:
+        return "A"
+    elif score >= 85:
+        return "A-"
+    elif score >= 80:
+        return "B+"
+    elif score >= 75:
+        return "B"
+    elif score >= 70:
+        return "B-"
+    elif score >= 65:
+        return "C+"
+    elif score >= 60:
+        return "C"
+    elif score >= 55:
+        return "C-"
+    elif score >= 50:
+        return "D"
+    else:
+        return "F"
+
+
+def _build_report_card(regime_scores: Dict) -> Dict:
+    """Build a summary report card across all regimes."""
+    cards = []
+    for regime_id, score_data in regime_scores.items():
+        if "error" in score_data:
+            cards.append({
+                "regime_id": regime_id,
+                "error": score_data["error"],
+            })
+            continue
+        cards.append({
+            "regime_id": score_data.get("regime_id", regime_id),
+            "regime_name": score_data.get("regime_name", regime_id),
+            "score": score_data.get("overall_score", 0),
+            "grade": score_data.get("grade", "F"),
+            "passed": score_data.get("summary", {}).get("passed", 0),
+            "failed": score_data.get("summary", {}).get("failed", 0),
+            "total": score_data.get("summary", {}).get("total_controls", 0),
+        })
+
+    return {
+        "cards": cards,
+        "best_regime": max(cards, key=lambda c: c.get("score", 0)) if cards else None,
+        "worst_regime": min(cards, key=lambda c: c.get("score", 0)) if cards else None,
+    }
+
+
+def _store_results(result: Dict) -> None:
+    """Persist audit results to database (append-only)."""
+    if not get_connection:
+        return
+
+    with get_connection() as conn:
+        # Ensure tables exist
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS audit_engine_results (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                target_path TEXT NOT NULL,
+                target_type TEXT NOT NULL,
+                project_id TEXT,
+                timestamp TEXT NOT NULL,
+                regimes_evaluated INTEGER,
+                scan_summary TEXT,
+                regime_scores TEXT,
+                report_card TEXT,
+                classification TEXT DEFAULT 'CUI'
+            )
+        """)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS audit_engine_scores (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                audit_id INTEGER,
+                regime_id TEXT NOT NULL,
+                regime_name TEXT,
+                overall_score REAL,
+                grade TEXT,
+                passed INTEGER,
+                partial INTEGER,
+                failed INTEGER,
+                not_applicable INTEGER,
+                total_controls INTEGER,
+                timestamp TEXT NOT NULL,
+                classification TEXT DEFAULT 'CUI'
+            )
+        """)
+
+        # Insert main result
+        cursor = conn.execute(
+            """INSERT INTO audit_engine_results
+               (target_path, target_type, project_id, timestamp,
+                regimes_evaluated, scan_summary, regime_scores, report_card, classification)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                result["target_path"],
+                result["target_type"],
+                result.get("project_id"),
+                result["timestamp"],
+                result["regimes_evaluated"],
+                json.dumps(result["scan_summary"]),
+                json.dumps(result["regime_scores"]),
+                json.dumps(result["report_card"]),
+                "CUI",
+            ),
+        )
+        audit_id = cursor.lastrowid
+
+        # Insert per-regime scores
+        for regime_id, score_data in result.get("regime_scores", {}).items():
+            if "error" in score_data:
+                continue
+            summary = score_data.get("summary", {})
+            conn.execute(
+                """INSERT INTO audit_engine_scores
+                   (audit_id, regime_id, regime_name, overall_score, grade,
+                    passed, partial, failed, not_applicable, total_controls,
+                    timestamp, classification)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    audit_id,
+                    score_data.get("regime_id", regime_id),
+                    score_data.get("regime_name", ""),
+                    score_data.get("overall_score", 0),
+                    score_data.get("grade", "F"),
+                    summary.get("passed", 0),
+                    summary.get("partial", 0),
+                    summary.get("failed", 0),
+                    summary.get("not_applicable", 0),
+                    summary.get("total_controls", 0),
+                    result["timestamp"],
+                    "CUI",
+                ),
+            )
+
+        conn.commit()
+
+        # Log audit event
+        try:
+            conn.execute(
+                """INSERT INTO audit_trail
+                   (project_id, event_type, actor, action, details,
+                    affected_files, classification)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    result.get("project_id", "audit-engine"),
+                    "audit_engine_scan",
+                    "icdev-audit-engine",
+                    f"Audit scan completed: {result['regimes_evaluated']} regimes",
+                    json.dumps(result.get("report_card", {})),
+                    json.dumps([result["target_path"]]),
+                    "CUI",
+                ),
+            )
+            conn.commit()
+        except Exception:
+            pass  # Audit trail table may not exist for BYOS
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="ICDEV Audit Engine")
+    parser.add_argument("--target", type=str, default=".", help="Target path to audit")
+    parser.add_argument("--type", type=str, default="auto",
+                        choices=["icdev", "module", "child_app", "byos", "auto"])
+    parser.add_argument("--project-id", type=str, help="ICDEV project ID")
+    parser.add_argument("--regimes", type=str, nargs="*", help="Regime IDs to evaluate")
+    parser.add_argument("--skip", type=str, nargs="*", help="Scanners to skip")
+    parser.add_argument("--no-store", action="store_true", help="Don't persist results")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    result = run_audit(
+        target_path=args.target,
+        target_type=args.type,
+        project_id=args.project_id,
+        regime_ids=args.regimes,
+        skip_tools=args.skip,
+        store_results=not args.no_store,
+    )
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(f"\n{'='*60}")
+        print(f"  ICDEV AUDIT ENGINE — REPORT CARD")
+        print(f"{'='*60}")
+        print(f"  Target: {result['target_path']}")
+        print(f"  Type:   {result['target_type']}")
+        print(f"  Date:   {result['timestamp'][:10]}")
+        print(f"{'='*60}\n")
+
+        for card in result.get("report_card", {}).get("cards", []):
+            if "error" in card:
+                print(f"  {card['regime_id']:30s}  ERROR: {card['error']}")
+                continue
+            score = card.get("score", 0)
+            grade = card.get("grade", "?")
+            bar_len = int(score / 2)
+            bar = "█" * bar_len + "░" * (50 - bar_len)
+            color = ""
+            print(f"  {card.get('regime_name', card['regime_id']):30s}  [{bar}] {score:5.1f}%  {grade}")
+            print(f"    Passed: {card.get('passed', 0)} | Failed: {card.get('failed', 0)} | Total: {card.get('total', 0)}")
+            print()
+
+        print(f"{'='*60}")
+        print(f"  CUI // SP-CTI")
+        print(f"{'='*60}")