icdev 0.0.3__py3-none-any.whl

tools/harness/__init__.py

@@ -0,0 +1,6 @@
+# CUI // SP-CTI
+"""Harness Engineering — tools for AI agent loop detection, progress tracking,
+exit criteria evaluation, trace analysis, maturity assessment, and scaffolding.
+
+Architecture decisions: D-HARNESS-1 through D-HARNESS-8.
+"""

tools/harness/exit_criteria_evaluator.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Exit Criteria Evaluator — checks workflow completion against registered criteria.
+
+Loads args/exit_criteria.yaml and evaluates required gates against test state
+or DB records. Returns structured JSON report with per-criterion status.
+
+Architecture decision D-HARNESS-4: Exit criteria in args/ YAML (GOTCHA separation).
+
+CLI:
+  python tools/harness/exit_criteria_evaluator.py --list --json
+  python tools/harness/exit_criteria_evaluator.py --workflow build --json
+  python tools/harness/exit_criteria_evaluator.py --workflow build --check-state .tmp/test_runs/state.json --json
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+EXIT_CRITERIA_PATH = BASE_DIR / "args" / "exit_criteria.yaml"
+
+
+def _load_exit_criteria() -> dict:
+    """Load exit criteria from YAML config."""
+    if not EXIT_CRITERIA_PATH.exists():
+        return {}
+    try:
+        import yaml
+        with open(EXIT_CRITERIA_PATH) as f:
+            return yaml.safe_load(f) or {}
+    except ImportError:
+        return _parse_exit_criteria_minimal()
+
+
+def _parse_exit_criteria_minimal() -> dict:
+    """Parse exit criteria YAML without PyYAML dependency."""
+    if not EXIT_CRITERIA_PATH.exists():
+        return {}
+    text = EXIT_CRITERIA_PATH.read_text()
+    workflows = {}
+    current_workflow = None
+    current_section = None
+
+    for line in text.split("\n"):
+        stripped = line.strip()
+        if stripped.startswith("#") or not stripped:
+            continue
+
+        # Detect workflow name (2-space indent under workflows:)
+        if line.startswith("  ") and not line.startswith("    ") and stripped.endswith(":"):
+            key = stripped.rstrip(":")
+            if key in ("build", "test", "deploy", "comply", "secure", "review"):
+                current_workflow = key
+                workflows[current_workflow] = {
+                    "description": "",
+                    "required_gates": [],
+                    "acceptance_criteria": [],
+                    "blocking_failures": [],
+                }
+                current_section = None
+        elif current_workflow and line.startswith("    ") and not line.startswith("      "):
+            key = stripped.rstrip(":")
+            if key == "description":
+                # inline value
+                parts = stripped.split(":", 1)
+                if len(parts) == 2:
+                    workflows[current_workflow]["description"] = parts[1].strip().strip('"')
+                current_section = None
+            elif key in ("required_gates", "acceptance_criteria", "blocking_failures"):
+                current_section = key
+        elif current_workflow and current_section and stripped.startswith("- "):
+            val = stripped[2:].strip().strip('"')
+            workflows[current_workflow][current_section].append(val)
+
+    return {"workflows": workflows}
+
+
+def _load_test_state(state_path: str) -> dict:
+    """Load test run state from JSON file."""
+    p = Path(state_path)
+    if not p.is_absolute():
+        p = BASE_DIR / p
+    if not p.exists():
+        return {}
+    with open(p) as f:
+        return json.load(f)
+
+
+def _evaluate_gate(gate: str, test_state: dict) -> tuple:
+    """Evaluate a single gate against test state. Returns (status, evidence)."""
+    gate_map = {
+        "py_compile_pass": lambda s: ("met", "Compile check passed") if not s else ("unknown", "No compile data"),
+        "ruff_lint_pass": lambda s: ("met", "Lint clean") if not s else ("unknown", "No lint data"),
+        "pytest_pass": lambda s: (
+            ("met", f"pytest: {s.get('unit_passed', 0)} passed, {s.get('unit_failed', 0)} failed")
+            if s.get("unit_failed", -1) == 0
+            else ("not_met", f"pytest: {s.get('unit_failed', '?')} failures")
+            if s.get("unit_failed", 0) > 0
+            else ("unknown", "No pytest data")
+        ),
+        "bandit_sast_pass": lambda s: (
+            ("met", "SAST gate passed") if s.get("security_gate_passed")
+            else ("not_met", "SAST gate failed") if s.get("security_gate_passed") is False
+            else ("unknown", "No SAST data")
+        ),
+        "unit_tests_pass": lambda s: (
+            ("met", f"{s.get('unit_passed', 0)} unit tests passed")
+            if s.get("unit_failed", -1) == 0
+            else ("not_met", f"{s.get('unit_failed', '?')} unit test failures")
+            if s.get("unit_failed", 0) > 0
+            else ("unknown", "No unit test data")
+        ),
+        "bdd_tests_pass": lambda s: (
+            ("met", f"{s.get('bdd_passed', 0)} BDD scenarios passed")
+            if s.get("bdd_failed", -1) == 0
+            else ("not_met", f"{s.get('bdd_failed', '?')} BDD failures")
+            if s.get("bdd_failed", 0) > 0
+            else ("unknown", "No BDD data")
+        ),
+        "e2e_tests_pass": lambda s: (
+            ("met", f"{s.get('e2e_passed', 0)} E2E tests passed")
+            if s.get("e2e_failed", -1) == 0
+            else ("not_met", f"{s.get('e2e_failed', '?')} E2E failures")
+            if s.get("e2e_failed", 0) > 0
+            else ("unknown", "No E2E data")
+        ),
+        "security_scan_clean": lambda s: (
+            ("met", "Security gate passed") if s.get("security_gate_passed")
+            else ("not_met", "Security gate failed") if s.get("security_gate_passed") is False
+            else ("unknown", "No security data")
+        ),
+        "compliance_gate_pass": lambda s: (
+            ("met", "Compliance gate passed") if s.get("compliance_gate_passed")
+            else ("not_met", "Compliance gate failed") if s.get("compliance_gate_passed") is False
+            else ("unknown", "No compliance data")
+        ),
+    }
+    evaluator = gate_map.get(gate, lambda s: ("unknown", f"No evaluator for gate '{gate}'"))
+    return evaluator(test_state)
+
+
+def list_workflows() -> dict:
+    """List all workflow types and their exit criteria."""
+    criteria = _load_exit_criteria()
+    workflows = criteria.get("workflows", {})
+    return {
+        "workflows": [
+            {
+                "name": name,
+                "description": wf.get("description", ""),
+                "gate_count": len(wf.get("required_gates", [])),
+                "criteria_count": len(wf.get("acceptance_criteria", [])),
+                "blocking_count": len(wf.get("blocking_failures", [])),
+            }
+            for name, wf in workflows.items()
+        ],
+        "count": len(workflows),
+    }
+
+
+def evaluate_workflow(workflow: str, state_path: str = None) -> dict:
+    """Evaluate a workflow's exit criteria against current state."""
+    criteria = _load_exit_criteria()
+    workflows = criteria.get("workflows", {})
+
+    if workflow not in workflows:
+        return {"error": f"Unknown workflow: {workflow}", "available": list(workflows.keys())}
+
+    wf = workflows[workflow]
+    test_state = _load_test_state(state_path) if state_path else {}
+
+    results = []
+    blocking_active = []
+
+    for i, gate in enumerate(wf.get("required_gates", [])):
+        status, evidence = _evaluate_gate(gate, test_state)
+        criterion_text = wf.get("acceptance_criteria", [])[i] if i < len(wf.get("acceptance_criteria", [])) else gate
+        results.append({
+            "criterion": criterion_text,
+            "gate": gate,
+            "status": status,
+            "evidence": evidence,
+        })
+        if status == "not_met":
+            # Check if this maps to a blocking failure
+            for bf in wf.get("blocking_failures", []):
+                if bf not in blocking_active:
+                    blocking_active.append(bf)
+
+    all_met = all(r["status"] == "met" for r in results)
+    any_failed = any(r["status"] == "not_met" for r in results)
+
+    return {
+        "workflow": workflow,
+        "description": wf.get("description", ""),
+        "all_met": all_met,
+        "any_failed": any_failed,
+        "criteria": results,
+        "blocking_failures": blocking_active,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "classification": "CUI // SP-CTI",
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Exit Criteria Evaluator")
+    parser.add_argument("--list", action="store_true", help="List all workflow types")
+    parser.add_argument("--workflow", type=str, help="Evaluate specific workflow")
+    parser.add_argument("--check-state", type=str, help="Path to test state JSON file")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    if args.list:
+        result = list_workflows()
+    elif args.workflow:
+        result = evaluate_workflow(args.workflow, args.check_state)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tools/harness/maturity_assessor.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Harness Maturity Assessor — scores project harness maturity 0-4.
+
+Checks 6 dimensions with configurable weights, computes weighted composite
+score, maps to maturity level (None/Initial/Managed/Defined/Optimized).
+
+Read-only, advisory-only — never modifies project files (D-HARNESS-6).
+
+CLI:
+  python tools/harness/maturity_assessor.py --project-dir . --json
+  python tools/harness/maturity_assessor.py --project-dir . --detailed --json
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+DEFAULT_WEIGHTS = {
+    "hooks_present": 0.20,
+    "gates_configured": 0.20,
+    "exit_criteria_defined": 0.15,
+    "tracing_active": 0.15,
+    "loop_detection_enabled": 0.15,
+    "progress_tracking_enabled": 0.15,
+}
+
+DEFAULT_THRESHOLDS = {
+    "initial": 0.20,
+    "managed": 0.40,
+    "defined": 0.60,
+    "optimized": 0.80,
+}
+
+
+def _load_harness_config(project_dir: Path) -> dict:
+    """Load harness config for weights and thresholds."""
+    config_path = project_dir / "args" / "harness_config.yaml"
+    if not config_path.exists():
+        return {}
+    try:
+        import yaml
+        with open(config_path) as f:
+            return yaml.safe_load(f) or {}
+    except ImportError:
+        return {}
+
+
+def _check_hooks_present(project_dir: Path) -> dict:
+    """Check for pre_tool_use, post_tool_use, and stop hooks."""
+    hooks_dir = project_dir / ".claude" / "hooks"
+    required = ["pre_tool_use.py", "post_tool_use.py", "stop.py"]
+    found = []
+    missing = []
+    for hook in required:
+        if (hooks_dir / hook).exists():
+            found.append(hook)
+        else:
+            missing.append(hook)
+
+    score = len(found) / len(required) if required else 0.0
+    return {
+        "dimension": "hooks_present",
+        "present": score == 1.0,
+        "score": score,
+        "details": f"{len(found)}/{len(required)} hooks: {', '.join(found)}" if found else "No hooks found",
+        "recommendations": [f"Create .claude/hooks/{h}" for h in missing],
+    }
+
+
+def _check_gates_configured(project_dir: Path) -> dict:
+    """Check for security_gates.yaml with blocking rules."""
+    gates_path = project_dir / "args" / "security_gates.yaml"
+    if not gates_path.exists():
+        return {
+            "dimension": "gates_configured",
+            "present": False,
+            "score": 0.0,
+            "details": "No security_gates.yaml found",
+            "recommendations": ["Create args/security_gates.yaml with merge and deployment gates"],
+        }
+    try:
+        import yaml
+        with open(gates_path) as f:
+            gates = yaml.safe_load(f) or {}
+        # Check for blocking rules
+        has_blocking = False
+        gate_count = 0
+        for section in gates.values():
+            if isinstance(section, dict):
+                if "blocking" in section or "block_on" in section:
+                    has_blocking = True
+                    items = section.get("blocking", section.get("block_on", []))
+                    gate_count += len(items) if isinstance(items, list) else 0
+        score = 1.0 if has_blocking else 0.5
+        return {
+            "dimension": "gates_configured",
+            "present": has_blocking,
+            "score": score,
+            "details": f"Security gates configured with {gate_count} blocking rules",
+            "recommendations": [] if has_blocking else ["Add blocking rules to security_gates.yaml"],
+        }
+    except ImportError:
+        # Check file exists and has content
+        text = gates_path.read_text()
+        has_blocking = "blocking:" in text or "block_on:" in text
+        return {
+            "dimension": "gates_configured",
+            "present": has_blocking,
+            "score": 1.0 if has_blocking else 0.5,
+            "details": "Security gates file exists" + (" with blocking rules" if has_blocking else ""),
+            "recommendations": [] if has_blocking else ["Add blocking rules"],
+        }
+
+
+def _check_exit_criteria(project_dir: Path) -> dict:
+    """Check for exit_criteria.yaml with workflow definitions."""
+    ec_path = project_dir / "args" / "exit_criteria.yaml"
+    if not ec_path.exists():
+        return {
+            "dimension": "exit_criteria_defined",
+            "present": False,
+            "score": 0.0,
+            "details": "No exit_criteria.yaml found",
+            "recommendations": ["Create args/exit_criteria.yaml with workflow exit conditions"],
+        }
+    try:
+        import yaml
+        with open(ec_path) as f:
+            ec = yaml.safe_load(f) or {}
+        workflows = ec.get("workflows", {})
+        count = len(workflows)
+        score = min(1.0, count / 3.0)  # 3+ workflows = full score
+        return {
+            "dimension": "exit_criteria_defined",
+            "present": count > 0,
+            "score": score,
+            "details": f"{count} workflow exit criteria defined",
+            "recommendations": [] if count >= 3 else [f"Define more workflows (have {count}, recommend 3+)"],
+        }
+    except ImportError:
+        text = ec_path.read_text()
+        wf_count = text.count("required_gates:")
+        return {
+            "dimension": "exit_criteria_defined",
+            "present": wf_count > 0,
+            "score": min(1.0, wf_count / 3.0),
+            "details": f"~{wf_count} workflows detected",
+            "recommendations": [],
+        }
+
+
+def _check_tracing_active(project_dir: Path) -> dict:
+    """Check for observability tracer module."""
+    tracer_paths = [
+        project_dir / "tools" / "observability" / "tracer.py",
+        project_dir / "tools" / "observability" / "sqlite_tracer.py",
+        project_dir / "tools" / "observability" / "otel_tracer.py",
+    ]
+    found = [p.name for p in tracer_paths if p.exists()]
+    score = min(1.0, len(found) / 2.0)  # 2+ tracers = full score
+    return {
+        "dimension": "tracing_active",
+        "present": len(found) > 0,
+        "score": score,
+        "details": f"Tracer modules found: {', '.join(found)}" if found else "No tracer modules",
+        "recommendations": [] if found else ["Add tools/observability/tracer.py"],
+    }
+
+
+def _check_loop_detection(project_dir: Path) -> dict:
+    """Check if loop detection is enabled in harness config."""
+    config_path = project_dir / "args" / "harness_config.yaml"
+    if not config_path.exists():
+        return {
+            "dimension": "loop_detection_enabled",
+            "present": False,
+            "score": 0.0,
+            "details": "No harness_config.yaml found",
+            "recommendations": ["Create args/harness_config.yaml with loop_detection.enabled: true"],
+        }
+    try:
+        import yaml
+        with open(config_path) as f:
+            config = yaml.safe_load(f) or {}
+        enabled = config.get("loop_detection", {}).get("enabled", False)
+        return {
+            "dimension": "loop_detection_enabled",
+            "present": enabled,
+            "score": 1.0 if enabled else 0.0,
+            "details": "Loop detection enabled" if enabled else "Loop detection disabled",
+            "recommendations": [] if enabled else ["Set loop_detection.enabled: true in harness_config.yaml"],
+        }
+    except ImportError:
+        text = config_path.read_text()
+        import re
+        enabled = bool(re.search(r"loop_detection:\s*\n\s+enabled:\s*true", text, re.IGNORECASE))
+        return {
+            "dimension": "loop_detection_enabled",
+            "present": enabled,
+            "score": 1.0 if enabled else 0.0,
+            "details": "Loop detection " + ("enabled" if enabled else "disabled"),
+            "recommendations": [],
+        }
+
+
+def _check_progress_tracking(project_dir: Path) -> dict:
+    """Check if progress tracking is enabled in harness config."""
+    config_path = project_dir / "args" / "harness_config.yaml"
+    if not config_path.exists():
+        return {
+            "dimension": "progress_tracking_enabled",
+            "present": False,
+            "score": 0.0,
+            "details": "No harness_config.yaml found",
+            "recommendations": ["Create args/harness_config.yaml with progress_tracking.enabled: true"],
+        }
+    try:
+        import yaml
+        with open(config_path) as f:
+            config = yaml.safe_load(f) or {}
+        enabled = config.get("progress_tracking", {}).get("enabled", False)
+        return {
+            "dimension": "progress_tracking_enabled",
+            "present": enabled,
+            "score": 1.0 if enabled else 0.0,
+            "details": "Progress tracking enabled" if enabled else "Progress tracking disabled",
+            "recommendations": [] if enabled else ["Set progress_tracking.enabled: true"],
+        }
+    except ImportError:
+        text = config_path.read_text()
+        import re
+        enabled = bool(re.search(r"progress_tracking:\s*\n\s+enabled:\s*true", text, re.IGNORECASE))
+        return {
+            "dimension": "progress_tracking_enabled",
+            "present": enabled,
+            "score": 1.0 if enabled else 0.0,
+            "details": "Progress tracking " + ("enabled" if enabled else "disabled"),
+            "recommendations": [],
+        }
+
+
+def assess_maturity(project_dir: str, detailed: bool = False) -> dict:
+    """Run full maturity assessment on a project directory."""
+    p = Path(project_dir).resolve()
+    config = _load_harness_config(p)
+    maturity_cfg = config.get("maturity", {})
+
+    # Get weights (from config or defaults)
+    weights = {}
+    dims_cfg = maturity_cfg.get("dimensions", {})
+    for dim, default_weight in DEFAULT_WEIGHTS.items():
+        if isinstance(dims_cfg.get(dim), dict):
+            weights[dim] = dims_cfg[dim].get("weight", default_weight)
+        else:
+            weights[dim] = default_weight
+
+    thresholds = maturity_cfg.get("thresholds", DEFAULT_THRESHOLDS)
+
+    # Run dimension checks
+    checkers = {
+        "hooks_present": _check_hooks_present,
+        "gates_configured": _check_gates_configured,
+        "exit_criteria_defined": _check_exit_criteria,
+        "tracing_active": _check_tracing_active,
+        "loop_detection_enabled": _check_loop_detection,
+        "progress_tracking_enabled": _check_progress_tracking,
+    }
+
+    dimensions = []
+    for dim_name, checker in checkers.items():
+        result = checker(p)
+        result["weight"] = weights.get(dim_name, 0.0)
+        dimensions.append(result)
+
+    # Compute composite score
+    composite = sum(d["score"] * d["weight"] for d in dimensions)
+
+    # Map to level
+    if composite >= thresholds.get("optimized", 0.80):
+        level, level_name = 4, "Optimized"
+    elif composite >= thresholds.get("defined", 0.60):
+        level, level_name = 3, "Defined"
+    elif composite >= thresholds.get("managed", 0.40):
+        level, level_name = 2, "Managed"
+    elif composite >= thresholds.get("initial", 0.20):
+        level, level_name = 1, "Initial"
+    else:
+        level, level_name = 0, "None"
+
+    # Top recommendations
+    all_recs = []
+    for d in dimensions:
+        all_recs.extend(d.get("recommendations", []))
+
+    result = {
+        "project_dir": str(p),
+        "level": level,
+        "level_name": level_name,
+        "composite_score": round(composite, 3),
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "classification": "CUI // SP-CTI",
+    }
+
+    if detailed:
+        result["dimensions"] = dimensions
+        result["recommendations"] = all_recs[:10]
+    else:
+        result["dimensions_met"] = sum(1 for d in dimensions if d["present"])
+        result["dimensions_total"] = len(dimensions)
+        result["top_recommendations"] = all_recs[:3]
+
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Harness Maturity Assessor")
+    parser.add_argument("--project-dir", type=str, required=True, help="Project directory to assess")
+    parser.add_argument("--detailed", action="store_true", help="Include per-dimension details")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    result = assess_maturity(args.project_dir, args.detailed)
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(f"Harness Maturity: Level {result['level']} — {result['level_name']}")
+        print(f"Composite Score: {result['composite_score']}")
+        if "dimensions" in result:
+            for d in result["dimensions"]:
+                status = "OK" if d["present"] else "MISSING"
+                print(f"  [{status}] {d['dimension']}: {d['details']}")
+        if result.get("top_recommendations") or result.get("recommendations"):
+            recs = result.get("recommendations", result.get("top_recommendations", []))
+            if recs:
+                print("\nRecommendations:")
+                for r in recs:
+                    print(f"  - {r}")
+
+
+if __name__ == "__main__":
+    main()