icdev 0.0.3__py3-none-any.whl

tools/architecture/adr_extractor.py

@@ -0,0 +1,393 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""
+ADR Extractor — Extract Architecture Decision Records from CLAUDE.md
+
+Reads CLAUDE.md, extracts all decision lines matching ^- \\*\\*D[-\\w]+:
+pattern, groups them by prefix, and generates grouped ADR markdown files
+in docs/adr/.
+
+Usage:
+    python tools/architecture/adr_extractor.py --extract --json
+    python tools/architecture/adr_extractor.py --list --json
+    python tools/architecture/adr_extractor.py --extract --output-dir docs/adr
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+
+# Resolve project root (two levels up from this file)
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+CLAUDE_MD = PROJECT_ROOT / "CLAUDE.md"
+DEFAULT_OUTPUT_DIR = PROJECT_ROOT / "docs" / "adr"
+
+# ADR line pattern: - **D[-\w]+:** description text
+ADR_PATTERN = re.compile(
+    r"^- \*\*(?P<id>D[-\w]+):\*\*\s+(?P<description>.+)$"
+)
+
+# Group prefix mapping
+GROUP_CONFIG = {
+    "core": {
+        "title": "Core Architecture",
+        "file": "core-decisions.md",
+        "prefix_match": lambda d: re.match(r"^D\d+$", d),
+    },
+    "db": {
+        "title": "Database & Storage",
+        "file": "db-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-DB-"),
+    },
+    "cf": {
+        "title": "Connector Forge & CloudForge",
+        "file": "connector-forge-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-CF-"),
+    },
+    "inv": {
+        "title": "Innovation Features",
+        "file": "innovation-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-INV-"),
+    },
+    "wg": {
+        "title": "WriteGuard",
+        "file": "writeguard-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-WG-"),
+    },
+    "mkt": {
+        "title": "Marketplace",
+        "file": "marketplace-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-MKT-"),
+    },
+    "sc": {
+        "title": "Scale Engine",
+        "file": "scale-engine-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-SC-"),
+    },
+    "harness": {
+        "title": "Harness Engineering",
+        "file": "harness-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-HARNESS-"),
+    },
+    "sbd": {
+        "title": "Secure by Design",
+        "file": "sbd-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-SBD-"),
+    },
+    "arch": {
+        "title": "Architecture Patterns",
+        "file": "arch-decisions.md",
+        "prefix_match": lambda d: d.startswith("D-ARCH-"),
+    },
+}
+
+
+def extract_adrs(claude_md_path=None):
+    """Extract all ADR entries from CLAUDE.md.
+
+    Returns:
+        list[dict]: Each dict has keys: id, description, group, status, is_superseded
+    """
+    path = Path(claude_md_path) if claude_md_path else CLAUDE_MD
+    if not path.exists():
+        raise FileNotFoundError(f"CLAUDE.md not found at {path}")
+
+    adrs = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.rstrip()
+            m = ADR_PATTERN.match(line)
+            if not m:
+                continue
+
+            adr_id = m.group("id")
+            raw_desc = m.group("description")
+
+            # Detect superseded (strikethrough)
+            is_superseded = "~~" in raw_desc
+            # Clean markdown formatting from description
+            clean_desc = re.sub(r"~~(.+?)~~", r"\1 [SUPERSEDED]", raw_desc)
+            clean_desc = clean_desc.replace("``", "`")
+
+            # Determine group
+            group = "core"  # default
+            for gkey, gcfg in GROUP_CONFIG.items():
+                if gcfg["prefix_match"](adr_id):
+                    group = gkey
+                    break
+
+            adrs.append({
+                "id": adr_id,
+                "description": clean_desc,
+                "raw_description": raw_desc,
+                "group": group,
+                "status": "Superseded" if is_superseded else "Accepted",
+            })
+
+    return adrs
+
+
+def group_adrs(adrs):
+    """Group ADRs by their prefix group.
+
+    Returns:
+        dict[str, list[dict]]: group_key -> list of ADR dicts
+    """
+    groups = {}
+    for adr in adrs:
+        g = adr["group"]
+        if g not in groups:
+            groups[g] = []
+        groups[g].append(adr)
+    return groups
+
+
+def _derive_title(adr_id, description):
+    """Derive a short title from the description."""
+    # Take first clause (up to first --, ;, or parenthetical)
+    title = re.split(r"\s+--\s+|;\s+|\(", description)[0]
+    # Truncate if too long
+    if len(title) > 80:
+        title = title[:77] + "..."
+    return title
+
+
+def _derive_context(adr_id, description):
+    """Generate a context statement from the decision description."""
+    desc_lower = description.lower()
+    if "air-gap" in desc_lower or "zero deps" in desc_lower:
+        return "The system must operate in air-gapped environments without external dependencies."
+    if "nist" in desc_lower or "compliance" in desc_lower or "au " in desc_lower:
+        return "Federal compliance requirements (NIST 800-53) mandate specific controls."
+    if "deterministic" in desc_lower:
+        return "Probabilistic behavior in business logic leads to unreliable results across runs."
+    if "security" in desc_lower or "trust" in desc_lower or "injection" in desc_lower:
+        return "Security posture must be maintained across all system boundaries."
+    if "marketplace" in desc_lower or "community" in desc_lower:
+        return "The marketplace and community features require clear operational boundaries."
+    if "mcp" in desc_lower or "agent" in desc_lower:
+        return "Multi-agent communication requires well-defined protocols and boundaries."
+    if "database" in desc_lower or "db" in desc_lower or "sql" in desc_lower or "postgre" in desc_lower:
+        return "Data persistence strategy must balance portability, performance, and compliance."
+    if "scaffold" in desc_lower or "template" in desc_lower or "yaml" in desc_lower:
+        return "Configuration and templates must follow GOTCHA separation of concerns."
+    return "System architecture requires a clear, documented decision for this concern."
+
+
+def _derive_consequences(adr_id, description):
+    """Generate consequences from the decision description."""
+    consequences = []
+    desc_lower = description.lower()
+    if "air-gap" in desc_lower or "zero deps" in desc_lower:
+        consequences.append("No external network dependencies required at runtime.")
+    if "append-only" in desc_lower or "immutable" in desc_lower:
+        consequences.append("Historical records cannot be modified or deleted.")
+    if "deterministic" in desc_lower:
+        consequences.append("Results are reproducible across runs.")
+    if "superseded" in description:
+        consequences.append("This decision has been replaced by a newer decision.")
+    if not consequences:
+        consequences.append("All components must conform to this architectural constraint.")
+    return consequences
+
+
+def generate_group_file(group_key, adrs, date_str="2026-03-08"):
+    """Generate markdown content for one group file.
+
+    Returns:
+        str: Markdown content
+    """
+    cfg = GROUP_CONFIG.get(group_key)
+    if not cfg:
+        return ""
+
+    title = cfg["title"]
+    lines = []
+    lines.append("# CUI // SP-CTI")
+    lines.append(f"# {title} — Architecture Decision Records")
+    lines.append("")
+    lines.append(f"**Total decisions:** {len(adrs)}")
+    lines.append(f"**Last updated:** {date_str}")
+    lines.append("")
+
+    # Summary table
+    lines.append("| ID | Status | Decision |")
+    lines.append("|----|--------|----------|")
+    for adr in adrs:
+        brief = adr["description"]
+        if len(brief) > 100:
+            brief = brief[:97] + "..."
+        lines.append(f"| {adr['id']} | {adr['status']} | {brief} |")
+
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Individual sections
+    for adr in adrs:
+        short_title = _derive_title(adr["id"], adr["description"])
+        context = _derive_context(adr["id"], adr["description"])
+        consequences = _derive_consequences(adr["id"], adr["description"])
+
+        lines.append(f"## {adr['id']}: {short_title}")
+        lines.append("")
+        lines.append(f"**Status:** {adr['status']}")
+        lines.append(f"**Date:** {date_str}")
+        lines.append(f"**Context:** {context}")
+        lines.append(f"**Decision:** {adr['description']}")
+        lines.append(f"**Consequences:**")
+        for c in consequences:
+            lines.append(f"- {c}")
+        lines.append("")
+        lines.append("---")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def generate_readme(grouped_adrs, output_dir, date_str="2026-03-08"):
+    """Generate README.md index for the ADR directory."""
+    total = sum(len(v) for v in grouped_adrs.values())
+    lines = []
+    lines.append("# CUI // SP-CTI")
+    lines.append("# Architecture Decision Records — Index")
+    lines.append("")
+    lines.append(f"**Total decisions:** {total}")
+    lines.append(f"**Last updated:** {date_str}")
+    lines.append("")
+    lines.append("## Decision Groups")
+    lines.append("")
+    lines.append("| Group | File | Count | Description |")
+    lines.append("|-------|------|-------|-------------|")
+
+    for gkey in [
+        "core", "db", "cf", "inv", "wg", "mkt", "sc", "harness", "sbd", "arch"
+    ]:
+        if gkey not in grouped_adrs:
+            continue
+        cfg = GROUP_CONFIG[gkey]
+        count = len(grouped_adrs[gkey])
+        lines.append(
+            f"| {cfg['title']} | [{cfg['file']}]({cfg['file']}) | {count} | {cfg['title']} decisions |"
+        )
+
+    lines.append("")
+    lines.append("## How to Use")
+    lines.append("")
+    lines.append("These ADRs are extracted from `CLAUDE.md` Key Architecture Decisions section.")
+    lines.append("To regenerate:")
+    lines.append("```bash")
+    lines.append("python tools/architecture/adr_extractor.py --extract --json")
+    lines.append("```")
+    lines.append("")
+    lines.append("To list all decisions:")
+    lines.append("```bash")
+    lines.append("python tools/architecture/adr_extractor.py --list --json")
+    lines.append("```")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def cmd_extract(args):
+    """Extract ADRs and generate group files."""
+    adrs = extract_adrs(args.claude_md)
+    grouped = group_adrs(adrs)
+    output_dir = Path(args.output_dir) if args.output_dir else DEFAULT_OUTPUT_DIR
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    date_str = datetime.now().strftime("%Y-%m-%d")
+    results = []
+
+    for gkey, group_adrs_list in grouped.items():
+        cfg = GROUP_CONFIG.get(gkey)
+        if not cfg:
+            continue
+        content = generate_group_file(gkey, group_adrs_list, date_str)
+        fpath = output_dir / cfg["file"]
+        fpath.write_text(content, encoding="utf-8")
+        results.append({
+            "group": gkey,
+            "file": str(fpath),
+            "count": len(group_adrs_list),
+        })
+
+    # Generate README
+    readme_content = generate_readme(grouped, output_dir, date_str)
+    readme_path = output_dir / "README.md"
+    readme_path.write_text(readme_content, encoding="utf-8")
+    results.append({"group": "index", "file": str(readme_path), "count": 0})
+
+    total = sum(len(v) for v in grouped.values())
+
+    if args.json:
+        print(json.dumps({
+            "status": "success",
+            "total_adrs": total,
+            "groups": len(grouped),
+            "files_generated": results,
+        }, indent=2))
+    else:
+        print(f"Extracted {total} ADRs into {len(grouped)} group files in {output_dir}")
+        for r in results:
+            print(f"  {r['group']}: {r['count']} decisions -> {r['file']}")
+
+
+def cmd_list(args):
+    """List all extracted ADRs."""
+    adrs = extract_adrs(args.claude_md)
+    grouped = group_adrs(adrs)
+
+    if args.json:
+        print(json.dumps({
+            "status": "success",
+            "total_adrs": len(adrs),
+            "groups": {
+                gkey: {
+                    "title": GROUP_CONFIG[gkey]["title"],
+                    "file": GROUP_CONFIG[gkey]["file"],
+                    "count": len(gadrs),
+                    "decisions": [
+                        {"id": a["id"], "status": a["status"], "description": a["description"]}
+                        for a in gadrs
+                    ],
+                }
+                for gkey, gadrs in grouped.items()
+            },
+        }, indent=2))
+    else:
+        print(f"Total ADRs: {len(adrs)}")
+        for gkey, gadrs in grouped.items():
+            cfg = GROUP_CONFIG[gkey]
+            print(f"\n{cfg['title']} ({len(gadrs)} decisions):")
+            for a in gadrs:
+                print(f"  [{a['status']}] {a['id']}: {a['description'][:80]}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ADR Extractor — Extract Architecture Decision Records from CLAUDE.md"
+    )
+    parser.add_argument("--extract", action="store_true", help="Extract ADRs and generate group files")
+    parser.add_argument("--list", action="store_true", dest="list_adrs", help="List all ADRs")
+    parser.add_argument("--json", action="store_true", help="Output in JSON format")
+    parser.add_argument("--claude-md", type=str, default=None, help="Path to CLAUDE.md (default: project root)")
+    parser.add_argument("--output-dir", type=str, default=None, help="Output directory (default: docs/adr)")
+
+    args = parser.parse_args()
+
+    if args.extract:
+        cmd_extract(args)
+    elif args.list_adrs:
+        cmd_list(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/audit/__init__.py

@@ -0,0 +1 @@
+# [TEMPLATE: CUI // SP-CTI]

tools/audit/audit_logger.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Append-only audit trail writer. Satisfies NIST 800-53 AU controls.
+No UPDATE or DELETE operations — all entries are immutable."""
+
+import argparse
+import json
+import sqlite3
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+try:
+    from tools.compat.db_utils import get_db_connection
+except ImportError:
+    get_db_connection = None
+
+from tools.db.storage import get_connection
+DB_PATH = None  # Storage layer handles path resolution (D-DB-20)
+
+VALID_EVENT_TYPES = (
+    "project_created", "project_updated",
+    "code_generated", "code_reviewed", "code_approved", "code_rejected",
+    "test_written", "test_executed", "test_passed", "test_failed",
+    "security_scan", "vulnerability_found", "vulnerability_resolved",
+    "compliance_check", "ssp_generated", "poam_generated", "stig_checked", "sbom_generated",
+    "deployment_initiated", "deployment_succeeded", "deployment_failed", "rollback_executed",
+    "decision_made", "approval_granted", "approval_denied",
+    "agent_task_submitted", "agent_task_completed", "agent_task_failed",
+    "self_heal_triggered", "pattern_detected", "knowledge_recorded",
+    "config_changed", "secret_rotated",
+    # RICOAS events (Phase 20)
+    "intake_session_created", "intake_session_resumed", "intake_session_completed",
+    "requirement_captured", "requirement_refined", "requirement_approved",
+    "gap_detected", "ambiguity_detected",
+    "readiness_scored", "decomposition_generated",
+    "document_uploaded", "document_extracted",
+    "bdd_criteria_generated",
+    # Boundary & Supply Chain events
+    "boundary_assessed", "boundary_impact_red", "boundary_alternative_generated",
+    "ato_system_registered", "isa_created", "isa_expired", "isa_renewed",
+    "scrm_assessed", "cve_triaged", "cve_impact_propagated",
+    "supply_chain_risk_escalated",
+    # Simulation & COA events
+    "simulation_created", "simulation_completed", "monte_carlo_completed",
+    "coa_generated", "coa_alternative_generated", "coa_compared",
+    "coa_selected", "coa_rejected", "coa_presented",
+    # Integration events
+    "integration_configured", "integration_sync_push", "integration_sync_pull",
+    "integration_sync_error", "reqif_exported",
+    "approval_submitted", "approval_reviewed", "approval_approved",
+    "approval_rejected", "approval_escalated",
+    "rtm_generated", "rtm_gap_detected",
+    # Observability events (Phase 39)
+    "hook_event_logged", "agent_execution_started", "agent_execution_completed",
+    "agent_execution_failed", "agent_execution_retried",
+    # NLQ events (Phase 40)
+    "nlq_query_executed", "nlq_query_blocked",
+    # Worktree & GitLab events (Phase 41)
+    "worktree_created", "worktree_cleaned",
+    "gitlab_task_claimed", "gitlab_task_completed", "gitlab_task_failed",
+    # Agent Orchestration events (Opus 4.6 Multi-Agent)
+    "bedrock_invoked", "bedrock_fallback", "bedrock_rate_limited",
+    "workflow_created", "workflow_completed", "workflow_failed",
+    "subtask_dispatched", "subtask_completed", "subtask_failed",
+    "agent_health_stale",
+    "agent_veto_issued", "agent_veto_overridden",
+    "agent_collaboration_started", "agent_collaboration_completed",
+    "agent_message_sent", "agent_memory_stored", "agent_memory_recalled",
+    "agent_escalation_created",
+    # Remote Command Gateway events (Phase 28)
+    "remote_binding_created", "remote_binding_provisioned", "remote_binding_revoked",
+    "remote_command_received", "remote_command_rejected", "remote_command_completed",
+    "remote_response_filtered",
+    # Spec-kit events (D156-D161)
+    "spec_quality_check", "spec_consistency_check",
+    "constitution_added", "constitution_removed", "constitution_defaults_loaded",
+    "clarification_analyzed",
+    "spec.init", "spec.register",
+    # Phase 29: Heartbeat + Auto-Resolution (D141-D145)
+    "heartbeat_check_warning", "heartbeat_check_critical",
+    "auto_resolution_started", "auto_resolution_completed",
+    "auto_resolution_failed", "auto_resolution_escalated",
+    # Cross-Language Translation events (Phase 43)
+    "translation.job_created", "translation.job_completed", "translation.job_failed",
+    "translation.extract", "translation.type_check",
+    "translation.unit_translated", "translation.unit_mocked", "translation.unit_failed",
+    "translation.repair_attempted", "translation.repair_succeeded",
+    "translation.validation_passed", "translation.validation_failed",
+    "translation.compliance_checked", "translation.assembly_completed",
+    # Multi-Stream Chat events (Phase 44 — D257-D260)
+    "chat.context_created", "chat.context_closed", "chat.context_archived",
+    "chat.message_sent", "chat.message_received",
+    "chat.intervention_requested", "chat.intervention_applied",
+    "chat.agent_loop_started", "chat.agent_loop_completed", "chat.agent_loop_error",
+    # Extension Hook events (Phase 44 — D261-D264)
+    "extension.registered", "extension.unregistered",
+    "extension.dispatched", "extension.completed",
+    "extension.error", "extension.timeout",
+    # Memory Consolidation events (Phase 44 — D276)
+    "memory.consolidation_checked", "memory.consolidation_executed",
+    "memory.consolidation_skipped",
+    # Observability & Explainability events (Phase 46 — D280-D289)
+    "trace.span_exported", "trace.batch_exported",
+    "prov.entity_created", "prov.activity_recorded", "prov.relation_established",
+    "shap.analysis_completed", "shap.analysis_failed",
+    "xai.assessment_completed", "xai.gate_evaluated",
+    # ATLAS Critique Phase (Phase 61 — Feature 3)
+    "critique_session_created", "critique_completed",
+    "critique_revision_requested",
+    # Session Purpose (Phase 61 — D-ORCH-5)
+    "session.purpose_declared",
+    # Child App Generation (Phase 19/36)
+    "child_app_generated", "child_app_registered",
+)
+
+
+def log_event(
+    event_type: str,
+    actor: str,
+    action: str,
+    project_id: str = None,
+    details: dict = None,
+    affected_files: list = None,
+    classification: str = "CUI",
+    ip_address: str = None,
+    session_id: str = None,
+    db_path: Path = None,
+) -> int:
+    """Write an immutable audit trail entry. Returns the entry ID."""
+    if event_type not in VALID_EVENT_TYPES:
+        raise ValueError(f"Invalid event_type '{event_type}'. Valid: {VALID_EVENT_TYPES}")
+
+    # Auto-populate session_id from correlation context (D149)
+    if session_id is None:
+        try:
+            from tools.resilience.correlation import get_correlation_id
+            session_id = get_correlation_id()
+        except ImportError:
+            pass
+
+    path = db_path or DB_PATH
+    if get_db_connection:
+        conn = get_db_connection(path, row_factory=False)
+    else:
+        conn = sqlite3.connect(str(path))
+    c = conn.cursor()
+    c.execute(
+        """INSERT INTO audit_trail
+           (project_id, event_type, actor, action, details, affected_files,
+            classification, ip_address, session_id)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+        (
+            project_id,
+            event_type,
+            actor,
+            action,
+            json.dumps(details) if details else None,
+            json.dumps(affected_files) if affected_files else None,
+            classification,
+            ip_address,
+            session_id,
+        ),
+    )
+    conn.commit()
+    entry_id = c.lastrowid
+    conn.close()
+    return entry_id
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Log an audit trail event")
+    parser.add_argument("--event", required=True, choices=VALID_EVENT_TYPES, help="Event type")
+    parser.add_argument("--actor", required=True, help="Who performed the action")
+    parser.add_argument("--action", required=True, help="Human-readable description")
+    parser.add_argument("--project-id", "--project", help="Project ID", dest="project_id")
+    parser.add_argument("--details", help="JSON details string")
+    parser.add_argument("--files", help="Comma-separated affected file paths")
+    parser.add_argument("--classification", default="CUI", help="Classification marking")
+    parser.add_argument("--json", action="store_true", dest="json_output", help="JSON output")
+    args = parser.parse_args()
+
+    details = json.loads(args.details) if args.details else None
+    affected_files = args.files.split(",") if args.files else None
+
+    entry_id = log_event(
+        event_type=args.event,
+        actor=args.actor,
+        action=args.action,
+        project_id=args.project_id,
+        details=details,
+        affected_files=affected_files,
+        classification=args.classification,
+    )
+    print(f"Audit entry #{entry_id} logged: [{args.event}] {args.action}")
+
+
+if __name__ == "__main__":
+    main()