icdev 0.0.3__py3-none-any.whl

tools/ci/workflows/icdev_comply.py

@@ -0,0 +1,284 @@
+# [TEMPLATE: CUI // SP-CTI]
+# SPARKPILOT Comply — Compliance artifact generation workflow
+# Runs SSP, POAM, STIG, SBOM, CUI marker and evaluates compliance gate
+
+"""
+SPARKPILOT Comply — Generate ATO compliance artifacts.
+
+Usage:
+    python tools/ci/workflows/icdev_comply.py <issue-number> <run-id>
+
+Workflow:
+    1. Load state from previous phase
+    2. Generate SSP (System Security Plan)
+    3. Generate POAM (Plan of Action and Milestones)
+    4. Run STIG checklist
+    5. Generate SBOM (Software Bill of Materials)
+    6. Apply CUI markings
+    7. Evaluate compliance gate
+    8. Commit artifacts and post results
+"""
+
+import logging
+import sys
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+from tools.ci.modules.state import ICDevState
+from tools.ci.modules.git_ops import commit_changes
+from tools.ci.modules.vcs import VCS
+from tools.ci.modules.workflow_ops import (
+    BOT_IDENTIFIER,
+)
+from tools.testing.utils import setup_logger
+
+AGENT_COMPLIANCE = "icdev_compliance"
+
+
+def run_compliance_artifacts(
+    run_id: str, issue_number: str, logger: logging.Logger
+) -> dict:
+    """Generate all compliance artifacts."""
+    results = {
+        "ssp": {"status": "skipped", "path": ""},
+        "poam": {"status": "skipped", "path": ""},
+        "stig": {"status": "skipped", "findings": {}},
+        "sbom": {"status": "skipped", "path": ""},
+        "cui_markings": {"status": "skipped"},
+        "gate_passed": False,
+        "errors": [],
+    }
+
+    project_id = f"issue-{issue_number}"
+
+    # 1. Generate SSP
+    logger.info("Generating SSP...")
+    try:
+        from tools.compliance.ssp_generator import generate_ssp
+        ssp_result = generate_ssp(project_id)
+        results["ssp"] = {
+            "status": "generated",
+            "path": ssp_result.get("output_path", ""),
+        }
+        logger.info(f"SSP generated: {results['ssp']['path']}")
+    except ImportError:
+        logger.warning("ssp_generator not available, using agent fallback")
+        _run_agent_comply(run_id, "ssp", logger)
+        results["ssp"]["status"] = "agent_generated"
+    except Exception as e:
+        logger.error(f"SSP generation failed: {e}")
+        results["errors"].append(f"SSP: {e}")
+
+    # 2. Generate POAM
+    logger.info("Generating POAM...")
+    try:
+        from tools.compliance.poam_generator import generate_poam
+        poam_result = generate_poam(project_id)
+        results["poam"] = {
+            "status": "generated",
+            "path": poam_result.get("output_path", ""),
+        }
+        logger.info(f"POAM generated: {results['poam']['path']}")
+    except ImportError:
+        logger.warning("poam_generator not available, using agent fallback")
+        _run_agent_comply(run_id, "poam", logger)
+        results["poam"]["status"] = "agent_generated"
+    except Exception as e:
+        logger.error(f"POAM generation failed: {e}")
+        results["errors"].append(f"POAM: {e}")
+
+    # 3. Run STIG checklist
+    logger.info("Running STIG checklist...")
+    try:
+        from tools.compliance.stig_checker import check_stig
+        stig_result = check_stig(project_id)
+        findings = stig_result.get("findings", {})
+        cat1_count = findings.get("cat1", 0)
+        results["stig"] = {
+            "status": "checked",
+            "findings": findings,
+            "cat1_count": cat1_count,
+        }
+        if cat1_count > 0:
+            logger.warning(f"STIG: {cat1_count} CAT1 findings (blocking)")
+        else:
+            logger.info("STIG: No CAT1 findings")
+    except ImportError:
+        logger.warning("stig_checker not available, using agent fallback")
+        _run_agent_comply(run_id, "stig", logger)
+        results["stig"]["status"] = "agent_checked"
+    except Exception as e:
+        logger.error(f"STIG check failed: {e}")
+        results["errors"].append(f"STIG: {e}")
+
+    # 4. Generate SBOM
+    logger.info("Generating SBOM...")
+    try:
+        from tools.compliance.sbom_generator import generate_sbom
+        sbom_result = generate_sbom(project_dir=str(PROJECT_ROOT))
+        results["sbom"] = {
+            "status": "generated",
+            "path": sbom_result.get("output_path", ""),
+        }
+        logger.info(f"SBOM generated: {results['sbom']['path']}")
+    except ImportError:
+        logger.warning("sbom_generator not available, using agent fallback")
+        _run_agent_comply(run_id, "sbom", logger)
+        results["sbom"]["status"] = "agent_generated"
+    except Exception as e:
+        logger.error(f"SBOM generation failed: {e}")
+        results["errors"].append(f"SBOM: {e}")
+
+    # 5. Apply CUI markings check
+    logger.info("Verifying CUI markings...")
+    try:
+        from tools.compliance.cui_marker import verify_cui_markings
+        cui_result = verify_cui_markings(project_dir=str(PROJECT_ROOT))
+        results["cui_markings"] = {
+            "status": "verified",
+            "files_checked": cui_result.get("files_checked", 0),
+            "missing_markings": cui_result.get("missing_markings", 0),
+        }
+    except (ImportError, Exception) as e:
+        logger.warning(f"CUI marking check skipped: {e}")
+        results["cui_markings"]["status"] = "skipped"
+
+    # 6. Evaluate compliance gate
+    results["gate_passed"] = _evaluate_compliance_gate(results, logger)
+
+    return results
+
+
+def _run_agent_comply(run_id: str, artifact_type: str, logger: logging.Logger):
+    """Fallback: use Claude Code agent to generate compliance artifact."""
+    try:
+        from tools.ci.modules.agent import execute_template
+        from tools.testing.data_types import AgentTemplateRequest
+
+        request = AgentTemplateRequest(
+            agent_name=AGENT_COMPLIANCE,
+            slash_command="/sparkpilot-comply",
+            args=[artifact_type],
+            run_id=run_id,
+        )
+        response = execute_template(request)
+        if response.success:
+            logger.info(f"Agent generated {artifact_type} artifact")
+        else:
+            logger.warning(f"Agent {artifact_type} generation produced output but may have issues")
+    except Exception as e:
+        logger.error(f"Agent fallback for {artifact_type} failed: {e}")
+
+
+def _evaluate_compliance_gate(results: dict, logger: logging.Logger) -> bool:
+    """Evaluate whether compliance gate passes.
+
+    Gate criteria (from security_gates.yaml):
+    - 0 CAT1 STIG findings
+    - SSP generated or agent-generated
+    - SBOM generated or agent-generated
+    - CUI markings present (warning only)
+    """
+    gate_passed = True
+    reasons = []
+
+    # STIG CAT1 check (blocking)
+    stig = results.get("stig", {})
+    cat1_count = stig.get("cat1_count", 0)
+    if cat1_count > 0:
+        gate_passed = False
+        reasons.append(f"{cat1_count} CAT1 STIG findings")
+
+    # SSP check (blocking)
+    ssp_status = results.get("ssp", {}).get("status", "skipped")
+    if ssp_status in ("skipped",):
+        gate_passed = False
+        reasons.append("SSP not generated")
+
+    # SBOM check (blocking)
+    sbom_status = results.get("sbom", {}).get("status", "skipped")
+    if sbom_status in ("skipped",):
+        gate_passed = False
+        reasons.append("SBOM not generated")
+
+    # CUI markings (warning only)
+    cui = results.get("cui_markings", {})
+    missing = cui.get("missing_markings", 0)
+    if missing > 0:
+        logger.warning(f"CUI markings missing on {missing} files (non-blocking)")
+
+    if gate_passed:
+        logger.info("Compliance gate: PASSED")
+    else:
+        logger.error(f"Compliance gate: FAILED — {'; '.join(reasons)}")
+
+    return gate_passed
+
+
+def main():
+    """Main entry point."""
+    if len(sys.argv) < 3:
+        print("CUI // SP-CTI")
+        print("Usage: python tools/ci/workflows/icdev_comply.py <issue-number> <run-id>")
+        print("\nGenerates compliance artifacts: SSP, POAM, STIG, SBOM, CUI markings")
+        sys.exit(1)
+
+    issue_number = sys.argv[1]
+    run_id = sys.argv[2]
+
+    print("CUI // SP-CTI")
+    print(f"SPARKPILOT Comply — run_id: {run_id}, issue: #{issue_number}")
+
+    # Set up
+    logger = setup_logger(run_id, "icdev_comply")
+    state = ICDevState.load(run_id)
+
+    # Run compliance artifacts
+    results = run_compliance_artifacts(run_id, issue_number, logger)
+
+    # Commit artifacts
+    logger.info("Committing compliance artifacts...")
+    commit_msg = f"icdev_compliance: comply: Generate ATO artifacts (run {run_id})"
+    success, error = commit_changes(commit_msg)
+    if not success:
+        logger.warning(f"Commit failed (may be no changes): {error}")
+
+    # Post results to issue
+    try:
+        platform = state.get("platform", "github")
+        vcs = VCS(platform=platform)
+
+        gate_emoji = "PASS" if results["gate_passed"] else "FAIL"
+        summary = (
+            f"{BOT_IDENTIFIER} **Compliance Phase** — {gate_emoji}\n\n"
+            f"| Artifact | Status |\n"
+            f"|----------|--------|\n"
+            f"| SSP | {results['ssp']['status']} |\n"
+            f"| POAM | {results['poam']['status']} |\n"
+            f"| STIG | {results['stig']['status']} |\n"
+            f"| SBOM | {results['sbom']['status']} |\n"
+            f"| CUI Markings | {results['cui_markings']['status']} |\n\n"
+            f"Run ID: `{run_id}`"
+        )
+
+        if results.get("errors"):
+            summary += "\n\nErrors:\n" + "\n".join(
+                f"- {e}" for e in results["errors"]
+            )
+
+        vcs.comment_on_issue(int(issue_number), summary)
+    except Exception as e:
+        logger.warning(f"Failed to post compliance results: {e}")
+
+    # Exit code
+    if not results["gate_passed"]:
+        logger.error("Compliance gate FAILED — pipeline blocked")
+        sys.exit(1)
+
+    logger.info("Compliance phase completed successfully")
+
+
+if __name__ == "__main__":
+    main()

tools/ci/workflows/icdev_document.py

@@ -0,0 +1,152 @@
+# [TEMPLATE: CUI // SP-CTI]
+# SPARKPILOT Document — Documentation generation workflow
+# Adapted from ADW adw_document.py with dual platform support
+
+"""
+SPARKPILOT Document — Generate documentation for implemented features.
+
+Usage:
+    python tools/ci/workflows/icdev_document.py <issue-number> <run-id>
+
+Requires:
+    - run-id from a previous workflow run
+    - Branch name and plan file in state
+
+Workflow:
+    1. Load state from previous phase
+    2. Check for changes against main branch
+    3. Generate feature documentation via Claude Code
+    4. Commit documentation
+    5. Push and update PR/MR
+"""
+
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+from tools.ci.modules.state import ICDevState
+from tools.ci.modules.git_ops import create_branch, commit_changes, finalize_git_operations
+from tools.ci.modules.vcs import VCS
+from tools.ci.modules.agent import execute_template
+from tools.ci.modules.workflow_ops import (
+    format_issue_message,
+)
+from tools.testing.data_types import AgentTemplateRequest
+from tools.testing.utils import setup_logger
+
+AGENT_DOCUMENTER = "icdev_documenter"
+
+
+def check_for_changes(logger: logging.Logger) -> bool:
+    """Check if there are changes between current branch and main."""
+    try:
+        result = subprocess.run(
+            ["git", "diff", "origin/main", "--stat"],
+            capture_output=True, text=True,
+            cwd=str(PROJECT_ROOT),
+        )
+        has_changes = bool(result.stdout.strip())
+        if not has_changes:
+            logger.info("No changes detected against origin/main")
+        return has_changes
+    except Exception:
+        return True  # Assume changes if check fails
+
+
+def main():
+    """Main entry point."""
+    if len(sys.argv) < 3:
+        print("Usage: python tools/ci/workflows/icdev_document.py <issue-number> <run-id>")
+        print("\nRequires run-id from a previous workflow run.")
+        sys.exit(1)
+
+    issue_number = sys.argv[1]
+    run_id = sys.argv[2]
+
+    state = ICDevState.load(run_id)
+    logger = setup_logger(run_id, "icdev_document")
+    logger.info(f"SPARKPILOT Document starting — run_id: {run_id}, issue: #{issue_number}")
+
+    try:
+        vcs = VCS()
+    except ValueError as e:
+        logger.error(f"VCS initialization failed: {e}")
+        sys.exit(1)
+
+    # Get branch from state
+    branch_name = state.get("branch_name")
+    if not branch_name:
+        logger.error("No branch_name in state. Run icdev_plan first.")
+        sys.exit(1)
+
+    # Checkout branch
+    success, _ = create_branch(branch_name)
+    if not success:
+        logger.error(f"Failed to checkout branch: {branch_name}")
+        sys.exit(1)
+
+    # Check for changes
+    if not check_for_changes(logger):
+        vcs.comment_on_issue(
+            int(issue_number),
+            format_issue_message(run_id, "ops", "No changes to document — skipping"),
+        )
+        logger.info("No changes to document")
+        state.save("icdev_document")
+        return
+
+    vcs.comment_on_issue(
+        int(issue_number),
+        format_issue_message(run_id, AGENT_DOCUMENTER, "Generating documentation"),
+    )
+
+    # Generate documentation
+    spec_path = state.get("plan_file", "")
+
+    request = AgentTemplateRequest(
+        agent_name=AGENT_DOCUMENTER,
+        slash_command="/document",
+        args=[run_id, spec_path],
+        run_id=run_id,
+    )
+
+    response = execute_template(request)
+
+    if response.success:
+        doc_path = response.output.strip()
+        logger.info(f"Documentation generated: {doc_path}")
+
+        # Commit
+        commit_msg = f"{AGENT_DOCUMENTER}: document feature for issue #{issue_number}"
+        success, error = commit_changes(commit_msg)
+        if success:
+            finalize_git_operations(state, logger, vcs)
+
+        vcs.comment_on_issue(
+            int(issue_number),
+            format_issue_message(
+                run_id, AGENT_DOCUMENTER,
+                f"Documentation created at `{doc_path}` and committed"
+            ),
+        )
+    else:
+        logger.error(f"Documentation generation failed: {response.output}")
+        vcs.comment_on_issue(
+            int(issue_number),
+            format_issue_message(
+                run_id, AGENT_DOCUMENTER,
+                f"Documentation generation failed: {response.output[:500]}"
+            ),
+        )
+        sys.exit(1)
+
+    state.save("icdev_document")
+    logger.info("Documentation phase completed")
+
+
+if __name__ == "__main__":
+    main()

tools/ci/workflows/icdev_e2e.py

@@ -0,0 +1,188 @@
+# [TEMPLATE: CUI // SP-CTI]
+# SPARKPILOT E2E — Playwright browser verification workflow
+
+"""
+SPARKPILOT E2E — Run Playwright E2E tests with screenshot validation.
+
+Usage:
+    python tools/ci/workflows/icdev_e2e.py <issue-number> <run-id>
+
+Workflow:
+    1. Start dashboard if not running
+    2. Discover E2E test specs in .claude/commands/e2e/
+    3. Run all E2E tests via e2e_runner.py
+    4. Validate screenshots with vision model
+    5. Report results
+"""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+
+
+def check_dashboard_running(base_url: str = "http://localhost:5000") -> bool:
+    """Check if the dashboard is reachable."""
+    try:
+        import urllib.request
+        req = urllib.request.Request(f"{base_url}/", method="HEAD")
+        urllib.request.urlopen(req, timeout=5)
+        return True
+    except Exception:
+        return False
+
+
+def start_dashboard() -> subprocess.Popen | None:
+    """Start the dashboard in the background."""
+    app_path = PROJECT_ROOT / "tools" / "dashboard" / "app.py"
+    if not app_path.exists():
+        return None
+
+    proc = subprocess.Popen(
+        [sys.executable, str(app_path)],
+        cwd=str(PROJECT_ROOT),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        stdin=subprocess.DEVNULL,
+    )
+
+    # Wait for dashboard to be ready (max 15 seconds)
+    import time
+    for _ in range(30):
+        time.sleep(0.5)
+        if check_dashboard_running():
+            return proc
+
+    print("WARNING: Dashboard did not start within 15 seconds")
+    proc.terminate()
+    return None
+
+
+def run_e2e_tests(validate_screenshots: bool = True) -> dict:
+    """Run E2E tests via the e2e_runner."""
+    runner_path = PROJECT_ROOT / "tools" / "testing" / "e2e_runner.py"
+    if not runner_path.exists():
+        return {"status": "skipped", "reason": "e2e_runner.py not found"}
+
+    cmd = [sys.executable, str(runner_path), "--run-all"]
+    if validate_screenshots:
+        cmd.append("--validate-screenshots")
+
+    result = subprocess.run(
+        cmd, cwd=str(PROJECT_ROOT),
+        capture_output=True, text=True, timeout=600,
+        stdin=subprocess.DEVNULL,
+    )
+
+    # Try to parse JSON output
+    try:
+        output = json.loads(result.stdout)
+    except (json.JSONDecodeError, ValueError):
+        output = {
+            "stdout": result.stdout[-2000:] if result.stdout else "",
+            "stderr": result.stderr[-2000:] if result.stderr else "",
+        }
+
+    return {
+        "status": "passed" if result.returncode == 0 else "failed",
+        "returncode": result.returncode,
+        "output": output,
+    }
+
+
+def discover_e2e_specs() -> list:
+    """List available E2E test spec files."""
+    e2e_dir = PROJECT_ROOT / ".claude" / "commands" / "e2e"
+    if not e2e_dir.exists():
+        return []
+    return sorted(e2e_dir.glob("*.md"))
+
+
+def main():
+    """Main entry point."""
+    if len(sys.argv) < 2:
+        print("CUI // SP-CTI")
+        print("Usage: python tools/ci/workflows/icdev_e2e.py <issue-number> [run-id]")
+        print("\nRuns Playwright E2E browser verification:")
+        print("  1. Start dashboard (if not running)")
+        print("  2. Discover E2E specs in .claude/commands/e2e/")
+        print("  3. Run all E2E tests with screenshot validation")
+        print("  4. Report results")
+        sys.exit(1)
+
+    issue_number = sys.argv[1]
+    run_id = sys.argv[2] if len(sys.argv) > 2 and not sys.argv[2].startswith("--") else "unknown"
+
+    print("CUI // SP-CTI")
+    print(f"SPARKPILOT E2E — run_id: {run_id}, issue: #{issue_number}")
+    print()
+
+    # Step 1: Discover E2E specs
+    specs = discover_e2e_specs()
+    print(f"E2E test specs found: {len(specs)}")
+    for spec in specs:
+        print(f"  - {spec.name}")
+
+    if not specs:
+        print("\nNo E2E test specs found in .claude/commands/e2e/")
+        print("E2E phase: SKIPPED (no specs)")
+        sys.exit(0)
+
+    # Step 2: Check/start dashboard
+    dashboard_proc = None
+    if not check_dashboard_running():
+        print("\nDashboard not running — starting...")
+        dashboard_proc = start_dashboard()
+        if dashboard_proc:
+            print("Dashboard started (PID: {})".format(dashboard_proc.pid))
+        else:
+            print("WARNING: Could not start dashboard — E2E tests may fail")
+    else:
+        print("\nDashboard already running")
+
+    # Step 3: Run E2E tests
+    print(f"\n{'='*40}")
+    print("  Running E2E Tests")
+    print(f"{'='*40}")
+
+    try:
+        results = run_e2e_tests(validate_screenshots=True)
+    except subprocess.TimeoutExpired:
+        results = {"status": "failed", "reason": "E2E tests timed out (600s)"}
+
+    # Step 4: Report results
+    print(f"\nE2E Result: {results['status'].upper()}")
+    if results.get("output") and isinstance(results["output"], dict):
+        if "total" in results["output"]:
+            print(f"  Total: {results['output'].get('total', 'N/A')}")
+            print(f"  Passed: {results['output'].get('passed', 'N/A')}")
+            print(f"  Failed: {results['output'].get('failed', 'N/A')}")
+
+    # Cleanup
+    if dashboard_proc:
+        print("\nStopping dashboard...")
+        dashboard_proc.terminate()
+        try:
+            dashboard_proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            dashboard_proc.kill()
+
+    # Exit code
+    if results["status"] == "passed":
+        print("\nE2E phase: PASSED")
+        sys.exit(0)
+    elif results["status"] == "skipped":
+        print(f"\nE2E phase: SKIPPED ({results.get('reason', 'unknown')})")
+        sys.exit(0)
+    else:
+        print("\nE2E phase: FAILED — review screenshots before merge")
+        # Non-blocking: exit 0 with warning (SDLC logs warning but continues)
+        # Change to sys.exit(1) to make E2E failures blocking
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()