icdev 0.0.3__py3-none-any.whl

tools/ci/pipeline_config_generator.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Generate CI/CD pipeline configs from sparkpilot.yaml (D192).
+
+Reads the project manifest and generates platform-specific CI/CD config
+files (GitHub Actions or GitLab CI) using a declarative CHECK_REGISTRY.
+Add new checks without code changes.
+
+Different from tools/infra/pipeline_generator.py (DB-driven, 830 lines,
+generates 9-stage GitLab CI for the SPARKPILOT framework itself). This generates
+lightweight customer project pipelines from declarative sparkpilot.yaml.
+
+Usage:
+    python tools/ci/pipeline_config_generator.py --dir /path --platform auto --dry-run --json
+    python tools/ci/pipeline_config_generator.py --dir /path --platform github --write
+    python tools/ci/pipeline_config_generator.py --dir /path --platform gitlab --write --json
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+sys.path.insert(0, str(BASE_DIR))
+from tools.project.manifest_loader import load_manifest, detect_vcs_platform
+
+
+# ── CUI Header ──────────────────────────────────────────────────────────
+
+CUI_HEADER = "# CUI // SP-CTI — Auto-generated by SPARKPILOT pipeline_config_generator"
+
+# ── Declarative Check Registry (D192) ───────────────────────────────────
+
+CHECK_REGISTRY = {
+    "sast": {
+        "name": "SAST Scan",
+        "command": 'python tools/security/sast_runner.py --project-dir . --json > .tmp/sast.json',
+        "stage": "security",
+        "artifact": ".tmp/sast.json",
+    },
+    "dependency_audit": {
+        "name": "Dependency Audit",
+        "command": 'python tools/security/dependency_auditor.py --project-dir . --json > .tmp/deps.json',
+        "stage": "security",
+        "artifact": ".tmp/deps.json",
+    },
+    "secret_detection": {
+        "name": "Secret Detection",
+        "command": 'python tools/security/secret_detector.py --project-dir . --json > .tmp/secrets.json',
+        "stage": "security",
+        "artifact": ".tmp/secrets.json",
+    },
+    "cui_check": {
+        "name": "CUI Marking Validation",
+        "command": 'python tools/compliance/cui_marker.py --validate --project-dir . --json > .tmp/cui.json',
+        "stage": "compliance",
+        "artifact": ".tmp/cui.json",
+    },
+    "stig_check": {
+        "name": "STIG Compliance Check",
+        "command": 'python tools/compliance/stig_checker.py --project-id "$PROJECT_ID" --json > .tmp/stig.json',
+        "stage": "compliance",
+        "artifact": ".tmp/stig.json",
+    },
+    "unit_tests": {
+        "name": "Unit Tests",
+        "command": "pytest tests/ -v --tb=short --junitxml=.tmp/test-results.xml",
+        "stage": "test",
+        "artifact": ".tmp/test-results.xml",
+    },
+    "bdd_tests": {
+        "name": "BDD Tests",
+        "command": "behave features/ --format json -o .tmp/bdd-results.json || true",
+        "stage": "test",
+        "artifact": ".tmp/bdd-results.json",
+    },
+    "lint": {
+        "name": "Lint",
+        "command": "ruff check .",
+        "stage": "test",
+        "artifact": None,
+    },
+    "format_check": {
+        "name": "Format Check",
+        "command": "ruff format --check .",
+        "stage": "test",
+        "artifact": None,
+    },
+    "ssp_generate": {
+        "name": "Generate SSP",
+        "command": 'python tools/compliance/ssp_generator.py --project-id "$PROJECT_ID" --json',
+        "stage": "artifacts",
+        "artifact": None,
+    },
+    "sbom_generate": {
+        "name": "Generate SBOM",
+        "command": "python tools/compliance/sbom_generator.py --project-dir . --json",
+        "stage": "artifacts",
+        "artifact": None,
+    },
+    "cato_refresh": {
+        "name": "cATO Evidence Refresh",
+        "command": 'python tools/compliance/cato_monitor.py --project-id "$PROJECT_ID" --check-freshness',
+        "stage": "artifacts",
+        "artifact": None,
+    },
+    "cve_triage": {
+        "name": "CVE Triage",
+        "command": 'python tools/supply_chain/cve_triager.py --project-id "$PROJECT_ID" --sla-check --json',
+        "stage": "security",
+        "artifact": None,
+    },
+    # Platform One / Iron Bank stages (D350)
+    "ironbank_generate": {
+        "name": "Iron Bank Hardening Manifest",
+        "command": 'python tools/infra/ironbank_metadata_generator.py --project-id "$PROJECT_ID" --generate --output-dir .tmp/ironbank --json',
+        "stage": "artifacts",
+        "artifact": ".tmp/ironbank/hardening_manifest.yaml",
+    },
+    "ironbank_validate": {
+        "name": "Iron Bank Manifest Validation",
+        "command": 'python tools/infra/ironbank_metadata_generator.py --project-id "$PROJECT_ID" --validate --manifest-path .tmp/ironbank/hardening_manifest.yaml --json',
+        "stage": "artifacts",
+        "artifact": None,
+    },
+}
+
+# Stage ordering
+STAGE_ORDER = ["security", "compliance", "test", "artifacts"]
+
+
+# ── Core API ────────────────────────────────────────────────────────────
+
+def generate_pipeline(
+    directory: str = None,
+    platform: str = "auto",
+    write: bool = False,
+    dry_run: bool = False,
+) -> dict:
+    """Generate CI/CD pipeline config from sparkpilot.yaml.
+
+    Args:
+        directory: Project directory containing sparkpilot.yaml.
+        platform: 'auto', 'github', or 'gitlab'.
+        write: If True, write config file to disk.
+        dry_run: If True, generate but don't write (overrides write).
+
+    Returns:
+        dict with platform, output_path, content, checks_enabled, gate_config, written.
+    """
+    cwd = Path(directory) if directory else Path.cwd()
+
+    result = {
+        "platform": None,
+        "output_path": None,
+        "content": "",
+        "checks_enabled": {"on_pr": [], "on_merge": []},
+        "gate_config": {},
+        "written": False,
+        "errors": [],
+    }
+
+    # Load manifest
+    manifest = load_manifest(directory=str(cwd))
+    if not manifest["valid"] and not manifest["raw"]:
+        result["errors"].append("No valid sparkpilot.yaml found")
+        return result
+
+    config = manifest["normalized"]
+
+    # Detect platform
+    if platform == "auto":
+        yaml_platform = config.get("pipeline", {}).get("platform", "auto")
+        if yaml_platform != "auto":
+            platform = yaml_platform
+        else:
+            platform = detect_vcs_platform(str(cwd))
+            if platform == "unknown":
+                platform = "gitlab"  # Default
+    result["platform"] = platform
+
+    # Extract pipeline config
+    pipeline = config.get("pipeline", {})
+    on_pr = pipeline.get("on_pr", [])
+    on_merge = pipeline.get("on_merge", [])
+    on_schedule = pipeline.get("on_schedule", [])
+    gates = pipeline.get("gates", {})
+
+    result["checks_enabled"]["on_pr"] = on_pr
+    result["checks_enabled"]["on_merge"] = on_merge
+    result["gate_config"] = gates
+
+    # Generate platform-specific config
+    project_name = config.get("project", {}).get("name", "sparkpilot-project")
+    project_id = config.get("project", {}).get("id", "proj-unknown")
+
+    if platform == "github":
+        content = _generate_github_workflow(
+            project_name, project_id, on_pr, on_merge, on_schedule, gates, config
+        )
+        output_path = cwd / ".github" / "workflows" / "sparkpilot.yml"
+    else:
+        content = _generate_gitlab_ci(
+            project_name, project_id, on_pr, on_merge, on_schedule, gates, config
+        )
+        output_path = cwd / ".gitlab-ci.yml"
+
+    result["content"] = content
+    result["output_path"] = str(output_path)
+
+    # Write if requested and not dry-run
+    if write and not dry_run:
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(content, encoding="utf-8")
+        result["written"] = True
+
+    return result
+
+
+# ── GitHub Actions Generator ────────────────────────────────────────────
+
+def _generate_github_workflow(
+    project_name: str,
+    project_id: str,
+    on_pr: list,
+    on_merge: list,
+    on_schedule: list,
+    gates: dict,
+    config: dict,
+) -> str:
+    """Generate GitHub Actions workflow YAML."""
+    lines = [CUI_HEADER, ""]
+    lines.append(f"name: SPARKPILOT Compliance Pipeline — {project_name}")
+    lines.append("")
+    lines.append("on:")
+    lines.append("  pull_request:")
+    lines.append("    branches: [main, master]")
+    lines.append("  push:")
+    lines.append("    branches: [main, master]")
+
+    if on_schedule:
+        lines.append("  schedule:")
+        lines.append("    - cron: '0 6 * * *'  # Daily at 6 AM UTC")
+
+    lines.append("")
+    lines.append("env:")
+    lines.append(f'  PROJECT_ID: "{project_id}"')
+    lines.append("")
+
+    # PR checks job
+    if on_pr:
+        lines.append("jobs:")
+        lines.append("  sparkpilot-pr-checks:")
+        lines.append("    if: github.event_name == 'pull_request'")
+        lines.append("    runs-on: ubuntu-latest")
+        lines.append("    steps:")
+        lines.append("      - uses: actions/checkout@v4")
+        lines.append("")
+        lines.append("      - name: Set up Python")
+        lines.append("        uses: actions/setup-python@v5")
+        lines.append("        with:")
+        lines.append("          python-version: '3.11'")
+        lines.append("")
+        lines.append("      - name: Install SPARKPILOT dependencies")
+        lines.append("        run: pip install -r requirements.txt")
+        lines.append("")
+        lines.append("      - name: Initialize SPARKPILOT database")
+        lines.append("        run: python tools/db/init_icdev_db.py")
+        lines.append("")
+        lines.append("      - name: Create temp directory")
+        lines.append("        run: mkdir -p .tmp")
+        lines.append("")
+
+        for check_name in on_pr:
+            check = CHECK_REGISTRY.get(check_name)
+            if check:
+                lines.append(f"      - name: {check['name']}")
+                lines.append(f"        run: {check['command']}")
+                lines.append("")
+
+        # Gate evaluation
+        gate_script = _build_gate_evaluation_script(gates, "github")
+        if gate_script:
+            lines.append("      - name: Gate Evaluation")
+            lines.append("        run: |")
+            for gl in gate_script.split("\n"):
+                lines.append(f"          {gl}")
+            lines.append("")
+
+    # Merge artifacts job
+    if on_merge:
+        if not on_pr:
+            lines.append("jobs:")
+        lines.append("  sparkpilot-merge-artifacts:")
+        lines.append("    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')")
+        lines.append("    runs-on: ubuntu-latest")
+        lines.append("    steps:")
+        lines.append("      - uses: actions/checkout@v4")
+        lines.append("")
+        lines.append("      - name: Set up Python")
+        lines.append("        uses: actions/setup-python@v5")
+        lines.append("        with:")
+        lines.append("          python-version: '3.11'")
+        lines.append("")
+        lines.append("      - name: Install SPARKPILOT dependencies")
+        lines.append("        run: pip install -r requirements.txt")
+        lines.append("")
+        lines.append("      - name: Initialize SPARKPILOT")
+        lines.append("        run: python tools/db/init_icdev_db.py")
+        lines.append("")
+
+        for check_name in on_merge:
+            check = CHECK_REGISTRY.get(check_name)
+            if check:
+                lines.append(f"      - name: {check['name']}")
+                lines.append(f"        run: {check['command']}")
+                lines.append("")
+
+        lines.append("      - name: Upload Compliance Artifacts")
+        lines.append("        uses: actions/upload-artifact@v4")
+        lines.append("        with:")
+        lines.append("          name: compliance-artifacts")
+        lines.append("          path: artifacts/")
+        lines.append("")
+
+    # Scheduled job
+    if on_schedule:
+        lines.append("  sparkpilot-scheduled:")
+        lines.append("    if: github.event_name == 'schedule'")
+        lines.append("    runs-on: ubuntu-latest")
+        lines.append("    steps:")
+        lines.append("      - uses: actions/checkout@v4")
+        lines.append("")
+        lines.append("      - name: Set up Python")
+        lines.append("        uses: actions/setup-python@v5")
+        lines.append("        with:")
+        lines.append("          python-version: '3.11'")
+        lines.append("")
+        lines.append("      - name: Install SPARKPILOT dependencies")
+        lines.append("        run: pip install -r requirements.txt")
+        lines.append("")
+
+        for check_name in on_schedule:
+            check = CHECK_REGISTRY.get(check_name)
+            if check:
+                lines.append(f"      - name: {check['name']}")
+                lines.append(f"        run: {check['command']}")
+                lines.append("")
+
+    return "\n".join(lines)
+
+
+# ── GitLab CI Generator ─────────────────────────────────────────────────
+
+def _generate_gitlab_ci(
+    project_name: str,
+    project_id: str,
+    on_pr: list,
+    on_merge: list,
+    on_schedule: list,
+    gates: dict,
+    config: dict,
+) -> str:
+    """Generate GitLab CI YAML."""
+    lines = [CUI_HEADER, ""]
+
+    # Determine stages needed
+    stages_needed = set()
+    for check_name in on_pr + on_merge + on_schedule:
+        check = CHECK_REGISTRY.get(check_name)
+        if check:
+            stages_needed.add(check["stage"])
+    stages = [s for s in STAGE_ORDER if s in stages_needed]
+    if on_merge:
+        if "artifacts" not in stages:
+            stages.append("artifacts")
+
+    lines.append("stages:")
+    for s in stages:
+        lines.append(f"  - {s}")
+    lines.append("")
+
+    lines.append("variables:")
+    lines.append(f'  PROJECT_ID: "{project_id}"')
+    lines.append('  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"')
+    lines.append("")
+
+    lines.append("cache:")
+    lines.append("  paths:")
+    lines.append("    - .cache/pip/")
+    lines.append("")
+
+    # Setup anchor
+    lines.append(".sparkpilot-setup: &sparkpilot-setup")
+    lines.append("  image: python:3.11-slim")
+    lines.append("  before_script:")
+    lines.append("    - pip install -r requirements.txt")
+    lines.append("    - python tools/db/init_icdev_db.py")
+    lines.append("    - mkdir -p .tmp")
+    lines.append("")
+
+    # PR checks (all stages)
+    for check_name in on_pr:
+        check = CHECK_REGISTRY.get(check_name)
+        if not check:
+            continue
+        job_name = check_name.replace("_", "-")
+        lines.append(f"{check_name}:")
+        lines.append("  <<: *sparkpilot-setup")
+        lines.append(f"  stage: {check['stage']}")
+        lines.append("  script:")
+        lines.append(f"    - {check['command']}")
+
+        if check.get("artifact"):
+            lines.append("  artifacts:")
+            lines.append("    paths:")
+            lines.append(f"      - {check['artifact']}")
+
+        lines.append("")
+
+    # Gate evaluation (as separate job if gates exist)
+    gate_script = _build_gate_evaluation_script(gates, "gitlab")
+    if gate_script and on_pr:
+        lines.append("gate-evaluation:")
+        lines.append("  <<: *sparkpilot-setup")
+        lines.append(f"  stage: {stages[-1] if stages else 'test'}")
+        lines.append("  script:")
+        lines.append("    - |")
+        for gl in gate_script.split("\n"):
+            lines.append(f"      {gl}")
+        lines.append("  needs:")
+        # Depend on compliance/security checks
+        for check_name in on_pr:
+            check = CHECK_REGISTRY.get(check_name)
+            if check and check["stage"] in ("security", "compliance"):
+                lines.append(f"    - {check_name}")
+        lines.append("")
+
+    # Merge artifacts
+    for check_name in on_merge:
+        check = CHECK_REGISTRY.get(check_name)
+        if not check:
+            continue
+        lines.append(f"{check_name}:")
+        lines.append("  <<: *sparkpilot-setup")
+        lines.append(f"  stage: {check['stage']}")
+        lines.append("  script:")
+        lines.append(f"    - {check['command']}")
+        lines.append("  only:")
+        lines.append("    - main")
+        lines.append("    - master")
+
+        if check.get("artifact"):
+            lines.append("  artifacts:")
+            lines.append("    paths:")
+            lines.append(f"      - {check['artifact']}")
+
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+# ── Gate Evaluation Script ──────────────────────────────────────────────
+
+def _build_gate_evaluation_script(gates: dict, platform: str) -> str:
+    """Build an inline Python script to evaluate pipeline gates."""
+    if not gates:
+        return ""
+
+    checks = []
+    stig_max_cat1 = gates.get("stig_max_cat1", 0)
+    stig_max_cat2 = gates.get("stig_max_cat2", 0)
+    min_coverage = gates.get("min_coverage", 80)
+    max_critical_vulns = gates.get("max_critical_vulns", 0)
+
+    script_lines = [
+        "python -c \"",
+        "import json, sys, os",
+        "errors = []",
+    ]
+
+    # STIG gate
+    script_lines.append("try:")
+    script_lines.append("    stig = json.load(open('.tmp/stig.json'))")
+    script_lines.append(f"    cat1 = stig.get('summary', {{}}).get('cat1_count', 0)")
+    script_lines.append(f"    if cat1 > {stig_max_cat1}:")
+    script_lines.append(f"        errors.append(f'STIG: {{cat1}} CAT1 findings (max {stig_max_cat1})')")
+    if stig_max_cat2 > 0:
+        script_lines.append(f"    cat2 = stig.get('summary', {{}}).get('cat2_count', 0)")
+        script_lines.append(f"    if cat2 > {stig_max_cat2}:")
+        script_lines.append(f"        errors.append(f'STIG: {{cat2}} CAT2 findings (max {stig_max_cat2})')")
+    script_lines.append("except FileNotFoundError: pass")
+
+    # Vulnerability gate
+    script_lines.append("try:")
+    script_lines.append("    sast = json.load(open('.tmp/sast.json'))")
+    script_lines.append(f"    critical = sast.get('summary', {{}}).get('critical', 0)")
+    script_lines.append(f"    if critical > {max_critical_vulns}:")
+    script_lines.append(f"        errors.append(f'SAST: {{critical}} critical findings (max {max_critical_vulns})')")
+    script_lines.append("except FileNotFoundError: pass")
+
+    # Result
+    script_lines.append("if errors:")
+    script_lines.append("    for e in errors: print(f'BLOCKED: {e}')")
+    script_lines.append("    sys.exit(1)")
+    script_lines.append("print('All gates passed')")
+    script_lines.append('"')
+
+    return "\n".join(script_lines)
+
+
+# ── CLI ─────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate CI/CD pipeline configs from sparkpilot.yaml (D192)"
+    )
+    parser.add_argument("--dir", help="Project directory containing sparkpilot.yaml")
+    parser.add_argument("--platform", choices=["auto", "github", "gitlab"],
+                        default="auto", help="Target CI/CD platform")
+    parser.add_argument("--write", action="store_true",
+                        help="Write config file to disk")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Generate but don't write (preview)")
+    parser.add_argument("--json", action="store_true",
+                        help="Output JSON result")
+    args = parser.parse_args()
+
+    result = generate_pipeline(
+        directory=args.dir,
+        platform=args.platform,
+        write=args.write,
+        dry_run=args.dry_run,
+    )
+
+    if args.json:
+        print(json.dumps(result, indent=2, default=str))
+    else:
+        if result["errors"]:
+            for err in result["errors"]:
+                print(f"ERROR: {err}")
+            sys.exit(1)
+
+        if result["written"]:
+            print(f"Written: {result['output_path']}")
+        elif args.dry_run:
+            print(f"# Platform: {result['platform']}")
+            print(f"# Output path: {result['output_path']}")
+            print()
+            print(result["content"])
+        else:
+            print(result["content"])
+
+
+if __name__ == "__main__":
+    main()