cherry-docs 0.2.0__py3-none-any.whl

scripts/pr_gate_report.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+"""Render a markdown + JSON PR gate report from CI artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Any
+
+
+def _load_json(path: Path) -> Any:
+    return json.loads(path.read_text()) if path.exists() else None
+
+
+def _job_result(name: str) -> str:
+    return os.getenv(name, "missing").strip() or "missing"
+
+
+def _summarize_coverage(path: Path) -> dict[str, Any] | None:
+    if not path.exists():
+        return None
+    root = ET.fromstring(path.read_text())
+    return {
+        "line_rate": round(float(root.attrib.get("line-rate", 0.0)) * 100, 2),
+        "branch_rate": round(float(root.attrib.get("branch-rate", 0.0)) * 100, 2),
+        "lines_covered": int(root.attrib.get("lines-covered", 0)),
+        "lines_valid": int(root.attrib.get("lines-valid", 0)),
+    }
+
+
+def _summarize_coverage_regression(path: Path) -> dict[str, Any] | None:
+    data = _load_json(path)
+    if not data:
+        return None
+    return {
+        "baseline_line_rate": data.get("baseline_line_rate"),
+        "candidate_line_rate": data.get("candidate_line_rate"),
+        "line_rate_drop": data.get("line_rate_drop"),
+        "baseline_branch_rate": data.get("baseline_branch_rate"),
+        "candidate_branch_rate": data.get("candidate_branch_rate"),
+        "branch_rate_drop": data.get("branch_rate_drop"),
+        "errors": data.get("errors") or [],
+    }
+
+
+def _summarize_jscpd(path: Path) -> dict[str, Any] | None:
+    data = _load_json(path)
+    if not data:
+        return None
+    total = data.get("statistics", {}).get("total", {})
+    return {
+        "files": total.get("sources"),
+        "clones": total.get("clones"),
+        "percentage": round(float(total.get("percentage", 0.0)), 2),
+        "duplicated_lines": total.get("duplicatedLines"),
+    }
+
+
+def _summarize_bandit(path: Path) -> dict[str, Any] | None:
+    data = _load_json(path)
+    if not data:
+        return None
+    results = data.get("results", [])
+    metrics = data.get("metrics", {}).get("_totals", {})
+    return {
+        "findings": len(results),
+        "high": metrics.get("SEVERITY.HIGH", 0),
+        "medium": metrics.get("SEVERITY.MEDIUM", 0),
+        "low": metrics.get("SEVERITY.LOW", 0),
+    }
+
+
+def _summarize_pip_audit(path: Path) -> dict[str, Any] | None:
+    data = _load_json(path)
+    if data is None:
+        return None
+    dependencies = data.get("dependencies", []) if isinstance(data, dict) else data
+    vulns = 0
+    affected = 0
+    for dep in dependencies or []:
+        vulnerabilities = dep.get("vulns") or dep.get("vulnerabilities") or []
+        if vulnerabilities:
+            affected += 1
+            vulns += len(vulnerabilities)
+    return {"affected_dependencies": affected, "vulnerabilities": vulns}
+
+
+def _summarize_gitleaks(path: Path) -> dict[str, Any] | None:
+    data = _load_json(path)
+    if data is None:
+        return None
+    return {"findings": len(data)}
+
+
+def _summarize_semgrep(path: Path) -> dict[str, Any] | None:
+    data = _load_json(path)
+    if not data:
+        return None
+    results = data.get("results", [])
+    severities = {"ERROR": 0, "WARNING": 0, "INFO": 0}
+    for finding in results:
+        severity = ((finding.get("extra") or {}).get("severity") or "INFO").upper()
+        severities[severity] = severities.get(severity, 0) + 1
+    return {
+        "findings": len(results),
+        "errors": len(data.get("errors", [])),
+        "error": severities.get("ERROR", 0),
+        "warning": severities.get("WARNING", 0),
+        "info": severities.get("INFO", 0),
+    }
+
+
+def _summarize_promptfoo(root: Path) -> dict[str, Any] | None:
+    combined = _load_json(root / "combined-summary.json")
+    if not combined:
+        return None
+    reports = {}
+    failing_cases: list[dict[str, Any]] = []
+    for path in root.glob("*-regression.json"):
+        reports[path.name] = _load_json(path)
+    for suite_name, suite in (combined.get("suites") or {}).items():
+        suite_summary = suite.get("summary") or {}
+        for case in suite_summary.get("failing_cases") or []:
+            failing_cases.append({"suite": suite_name, **case})
+    return {
+        "overall_score": round(float(combined.get("overall_score", 0.0)), 3),
+        "passing_cases": combined.get("passing_cases", 0),
+        "failing_cases": combined.get("failing_cases", 0),
+        "suite_count": len(combined.get("suites", {})),
+        "regressions": {
+            name: len((report or {}).get("regressions", []))
+            for name, report in reports.items()
+        },
+        "regression_details": {
+            name: (report or {}).get("regressions", [])
+            for name, report in reports.items()
+        },
+        "failing_case_details": failing_cases,
+        "raw": combined,
+    }
+
+
+def _render_markdown(summary: dict[str, Any]) -> str:
+    lines = ["## CherryDocs PR Gate", ""]
+    lines.append("### Job results")
+    for label, value in summary["job_results"].items():
+        lines.append(f"- `{label}`: **{value}**")
+    lines.append("")
+
+    coverage = summary.get("coverage")
+    if coverage:
+        lines.append("### Tests / coverage")
+        lines.append(
+            f"- coverage: **{coverage['line_rate']:.2f}%** line, **{coverage['branch_rate']:.2f}%** branch"
+        )
+        lines.append(
+            f"- covered lines: `{coverage['lines_covered']}` / `{coverage['lines_valid']}`"
+        )
+        lines.append("")
+
+    coverage_regression = summary.get("coverage_regression")
+    if coverage_regression:
+        lines.append("### Coverage regression")
+        lines.append(
+            f"- baseline vs candidate: **{coverage_regression['baseline_line_rate']:.2f}%** -> **{coverage_regression['candidate_line_rate']:.2f}%** line"
+        )
+        lines.append(
+            f"- line delta: `{coverage_regression['line_rate_drop']:.2f}` points; branch delta: `{coverage_regression['branch_rate_drop']:.2f}` points"
+        )
+        if coverage_regression["errors"]:
+            lines.append("- blocking coverage errors:")
+            for error in coverage_regression["errors"]:
+                lines.append(f"  - {error}")
+        lines.append("")
+
+    promptfoo = summary.get("promptfoo")
+    if promptfoo:
+        lines.append("### Promptfoo")
+        lines.append(
+            f"- overall score: **{promptfoo['overall_score']:.3f}** across `{promptfoo['suite_count']}` suite(s)"
+        )
+        lines.append(
+            f"- cases: `{promptfoo['passing_cases']}` passing / `{promptfoo['failing_cases']}` failing"
+        )
+        if promptfoo["regressions"]:
+            for name, count in promptfoo["regressions"].items():
+                lines.append(f"- regressions `{name}`: `{count}`")
+        if promptfoo.get("failing_case_details"):
+            lines.append("- failing cases:")
+            for case in promptfoo["failing_case_details"][:5]:
+                reason = case.get("failure_reason") or "no reason captured"
+                lines.append(
+                    f"  - `{case.get('suite')}` / {case.get('description', 'unknown case')}: {reason}"
+                )
+        for name, regressions in (promptfoo.get("regression_details") or {}).items():
+            for regression in regressions[:5]:
+                lines.append(
+                    f"  - regression `{name}`: {regression.get('description', 'unknown case')}"
+                )
+        lines.append("")
+
+    duplication = summary.get("duplication")
+    if duplication:
+        lines.append("### Duplication")
+        lines.append(
+            f"- duplicated lines: **{duplication['percentage']:.2f}%** (`{duplication['duplicated_lines']}` lines across `{duplication['clones']}` clones)"
+        )
+        lines.append("")
+
+    security = summary.get("security")
+    if security:
+        lines.append("### Security")
+        bandit = security.get("bandit")
+        if bandit:
+            lines.append(
+                f"- bandit findings: `{bandit['findings']}` (high `{bandit['high']}`, medium `{bandit['medium']}`, low `{bandit['low']}`)"
+            )
+        pip_audit = security.get("pip_audit")
+        if pip_audit:
+            lines.append(
+                f"- pip-audit: `{pip_audit['affected_dependencies']}` affected dependencies / `{pip_audit['vulnerabilities']}` vulnerabilities"
+            )
+        gitleaks = security.get("gitleaks")
+        if gitleaks:
+            lines.append(f"- gitleaks findings: `{gitleaks['findings']}`")
+        semgrep = security.get("semgrep")
+        if semgrep:
+            lines.append(
+                f"- semgrep findings: `{semgrep['findings']}` (error `{semgrep['error']}`, warning `{semgrep['warning']}`, info `{semgrep['info']}`)"
+            )
+        lines.append("")
+
+    lines.append("_Generated by the canonical CherryDocs CI workflow._")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--artifacts-root", required=True)
+    parser.add_argument("--output-md", required=True)
+    parser.add_argument("--output-json", required=True)
+    args = parser.parse_args()
+
+    root = Path(args.artifacts_root)
+    summary = {
+        "job_results": {
+            "quality_fast": _job_result("QUALITY_FAST_RESULT"),
+            "quality_integration": _job_result("QUALITY_INTEGRATION_RESULT"),
+            "security_scan": _job_result("SECURITY_SCAN_RESULT"),
+            "typecheck_targeted": _job_result("TYPECHECK_TARGETED_RESULT"),
+            "duplication_check": _job_result("DUPLICATION_CHECK_RESULT"),
+            "browser_e2e": _job_result("BROWSER_E2E_RESULT"),
+            "safety_e2e": _job_result("SAFETY_E2E_RESULT"),
+            "mcp_parity": _job_result("MCP_PARITY_RESULT"),
+            "eval_pr_local": _job_result("EVAL_PR_LOCAL_RESULT"),
+        },
+        "coverage": _summarize_coverage(root / "coverage-report" / "coverage.xml"),
+        "coverage_regression": _summarize_coverage_regression(
+            root / "coverage-report" / "coverage-regression.json"
+        ),
+        "duplication": _summarize_jscpd(root / "jscpd-report" / "jscpd-report.json"),
+        "promptfoo": _summarize_promptfoo(root / "promptfoo-pr-gate-results"),
+        "security": {
+            "bandit": _summarize_bandit(root / "security-scan-report" / "bandit-report.json"),
+            "pip_audit": _summarize_pip_audit(root / "security-scan-report" / "pip-audit-report.json"),
+            "gitleaks": _summarize_gitleaks(root / "security-scan-report" / "gitleaks-report.json"),
+            "semgrep": _summarize_semgrep(root / "security-scan-report" / "semgrep-report.json"),
+        },
+    }
+
+    Path(args.output_json).write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
+    Path(args.output_md).write_text(_render_markdown(summary))
+    print(Path(args.output_md).read_text())
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/promptfoo_regression_gate.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""Summarize Promptfoo output and enforce a regression gate against a baseline."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+def _load_json(path: str) -> dict[str, Any]:
+    return json.loads(Path(path).read_text())
+
+
+def _normalize_eval(path: str) -> dict[str, Any]:
+    data = _load_json(path)
+    if "cases" in data and "overall_score" in data:
+        return data
+
+    rows = data.get("results", {}).get("results", [])
+    if not isinstance(rows, list):
+        raise ValueError(f"{path} does not look like Promptfoo output.")
+
+    cases: list[dict[str, Any]] = []
+    total_score = 0.0
+    for index, row in enumerate(rows):
+        test_case = row.get("testCase") or {}
+        description = (
+            test_case.get("description")
+            or row.get("description")
+            or f"case-{index}"
+        )
+        score = float(row.get("score") or 0.0)
+        success = bool(row.get("success"))
+        failure_reason = row.get("failureReason") or ""
+        cases.append(
+            {
+                "description": description,
+                "success": success,
+                "score": score,
+                "failure_reason": failure_reason,
+            }
+        )
+        total_score += score
+
+    overall_score = total_score / len(cases) if cases else 0.0
+    passing = sum(1 for case in cases if case["success"])
+    failing = len(cases) - passing
+    return {
+        "overall_score": overall_score,
+        "passing": passing,
+        "failing": failing,
+        "case_count": len(cases),
+        "cases": cases,
+    }
+
+
+def _write_json(path: str | None, payload: dict[str, Any]) -> None:
+    if not path:
+        return
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def _compare(
+    baseline: dict[str, Any],
+    candidate: dict[str, Any],
+    *,
+    min_score: float,
+    max_score_drop: float,
+) -> tuple[dict[str, Any], list[str]]:
+    errors: list[str] = []
+    baseline_cases = {case["description"]: case for case in baseline["cases"]}
+    candidate_cases = {case["description"]: case for case in candidate["cases"]}
+
+    overall_drop = float(baseline["overall_score"]) - float(candidate["overall_score"])
+    if float(candidate["overall_score"]) < min_score:
+        errors.append(
+            f"overall score {candidate['overall_score']:.3f} is below minimum {min_score:.3f}"
+        )
+    if overall_drop > max_score_drop:
+        errors.append(
+            f"overall score dropped by {overall_drop:.3f}, exceeding max drop {max_score_drop:.3f}"
+        )
+
+    regressions: list[dict[str, Any]] = []
+    for description, baseline_case in baseline_cases.items():
+        candidate_case = candidate_cases.get(description)
+        if candidate_case is None:
+            regressions.append(
+                {
+                    "description": description,
+                    "reason": "missing from candidate results",
+                }
+            )
+            continue
+        if baseline_case["success"] and not candidate_case["success"]:
+            regressions.append(
+                {
+                    "description": description,
+                    "reason": "pass->fail regression",
+                    "baseline_score": baseline_case["score"],
+                    "candidate_score": candidate_case["score"],
+                    "candidate_failure_reason": candidate_case["failure_reason"],
+                }
+            )
+
+    if regressions:
+        errors.append(f"{len(regressions)} regression case(s) detected")
+
+    report = {
+        "baseline_overall_score": baseline["overall_score"],
+        "candidate_overall_score": candidate["overall_score"],
+        "overall_drop": overall_drop,
+        "min_score": min_score,
+        "max_score_drop": max_score_drop,
+        "baseline_passing": baseline["passing"],
+        "candidate_passing": candidate["passing"],
+        "baseline_failing": baseline["failing"],
+        "candidate_failing": candidate["failing"],
+        "regressions": regressions,
+        "errors": errors,
+    }
+    return report, errors
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--candidate", required=True, help="Promptfoo result JSON to summarize/compare.")
+    parser.add_argument("--baseline", help="Baseline summary JSON to compare against.")
+    parser.add_argument("--summary-output", help="Where to write the normalized summary JSON.")
+    parser.add_argument("--report-output", help="Where to write the comparison report JSON.")
+    parser.add_argument("--min-score", type=float, default=0.8)
+    parser.add_argument("--max-score-drop", type=float, default=0.05)
+    args = parser.parse_args()
+
+    candidate = _normalize_eval(args.candidate)
+    _write_json(args.summary_output, candidate)
+
+    print(
+        "Promptfoo summary:",
+        f"score={candidate['overall_score']:.3f}",
+        f"passing={candidate['passing']}",
+        f"failing={candidate['failing']}",
+    )
+
+    if not args.baseline:
+        return 0
+
+    baseline = _normalize_eval(args.baseline)
+    report, errors = _compare(
+        baseline,
+        candidate,
+        min_score=args.min_score,
+        max_score_drop=args.max_score_drop,
+    )
+    _write_json(args.report_output, report)
+
+    print(
+        "Promptfoo regression check:",
+        f"baseline={report['baseline_overall_score']:.3f}",
+        f"candidate={report['candidate_overall_score']:.3f}",
+        f"drop={report['overall_drop']:.3f}",
+        f"regressions={len(report['regressions'])}",
+    )
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/render_agent_prompts.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""Render or check generated agent prompt files from the canonical protocol source."""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from app.services.agent_protocol import build_prompt_file_map  # noqa: E402
+
+
+def write_outputs(repo_root: Path) -> None:
+    for relative_path, content in build_prompt_file_map().items():
+        target = repo_root / relative_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(content, encoding="utf-8")
+        print(f"wrote {relative_path}")
+
+
+def check_outputs(repo_root: Path) -> int:
+    mismatches = []
+    for relative_path, expected in build_prompt_file_map().items():
+        target = repo_root / relative_path
+        actual = target.read_text(encoding="utf-8") if target.exists() else None
+        if actual != expected:
+            mismatches.append(relative_path)
+    if mismatches:
+        print("Generated prompt files are out of date:", file=sys.stderr)
+        for path in mismatches:
+            print(f"  - {path}", file=sys.stderr)
+        print("Run: python scripts/render_agent_prompts.py --write", file=sys.stderr)
+        return 1
+    print("Generated prompt files are in sync.")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--write", action="store_true", help="Write generated prompt files to the repo.")
+    parser.add_argument("--check", action="store_true", help="Fail if generated prompt files are out of sync.")
+    args = parser.parse_args()
+
+    if args.write == args.check:
+        parser.error("Choose exactly one of --write or --check")
+
+    if args.write:
+        write_outputs(REPO_ROOT)
+        return 0
+    return check_outputs(REPO_ROOT)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())