alignmenter 0.3.0__tar.gz → 0.3.1__tar.gz

{alignmenter-0.3.0/src/alignmenter.egg-info → alignmenter-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alignmenter
-Version: 0.3.0
+Version: 0.3.1
 Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
 Author: Alignmenter
 License-Expression: Apache-2.0

{alignmenter-0.3.0 → alignmenter-0.3.1}/src/alignmenter/_version.py

@@ -1,3 +1,3 @@
 """Single source for distribution and runtime version metadata."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{alignmenter-0.3.0 → alignmenter-0.3.1}/src/alignmenter/release_cli.py

@@ -22,6 +22,18 @@ from alignmenter.schemas.gates import GatePolicy
 EXIT_CODES = {"pass": 0, "fail": 2, "inconclusive": 3}
 
 
+def _exit_code(decision: str, *, allow_inconclusive: bool = False) -> int:
+    """Map a decision to a process exit code.
+
+    With ``allow_inconclusive`` an inconclusive decision (e.g. an unreviewed
+    ``draft`` spec that met every applicable criterion) exits 0 so an automated
+    gate is not blocked; a genuine ``fail`` still exits non-zero.
+    """
+    if allow_inconclusive and decision == "inconclusive":
+        return 0
+    return EXIT_CODES[decision]
+
+
 def register_release_commands(app):
     @app.command("archive-export")
     def archive_export(
@@ -67,6 +79,9 @@ def register_release_commands(app):
         suite: Path = typer.Argument(..., exists=True, dir_okay=False),
         out: Path = typer.Option(Path("reports"), "--out"),
         resume: Path | None = typer.Option(None, "--resume", exists=True, file_okay=False),
+        allow_inconclusive: bool = typer.Option(
+            False, "--allow-inconclusive",
+            help="Exit 0 on an inconclusive decision (a fail still exits non-zero)."),
     ):
         """Capture, evaluate, compare, and write CI artifacts under a frozen suite config."""
         try:
@@ -74,7 +89,9 @@ def register_release_commands(app):
         except Exception as exc:
             raise typer.BadParameter(str(exc)) from exc
         typer.echo(json.dumps(result, indent=2))
-        raise typer.Exit(EXIT_CODES[result["decision"]])
+        if allow_inconclusive and result["decision"] == "inconclusive":
+            typer.echo("Inconclusive tolerated (--allow-inconclusive): exiting 0.")
+        raise typer.Exit(_exit_code(result["decision"], allow_inconclusive=allow_inconclusive))
 
     @app.command("review-export")
     def review_export(
@@ -137,6 +154,9 @@ def register_release_commands(app):
         baseline: Path | None = typer.Option(None, "--baseline", exists=True, file_okay=False),
         baseline_id: UUID | None = typer.Option(None, "--baseline-id"),
         force: bool = typer.Option(False, "--force"),
+        allow_inconclusive: bool = typer.Option(
+            False, "--allow-inconclusive",
+            help="Exit 0 on an inconclusive decision (a fail still exits non-zero)."),
     ):
         """Check saved results and export CI artifacts without invoking any provider."""
         try:
@@ -147,7 +167,7 @@ def register_release_commands(app):
             raise typer.BadParameter(str(exc)) from exc
         decision = report["gate_report"]["decision"]
         typer.echo(f"Decision: {decision}\nArtifacts: {out.resolve()}")
-        raise typer.Exit(EXIT_CODES[decision])
+        raise typer.Exit(_exit_code(decision, allow_inconclusive=allow_inconclusive))
 
     @app.command("compare")
     def compare(

{alignmenter-0.3.0 → alignmenter-0.3.1}/src/alignmenter/reporting/durable.py

@@ -12,6 +12,7 @@ from xml.etree import ElementTree as ET
 
 from alignmenter.execution.evaluation import evaluation_summary
 from alignmenter.execution.gates import gate_report
+from alignmenter.reporting.github_comment import render_github_comment
 
 
 def _escape(value):
@@ -167,5 +168,6 @@ def export_evaluation(run_dir, out_dir, *, evaluation_id=None, policy=None, comp
     report["comparison"] = comparison
     report["review"] = qualification_report(run_dir, report["evaluation_id"])
     write_artifacts(out_dir, {"evaluation.json": _pretty(report) + "\n", "index.html": render_html(report),
-                              "junit.xml": render_junit(report), "summary.md": render_markdown(report)}, force=force)
+                              "junit.xml": render_junit(report), "summary.md": render_markdown(report),
+                              "comment.md": render_github_comment(report)}, force=force)
     return report

alignmenter-0.3.1/src/alignmenter/reporting/github_comment.py

@@ -0,0 +1,140 @@
+"""Sticky GitHub pull-request comment over one saved decision; never executes a scorer.
+
+Renders the same ``report`` structure the other durable reporters consume
+(:mod:`alignmenter.reporting.durable`) into Markdown tuned for a PR comment: a
+verdict badge, blocking issues first, a gate table, metrics (with baseline
+deltas when a comparison is present), and collapsed breakdowns. The leading
+marker lets a CI step find and update one sticky comment instead of appending.
+"""
+
+from __future__ import annotations
+
+MARKER = "<!-- alignmenter:report -->"
+
+_DECISION_BADGE = {"pass": "✅ Pass", "fail": "❌ Fail", "inconclusive": "⚠️ Inconclusive"}
+_CHECK_ICON = {"pass": "✅", "fail": "❌", "inconclusive": "➖"}
+_OPERATOR = {"at_least": "≥", "at_most": "≤"}
+
+
+def _cell(value):
+    """Escape a value for a single Markdown table cell."""
+    if value is None:
+        return "—"
+    return (str(value).replace("\\", "\\\\").replace("|", "\\|")
+            .replace("\n", " ").replace("<", "&lt;").replace(">", "&gt;"))
+
+
+def _num(value, digits=3):
+    if value is None:
+        return "—"
+    if isinstance(value, bool):
+        return str(value)
+    if isinstance(value, float):
+        return f"{value:.{digits}f}".rstrip("0").rstrip(".") or "0"
+    return str(value)
+
+
+def _signed(value, digits=3):
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return ("+" if value > 0 else "") + _num(value, digits)
+    return _num(value, digits)
+
+
+def _icon(decision):
+    return _CHECK_ICON.get(decision, _cell(decision))
+
+
+def _gate_detail(check):
+    metric = check.get("metric")
+    if metric is None:
+        return _cell(check.get("reason", ""))
+    operator = _OPERATOR.get(check.get("operator"), check.get("operator") or "")
+    return f"`{_cell(metric)}` {operator} {_num(check.get('threshold'))} · got **{_num(check.get('value'))}**"
+
+
+def _blocking_lines(report):
+    lines = []
+    for check in report["gate_report"].get("checks", []):
+        # Skip the structural "evaluation"/"comparison" umbrella checks (no metric);
+        # the ❌ badge and the violated cases below already convey those.
+        if check["decision"] == "fail" and check.get("metric") is not None:
+            lines.append(f"- **{_cell(check['id'])}** — {_gate_detail(check)}")
+    inputs = {item["key"]: item for item in report.get("inputs", [])}
+    violated = [(inputs.get(r["input_key"]), r) for r in report.get("results", [])
+                if r.get("status") == "violated"]
+    for item, result in violated[:5]:
+        item = item or {}
+        label = _cell(item.get("case_id") or item.get("session_id") or "case")
+        lines.append(f"- case **{label}** / {_cell(item.get('criterion_id'))}: "
+                     f"{_cell(result.get('reason') or 'violated')}")
+    if len(violated) > 5:
+        lines.append(f"- …and {len(violated) - 5} more violated case(s)")
+    return lines
+
+
+def _gate_table(report):
+    checks = report["gate_report"].get("checks", [])
+    if not checks:
+        return []
+    rows = ["| Gate | Result | Detail |", "| --- | :---: | --- |"]
+    rows += [f"| {_cell(c['id'])} | {_icon(c['decision'])} | {_gate_detail(c)} |" for c in checks]
+    return ["#### Gates", *rows, ""]
+
+
+def _metrics_table(report):
+    comparison = report.get("comparison")
+    if comparison and comparison.get("metrics"):
+        rows = ["| Metric | Baseline | Candidate | Δ | Note |",
+                "| --- | ---: | ---: | ---: | :---: |"]
+        for name, metric in comparison["metrics"].items():
+            note = "⚠️ unavailable" if metric.get("unavailable") else ""
+            rows.append(f"| `{_cell(name)}` | {_num(metric.get('baseline'))} | "
+                        f"{_num(metric.get('candidate'))} | {_signed(metric.get('delta'))} | {note} |")
+        return ["#### Metrics (vs baseline)", *rows, ""]
+    metrics = report.get("metrics") or {}
+    if not metrics:
+        return []
+    rows = ["| Metric | Value | Denominator |", "| --- | ---: | ---: |"]
+    rows += [f"| `{_cell(name)}` | {_num(m.get('value'))} | {_num(m.get('denominator'))} |"
+             for name, m in metrics.items()]
+    return ["#### Metrics", *rows, ""]
+
+
+def _details(summary, body):
+    return [f"<details><summary>{_cell(summary)}</summary>", "", body, "", "</details>", ""]
+
+
+def _breakdown(report):
+    lines = []
+    criteria = report.get("criteria") or {}
+    if criteria:
+        rows = ["| Criterion | met | violated | n/a | decision |",
+                "| --- | ---: | ---: | ---: | :---: |"]
+        for cid, summary in criteria.items():
+            counts = summary.get("counts", {})
+            rows.append(f"| `{_cell(cid)}` | {counts.get('met', 0)} | {counts.get('violated', 0)} | "
+                        f"{counts.get('not_applicable', 0)} | {_icon(summary.get('decision'))} |")
+        lines += _details("Per-criterion breakdown", "\n".join(rows))
+    counts = report.get("counts") or {}
+    if counts:
+        lines += _details("Outcome counts", ", ".join(f"{k}: {v}" for k, v in counts.items()))
+    return lines
+
+
+def render_github_comment(report, *, title="Alignmenter"):
+    """Return a sticky PR-comment Markdown body for one saved evaluation ``report``."""
+    decision = report["gate_report"]["decision"]
+    spec = report["spec"]
+    header = (f"`{_cell(spec['id'])} @ {_cell(spec['revision'])}` · "
+              f"qualification `{_cell(spec['qualification'])}` · "
+              f"{report['judged']}/{report['applicable']} applicable criteria evaluated")
+    if report.get("unavailable"):
+        header += f" · {report['unavailable']} unavailable"
+    lines = [MARKER, f"### {_DECISION_BADGE.get(decision, _cell(decision))} — {_cell(title)}", header, ""]
+    blocking = _blocking_lines(report)
+    if blocking:
+        lines += ["#### ❌ Blocking", *blocking, ""]
+    lines += _gate_table(report)
+    lines += _metrics_table(report)
+    lines += _breakdown(report)
+    return "\n".join(lines).rstrip() + "\n"

{alignmenter-0.3.0 → alignmenter-0.3.1/src/alignmenter.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alignmenter
-Version: 0.3.0
+Version: 0.3.1
 Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
 Author: Alignmenter
 License-Expression: Apache-2.0

{alignmenter-0.3.0 → alignmenter-0.3.1}/src/alignmenter.egg-info/SOURCES.txt

@@ -77,6 +77,7 @@ src/alignmenter/providers/local.py
 src/alignmenter/providers/openai.py
 src/alignmenter/reporting/__init__.py
 src/alignmenter/reporting/durable.py
+src/alignmenter/reporting/github_comment.py
 src/alignmenter/reporting/html.py
 src/alignmenter/reporting/json_out.py
 src/alignmenter/schemas/__init__.py
@@ -123,6 +124,7 @@ tests/test_config.py
 tests/test_durable_evaluations.py
 tests/test_durable_execution.py
 tests/test_faithfulness.py
+tests/test_github_comment.py
 tests/test_grounding.py
 tests/test_html_report.py
 tests/test_judge_providers.py

alignmenter-0.3.1/tests/test_github_comment.py

@@ -0,0 +1,116 @@
+"""GitHub PR-comment reporter and the --allow-inconclusive CI exit flag."""
+
+from __future__ import annotations
+
+from typer.testing import CliRunner
+
+from alignmenter.cli import app
+from alignmenter.execution.evaluation import evaluate_saved
+from alignmenter.release_cli import _exit_code
+from alignmenter.reporting.durable import export_evaluation
+from alignmenter.reporting.github_comment import MARKER, render_github_comment
+from alignmenter.schemas.evaluation import JudgeBudget
+
+from .test_durable_evaluations import Judge, captured, spec
+
+
+def make_report(*, decision="pass", checks=None, metrics=None, comparison=None,
+                criteria=None, counts=None, inputs=None, results=None,
+                judged=2, applicable=2, unavailable=0, qualification="reviewed"):
+    return {
+        "spec": {"id": "avercare", "revision": "v1", "qualification": qualification},
+        "judged": judged, "applicable": applicable, "unavailable": unavailable,
+        "metrics": metrics or {}, "comparison": comparison,
+        "criteria": criteria or {}, "counts": counts or {},
+        "inputs": inputs or [], "results": results or [],
+        "gate_report": {"decision": decision, "checks": checks or [
+            {"id": "evaluation", "decision": decision, "reason": "All required outcomes."}]},
+    }
+
+
+def test_pass_comment_has_marker_badge_and_header():
+    body = render_github_comment(make_report(decision="pass"))
+    assert body.splitlines()[0] == MARKER  # first line so a sticky-comment step can match it
+    assert "✅ Pass" in body
+    assert "`avercare @ v1`" in body and "2/2 applicable criteria" in body
+
+
+def test_failing_gate_and_violated_case_surface_in_blocking_section():
+    report = make_report(
+        decision="fail",
+        checks=[
+            {"id": "evaluation", "decision": "fail", "reason": "A criterion was violated."},
+            {"id": "dangerous_zero", "decision": "fail", "metric": "faithfulness.dangerous_answers",
+             "operator": "at_most", "threshold": 0, "value": 2},
+        ],
+        inputs=[{"key": "k1", "case_id": "hydration-01", "criterion_id": "faithfulness"}],
+        results=[{"input_key": "k1", "status": "violated", "reason": "Unsupported dosage claim."}],
+    )
+    body = render_github_comment(report)
+    blocking = body.split("#### ❌ Blocking", 1)[1]
+    assert "❌ Fail" in body
+    assert "**dangerous_zero**" in blocking and "≤ 0" in blocking and "got **2**" in blocking
+    assert "hydration-01" in blocking and "Unsupported dosage claim." in blocking
+
+
+def test_inconclusive_badge_and_no_blocking_section():
+    body = render_github_comment(make_report(decision="inconclusive", qualification="draft"))
+    assert "⚠️ Inconclusive" in body
+    assert "#### ❌ Blocking" not in body
+
+
+def test_comparison_renders_baseline_delta_columns():
+    report = make_report(
+        decision="pass",
+        comparison={"metrics": {
+            "evaluation.met_rate": {"baseline": 0.8, "candidate": 0.9, "delta": 0.1, "unavailable": False},
+            "grounding.citation_resolution": {"baseline": None, "candidate": None, "delta": None, "unavailable": True},
+        }},
+    )
+    body = render_github_comment(report)
+    assert "Metrics (vs baseline)" in body
+    assert "+0.1" in body                     # signed delta
+    assert "⚠️ unavailable" in body           # unavailable metric flagged, not dropped
+
+
+def test_table_cells_are_escaped_and_do_not_break_the_marker():
+    report = make_report(
+        decision="fail",
+        checks=[{"id": "evaluation", "decision": "fail", "reason": "pipe | and\nnewline <b>"}],
+        inputs=[{"key": "k1", "case_id": "a|b", "criterion_id": "c"}],
+        results=[{"input_key": "k1", "status": "violated", "reason": "x | y\nz"}],
+    )
+    body = render_github_comment(report)
+    assert body.splitlines()[0] == MARKER
+    # no raw pipe/newline leaks into a table cell that would break rendering
+    assert "a\\|b" in body and "x \\| y z" in body
+    assert "<b>" not in body
+
+
+def test_exit_code_mapping_and_allow_inconclusive():
+    assert _exit_code("pass") == 0
+    assert _exit_code("fail") == 2
+    assert _exit_code("inconclusive") == 3
+    assert _exit_code("inconclusive", allow_inconclusive=True) == 0
+    assert _exit_code("fail", allow_inconclusive=True) == 2  # a real failure still blocks
+
+
+def test_export_writes_comment_alongside_other_artifacts(tmp_path):
+    run_dir = captured(tmp_path)
+    evaluate_saved(run_dir, spec(), Judge(), budget=JudgeBudget(max_calls=10))  # reviewed + met => pass
+    out = tmp_path / "reports"
+    export_evaluation(run_dir, out)
+    for name in ("evaluation.json", "index.html", "junit.xml", "summary.md", "comment.md"):
+        assert (out / name).exists(), name
+    comment = (out / "comment.md").read_text()
+    assert comment.startswith(MARKER) and "✅ Pass" in comment
+
+
+def test_check_cli_allow_inconclusive_flips_exit_but_not_failure(tmp_path):
+    run_dir = captured(tmp_path)
+    evaluate_saved(run_dir, spec(qualification="draft"), Judge(), budget=JudgeBudget(max_calls=10))  # met, but draft => inconclusive
+    default = CliRunner().invoke(app, ["check", str(run_dir), "--out", str(tmp_path / "r1")])
+    assert default.exit_code == 3, default.output
+    allowed = CliRunner().invoke(app, ["check", str(run_dir), "--out", str(tmp_path / "r2"), "--allow-inconclusive"])
+    assert allowed.exit_code == 0, allowed.output
+    assert (tmp_path / "r2" / "comment.md").exists()