alignmenter 0.3.0__tar.gz → 0.3.2__tar.gz

{alignmenter-0.3.0/src/alignmenter.egg-info → alignmenter-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alignmenter
-Version: 0.3.0
+Version: 0.3.2
 Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
 Author: Alignmenter
 License-Expression: Apache-2.0

{alignmenter-0.3.0 → alignmenter-0.3.2}/datasets/README.md

@@ -11,6 +11,28 @@ Alignmenter datasets serve two primary purposes:
 
 All datasets in this repository follow strict data hygiene practices to protect user privacy and enable reproducible research.
 
+## Dataset management commands
+
+The `alignmenter dataset` sub-app manages datasets as content-addressed, versioned
+artifacts:
+
+```bash
+alignmenter dataset stats data.jsonl [--json]        # coverage: counts, roles, tag/persona histograms
+alignmenter dataset validate data.jsonl [--strict]   # canonical schema check (exit 1 on errors)
+alignmenter dataset dedupe data.jsonl --out out.jsonl # drop content-identical records
+alignmenter dataset merge a.jsonl b.jsonl --out m.jsonl [--dedupe] [--namespace-sessions]
+alignmenter dataset split data.jsonl --out dir/ --holdout 0.2 --by split_group  # group-aware train/holdout
+alignmenter dataset manifest data.jsonl --out manifest.json   # content-addressed manifest + provenance
+alignmenter dataset manifest data.jsonl --verify manifest.json # confirm the data still matches (exit 2 on drift)
+```
+
+`validate` checks the row schema below (lenient by default, `--strict` for the fuller
+contract); `lint` additionally checks turn sequencing, scenario-tag coverage, and
+persona files. `split` keeps a
+case and its counterfactual variants on the same side of the boundary via the
+grouping unit (`split_group` / `group:` tag / session / persona). `manifest`'s digest
+is order-independent (reordering rows does not change the dataset's identity).
+
 ## Dataset Format
 
 ### Conversation JSONL Schema

{alignmenter-0.3.0 → alignmenter-0.3.2}/src/alignmenter/_version.py

@@ -1,3 +1,3 @@
 """Single source for distribution and runtime version metadata."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"

{alignmenter-0.3.0 → alignmenter-0.3.2}/src/alignmenter/cli.py

@@ -21,6 +21,7 @@ import typer
 import yaml
 
 from alignmenter.config import get_settings
+from alignmenter.dataset_cli import register_dataset_commands
 from alignmenter.evaluators.custom import evaluator_registry, load_evaluators
 from alignmenter.execution.evaluation import evaluate_saved, evaluation_summary
 from alignmenter.execution.recovery import resume_capture
@@ -58,6 +59,8 @@ app.add_typer(dataset_app, name="dataset")
 app.add_typer(import_app, name="import")
 app.add_typer(calibrate_app, name="calibrate")
 
+register_dataset_commands(dataset_app)
+
 
 def _show_version(value):
     if value:

alignmenter-0.3.2/src/alignmenter/dataset_cli.py

@@ -0,0 +1,205 @@
+"""Dataset-management commands: stats, validate, dedupe, merge, split, manifest.
+
+Registered onto the existing `dataset` sub-app (alongside lint/sanitize/bootstrap)
+via register_dataset_commands(dataset_app). Builds on the shared primitives
+(content_digest, read/write_jsonl) and schemas/dataset.py.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import typer
+
+from alignmenter.schemas.dataset import (
+    ProvenanceEntry,
+    build_manifest,
+    dataset_digest,
+    validate_records,
+)
+from alignmenter.schemas.execution import content_digest
+from alignmenter.utils.io import read_jsonl, write_json, write_jsonl
+
+
+def _group_key(record: dict, by: str) -> str:
+    """Grouping key for split — keeps a case + its variants on the same side."""
+    if not isinstance(record, dict):
+        return ""  # non-dict rows group together (like the other commands, don't crash)
+    metadata = record.get("metadata") or {}
+    session = record.get("session_id") or ""
+    if by == "split_group":
+        return str(metadata.get("split_group") or session)
+    if by == "group":
+        for tag in record.get("tags") or []:
+            if isinstance(tag, str) and tag.startswith("group:"):
+                return tag
+        return session
+    if by == "persona":
+        return str(record.get("persona_id") or session)
+    return session
+
+
+def register_dataset_commands(dataset_app: typer.Typer) -> None:
+    @dataset_app.command("stats")
+    def stats(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        as_json: bool = typer.Option(False, "--json", help="Emit the coverage report as JSON."),
+    ):
+        """Coverage report — record/session counts, role balance, tag + persona histograms."""
+        records = read_jsonl(path)
+        manifest = build_manifest(records, id=path.stem, revision="stats")
+        roles = Counter(r.get("role") for r in records if isinstance(r, dict))
+        with_context = sum(
+            1 for r in records
+            if isinstance(r, dict) and isinstance(r.get("metadata"), dict) and "context" in r["metadata"]
+        )
+        if as_json:
+            typer.echo(json.dumps({
+                "records": manifest.record_count, "sessions": manifest.session_count,
+                "roles": {k: v for k, v in roles.items()}, "with_context": with_context,
+                "tags": manifest.tags, "personas": manifest.personas,
+                "content_digest": manifest.content_digest,
+            }, indent=2))
+            return
+        typer.echo(f"records {manifest.record_count} · sessions {manifest.session_count} · turns-with-context {with_context}")
+        typer.echo("roles: " + ", ".join(f"{k}={v}" for k, v in sorted(roles.items(), key=lambda kv: str(kv[0]))))
+        if manifest.tags:
+            typer.echo("tags:  " + ", ".join(f"{k}={v}" for k, v in sorted(manifest.tags.items())))
+        if manifest.personas:
+            typer.echo("personas: " + ", ".join(f"{k}={v}" for k, v in sorted(manifest.personas.items())))
+        typer.echo(f"content_digest: {manifest.content_digest[:12]}…")
+
+    @dataset_app.command("validate")
+    def validate(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        strict: bool = typer.Option(False, "--strict", help="Require turn_index+tags+persona_id and an assistant turn per session."),
+    ):
+        """Schema validator — lenient by default, --strict adds turn_index+tags+persona_id + an assistant turn per session.
+
+        Complements `dataset lint`, which additionally checks turn-index contiguity,
+        scenario-tag coverage, and persona-file existence.
+        """
+        records = read_jsonl(path)
+        errors = validate_records(records, strict=strict)
+        for error in errors:
+            typer.echo(error, err=True)
+        typer.echo(f"{len(records)} records, {len(errors)} error(s)")
+        raise typer.Exit(1 if errors else 0)
+
+    @dataset_app.command("dedupe")
+    def dedupe(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path | None = typer.Option(None, "--out", help="Output path (default: <stem>.dedup.jsonl)."),
+        in_place: bool = typer.Option(False, "--in-place"),
+    ):
+        """Drop content-identical duplicate records (order preserved)."""
+        records = read_jsonl(path)
+        seen: set[str] = set()
+        kept = []
+        for record in records:
+            digest = content_digest(record)
+            if digest in seen:
+                continue
+            seen.add(digest)
+            kept.append(record)
+        destination = path if in_place else (out or path.with_name(f"{path.stem}.dedup.jsonl"))
+        write_jsonl(destination, kept)
+        typer.echo(f"removed {len(records) - len(kept)} duplicate(s); wrote {len(kept)} -> {destination}")
+
+    @dataset_app.command("merge")
+    def merge(
+        paths: list[Path] = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path = typer.Option(..., "--out"),
+        dedupe: bool = typer.Option(False, "--dedupe", help="Drop content-identical records after merging."),
+        namespace_sessions: bool = typer.Option(False, "--namespace-sessions", help="Prefix session_id with the source file stem to avoid collisions."),
+    ):
+        """Concatenate datasets into one (optionally dedupe / namespace session ids)."""
+        merged: list[dict] = []
+        for source in paths:
+            records = read_jsonl(source)
+            if namespace_sessions:
+                for record in records:
+                    if isinstance(record, dict) and record.get("session_id"):
+                        record["session_id"] = f"{source.stem}:{record['session_id']}"
+            merged.extend(records)
+        if dedupe:
+            seen: set[str] = set()
+            deduped = []
+            for record in merged:
+                digest = content_digest(record)
+                if digest in seen:
+                    continue
+                seen.add(digest)
+                deduped.append(record)
+            merged = deduped
+        write_jsonl(out, merged)
+        typer.echo(f"merged {len(paths)} file(s) -> {len(merged)} records -> {out}")
+
+    @dataset_app.command("split")
+    def split(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path = typer.Option(..., "--out", help="Output directory for train.jsonl + holdout.jsonl."),
+        holdout: float = typer.Option(0.2, "--holdout", min=0.0, max=1.0, help="Target holdout fraction of records."),
+        by: str = typer.Option("split_group", "--by", help="Grouping unit kept together: split_group | group | session | persona."),
+        seed: int = typer.Option(42, "--seed"),
+    ):
+        """Group-aware train/holdout split — a case and its variants never straddle the boundary."""
+        if by not in {"split_group", "group", "session", "persona"}:
+            raise typer.BadParameter("--by must be split_group, group, session, or persona")
+        records = read_jsonl(path)
+        groups: dict[str, list[dict]] = defaultdict(list)
+        for record in records:
+            groups[_group_key(record, by)].append(record)
+        names = sorted(groups)
+        random.Random(seed).shuffle(names)
+        target = holdout * len(records)
+        held: set[str] = set()
+        accumulated = 0
+        for name in names:
+            if accumulated >= target:
+                break
+            held.add(name)
+            accumulated += len(groups[name])
+        train = [r for r in records if _group_key(r, by) not in held]
+        holdout_records = [r for r in records if _group_key(r, by) in held]
+        write_jsonl(Path(out) / "train.jsonl", train)
+        write_jsonl(Path(out) / "holdout.jsonl", holdout_records)
+        typer.echo(
+            f"train {len(train)} · holdout {len(holdout_records)} "
+            f"({len(held)}/{len(names)} '{by}' groups) -> {out}"
+        )
+
+    @dataset_app.command("manifest")
+    def manifest(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path | None = typer.Option(None, "--out", help="Write the manifest JSON here (default: stdout)."),
+        identifier: str | None = typer.Option(None, "--id", help="Dataset id (default: file stem)."),
+        revision: str = typer.Option("v1", "--revision"),
+        verify: Path | None = typer.Option(None, "--verify", help="Verify the data still matches this manifest's content_digest."),
+    ):
+        """Build (or --verify) a content-addressed dataset manifest with provenance."""
+        records = read_jsonl(path)
+        if verify is not None:
+            try:
+                expected = json.loads(Path(verify).read_text()).get("content_digest")
+            except (json.JSONDecodeError, OSError) as exc:
+                raise typer.BadParameter(f"Could not read manifest {verify}: {exc}") from exc
+            actual = dataset_digest(records)
+            match = expected == actual
+            typer.echo(f"{'OK' if match else 'MISMATCH'}: manifest {str(expected)[:12]} vs data {actual[:12]}")
+            raise typer.Exit(0 if match else 2)
+        built = build_manifest(
+            records,
+            id=identifier or path.stem,
+            revision=revision,
+            provenance=(ProvenanceEntry(kind="authored", ref=str(path), count=len(records), digest=dataset_digest(records)),),
+        )
+        payload = built.model_dump(mode="json")
+        if out is not None:
+            write_json(out, payload)
+            typer.echo(f"wrote manifest -> {out}")
+        else:
+            typer.echo(json.dumps(payload, indent=2))

{alignmenter-0.3.0 → alignmenter-0.3.2}/src/alignmenter/release_cli.py

@@ -22,6 +22,18 @@ from alignmenter.schemas.gates import GatePolicy
 EXIT_CODES = {"pass": 0, "fail": 2, "inconclusive": 3}
 
 
+def _exit_code(decision: str, *, allow_inconclusive: bool = False) -> int:
+    """Map a decision to a process exit code.
+
+    With ``allow_inconclusive`` an inconclusive decision (e.g. an unreviewed
+    ``draft`` spec that met every applicable criterion) exits 0 so an automated
+    gate is not blocked; a genuine ``fail`` still exits non-zero.
+    """
+    if allow_inconclusive and decision == "inconclusive":
+        return 0
+    return EXIT_CODES[decision]
+
+
 def register_release_commands(app):
     @app.command("archive-export")
     def archive_export(
@@ -67,6 +79,9 @@ def register_release_commands(app):
         suite: Path = typer.Argument(..., exists=True, dir_okay=False),
         out: Path = typer.Option(Path("reports"), "--out"),
         resume: Path | None = typer.Option(None, "--resume", exists=True, file_okay=False),
+        allow_inconclusive: bool = typer.Option(
+            False, "--allow-inconclusive",
+            help="Exit 0 on an inconclusive decision (a fail still exits non-zero)."),
     ):
         """Capture, evaluate, compare, and write CI artifacts under a frozen suite config."""
         try:
@@ -74,7 +89,9 @@ def register_release_commands(app):
         except Exception as exc:
             raise typer.BadParameter(str(exc)) from exc
         typer.echo(json.dumps(result, indent=2))
-        raise typer.Exit(EXIT_CODES[result["decision"]])
+        if allow_inconclusive and result["decision"] == "inconclusive":
+            typer.echo("Inconclusive tolerated (--allow-inconclusive): exiting 0.")
+        raise typer.Exit(_exit_code(result["decision"], allow_inconclusive=allow_inconclusive))
 
     @app.command("review-export")
     def review_export(
@@ -137,6 +154,9 @@ def register_release_commands(app):
         baseline: Path | None = typer.Option(None, "--baseline", exists=True, file_okay=False),
         baseline_id: UUID | None = typer.Option(None, "--baseline-id"),
         force: bool = typer.Option(False, "--force"),
+        allow_inconclusive: bool = typer.Option(
+            False, "--allow-inconclusive",
+            help="Exit 0 on an inconclusive decision (a fail still exits non-zero)."),
     ):
         """Check saved results and export CI artifacts without invoking any provider."""
         try:
@@ -147,7 +167,7 @@ def register_release_commands(app):
             raise typer.BadParameter(str(exc)) from exc
         decision = report["gate_report"]["decision"]
         typer.echo(f"Decision: {decision}\nArtifacts: {out.resolve()}")
-        raise typer.Exit(EXIT_CODES[decision])
+        raise typer.Exit(_exit_code(decision, allow_inconclusive=allow_inconclusive))
 
     @app.command("compare")
     def compare(

{alignmenter-0.3.0 → alignmenter-0.3.2}/src/alignmenter/reporting/durable.py

@@ -12,6 +12,7 @@ from xml.etree import ElementTree as ET
 
 from alignmenter.execution.evaluation import evaluation_summary
 from alignmenter.execution.gates import gate_report
+from alignmenter.reporting.github_comment import render_github_comment
 
 
 def _escape(value):
@@ -167,5 +168,6 @@ def export_evaluation(run_dir, out_dir, *, evaluation_id=None, policy=None, comp
     report["comparison"] = comparison
     report["review"] = qualification_report(run_dir, report["evaluation_id"])
     write_artifacts(out_dir, {"evaluation.json": _pretty(report) + "\n", "index.html": render_html(report),
-                              "junit.xml": render_junit(report), "summary.md": render_markdown(report)}, force=force)
+                              "junit.xml": render_junit(report), "summary.md": render_markdown(report),
+                              "comment.md": render_github_comment(report)}, force=force)
     return report

alignmenter-0.3.2/src/alignmenter/reporting/github_comment.py

@@ -0,0 +1,140 @@
+"""Sticky GitHub pull-request comment over one saved decision; never executes a scorer.
+
+Renders the same ``report`` structure the other durable reporters consume
+(:mod:`alignmenter.reporting.durable`) into Markdown tuned for a PR comment: a
+verdict badge, blocking issues first, a gate table, metrics (with baseline
+deltas when a comparison is present), and collapsed breakdowns. The leading
+marker lets a CI step find and update one sticky comment instead of appending.
+"""
+
+from __future__ import annotations
+
+MARKER = "<!-- alignmenter:report -->"
+
+_DECISION_BADGE = {"pass": "✅ Pass", "fail": "❌ Fail", "inconclusive": "⚠️ Inconclusive"}
+_CHECK_ICON = {"pass": "✅", "fail": "❌", "inconclusive": "➖"}
+_OPERATOR = {"at_least": "≥", "at_most": "≤"}
+
+
+def _cell(value):
+    """Escape a value for a single Markdown table cell."""
+    if value is None:
+        return "—"
+    return (str(value).replace("\\", "\\\\").replace("|", "\\|")
+            .replace("\n", " ").replace("<", "&lt;").replace(">", "&gt;"))
+
+
+def _num(value, digits=3):
+    if value is None:
+        return "—"
+    if isinstance(value, bool):
+        return str(value)
+    if isinstance(value, float):
+        return f"{value:.{digits}f}".rstrip("0").rstrip(".") or "0"
+    return str(value)
+
+
+def _signed(value, digits=3):
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return ("+" if value > 0 else "") + _num(value, digits)
+    return _num(value, digits)
+
+
+def _icon(decision):
+    return _CHECK_ICON.get(decision, _cell(decision))
+
+
+def _gate_detail(check):
+    metric = check.get("metric")
+    if metric is None:
+        return _cell(check.get("reason", ""))
+    operator = _OPERATOR.get(check.get("operator"), check.get("operator") or "")
+    return f"`{_cell(metric)}` {operator} {_num(check.get('threshold'))} · got **{_num(check.get('value'))}**"
+
+
+def _blocking_lines(report):
+    lines = []
+    for check in report["gate_report"].get("checks", []):
+        # Skip the structural "evaluation"/"comparison" umbrella checks (no metric);
+        # the ❌ badge and the violated cases below already convey those.
+        if check["decision"] == "fail" and check.get("metric") is not None:
+            lines.append(f"- **{_cell(check['id'])}** — {_gate_detail(check)}")
+    inputs = {item["key"]: item for item in report.get("inputs", [])}
+    violated = [(inputs.get(r["input_key"]), r) for r in report.get("results", [])
+                if r.get("status") == "violated"]
+    for item, result in violated[:5]:
+        item = item or {}
+        label = _cell(item.get("case_id") or item.get("session_id") or "case")
+        lines.append(f"- case **{label}** / {_cell(item.get('criterion_id'))}: "
+                     f"{_cell(result.get('reason') or 'violated')}")
+    if len(violated) > 5:
+        lines.append(f"- …and {len(violated) - 5} more violated case(s)")
+    return lines
+
+
+def _gate_table(report):
+    checks = report["gate_report"].get("checks", [])
+    if not checks:
+        return []
+    rows = ["| Gate | Result | Detail |", "| --- | :---: | --- |"]
+    rows += [f"| {_cell(c['id'])} | {_icon(c['decision'])} | {_gate_detail(c)} |" for c in checks]
+    return ["#### Gates", *rows, ""]
+
+
+def _metrics_table(report):
+    comparison = report.get("comparison")
+    if comparison and comparison.get("metrics"):
+        rows = ["| Metric | Baseline | Candidate | Δ | Note |",
+                "| --- | ---: | ---: | ---: | :---: |"]
+        for name, metric in comparison["metrics"].items():
+            note = "⚠️ unavailable" if metric.get("unavailable") else ""
+            rows.append(f"| `{_cell(name)}` | {_num(metric.get('baseline'))} | "
+                        f"{_num(metric.get('candidate'))} | {_signed(metric.get('delta'))} | {note} |")
+        return ["#### Metrics (vs baseline)", *rows, ""]
+    metrics = report.get("metrics") or {}
+    if not metrics:
+        return []
+    rows = ["| Metric | Value | Denominator |", "| --- | ---: | ---: |"]
+    rows += [f"| `{_cell(name)}` | {_num(m.get('value'))} | {_num(m.get('denominator'))} |"
+             for name, m in metrics.items()]
+    return ["#### Metrics", *rows, ""]
+
+
+def _details(summary, body):
+    return [f"<details><summary>{_cell(summary)}</summary>", "", body, "", "</details>", ""]
+
+
+def _breakdown(report):
+    lines = []
+    criteria = report.get("criteria") or {}
+    if criteria:
+        rows = ["| Criterion | met | violated | n/a | decision |",
+                "| --- | ---: | ---: | ---: | :---: |"]
+        for cid, summary in criteria.items():
+            counts = summary.get("counts", {})
+            rows.append(f"| `{_cell(cid)}` | {counts.get('met', 0)} | {counts.get('violated', 0)} | "
+                        f"{counts.get('not_applicable', 0)} | {_icon(summary.get('decision'))} |")
+        lines += _details("Per-criterion breakdown", "\n".join(rows))
+    counts = report.get("counts") or {}
+    if counts:
+        lines += _details("Outcome counts", ", ".join(f"{k}: {v}" for k, v in counts.items()))
+    return lines
+
+
+def render_github_comment(report, *, title="Alignmenter"):
+    """Return a sticky PR-comment Markdown body for one saved evaluation ``report``."""
+    decision = report["gate_report"]["decision"]
+    spec = report["spec"]
+    header = (f"`{_cell(spec['id'])} @ {_cell(spec['revision'])}` · "
+              f"qualification `{_cell(spec['qualification'])}` · "
+              f"{report['judged']}/{report['applicable']} applicable criteria evaluated")
+    if report.get("unavailable"):
+        header += f" · {report['unavailable']} unavailable"
+    lines = [MARKER, f"### {_DECISION_BADGE.get(decision, _cell(decision))} — {_cell(title)}", header, ""]
+    blocking = _blocking_lines(report)
+    if blocking:
+        lines += ["#### ❌ Blocking", *blocking, ""]
+    lines += _gate_table(report)
+    lines += _metrics_table(report)
+    lines += _breakdown(report)
+    return "\n".join(lines).rstrip() + "\n"

alignmenter-0.3.2/src/alignmenter/schemas/dataset.py

@@ -0,0 +1,139 @@
+"""Dataset manifest + canonical record validation for dataset management.
+
+Datasets are flat JSONL of turn records (see datasets/README.md). `validate_records`
+is a schema validator: lenient by default (matching what capture requires —
+session_id + role + text, unique turn_index per session), `strict=True` adds
+turn_index + tags + persona_id and an assistant turn per session. It complements
+`dataset lint`, which additionally checks turn-index contiguity, scenario-tag
+coverage, and persona-file existence. `DatasetManifest` gives a dataset a
+content-addressed identity + provenance so a curated set can be versioned and its
+lineage tracked.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+
+from pydantic import AwareDatetime, Field
+
+from alignmenter.schemas.evaluation import NonBlank
+from alignmenter.schemas.execution import (
+    Digest,
+    NonNegativeInt,
+    VersionedRecord,
+    content_digest,
+    utc_now,
+)
+
+ROLES = {"user", "assistant"}
+
+
+def dataset_digest(records) -> str:
+    """Order-independent content hash of a dataset (same records, any order → same digest).
+
+    content_digest sorts dict keys but preserves list order, so hashing the record
+    list directly would change when rows are merely reordered. Hashing the sorted
+    per-record digests gives a stable content identity for versioning/verification.
+
+    Note: this is distinct from RunManifest.dataset_digest (runner.py), which is the
+    order-SENSITIVE content_digest(records) used for run reproducibility — don't
+    compare the two.
+    """
+    return content_digest(sorted(content_digest(r) for r in records))
+
+
+class ProvenanceEntry(VersionedRecord):
+    kind: NonBlank  # authored | bootstrap | import | merge | dedupe | split | promote
+    ref: str = ""  # source path / url / description
+    count: NonNegativeInt = 0
+    digest: str | None = None
+
+
+class DatasetManifest(VersionedRecord):
+    id: NonBlank
+    revision: NonBlank
+    content_digest: Digest
+    record_count: NonNegativeInt
+    session_count: NonNegativeInt
+    created_at: AwareDatetime = Field(default_factory=utc_now)
+    tags: dict[str, int] = Field(default_factory=dict)
+    personas: dict[str, int] = Field(default_factory=dict)
+    provenance: tuple[ProvenanceEntry, ...] = ()
+
+
+def build_manifest(records, *, id: str, revision: str, provenance=()) -> DatasetManifest:
+    """Content-address a record list into a DatasetManifest (order-independent digest)."""
+    dicts = [r for r in records if isinstance(r, dict)]
+    sessions = {r.get("session_id") for r in dicts if r.get("session_id")}
+    tags = Counter(t for r in dicts for t in (r.get("tags") or []) if isinstance(t, str))
+    personas = Counter(r["persona_id"] for r in dicts if isinstance(r.get("persona_id"), str))
+    return DatasetManifest(
+        id=id,
+        revision=revision,
+        content_digest=dataset_digest(records),
+        record_count=len(records),
+        session_count=len(sessions),
+        tags=dict(tags),
+        personas=dict(personas),
+        provenance=tuple(provenance),
+    )
+
+
+def validate_records(records, *, strict: bool = False) -> list[str]:
+    """Return a list of human-readable schema errors (empty == valid).
+
+    Default rules match ingest (session_id + role + text; turn_index optional but,
+    when present, a non-negative int unique within its session). ``strict`` adds the
+    lint contract: turn_index + tags + persona_id required, and every session must
+    have an assistant turn.
+    """
+    errors: list[str] = []
+    seen_turns: dict[str, set[int]] = defaultdict(set)
+    session_roles: dict[str, set] = defaultdict(set)
+
+    for i, r in enumerate(records):
+        loc = f"record {i}"
+        if not isinstance(r, dict):
+            errors.append(f"{loc}: not a JSON object")
+            continue
+        sid = r.get("session_id")
+        if not isinstance(sid, str) or not sid.strip():
+            errors.append(f"{loc}: missing or blank session_id")
+        role = r.get("role")
+        if role not in ROLES:
+            errors.append(f"{loc}: role must be one of {sorted(ROLES)} (got {role!r})")
+        if not isinstance(r.get("text"), str):
+            errors.append(f"{loc}: text must be a string")
+
+        ti = r.get("turn_index")
+        if ti is not None:
+            if isinstance(ti, bool) or not isinstance(ti, int) or ti < 0:
+                errors.append(f"{loc}: turn_index must be a non-negative int")
+            elif isinstance(sid, str):
+                if ti in seen_turns[sid]:
+                    errors.append(f"{loc}: duplicate turn_index {ti} in session {sid!r}")
+                seen_turns[sid].add(ti)
+
+        tags = r.get("tags")
+        if tags is not None and not (isinstance(tags, list) and all(isinstance(t, str) for t in tags)):
+            errors.append(f"{loc}: tags must be a list of strings")
+        md = r.get("metadata")
+        if md is not None and not isinstance(md, dict):
+            errors.append(f"{loc}: metadata must be an object")
+
+        if isinstance(sid, str) and role in ROLES:
+            session_roles[sid].add(role)
+
+        if strict:
+            if ti is None:
+                errors.append(f"{loc}: turn_index is required in --strict mode")
+            if not isinstance(tags, list):
+                errors.append(f"{loc}: tags is required in --strict mode")
+            if not isinstance(r.get("persona_id"), str):
+                errors.append(f"{loc}: persona_id is required in --strict mode")
+
+    if strict:
+        for sid, roles in session_roles.items():
+            if "assistant" not in roles:
+                errors.append(f"session {sid!r}: has no assistant turn (--strict)")
+    return errors

{alignmenter-0.3.0 → alignmenter-0.3.2/src/alignmenter.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alignmenter
-Version: 0.3.0
+Version: 0.3.2
 Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
 Author: Alignmenter
 License-Expression: Apache-2.0

{alignmenter-0.3.0 → alignmenter-0.3.2}/src/alignmenter.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/alignmenter/__init__.py
 src/alignmenter/_version.py
 src/alignmenter/cli.py
 src/alignmenter/config.py
+src/alignmenter/dataset_cli.py
 src/alignmenter/release_cli.py
 src/alignmenter/run_config.py
 src/alignmenter/runner.py
@@ -77,9 +78,11 @@ src/alignmenter/providers/local.py
 src/alignmenter/providers/openai.py
 src/alignmenter/reporting/__init__.py
 src/alignmenter/reporting/durable.py
+src/alignmenter/reporting/github_comment.py
 src/alignmenter/reporting/html.py
 src/alignmenter/reporting/json_out.py
 src/alignmenter/schemas/__init__.py
+src/alignmenter/schemas/dataset.py
 src/alignmenter/schemas/evaluation.py
 src/alignmenter/schemas/execution.py
 src/alignmenter/schemas/gates.py
@@ -120,9 +123,11 @@ tests/test_cli_import.py
 tests/test_cli_init.py
 tests/test_cli_run_config.py
 tests/test_config.py
+tests/test_dataset_management.py
 tests/test_durable_evaluations.py
 tests/test_durable_execution.py
 tests/test_faithfulness.py
+tests/test_github_comment.py
 tests/test_grounding.py
 tests/test_html_report.py
 tests/test_judge_providers.py