alignmenter 0.3.1__tar.gz → 0.3.2__tar.gz

{alignmenter-0.3.1/src/alignmenter.egg-info → alignmenter-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alignmenter
-Version: 0.3.1
+Version: 0.3.2
 Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
 Author: Alignmenter
 License-Expression: Apache-2.0

{alignmenter-0.3.1 → alignmenter-0.3.2}/datasets/README.md

@@ -11,6 +11,28 @@ Alignmenter datasets serve two primary purposes:
 
 All datasets in this repository follow strict data hygiene practices to protect user privacy and enable reproducible research.
 
+## Dataset management commands
+
+The `alignmenter dataset` sub-app manages datasets as content-addressed, versioned
+artifacts:
+
+```bash
+alignmenter dataset stats data.jsonl [--json]        # coverage: counts, roles, tag/persona histograms
+alignmenter dataset validate data.jsonl [--strict]   # canonical schema check (exit 1 on errors)
+alignmenter dataset dedupe data.jsonl --out out.jsonl # drop content-identical records
+alignmenter dataset merge a.jsonl b.jsonl --out m.jsonl [--dedupe] [--namespace-sessions]
+alignmenter dataset split data.jsonl --out dir/ --holdout 0.2 --by split_group  # group-aware train/holdout
+alignmenter dataset manifest data.jsonl --out manifest.json   # content-addressed manifest + provenance
+alignmenter dataset manifest data.jsonl --verify manifest.json # confirm the data still matches (exit 2 on drift)
+```
+
+`validate` checks the row schema below (lenient by default, `--strict` for the fuller
+contract); `lint` additionally checks turn sequencing, scenario-tag coverage, and
+persona files. `split` keeps a
+case and its counterfactual variants on the same side of the boundary via the
+grouping unit (`split_group` / `group:` tag / session / persona). `manifest`'s digest
+is order-independent (reordering rows does not change the dataset's identity).
+
 ## Dataset Format
 
 ### Conversation JSONL Schema

{alignmenter-0.3.1 → alignmenter-0.3.2}/src/alignmenter/_version.py

@@ -1,3 +1,3 @@
 """Single source for distribution and runtime version metadata."""
 
-__version__ = "0.3.1"
+__version__ = "0.3.2"

{alignmenter-0.3.1 → alignmenter-0.3.2}/src/alignmenter/cli.py

@@ -21,6 +21,7 @@ import typer
 import yaml
 
 from alignmenter.config import get_settings
+from alignmenter.dataset_cli import register_dataset_commands
 from alignmenter.evaluators.custom import evaluator_registry, load_evaluators
 from alignmenter.execution.evaluation import evaluate_saved, evaluation_summary
 from alignmenter.execution.recovery import resume_capture
@@ -58,6 +59,8 @@ app.add_typer(dataset_app, name="dataset")
 app.add_typer(import_app, name="import")
 app.add_typer(calibrate_app, name="calibrate")
 
+register_dataset_commands(dataset_app)
+
 
 def _show_version(value):
     if value:

alignmenter-0.3.2/src/alignmenter/dataset_cli.py

@@ -0,0 +1,205 @@
+"""Dataset-management commands: stats, validate, dedupe, merge, split, manifest.
+
+Registered onto the existing `dataset` sub-app (alongside lint/sanitize/bootstrap)
+via register_dataset_commands(dataset_app). Builds on the shared primitives
+(content_digest, read/write_jsonl) and schemas/dataset.py.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import typer
+
+from alignmenter.schemas.dataset import (
+    ProvenanceEntry,
+    build_manifest,
+    dataset_digest,
+    validate_records,
+)
+from alignmenter.schemas.execution import content_digest
+from alignmenter.utils.io import read_jsonl, write_json, write_jsonl
+
+
+def _group_key(record: dict, by: str) -> str:
+    """Grouping key for split — keeps a case + its variants on the same side."""
+    if not isinstance(record, dict):
+        return ""  # non-dict rows group together (like the other commands, don't crash)
+    metadata = record.get("metadata") or {}
+    session = record.get("session_id") or ""
+    if by == "split_group":
+        return str(metadata.get("split_group") or session)
+    if by == "group":
+        for tag in record.get("tags") or []:
+            if isinstance(tag, str) and tag.startswith("group:"):
+                return tag
+        return session
+    if by == "persona":
+        return str(record.get("persona_id") or session)
+    return session
+
+
+def register_dataset_commands(dataset_app: typer.Typer) -> None:
+    @dataset_app.command("stats")
+    def stats(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        as_json: bool = typer.Option(False, "--json", help="Emit the coverage report as JSON."),
+    ):
+        """Coverage report — record/session counts, role balance, tag + persona histograms."""
+        records = read_jsonl(path)
+        manifest = build_manifest(records, id=path.stem, revision="stats")
+        roles = Counter(r.get("role") for r in records if isinstance(r, dict))
+        with_context = sum(
+            1 for r in records
+            if isinstance(r, dict) and isinstance(r.get("metadata"), dict) and "context" in r["metadata"]
+        )
+        if as_json:
+            typer.echo(json.dumps({
+                "records": manifest.record_count, "sessions": manifest.session_count,
+                "roles": {k: v for k, v in roles.items()}, "with_context": with_context,
+                "tags": manifest.tags, "personas": manifest.personas,
+                "content_digest": manifest.content_digest,
+            }, indent=2))
+            return
+        typer.echo(f"records {manifest.record_count} · sessions {manifest.session_count} · turns-with-context {with_context}")
+        typer.echo("roles: " + ", ".join(f"{k}={v}" for k, v in sorted(roles.items(), key=lambda kv: str(kv[0]))))
+        if manifest.tags:
+            typer.echo("tags:  " + ", ".join(f"{k}={v}" for k, v in sorted(manifest.tags.items())))
+        if manifest.personas:
+            typer.echo("personas: " + ", ".join(f"{k}={v}" for k, v in sorted(manifest.personas.items())))
+        typer.echo(f"content_digest: {manifest.content_digest[:12]}…")
+
+    @dataset_app.command("validate")
+    def validate(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        strict: bool = typer.Option(False, "--strict", help="Require turn_index+tags+persona_id and an assistant turn per session."),
+    ):
+        """Schema validator — lenient by default, --strict adds turn_index+tags+persona_id + an assistant turn per session.
+
+        Complements `dataset lint`, which additionally checks turn-index contiguity,
+        scenario-tag coverage, and persona-file existence.
+        """
+        records = read_jsonl(path)
+        errors = validate_records(records, strict=strict)
+        for error in errors:
+            typer.echo(error, err=True)
+        typer.echo(f"{len(records)} records, {len(errors)} error(s)")
+        raise typer.Exit(1 if errors else 0)
+
+    @dataset_app.command("dedupe")
+    def dedupe(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path | None = typer.Option(None, "--out", help="Output path (default: <stem>.dedup.jsonl)."),
+        in_place: bool = typer.Option(False, "--in-place"),
+    ):
+        """Drop content-identical duplicate records (order preserved)."""
+        records = read_jsonl(path)
+        seen: set[str] = set()
+        kept = []
+        for record in records:
+            digest = content_digest(record)
+            if digest in seen:
+                continue
+            seen.add(digest)
+            kept.append(record)
+        destination = path if in_place else (out or path.with_name(f"{path.stem}.dedup.jsonl"))
+        write_jsonl(destination, kept)
+        typer.echo(f"removed {len(records) - len(kept)} duplicate(s); wrote {len(kept)} -> {destination}")
+
+    @dataset_app.command("merge")
+    def merge(
+        paths: list[Path] = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path = typer.Option(..., "--out"),
+        dedupe: bool = typer.Option(False, "--dedupe", help="Drop content-identical records after merging."),
+        namespace_sessions: bool = typer.Option(False, "--namespace-sessions", help="Prefix session_id with the source file stem to avoid collisions."),
+    ):
+        """Concatenate datasets into one (optionally dedupe / namespace session ids)."""
+        merged: list[dict] = []
+        for source in paths:
+            records = read_jsonl(source)
+            if namespace_sessions:
+                for record in records:
+                    if isinstance(record, dict) and record.get("session_id"):
+                        record["session_id"] = f"{source.stem}:{record['session_id']}"
+            merged.extend(records)
+        if dedupe:
+            seen: set[str] = set()
+            deduped = []
+            for record in merged:
+                digest = content_digest(record)
+                if digest in seen:
+                    continue
+                seen.add(digest)
+                deduped.append(record)
+            merged = deduped
+        write_jsonl(out, merged)
+        typer.echo(f"merged {len(paths)} file(s) -> {len(merged)} records -> {out}")
+
+    @dataset_app.command("split")
+    def split(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path = typer.Option(..., "--out", help="Output directory for train.jsonl + holdout.jsonl."),
+        holdout: float = typer.Option(0.2, "--holdout", min=0.0, max=1.0, help="Target holdout fraction of records."),
+        by: str = typer.Option("split_group", "--by", help="Grouping unit kept together: split_group | group | session | persona."),
+        seed: int = typer.Option(42, "--seed"),
+    ):
+        """Group-aware train/holdout split — a case and its variants never straddle the boundary."""
+        if by not in {"split_group", "group", "session", "persona"}:
+            raise typer.BadParameter("--by must be split_group, group, session, or persona")
+        records = read_jsonl(path)
+        groups: dict[str, list[dict]] = defaultdict(list)
+        for record in records:
+            groups[_group_key(record, by)].append(record)
+        names = sorted(groups)
+        random.Random(seed).shuffle(names)
+        target = holdout * len(records)
+        held: set[str] = set()
+        accumulated = 0
+        for name in names:
+            if accumulated >= target:
+                break
+            held.add(name)
+            accumulated += len(groups[name])
+        train = [r for r in records if _group_key(r, by) not in held]
+        holdout_records = [r for r in records if _group_key(r, by) in held]
+        write_jsonl(Path(out) / "train.jsonl", train)
+        write_jsonl(Path(out) / "holdout.jsonl", holdout_records)
+        typer.echo(
+            f"train {len(train)} · holdout {len(holdout_records)} "
+            f"({len(held)}/{len(names)} '{by}' groups) -> {out}"
+        )
+
+    @dataset_app.command("manifest")
+    def manifest(
+        path: Path = typer.Argument(..., exists=True, dir_okay=False),
+        out: Path | None = typer.Option(None, "--out", help="Write the manifest JSON here (default: stdout)."),
+        identifier: str | None = typer.Option(None, "--id", help="Dataset id (default: file stem)."),
+        revision: str = typer.Option("v1", "--revision"),
+        verify: Path | None = typer.Option(None, "--verify", help="Verify the data still matches this manifest's content_digest."),
+    ):
+        """Build (or --verify) a content-addressed dataset manifest with provenance."""
+        records = read_jsonl(path)
+        if verify is not None:
+            try:
+                expected = json.loads(Path(verify).read_text()).get("content_digest")
+            except (json.JSONDecodeError, OSError) as exc:
+                raise typer.BadParameter(f"Could not read manifest {verify}: {exc}") from exc
+            actual = dataset_digest(records)
+            match = expected == actual
+            typer.echo(f"{'OK' if match else 'MISMATCH'}: manifest {str(expected)[:12]} vs data {actual[:12]}")
+            raise typer.Exit(0 if match else 2)
+        built = build_manifest(
+            records,
+            id=identifier or path.stem,
+            revision=revision,
+            provenance=(ProvenanceEntry(kind="authored", ref=str(path), count=len(records), digest=dataset_digest(records)),),
+        )
+        payload = built.model_dump(mode="json")
+        if out is not None:
+            write_json(out, payload)
+            typer.echo(f"wrote manifest -> {out}")
+        else:
+            typer.echo(json.dumps(payload, indent=2))

alignmenter-0.3.2/src/alignmenter/schemas/dataset.py

@@ -0,0 +1,139 @@
+"""Dataset manifest + canonical record validation for dataset management.
+
+Datasets are flat JSONL of turn records (see datasets/README.md). `validate_records`
+is a schema validator: lenient by default (matching what capture requires —
+session_id + role + text, unique turn_index per session), `strict=True` adds
+turn_index + tags + persona_id and an assistant turn per session. It complements
+`dataset lint`, which additionally checks turn-index contiguity, scenario-tag
+coverage, and persona-file existence. `DatasetManifest` gives a dataset a
+content-addressed identity + provenance so a curated set can be versioned and its
+lineage tracked.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+
+from pydantic import AwareDatetime, Field
+
+from alignmenter.schemas.evaluation import NonBlank
+from alignmenter.schemas.execution import (
+    Digest,
+    NonNegativeInt,
+    VersionedRecord,
+    content_digest,
+    utc_now,
+)
+
+ROLES = {"user", "assistant"}
+
+
+def dataset_digest(records) -> str:
+    """Order-independent content hash of a dataset (same records, any order → same digest).
+
+    content_digest sorts dict keys but preserves list order, so hashing the record
+    list directly would change when rows are merely reordered. Hashing the sorted
+    per-record digests gives a stable content identity for versioning/verification.
+
+    Note: this is distinct from RunManifest.dataset_digest (runner.py), which is the
+    order-SENSITIVE content_digest(records) used for run reproducibility — don't
+    compare the two.
+    """
+    return content_digest(sorted(content_digest(r) for r in records))
+
+
+class ProvenanceEntry(VersionedRecord):
+    kind: NonBlank  # authored | bootstrap | import | merge | dedupe | split | promote
+    ref: str = ""  # source path / url / description
+    count: NonNegativeInt = 0
+    digest: str | None = None
+
+
+class DatasetManifest(VersionedRecord):
+    id: NonBlank
+    revision: NonBlank
+    content_digest: Digest
+    record_count: NonNegativeInt
+    session_count: NonNegativeInt
+    created_at: AwareDatetime = Field(default_factory=utc_now)
+    tags: dict[str, int] = Field(default_factory=dict)
+    personas: dict[str, int] = Field(default_factory=dict)
+    provenance: tuple[ProvenanceEntry, ...] = ()
+
+
+def build_manifest(records, *, id: str, revision: str, provenance=()) -> DatasetManifest:
+    """Content-address a record list into a DatasetManifest (order-independent digest)."""
+    dicts = [r for r in records if isinstance(r, dict)]
+    sessions = {r.get("session_id") for r in dicts if r.get("session_id")}
+    tags = Counter(t for r in dicts for t in (r.get("tags") or []) if isinstance(t, str))
+    personas = Counter(r["persona_id"] for r in dicts if isinstance(r.get("persona_id"), str))
+    return DatasetManifest(
+        id=id,
+        revision=revision,
+        content_digest=dataset_digest(records),
+        record_count=len(records),
+        session_count=len(sessions),
+        tags=dict(tags),
+        personas=dict(personas),
+        provenance=tuple(provenance),
+    )
+
+
+def validate_records(records, *, strict: bool = False) -> list[str]:
+    """Return a list of human-readable schema errors (empty == valid).
+
+    Default rules match ingest (session_id + role + text; turn_index optional but,
+    when present, a non-negative int unique within its session). ``strict`` adds the
+    lint contract: turn_index + tags + persona_id required, and every session must
+    have an assistant turn.
+    """
+    errors: list[str] = []
+    seen_turns: dict[str, set[int]] = defaultdict(set)
+    session_roles: dict[str, set] = defaultdict(set)
+
+    for i, r in enumerate(records):
+        loc = f"record {i}"
+        if not isinstance(r, dict):
+            errors.append(f"{loc}: not a JSON object")
+            continue
+        sid = r.get("session_id")
+        if not isinstance(sid, str) or not sid.strip():
+            errors.append(f"{loc}: missing or blank session_id")
+        role = r.get("role")
+        if role not in ROLES:
+            errors.append(f"{loc}: role must be one of {sorted(ROLES)} (got {role!r})")
+        if not isinstance(r.get("text"), str):
+            errors.append(f"{loc}: text must be a string")
+
+        ti = r.get("turn_index")
+        if ti is not None:
+            if isinstance(ti, bool) or not isinstance(ti, int) or ti < 0:
+                errors.append(f"{loc}: turn_index must be a non-negative int")
+            elif isinstance(sid, str):
+                if ti in seen_turns[sid]:
+                    errors.append(f"{loc}: duplicate turn_index {ti} in session {sid!r}")
+                seen_turns[sid].add(ti)
+
+        tags = r.get("tags")
+        if tags is not None and not (isinstance(tags, list) and all(isinstance(t, str) for t in tags)):
+            errors.append(f"{loc}: tags must be a list of strings")
+        md = r.get("metadata")
+        if md is not None and not isinstance(md, dict):
+            errors.append(f"{loc}: metadata must be an object")
+
+        if isinstance(sid, str) and role in ROLES:
+            session_roles[sid].add(role)
+
+        if strict:
+            if ti is None:
+                errors.append(f"{loc}: turn_index is required in --strict mode")
+            if not isinstance(tags, list):
+                errors.append(f"{loc}: tags is required in --strict mode")
+            if not isinstance(r.get("persona_id"), str):
+                errors.append(f"{loc}: persona_id is required in --strict mode")
+
+    if strict:
+        for sid, roles in session_roles.items():
+            if "assistant" not in roles:
+                errors.append(f"session {sid!r}: has no assistant turn (--strict)")
+    return errors

{alignmenter-0.3.1 → alignmenter-0.3.2/src/alignmenter.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alignmenter
-Version: 0.3.1
+Version: 0.3.2
 Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
 Author: Alignmenter
 License-Expression: Apache-2.0

{alignmenter-0.3.1 → alignmenter-0.3.2}/src/alignmenter.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/alignmenter/__init__.py
 src/alignmenter/_version.py
 src/alignmenter/cli.py
 src/alignmenter/config.py
+src/alignmenter/dataset_cli.py
 src/alignmenter/release_cli.py
 src/alignmenter/run_config.py
 src/alignmenter/runner.py
@@ -81,6 +82,7 @@ src/alignmenter/reporting/github_comment.py
 src/alignmenter/reporting/html.py
 src/alignmenter/reporting/json_out.py
 src/alignmenter/schemas/__init__.py
+src/alignmenter/schemas/dataset.py
 src/alignmenter/schemas/evaluation.py
 src/alignmenter/schemas/execution.py
 src/alignmenter/schemas/gates.py
@@ -121,6 +123,7 @@ tests/test_cli_import.py
 tests/test_cli_init.py
 tests/test_cli_run_config.py
 tests/test_config.py
+tests/test_dataset_management.py
 tests/test_durable_evaluations.py
 tests/test_durable_execution.py
 tests/test_faithfulness.py

alignmenter-0.3.2/tests/test_dataset_management.py

@@ -0,0 +1,142 @@
+"""Dataset-management: canonical validation, manifest, and the dataset CLI commands."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from typer.testing import CliRunner
+
+from alignmenter.cli import app
+from alignmenter.schemas.dataset import build_manifest, validate_records
+from alignmenter.utils.io import read_jsonl, write_jsonl
+
+runner = CliRunner()
+
+
+def _records():
+    return [
+        {"session_id": "a", "turn_index": 0, "role": "user", "text": "hi", "tags": ["family:g"], "persona_id": "aver"},
+        {"session_id": "a", "turn_index": 1, "role": "assistant", "text": "hello", "tags": ["family:g"], "persona_id": "aver"},
+        {"session_id": "b", "turn_index": 0, "role": "user", "text": "q", "tags": ["group:x"], "persona_id": "aver"},
+        {"session_id": "b", "turn_index": 1, "role": "assistant", "text": "a", "tags": ["group:x"], "persona_id": "aver"},
+    ]
+
+
+def _write(tmp_path, records, name="data.jsonl") -> Path:
+    p = tmp_path / name
+    write_jsonl(p, records)
+    return p
+
+
+# ---- schema / manifest ----
+
+def test_validate_accepts_good_records():
+    assert validate_records(_records()) == []
+
+
+def test_validate_flags_missing_session_bad_role_and_dup_turn():
+    bad = [
+        {"role": "user", "text": "x"},                                  # missing session_id
+        {"session_id": "s", "role": "bot", "text": "x"},               # bad role
+        {"session_id": "s", "turn_index": 3, "role": "user", "text": "x"},
+        {"session_id": "s", "turn_index": 3, "role": "assistant", "text": "x"},  # dup turn_index
+    ]
+    errors = validate_records(bad)
+    assert any("session_id" in e for e in errors)
+    assert any("role must be one of" in e for e in errors)
+    assert any("duplicate turn_index" in e for e in errors)
+
+
+def test_validate_strict_requires_persona_and_assistant_turn():
+    records = [{"session_id": "s", "turn_index": 0, "role": "user", "text": "x", "tags": []}]  # no persona, no assistant
+    errors = validate_records(records, strict=True)
+    assert any("persona_id is required" in e for e in errors)
+    assert any("has no assistant turn" in e for e in errors)
+
+
+def test_manifest_digest_is_order_independent_and_counts_are_right():
+    m1 = build_manifest(_records(), id="d", revision="v1")
+    m2 = build_manifest(list(reversed(_records())), id="d", revision="v1")
+    assert m1.content_digest == m2.content_digest       # canonical, order-independent
+    assert m1.record_count == 4 and m1.session_count == 2
+    assert m1.personas == {"aver": 4}
+    assert m1.tags["family:g"] == 2 and m1.tags["group:x"] == 2
+
+
+# ---- CLI ----
+
+def test_cli_stats_json(tmp_path):
+    p = _write(tmp_path, _records())
+    res = runner.invoke(app, ["dataset", "stats", str(p), "--json"])
+    assert res.exit_code == 0, res.output
+    report = json.loads(res.output)
+    assert report["records"] == 4 and report["sessions"] == 2 and report["roles"]["assistant"] == 2
+
+
+def test_cli_validate_exit_codes(tmp_path):
+    good = _write(tmp_path, _records(), "good.jsonl")
+    assert runner.invoke(app, ["dataset", "validate", str(good)]).exit_code == 0
+    bad = _write(tmp_path, [{"role": "user", "text": "x"}], "bad.jsonl")
+    assert runner.invoke(app, ["dataset", "validate", str(bad)]).exit_code == 1
+
+
+def test_cli_dedupe(tmp_path):
+    p = _write(tmp_path, _records() + _records()[:1])  # one duplicate
+    out = tmp_path / "dd.jsonl"
+    res = runner.invoke(app, ["dataset", "dedupe", str(p), "--out", str(out)])
+    assert res.exit_code == 0 and "removed 1 duplicate" in res.output
+    assert len(read_jsonl(out)) == 4
+
+
+def test_cli_merge_namespaces_sessions(tmp_path):
+    a = _write(tmp_path, _records()[:2], "a.jsonl")
+    b = _write(tmp_path, _records()[2:], "b.jsonl")
+    out = tmp_path / "m.jsonl"
+    res = runner.invoke(app, ["dataset", "merge", str(a), str(b), "--out", str(out), "--namespace-sessions"])
+    assert res.exit_code == 0
+    sessions = {r["session_id"] for r in read_jsonl(out)}
+    assert sessions == {"a:a", "b:b"}
+
+
+def test_cli_split_keeps_groups_together(tmp_path):
+    # two group:* groups; a holdout must contain whole groups, never split one across sides
+    records = []
+    for g in ("g1", "g2"):
+        for i in range(3):
+            records.append({"session_id": f"{g}-{i}", "role": "assistant", "text": "x", "tags": [f"group:{g}"]})
+    p = _write(tmp_path, records)
+    out = tmp_path / "split"
+    res = runner.invoke(app, ["dataset", "split", str(p), "--out", str(out), "--by", "group", "--holdout", "0.5"])
+    assert res.exit_code == 0, res.output
+    train = {t for r in read_jsonl(out / "train.jsonl") for t in r["tags"]}
+    hold = {t for r in read_jsonl(out / "holdout.jsonl") for t in r["tags"]}
+    assert train and hold and train.isdisjoint(hold)   # no group straddles the boundary
+
+
+def test_cli_split_tolerates_non_dict_rows(tmp_path):
+    p = tmp_path / "mixed.jsonl"
+    write_jsonl(p, _records() + [[1, 2, 3]])  # a non-dict row must not crash split
+    out = tmp_path / "s"
+    res = runner.invoke(app, ["dataset", "split", str(p), "--out", str(out), "--by", "session", "--holdout", "0.5"])
+    assert res.exit_code == 0, res.output
+
+
+def test_cli_manifest_verify_bad_json_errors_friendly(tmp_path):
+    p = _write(tmp_path, _records())
+    bad = tmp_path / "bad-manifest.json"
+    bad.write_text("not json at all")
+    res = runner.invoke(app, ["dataset", "manifest", str(p), "--verify", str(bad)])
+    assert res.exit_code != 0
+    assert "Could not read manifest" in res.output
+
+
+def test_cli_manifest_build_and_verify(tmp_path):
+    p = _write(tmp_path, _records())
+    man = tmp_path / "manifest.json"
+    assert runner.invoke(app, ["dataset", "manifest", str(p), "--out", str(man)]).exit_code == 0
+    assert json.loads(man.read_text())["record_count"] == 4
+    assert runner.invoke(app, ["dataset", "manifest", str(p), "--verify", str(man)]).exit_code == 0
+    # mutate the data → verify must fail
+    write_jsonl(p, _records() + [{"session_id": "z", "role": "user", "text": "new"}])
+    assert runner.invoke(app, ["dataset", "manifest", str(p), "--verify", str(man)]).exit_code == 2