robot-data-diff 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,4 +3,28 @@
3
3
  Answers: "we retrained the policy and it got worse — what changed in the data?"
4
4
  """
5
5
 
6
- __version__ = "0.1.0"
6
+ from importlib.metadata import PackageNotFoundError, version
7
+
8
+ try:
9
+ __version__ = version("robot-data-diff")
10
+ except PackageNotFoundError: # running from a source tree without installing
11
+ __version__ = "0.0.0+unknown"
12
+
13
+ from .differ import SCHEMA_VERSION, DiffResult, EpisodeDelta, diff_datasets # noqa: E402
14
+ from .loader import DatasetError, DatasetView, EpisodeInfo, load_dataset # noqa: E402
15
+ from .report import render_html, render_markdown, render_text # noqa: E402
16
+
17
+ __all__ = [
18
+ "SCHEMA_VERSION",
19
+ "DatasetError",
20
+ "DatasetView",
21
+ "DiffResult",
22
+ "EpisodeDelta",
23
+ "EpisodeInfo",
24
+ "__version__",
25
+ "diff_datasets",
26
+ "load_dataset",
27
+ "render_html",
28
+ "render_markdown",
29
+ "render_text",
30
+ ]
robot_data_diff/cli.py CHANGED
@@ -1,13 +1,18 @@
1
- """CLI entry point: robot-data-diff diff <A> <B> [--html OUT] [--checkpoint LBL]."""
1
+ """CLI entry point: robot-data-diff diff <A> <B> [--format F] [--html OUT] [--json OUT]."""
2
2
  from __future__ import annotations
3
3
 
4
4
  import argparse
5
+ import json
5
6
  import sys
7
+ import traceback
6
8
  from pathlib import Path
7
9
 
10
+ from . import __version__
8
11
  from .differ import diff_datasets
12
+ from .filters import parse_episodes, select_episodes
13
+ from .gate import evaluate, parse_fail_on
9
14
  from .loader import load_dataset
10
- from .report import render_html, render_text
15
+ from .report import render_html, render_markdown, render_text
11
16
 
12
17
 
13
18
  def build_parser() -> argparse.ArgumentParser:
@@ -16,45 +21,123 @@ def build_parser() -> argparse.ArgumentParser:
16
21
  description="Episode-aware diff for LeRobot robot-training datasets. "
17
22
  "Answers: what changed in the data since the last good run?",
18
23
  )
24
+ p.add_argument("--version", action="version",
25
+ version=f"%(prog)s {__version__}")
19
26
  sub = p.add_subparsers(dest="cmd", required=True)
20
27
  d = sub.add_parser("diff", help="diff two dataset revisions")
21
28
  d.add_argument("dataset_a", help="local path or HF id (org/repo[@rev])")
22
29
  d.add_argument("dataset_b", help="local path or HF id (org/repo[@rev])")
30
+ d.add_argument("--format", choices=["text", "json", "markdown"], default="text",
31
+ help="what to print on stdout (default: text)")
23
32
  d.add_argument("--html", metavar="OUT",
24
33
  help="also write an HTML report to OUT")
34
+ d.add_argument("--json", metavar="OUT", dest="json_out",
35
+ help="also write the machine-readable JSON result to OUT")
25
36
  d.add_argument("--checkpoint", metavar="LABEL",
26
37
  help="policy checkpoint name to link this diff to")
27
38
  d.add_argument("--no-quality", action="store_true",
28
39
  help="skip per-episode quality stats (metadata-only, faster)")
40
+ d.add_argument("--verify-stats", action="store_true",
41
+ help="recompute per-episode min/max/mean/std from data/ and warn "
42
+ "where the metadata stats disagree (stale metadata)")
43
+ d.add_argument("--match-by", choices=["index", "content"], default="index",
44
+ help="pair episodes by episode_index (default) or by a hash "
45
+ "of their action/state frames, which reports reordered "
46
+ "episodes as moved instead of added + removed")
47
+ d.add_argument("--episodes", metavar="SPEC",
48
+ help="only diff these episode indices, e.g. '0-50,99'")
49
+ d.add_argument("--task", metavar="TEXT", action="append",
50
+ help="only diff episodes whose task (in A or B) contains TEXT "
51
+ "(case-insensitive; repeatable)")
52
+ d.add_argument("--max-show", metavar="N", type=int, default=None,
53
+ help="show at most N changed episodes (and N added/removed "
54
+ "indices) in the text/HTML reports")
55
+ d.add_argument("--fail-on", metavar="RULES",
56
+ help="exit 1 if any rule hits, e.g. "
57
+ "'schema,fps,removed,changed>5%%,idle-increase>0.05' "
58
+ "(see README: Use in CI)")
59
+ d.add_argument("--verbose", action="store_true",
60
+ help="show full tracebacks on errors")
29
61
  d.add_argument("--cache-dir", metavar="DIR", default=None,
30
62
  help="where to cache Hugging Face downloads")
31
63
  return p
32
64
 
33
65
 
34
66
  def cmd_diff(args) -> int:
67
+ rules = parse_fail_on(args.fail_on) if args.fail_on else []
68
+ episodes = parse_episodes(args.episodes) if args.episodes else None
69
+ if args.max_show is not None and args.max_show < 0:
70
+ raise ValueError("--max-show must be >= 0")
35
71
  print("loading A ...", file=sys.stderr)
36
72
  a = load_dataset(args.dataset_a, cache_dir=args.cache_dir,
37
- with_quality=not args.no_quality)
73
+ with_quality=not args.no_quality,
74
+ verify_stats=args.verify_stats,
75
+ content_hash=args.match_by == "content")
38
76
  print(f" {a.version}, {len(a.episodes)} episodes", file=sys.stderr)
39
77
  print("loading B ...", file=sys.stderr)
40
78
  b = load_dataset(args.dataset_b, cache_dir=args.cache_dir,
41
- with_quality=not args.no_quality)
79
+ with_quality=not args.no_quality,
80
+ verify_stats=args.verify_stats,
81
+ content_hash=args.match_by == "content")
42
82
  print(f" {b.version}, {len(b.episodes)} episodes", file=sys.stderr)
43
83
 
44
- result = diff_datasets(a, b, checkpoint=args.checkpoint)
45
- print(render_text(result))
46
- if args.html:
47
- out = Path(args.html)
48
- out.parent.mkdir(parents=True, exist_ok=True)
49
- out.write_text(render_html(result))
50
- print(f"\nHTML report written to {args.html}", file=sys.stderr)
51
- return 0
84
+ for label, ds in (("A", a), ("B", b)):
85
+ if ds.stats_mismatches:
86
+ n = len({m["episode"] for m in ds.stats_mismatches})
87
+ print(f"warning: {label}: metadata stats disagree with data/ for {n} "
88
+ "episode(s); metadata may be stale", file=sys.stderr)
89
+
90
+ if episodes is not None or args.task:
91
+ a, b = select_episodes(a, b, episodes=episodes, tasks=args.task)
92
+ print(f"filtered to {len(a.episodes)} (A) / {len(b.episodes)} (B) episodes",
93
+ file=sys.stderr)
94
+
95
+ result = diff_datasets(a, b, checkpoint=args.checkpoint, match_by=args.match_by)
96
+ hits = evaluate(rules, result) if rules else []
97
+ data = result.to_dict()
98
+ if rules:
99
+ data["fail_on"] = {"rules": [r.text for r in rules], "failed": hits}
100
+ as_json = json.dumps(data, indent=2, allow_nan=False)
101
+ if args.format == "json":
102
+ print(as_json)
103
+ elif args.format == "markdown":
104
+ print(render_markdown(result, args.max_show))
105
+ else:
106
+ print(render_text(result, args.max_show))
107
+ for path, render, what in ((args.html, lambda r: render_html(r, args.max_show),
108
+ "HTML report"),
109
+ (args.json_out, lambda _: as_json, "JSON result")):
110
+ if path:
111
+ out = Path(path)
112
+ out.parent.mkdir(parents=True, exist_ok=True)
113
+ out.write_text(render(result) + "\n", encoding="utf-8")
114
+ print(f"{what} written to {path}", file=sys.stderr)
115
+ for hit in hits:
116
+ print(f"fail-on: {hit}", file=sys.stderr)
117
+ return 1 if hits else 0
52
118
 
53
119
 
54
120
  def main(argv=None) -> int:
121
+ """Exit codes: 0 OK, 1 a --fail-on rule hit, 2 error (bad input,
122
+ unreadable dataset, ...)."""
55
123
  args = build_parser().parse_args(argv)
56
- if args.cmd == "diff":
57
- return cmd_diff(args)
124
+ # Reports contain non-ASCII (arrows, task names); a redirected stdout on
125
+ # Windows would otherwise use the ANSI code page and crash.
126
+ if hasattr(sys.stdout, "reconfigure") and sys.stdout.encoding.lower() != "utf-8":
127
+ sys.stdout.reconfigure(encoding="utf-8")
128
+ try:
129
+ if args.cmd == "diff":
130
+ return cmd_diff(args)
131
+ except KeyboardInterrupt:
132
+ print("interrupted", file=sys.stderr)
133
+ return 130
134
+ except Exception as e:
135
+ if getattr(args, "verbose", False):
136
+ traceback.print_exc()
137
+ else:
138
+ msg = (str(e).strip().splitlines() or [type(e).__name__])[0]
139
+ print(f"error: {msg}", file=sys.stderr)
140
+ return 2
58
141
  return 1
59
142
 
60
143
 
robot_data_diff/differ.py CHANGED
@@ -1,26 +1,36 @@
1
1
  """Episode-aware diff between two DatasetViews."""
2
2
  from __future__ import annotations
3
3
 
4
+ import math
4
5
  from dataclasses import dataclass, field
5
6
 
6
7
  from .loader import DatasetView
7
- from .stats import fingerprint
8
+ from .stats import STAT_KEYS, fingerprint, rounded
9
+
10
+ # Bump when a key in DiffResult.to_dict() is renamed, removed or changes
11
+ # meaning. Adding keys does not bump it.
12
+ SCHEMA_VERSION = 1
8
13
 
9
14
 
10
15
  @dataclass
11
16
  class EpisodeDelta:
12
- index: int
17
+ index: int # episode index in B
13
18
  old_length: int
14
19
  new_length: int
15
20
  d_length: int
16
21
  tasks_changed: bool
17
22
  old_tasks: list = field(default_factory=list)
18
23
  new_tasks: list = field(default_factory=list)
24
+ # feature -> {stat: per-dim new - old}; only stats that changed are kept.
25
+ # A value of None means the stat can't be subtracted (shape changed, or
26
+ # missing on one side).
27
+ feature_deltas: dict = field(default_factory=dict)
19
28
  d_action_min: list | None = None # per-dim new - old
20
29
  d_action_max: list | None = None
21
30
  d_action_mean: list | None = None
22
31
  d_idle_ratio: float | None = None
23
32
  d_spike_count: int | None = None
33
+ old_index: int | None = None # episode index in A (differs only if moved)
24
34
 
25
35
 
26
36
  @dataclass
@@ -37,47 +47,212 @@ class DiffResult:
37
47
  removed: list[int] = field(default_factory=list)
38
48
  changed: list[EpisodeDelta] = field(default_factory=list)
39
49
  unchanged: int = 0
50
+ # (index in A, index in B) for identical content at a new index;
51
+ # only filled with match_by="content"
52
+ moved: list = field(default_factory=list)
53
+ match_by: str = "index"
40
54
  task_counts: dict = field(default_factory=dict) # task -> (count_a, count_b)
55
+ # (field, value in A, value in B) for fps / robot_type / codebase_version
56
+ dataset_changes: list = field(default_factory=list)
57
+ a_robot_type: str | None = None
58
+ b_robot_type: str | None = None
41
59
  features_added: list = field(default_factory=list)
42
60
  features_removed: list = field(default_factory=list)
43
- features_changed: list = field(default_factory=list)
61
+ features_changed: list = field(default_factory=list) # "f: {..} -> {..}"
62
+ # (feature, spec in A, spec in B) for each entry of features_changed
63
+ features_changed_specs: list = field(default_factory=list)
64
+ a_stats_verified: bool = False
65
+ b_stats_verified: bool = False
66
+ a_stats_mismatches: list = field(default_factory=list)
67
+ b_stats_mismatches: list = field(default_factory=list)
44
68
  mean_idle_a: float | None = None
45
69
  mean_idle_b: float | None = None
46
70
  spikes_a: int | None = None
47
71
  spikes_b: int | None = None
48
72
  checkpoint: str | None = None
73
+ a_lengths: list = field(default_factory=list) # every episode's length
74
+ b_lengths: list = field(default_factory=list)
75
+
76
+ def to_dict(self) -> dict:
77
+ """JSON-ready dict with a stable layout (see docs/json-schema.md)."""
78
+ from . import __version__
79
+
80
+ def side(x):
81
+ lengths = getattr(self, f"{x}_lengths")
82
+ return {
83
+ "name": getattr(self, f"{x}_name"),
84
+ "codebase_version": getattr(self, f"{x}_version"),
85
+ "fps": getattr(self, f"{x}_fps"),
86
+ "robot_type": getattr(self, f"{x}_robot_type"),
87
+ "episodes": getattr(self, f"{x}_episodes"),
88
+ "frames": sum(lengths),
89
+ "mean_idle_ratio": getattr(self, f"mean_idle_{x}"),
90
+ "total_spikes": getattr(self, f"spikes_{x}"),
91
+ "stats_verified": getattr(self, f"{x}_stats_verified"),
92
+ "stats_mismatches": getattr(self, f"{x}_stats_mismatches"),
93
+ }
94
+
95
+ out = {
96
+ "schema_version": SCHEMA_VERSION,
97
+ "tool": {"name": "robot-data-diff", "version": __version__},
98
+ "checkpoint": self.checkpoint,
99
+ "match_by": self.match_by,
100
+ "a": side("a"),
101
+ "b": side("b"),
102
+ "summary": {"added": len(self.added), "removed": len(self.removed),
103
+ "changed": len(self.changed), "moved": len(self.moved),
104
+ "unchanged": self.unchanged},
105
+ "episodes": {
106
+ "added": self.added,
107
+ "removed": self.removed,
108
+ "moved": [{"from": ia, "to": ib} for ia, ib in self.moved],
109
+ "changed": [{
110
+ "index": d.index,
111
+ "old_index": d.old_index if d.old_index is not None else d.index,
112
+ "old_length": d.old_length, "new_length": d.new_length,
113
+ "d_length": d.d_length,
114
+ "tasks_changed": d.tasks_changed,
115
+ "old_tasks": d.old_tasks, "new_tasks": d.new_tasks,
116
+ "feature_deltas": d.feature_deltas,
117
+ "d_idle_ratio": d.d_idle_ratio,
118
+ "d_spike_count": d.d_spike_count,
119
+ } for d in self.changed],
120
+ },
121
+ "tasks": [{"task": t, "a": ca, "b": cb}
122
+ for t, (ca, cb) in self.task_counts.items()],
123
+ "dataset_changes": [{"field": n, "a": va, "b": vb}
124
+ for n, va, vb in self.dataset_changes],
125
+ "schema": {
126
+ "added": self.features_added,
127
+ "removed": self.features_removed,
128
+ "changed": [{"feature": f, "a": sa, "b": sb}
129
+ for f, sa, sb in self.features_changed_specs],
130
+ },
131
+ }
132
+ return _json_safe(out)
133
+
134
+
135
+ def _json_safe(x):
136
+ """Replace NaN/inf (not valid JSON) with None, recursively."""
137
+ if isinstance(x, float):
138
+ return x if math.isfinite(x) else None
139
+ if isinstance(x, dict):
140
+ return {k: _json_safe(v) for k, v in x.items()}
141
+ if isinstance(x, (list, tuple)):
142
+ return [_json_safe(v) for v in x]
143
+ return x
49
144
 
50
145
 
51
146
  def _sub(a, b):
52
147
  if a is None or b is None or len(a) != len(b):
53
148
  return None
54
- return [round(float(x) - float(y), 6) for x, y in zip(b, a)]
149
+ return [round(float(x) - float(y), 6) for x, y in zip(b, a, strict=True)]
150
+
151
+
152
+ def _stats_features(ds: DatasetView) -> set:
153
+ return {f for ep in ds.episodes.values() for f in ep.stats}
154
+
155
+
156
+ def _feature_deltas(ea, eb, features) -> dict:
157
+ out = {}
158
+ for f in features:
159
+ fd = {}
160
+ for k in STAT_KEYS:
161
+ va, vb = ea.stat(f, k), eb.stat(f, k)
162
+ if rounded(va) != rounded(vb):
163
+ fd[k] = _sub(va, vb)
164
+ if fd:
165
+ out[f] = fd
166
+ return out
167
+
168
+
169
+ def _pair_by_content(a: DatasetView, b: DatasetView) -> list[tuple[int, int]]:
170
+ """Pair A and B episodes: same index + same content first, then same
171
+ content at a different index (moved), then leftovers sharing an index
172
+ (edited in place). Unpaired episodes are added / removed."""
173
+ a_left, b_left = set(a.episodes), set(b.episodes)
174
+ pairs = []
175
+ for idx in sorted(a_left & b_left):
176
+ h = a.episodes[idx].content_hash
177
+ if h is not None and h == b.episodes[idx].content_hash:
178
+ pairs.append((idx, idx))
179
+ a_left -= {p[0] for p in pairs}
180
+ b_left -= {p[1] for p in pairs}
181
+
182
+ by_hash: dict[str, list[int]] = {}
183
+ for ia in sorted(a_left):
184
+ h = a.episodes[ia].content_hash
185
+ if h is not None:
186
+ by_hash.setdefault(h, []).append(ia)
187
+ for ib in sorted(b_left):
188
+ candidates = by_hash.get(b.episodes[ib].content_hash)
189
+ if candidates:
190
+ ia = candidates.pop(0)
191
+ pairs.append((ia, ib))
192
+ a_left.discard(ia)
193
+ b_left.discard(ib)
194
+
195
+ pairs.extend((idx, idx) for idx in sorted(a_left & b_left))
196
+ return sorted(pairs, key=lambda p: p[1])
55
197
 
56
198
 
57
199
  def diff_datasets(a: DatasetView, b: DatasetView,
58
- checkpoint: str | None = None) -> DiffResult:
200
+ checkpoint: str | None = None,
201
+ match_by: str = "index") -> DiffResult:
202
+ """Diff two dataset views.
203
+
204
+ match_by="index" (default) pairs episodes by episode_index.
205
+ match_by="content" pairs them by a hash of their action/state frames, so
206
+ reordered episodes are reported as moved instead of added + removed;
207
+ both views must be loaded with load_dataset(..., content_hash=True).
208
+ """
209
+ if match_by not in ("index", "content"):
210
+ raise ValueError(f"match_by must be 'index' or 'content', not {match_by!r}")
211
+ if match_by == "content":
212
+ for ds in (a, b):
213
+ if any(e.content_hash is None for e in ds.episodes.values()):
214
+ raise ValueError(f"match_by='content' needs {ds.name} loaded with "
215
+ "content_hash=True (and its data/ files present)")
59
216
  res = DiffResult(
60
217
  a_name=a.name, b_name=b.name,
61
218
  a_version=a.version, b_version=b.version,
62
219
  a_fps=a.fps, b_fps=b.fps,
63
220
  a_episodes=len(a.episodes), b_episodes=len(b.episodes),
64
- checkpoint=checkpoint,
221
+ a_robot_type=a.robot_type, b_robot_type=b.robot_type,
222
+ a_stats_verified=a.stats_verified, b_stats_verified=b.stats_verified,
223
+ a_stats_mismatches=list(a.stats_mismatches),
224
+ b_stats_mismatches=list(b.stats_mismatches),
225
+ checkpoint=checkpoint, match_by=match_by,
65
226
  )
66
- ai, bi = set(a.episodes), set(b.episodes)
67
- res.added = sorted(bi - ai)
68
- res.removed = sorted(ai - bi)
69
-
70
- for idx in sorted(ai & bi):
71
- ea, eb = a.episodes[idx], b.episodes[idx]
72
- if fingerprint(ea) == fingerprint(eb):
73
- res.unchanged += 1
227
+ for name, va, vb in (("fps", a.fps, b.fps),
228
+ ("robot_type", a.robot_type, b.robot_type),
229
+ ("codebase_version", a.version, b.version)):
230
+ if va != vb:
231
+ res.dataset_changes.append((name, va, vb))
232
+ if match_by == "content":
233
+ pairs = _pair_by_content(a, b)
234
+ else:
235
+ pairs = [(i, i) for i in sorted(set(a.episodes) & set(b.episodes))]
236
+ res.added = sorted(set(b.episodes) - {p[1] for p in pairs})
237
+ res.removed = sorted(set(a.episodes) - {p[0] for p in pairs})
238
+ res.moved = [(ia, ib) for ia, ib in pairs if ia != ib]
239
+
240
+ # only compare per-feature stats that both revisions have
241
+ common = sorted(_stats_features(a) & _stats_features(b))
242
+
243
+ for ia, ib in pairs:
244
+ ea, eb = a.episodes[ia], b.episodes[ib]
245
+ if fingerprint(ea, common) == fingerprint(eb, common):
246
+ if ia == ib:
247
+ res.unchanged += 1
74
248
  continue
75
249
  res.changed.append(EpisodeDelta(
76
- index=idx,
250
+ index=ib, old_index=ia,
77
251
  old_length=ea.length, new_length=eb.length,
78
252
  d_length=eb.length - ea.length,
79
253
  tasks_changed=sorted(ea.tasks) != sorted(eb.tasks),
80
254
  old_tasks=ea.tasks, new_tasks=eb.tasks,
255
+ feature_deltas=_feature_deltas(ea, eb, common),
81
256
  d_action_min=_sub(ea.action_min, eb.action_min),
82
257
  d_action_max=_sub(ea.action_max, eb.action_max),
83
258
  d_action_mean=_sub(ea.action_mean, eb.action_mean),
@@ -109,6 +284,7 @@ def diff_datasets(a: DatasetView, b: DatasetView,
109
284
  if a.features[f] != b.features[f]:
110
285
  res.features_changed.append(
111
286
  f"{f}: {a.features[f]} -> {b.features[f]}")
287
+ res.features_changed_specs.append((f, a.features[f], b.features[f]))
112
288
 
113
289
  # quality summary
114
290
  def qmean(ds: DatasetView):
@@ -121,4 +297,6 @@ def diff_datasets(a: DatasetView, b: DatasetView,
121
297
 
122
298
  res.mean_idle_a, res.spikes_a = qmean(a)
123
299
  res.mean_idle_b, res.spikes_b = qmean(b)
300
+ res.a_lengths = [e.length for e in a.episodes.values()]
301
+ res.b_lengths = [e.length for e in b.episodes.values()]
124
302
  return res
@@ -0,0 +1,54 @@
1
+ """--episodes / --task: restrict a diff to a subset of episodes."""
2
+ from __future__ import annotations
3
+
4
+ from dataclasses import replace
5
+
6
+ from .loader import DatasetView
7
+
8
+
9
+ def parse_episodes(spec: str) -> set[int]:
10
+ """'0-50,99' -> {0, ..., 50, 99}; ranges are inclusive."""
11
+ out: set[int] = set()
12
+ for part in spec.split(","):
13
+ part = part.strip()
14
+ try:
15
+ if "-" in part:
16
+ lo, hi = (int(x) for x in part.split("-", 1))
17
+ if lo > hi:
18
+ raise ValueError
19
+ out.update(range(lo, hi + 1))
20
+ else:
21
+ out.add(int(part))
22
+ except ValueError:
23
+ raise ValueError(f"bad --episodes item {part!r} "
24
+ "(expected e.g. '0-50,99')") from None
25
+ return out
26
+
27
+
28
+ def select_episodes(a: DatasetView, b: DatasetView, episodes: set[int] | None = None,
29
+ tasks: list[str] | None = None) -> tuple[DatasetView, DatasetView]:
30
+ """Keep only matching episode indices, on both sides.
31
+
32
+ tasks: case-insensitive substrings; an index is kept when its episode in
33
+ A *or* B has a matching task, so an episode whose task was renamed away
34
+ from the filter still shows up as changed rather than removed.
35
+ """
36
+ keep = set(a.episodes) | set(b.episodes)
37
+ if episodes is not None:
38
+ keep &= episodes
39
+ if tasks:
40
+ needles = [t.lower() for t in tasks]
41
+
42
+ def matches(ep) -> bool:
43
+ return any(n in t.lower() for n in needles for t in ep.tasks)
44
+
45
+ keep &= {i for ds in (a, b) for i, ep in ds.episodes.items() if matches(ep)}
46
+
47
+ def restrict(ds: DatasetView) -> DatasetView:
48
+ return replace(
49
+ ds,
50
+ episodes={i: ep for i, ep in ds.episodes.items() if i in keep},
51
+ stats_mismatches=[m for m in ds.stats_mismatches if m["episode"] in keep],
52
+ )
53
+
54
+ return restrict(a), restrict(b)
@@ -0,0 +1,114 @@
1
+ """--fail-on: turn a DiffResult into a CI pass/fail decision.
2
+
3
+ A spec is a comma-separated list of rules. Each rule is a name, optionally
4
+ followed by `>N`, `>=N` or (for episode counts) `>N%` of A's episodes:
5
+
6
+ schema, fps, robot_type, codebase_version, dataset, tasks, stale
7
+ fail if that kind of change is present (no threshold)
8
+ added, removed, changed, moved
9
+ fail if the count is > 0, or exceeds the threshold
10
+ idle-increase, spikes-increase
11
+ fail if the mean idle ratio / total spike count rose by more than
12
+ the threshold (default 0)
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import operator
17
+ import re
18
+ from dataclasses import dataclass
19
+
20
+ from .differ import DiffResult
21
+
22
+ FLAG_RULES = ("schema", "fps", "robot_type", "codebase_version", "dataset", "tasks",
23
+ "stale")
24
+ COUNT_RULES = ("added", "removed", "changed", "moved")
25
+ QUALITY_RULES = ("idle-increase", "spikes-increase")
26
+
27
+ _RULE = re.compile(r"^([a-z_-]+)\s*(?:(>=|>)\s*(\d+(?:\.\d+)?|\.\d+)\s*(%)?)?$")
28
+ _OPS = {">": operator.gt, ">=": operator.ge}
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class Rule:
33
+ name: str
34
+ op: str = ">"
35
+ threshold: float = 0.0
36
+ percent: bool = False
37
+ text: str = ""
38
+
39
+
40
+ def parse_fail_on(spec: str) -> list[Rule]:
41
+ """Parse 'schema,fps,changed>5%' into rules; ValueError on a bad spec."""
42
+ rules = []
43
+ for raw in spec.split(","):
44
+ text = raw.strip()
45
+ m = _RULE.match(text)
46
+ if not m:
47
+ raise ValueError(f"bad --fail-on rule {text!r} "
48
+ "(expected e.g. 'schema', 'removed', 'changed>5%')")
49
+ name, op, num, pct = m.groups()
50
+ if name not in FLAG_RULES + COUNT_RULES + QUALITY_RULES:
51
+ known = ", ".join(FLAG_RULES + COUNT_RULES + QUALITY_RULES)
52
+ raise ValueError(f"unknown --fail-on rule {name!r} (known: {known})")
53
+ if name in FLAG_RULES and op:
54
+ raise ValueError(f"--fail-on rule {name!r} takes no threshold")
55
+ if pct and name not in COUNT_RULES:
56
+ raise ValueError(f"--fail-on rule {name!r} can't use a percentage")
57
+ rules.append(Rule(name, op or ">", float(num) if num else 0.0, bool(pct), text))
58
+ return rules
59
+
60
+
61
+ def _flag(r: DiffResult, name: str) -> str | None:
62
+ changes = {n: (va, vb) for n, va, vb in r.dataset_changes}
63
+ if name == "schema":
64
+ n = len(r.features_added) + len(r.features_removed) + len(r.features_changed)
65
+ return f"{n} feature(s) added/removed/changed" if n else None
66
+ if name in ("fps", "robot_type", "codebase_version"):
67
+ if name in changes:
68
+ va, vb = changes[name]
69
+ return f"{va} -> {vb}"
70
+ return None
71
+ if name == "dataset":
72
+ return ", ".join(f"{n} {va} -> {vb}" for n, (va, vb) in changes.items()) or None
73
+ if name == "tasks":
74
+ diff = [t for t, (ca, cb) in r.task_counts.items() if ca != cb]
75
+ return f"episode counts changed for {len(diff)} task(s)" if diff else None
76
+ if name == "stale":
77
+ if not (r.a_stats_verified or r.b_stats_verified):
78
+ raise ValueError("--fail-on stale needs --verify-stats")
79
+ n = len(r.a_stats_mismatches) + len(r.b_stats_mismatches)
80
+ return f"{n} metadata stat(s) disagree with data/" if n else None
81
+ raise AssertionError(name)
82
+
83
+
84
+ def evaluate(rules: list[Rule], r: DiffResult) -> list[str]:
85
+ """Messages for the rules that fail; empty list means the gate passes."""
86
+ hits = []
87
+ for rule in rules:
88
+ if rule.name in FLAG_RULES:
89
+ msg = _flag(r, rule.name)
90
+ if msg:
91
+ hits.append(f"{rule.text}: {msg}")
92
+ continue
93
+ if rule.name in COUNT_RULES:
94
+ count = len(getattr(r, rule.name))
95
+ if rule.percent:
96
+ value = 100.0 * count / r.a_episodes if r.a_episodes else 0.0
97
+ detail = (f"{count} {rule.name} = {value:.1f}% of "
98
+ f"{r.a_episodes} episodes in A")
99
+ else:
100
+ value, detail = count, f"{count} {rule.name}"
101
+ else:
102
+ if r.mean_idle_a is None or r.mean_idle_b is None:
103
+ raise ValueError(f"--fail-on {rule.name} needs quality stats "
104
+ "(drop --no-quality)")
105
+ if rule.name == "idle-increase":
106
+ value = r.mean_idle_b - r.mean_idle_a
107
+ detail = (f"mean idle ratio {r.mean_idle_a:.4f} -> "
108
+ f"{r.mean_idle_b:.4f} ({value:+.4f})")
109
+ else:
110
+ value = r.spikes_b - r.spikes_a
111
+ detail = f"total spikes {r.spikes_a} -> {r.spikes_b} ({value:+d})"
112
+ if _OPS[rule.op](value, rule.threshold):
113
+ hits.append(f"{rule.text}: {detail}")
114
+ return hits