robot-data-diff 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robot_data_diff/__init__.py +25 -1
- robot_data_diff/cli.py +97 -14
- robot_data_diff/differ.py +193 -15
- robot_data_diff/filters.py +54 -0
- robot_data_diff/gate.py +114 -0
- robot_data_diff/html_report.py +320 -0
- robot_data_diff/loader.py +307 -103
- robot_data_diff/report.py +182 -76
- robot_data_diff/stats.py +14 -8
- robot_data_diff-0.2.0.dist-info/METADATA +351 -0
- robot_data_diff-0.2.0.dist-info/RECORD +16 -0
- {robot_data_diff-0.1.0.dist-info → robot_data_diff-0.2.0.dist-info}/WHEEL +1 -1
- robot_data_diff-0.2.0.dist-info/licenses/LICENSE +201 -0
- robot_data_diff-0.1.0.dist-info/METADATA +0 -127
- robot_data_diff-0.1.0.dist-info/RECORD +0 -12
- {robot_data_diff-0.1.0.dist-info → robot_data_diff-0.2.0.dist-info}/entry_points.txt +0 -0
- {robot_data_diff-0.1.0.dist-info → robot_data_diff-0.2.0.dist-info}/top_level.txt +0 -0
robot_data_diff/__init__.py
CHANGED
|
@@ -3,4 +3,28 @@
|
|
|
3
3
|
Answers: "we retrained the policy and it got worse — what changed in the data?"
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
__version__ = version("robot-data-diff")
|
|
10
|
+
except PackageNotFoundError: # running from a source tree without installing
|
|
11
|
+
__version__ = "0.0.0+unknown"
|
|
12
|
+
|
|
13
|
+
from .differ import SCHEMA_VERSION, DiffResult, EpisodeDelta, diff_datasets # noqa: E402
|
|
14
|
+
from .loader import DatasetError, DatasetView, EpisodeInfo, load_dataset # noqa: E402
|
|
15
|
+
from .report import render_html, render_markdown, render_text # noqa: E402
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"SCHEMA_VERSION",
|
|
19
|
+
"DatasetError",
|
|
20
|
+
"DatasetView",
|
|
21
|
+
"DiffResult",
|
|
22
|
+
"EpisodeDelta",
|
|
23
|
+
"EpisodeInfo",
|
|
24
|
+
"__version__",
|
|
25
|
+
"diff_datasets",
|
|
26
|
+
"load_dataset",
|
|
27
|
+
"render_html",
|
|
28
|
+
"render_markdown",
|
|
29
|
+
"render_text",
|
|
30
|
+
]
|
robot_data_diff/cli.py
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
|
-
"""CLI entry point: robot-data-diff diff <A> <B> [--html OUT] [--
|
|
1
|
+
"""CLI entry point: robot-data-diff diff <A> <B> [--format F] [--html OUT] [--json OUT]."""
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import argparse
|
|
5
|
+
import json
|
|
5
6
|
import sys
|
|
7
|
+
import traceback
|
|
6
8
|
from pathlib import Path
|
|
7
9
|
|
|
10
|
+
from . import __version__
|
|
8
11
|
from .differ import diff_datasets
|
|
12
|
+
from .filters import parse_episodes, select_episodes
|
|
13
|
+
from .gate import evaluate, parse_fail_on
|
|
9
14
|
from .loader import load_dataset
|
|
10
|
-
from .report import render_html, render_text
|
|
15
|
+
from .report import render_html, render_markdown, render_text
|
|
11
16
|
|
|
12
17
|
|
|
13
18
|
def build_parser() -> argparse.ArgumentParser:
|
|
@@ -16,45 +21,123 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
16
21
|
description="Episode-aware diff for LeRobot robot-training datasets. "
|
|
17
22
|
"Answers: what changed in the data since the last good run?",
|
|
18
23
|
)
|
|
24
|
+
p.add_argument("--version", action="version",
|
|
25
|
+
version=f"%(prog)s {__version__}")
|
|
19
26
|
sub = p.add_subparsers(dest="cmd", required=True)
|
|
20
27
|
d = sub.add_parser("diff", help="diff two dataset revisions")
|
|
21
28
|
d.add_argument("dataset_a", help="local path or HF id (org/repo[@rev])")
|
|
22
29
|
d.add_argument("dataset_b", help="local path or HF id (org/repo[@rev])")
|
|
30
|
+
d.add_argument("--format", choices=["text", "json", "markdown"], default="text",
|
|
31
|
+
help="what to print on stdout (default: text)")
|
|
23
32
|
d.add_argument("--html", metavar="OUT",
|
|
24
33
|
help="also write an HTML report to OUT")
|
|
34
|
+
d.add_argument("--json", metavar="OUT", dest="json_out",
|
|
35
|
+
help="also write the machine-readable JSON result to OUT")
|
|
25
36
|
d.add_argument("--checkpoint", metavar="LABEL",
|
|
26
37
|
help="policy checkpoint name to link this diff to")
|
|
27
38
|
d.add_argument("--no-quality", action="store_true",
|
|
28
39
|
help="skip per-episode quality stats (metadata-only, faster)")
|
|
40
|
+
d.add_argument("--verify-stats", action="store_true",
|
|
41
|
+
help="recompute per-episode min/max/mean/std from data/ and warn "
|
|
42
|
+
"where the metadata stats disagree (stale metadata)")
|
|
43
|
+
d.add_argument("--match-by", choices=["index", "content"], default="index",
|
|
44
|
+
help="pair episodes by episode_index (default) or by a hash "
|
|
45
|
+
"of their action/state frames, which reports reordered "
|
|
46
|
+
"episodes as moved instead of added + removed")
|
|
47
|
+
d.add_argument("--episodes", metavar="SPEC",
|
|
48
|
+
help="only diff these episode indices, e.g. '0-50,99'")
|
|
49
|
+
d.add_argument("--task", metavar="TEXT", action="append",
|
|
50
|
+
help="only diff episodes whose task (in A or B) contains TEXT "
|
|
51
|
+
"(case-insensitive; repeatable)")
|
|
52
|
+
d.add_argument("--max-show", metavar="N", type=int, default=None,
|
|
53
|
+
help="show at most N changed episodes (and N added/removed "
|
|
54
|
+
"indices) in the text/HTML reports")
|
|
55
|
+
d.add_argument("--fail-on", metavar="RULES",
|
|
56
|
+
help="exit 1 if any rule hits, e.g. "
|
|
57
|
+
"'schema,fps,removed,changed>5%%,idle-increase>0.05' "
|
|
58
|
+
"(see README: Use in CI)")
|
|
59
|
+
d.add_argument("--verbose", action="store_true",
|
|
60
|
+
help="show full tracebacks on errors")
|
|
29
61
|
d.add_argument("--cache-dir", metavar="DIR", default=None,
|
|
30
62
|
help="where to cache Hugging Face downloads")
|
|
31
63
|
return p
|
|
32
64
|
|
|
33
65
|
|
|
34
66
|
def cmd_diff(args) -> int:
|
|
67
|
+
rules = parse_fail_on(args.fail_on) if args.fail_on else []
|
|
68
|
+
episodes = parse_episodes(args.episodes) if args.episodes else None
|
|
69
|
+
if args.max_show is not None and args.max_show < 0:
|
|
70
|
+
raise ValueError("--max-show must be >= 0")
|
|
35
71
|
print("loading A ...", file=sys.stderr)
|
|
36
72
|
a = load_dataset(args.dataset_a, cache_dir=args.cache_dir,
|
|
37
|
-
with_quality=not args.no_quality
|
|
73
|
+
with_quality=not args.no_quality,
|
|
74
|
+
verify_stats=args.verify_stats,
|
|
75
|
+
content_hash=args.match_by == "content")
|
|
38
76
|
print(f" {a.version}, {len(a.episodes)} episodes", file=sys.stderr)
|
|
39
77
|
print("loading B ...", file=sys.stderr)
|
|
40
78
|
b = load_dataset(args.dataset_b, cache_dir=args.cache_dir,
|
|
41
|
-
with_quality=not args.no_quality
|
|
79
|
+
with_quality=not args.no_quality,
|
|
80
|
+
verify_stats=args.verify_stats,
|
|
81
|
+
content_hash=args.match_by == "content")
|
|
42
82
|
print(f" {b.version}, {len(b.episodes)} episodes", file=sys.stderr)
|
|
43
83
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
84
|
+
for label, ds in (("A", a), ("B", b)):
|
|
85
|
+
if ds.stats_mismatches:
|
|
86
|
+
n = len({m["episode"] for m in ds.stats_mismatches})
|
|
87
|
+
print(f"warning: {label}: metadata stats disagree with data/ for {n} "
|
|
88
|
+
"episode(s); metadata may be stale", file=sys.stderr)
|
|
89
|
+
|
|
90
|
+
if episodes is not None or args.task:
|
|
91
|
+
a, b = select_episodes(a, b, episodes=episodes, tasks=args.task)
|
|
92
|
+
print(f"filtered to {len(a.episodes)} (A) / {len(b.episodes)} (B) episodes",
|
|
93
|
+
file=sys.stderr)
|
|
94
|
+
|
|
95
|
+
result = diff_datasets(a, b, checkpoint=args.checkpoint, match_by=args.match_by)
|
|
96
|
+
hits = evaluate(rules, result) if rules else []
|
|
97
|
+
data = result.to_dict()
|
|
98
|
+
if rules:
|
|
99
|
+
data["fail_on"] = {"rules": [r.text for r in rules], "failed": hits}
|
|
100
|
+
as_json = json.dumps(data, indent=2, allow_nan=False)
|
|
101
|
+
if args.format == "json":
|
|
102
|
+
print(as_json)
|
|
103
|
+
elif args.format == "markdown":
|
|
104
|
+
print(render_markdown(result, args.max_show))
|
|
105
|
+
else:
|
|
106
|
+
print(render_text(result, args.max_show))
|
|
107
|
+
for path, render, what in ((args.html, lambda r: render_html(r, args.max_show),
|
|
108
|
+
"HTML report"),
|
|
109
|
+
(args.json_out, lambda _: as_json, "JSON result")):
|
|
110
|
+
if path:
|
|
111
|
+
out = Path(path)
|
|
112
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
113
|
+
out.write_text(render(result) + "\n", encoding="utf-8")
|
|
114
|
+
print(f"{what} written to {path}", file=sys.stderr)
|
|
115
|
+
for hit in hits:
|
|
116
|
+
print(f"fail-on: {hit}", file=sys.stderr)
|
|
117
|
+
return 1 if hits else 0
|
|
52
118
|
|
|
53
119
|
|
|
54
120
|
def main(argv=None) -> int:
|
|
121
|
+
"""Exit codes: 0 OK, 1 a --fail-on rule hit, 2 error (bad input,
|
|
122
|
+
unreadable dataset, ...)."""
|
|
55
123
|
args = build_parser().parse_args(argv)
|
|
56
|
-
|
|
57
|
-
|
|
124
|
+
# Reports contain non-ASCII (arrows, task names); a redirected stdout on
|
|
125
|
+
# Windows would otherwise use the ANSI code page and crash.
|
|
126
|
+
if hasattr(sys.stdout, "reconfigure") and sys.stdout.encoding.lower() != "utf-8":
|
|
127
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
128
|
+
try:
|
|
129
|
+
if args.cmd == "diff":
|
|
130
|
+
return cmd_diff(args)
|
|
131
|
+
except KeyboardInterrupt:
|
|
132
|
+
print("interrupted", file=sys.stderr)
|
|
133
|
+
return 130
|
|
134
|
+
except Exception as e:
|
|
135
|
+
if getattr(args, "verbose", False):
|
|
136
|
+
traceback.print_exc()
|
|
137
|
+
else:
|
|
138
|
+
msg = (str(e).strip().splitlines() or [type(e).__name__])[0]
|
|
139
|
+
print(f"error: {msg}", file=sys.stderr)
|
|
140
|
+
return 2
|
|
58
141
|
return 1
|
|
59
142
|
|
|
60
143
|
|
robot_data_diff/differ.py
CHANGED
|
@@ -1,26 +1,36 @@
|
|
|
1
1
|
"""Episode-aware diff between two DatasetViews."""
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
import math
|
|
4
5
|
from dataclasses import dataclass, field
|
|
5
6
|
|
|
6
7
|
from .loader import DatasetView
|
|
7
|
-
from .stats import fingerprint
|
|
8
|
+
from .stats import STAT_KEYS, fingerprint, rounded
|
|
9
|
+
|
|
10
|
+
# Bump when a key in DiffResult.to_dict() is renamed, removed or changes
|
|
11
|
+
# meaning. Adding keys does not bump it.
|
|
12
|
+
SCHEMA_VERSION = 1
|
|
8
13
|
|
|
9
14
|
|
|
10
15
|
@dataclass
|
|
11
16
|
class EpisodeDelta:
|
|
12
|
-
index: int
|
|
17
|
+
index: int # episode index in B
|
|
13
18
|
old_length: int
|
|
14
19
|
new_length: int
|
|
15
20
|
d_length: int
|
|
16
21
|
tasks_changed: bool
|
|
17
22
|
old_tasks: list = field(default_factory=list)
|
|
18
23
|
new_tasks: list = field(default_factory=list)
|
|
24
|
+
# feature -> {stat: per-dim new - old}; only stats that changed are kept.
|
|
25
|
+
# A value of None means the stat can't be subtracted (shape changed, or
|
|
26
|
+
# missing on one side).
|
|
27
|
+
feature_deltas: dict = field(default_factory=dict)
|
|
19
28
|
d_action_min: list | None = None # per-dim new - old
|
|
20
29
|
d_action_max: list | None = None
|
|
21
30
|
d_action_mean: list | None = None
|
|
22
31
|
d_idle_ratio: float | None = None
|
|
23
32
|
d_spike_count: int | None = None
|
|
33
|
+
old_index: int | None = None # episode index in A (differs only if moved)
|
|
24
34
|
|
|
25
35
|
|
|
26
36
|
@dataclass
|
|
@@ -37,47 +47,212 @@ class DiffResult:
|
|
|
37
47
|
removed: list[int] = field(default_factory=list)
|
|
38
48
|
changed: list[EpisodeDelta] = field(default_factory=list)
|
|
39
49
|
unchanged: int = 0
|
|
50
|
+
# (index in A, index in B) for identical content at a new index;
|
|
51
|
+
# only filled with match_by="content"
|
|
52
|
+
moved: list = field(default_factory=list)
|
|
53
|
+
match_by: str = "index"
|
|
40
54
|
task_counts: dict = field(default_factory=dict) # task -> (count_a, count_b)
|
|
55
|
+
# (field, value in A, value in B) for fps / robot_type / codebase_version
|
|
56
|
+
dataset_changes: list = field(default_factory=list)
|
|
57
|
+
a_robot_type: str | None = None
|
|
58
|
+
b_robot_type: str | None = None
|
|
41
59
|
features_added: list = field(default_factory=list)
|
|
42
60
|
features_removed: list = field(default_factory=list)
|
|
43
|
-
features_changed: list = field(default_factory=list)
|
|
61
|
+
features_changed: list = field(default_factory=list) # "f: {..} -> {..}"
|
|
62
|
+
# (feature, spec in A, spec in B) for each entry of features_changed
|
|
63
|
+
features_changed_specs: list = field(default_factory=list)
|
|
64
|
+
a_stats_verified: bool = False
|
|
65
|
+
b_stats_verified: bool = False
|
|
66
|
+
a_stats_mismatches: list = field(default_factory=list)
|
|
67
|
+
b_stats_mismatches: list = field(default_factory=list)
|
|
44
68
|
mean_idle_a: float | None = None
|
|
45
69
|
mean_idle_b: float | None = None
|
|
46
70
|
spikes_a: int | None = None
|
|
47
71
|
spikes_b: int | None = None
|
|
48
72
|
checkpoint: str | None = None
|
|
73
|
+
a_lengths: list = field(default_factory=list) # every episode's length
|
|
74
|
+
b_lengths: list = field(default_factory=list)
|
|
75
|
+
|
|
76
|
+
def to_dict(self) -> dict:
|
|
77
|
+
"""JSON-ready dict with a stable layout (see docs/json-schema.md)."""
|
|
78
|
+
from . import __version__
|
|
79
|
+
|
|
80
|
+
def side(x):
|
|
81
|
+
lengths = getattr(self, f"{x}_lengths")
|
|
82
|
+
return {
|
|
83
|
+
"name": getattr(self, f"{x}_name"),
|
|
84
|
+
"codebase_version": getattr(self, f"{x}_version"),
|
|
85
|
+
"fps": getattr(self, f"{x}_fps"),
|
|
86
|
+
"robot_type": getattr(self, f"{x}_robot_type"),
|
|
87
|
+
"episodes": getattr(self, f"{x}_episodes"),
|
|
88
|
+
"frames": sum(lengths),
|
|
89
|
+
"mean_idle_ratio": getattr(self, f"mean_idle_{x}"),
|
|
90
|
+
"total_spikes": getattr(self, f"spikes_{x}"),
|
|
91
|
+
"stats_verified": getattr(self, f"{x}_stats_verified"),
|
|
92
|
+
"stats_mismatches": getattr(self, f"{x}_stats_mismatches"),
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
out = {
|
|
96
|
+
"schema_version": SCHEMA_VERSION,
|
|
97
|
+
"tool": {"name": "robot-data-diff", "version": __version__},
|
|
98
|
+
"checkpoint": self.checkpoint,
|
|
99
|
+
"match_by": self.match_by,
|
|
100
|
+
"a": side("a"),
|
|
101
|
+
"b": side("b"),
|
|
102
|
+
"summary": {"added": len(self.added), "removed": len(self.removed),
|
|
103
|
+
"changed": len(self.changed), "moved": len(self.moved),
|
|
104
|
+
"unchanged": self.unchanged},
|
|
105
|
+
"episodes": {
|
|
106
|
+
"added": self.added,
|
|
107
|
+
"removed": self.removed,
|
|
108
|
+
"moved": [{"from": ia, "to": ib} for ia, ib in self.moved],
|
|
109
|
+
"changed": [{
|
|
110
|
+
"index": d.index,
|
|
111
|
+
"old_index": d.old_index if d.old_index is not None else d.index,
|
|
112
|
+
"old_length": d.old_length, "new_length": d.new_length,
|
|
113
|
+
"d_length": d.d_length,
|
|
114
|
+
"tasks_changed": d.tasks_changed,
|
|
115
|
+
"old_tasks": d.old_tasks, "new_tasks": d.new_tasks,
|
|
116
|
+
"feature_deltas": d.feature_deltas,
|
|
117
|
+
"d_idle_ratio": d.d_idle_ratio,
|
|
118
|
+
"d_spike_count": d.d_spike_count,
|
|
119
|
+
} for d in self.changed],
|
|
120
|
+
},
|
|
121
|
+
"tasks": [{"task": t, "a": ca, "b": cb}
|
|
122
|
+
for t, (ca, cb) in self.task_counts.items()],
|
|
123
|
+
"dataset_changes": [{"field": n, "a": va, "b": vb}
|
|
124
|
+
for n, va, vb in self.dataset_changes],
|
|
125
|
+
"schema": {
|
|
126
|
+
"added": self.features_added,
|
|
127
|
+
"removed": self.features_removed,
|
|
128
|
+
"changed": [{"feature": f, "a": sa, "b": sb}
|
|
129
|
+
for f, sa, sb in self.features_changed_specs],
|
|
130
|
+
},
|
|
131
|
+
}
|
|
132
|
+
return _json_safe(out)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _json_safe(x):
|
|
136
|
+
"""Replace NaN/inf (not valid JSON) with None, recursively."""
|
|
137
|
+
if isinstance(x, float):
|
|
138
|
+
return x if math.isfinite(x) else None
|
|
139
|
+
if isinstance(x, dict):
|
|
140
|
+
return {k: _json_safe(v) for k, v in x.items()}
|
|
141
|
+
if isinstance(x, (list, tuple)):
|
|
142
|
+
return [_json_safe(v) for v in x]
|
|
143
|
+
return x
|
|
49
144
|
|
|
50
145
|
|
|
51
146
|
def _sub(a, b):
|
|
52
147
|
if a is None or b is None or len(a) != len(b):
|
|
53
148
|
return None
|
|
54
|
-
return [round(float(x) - float(y), 6) for x, y in zip(b, a)]
|
|
149
|
+
return [round(float(x) - float(y), 6) for x, y in zip(b, a, strict=True)]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _stats_features(ds: DatasetView) -> set:
|
|
153
|
+
return {f for ep in ds.episodes.values() for f in ep.stats}
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _feature_deltas(ea, eb, features) -> dict:
|
|
157
|
+
out = {}
|
|
158
|
+
for f in features:
|
|
159
|
+
fd = {}
|
|
160
|
+
for k in STAT_KEYS:
|
|
161
|
+
va, vb = ea.stat(f, k), eb.stat(f, k)
|
|
162
|
+
if rounded(va) != rounded(vb):
|
|
163
|
+
fd[k] = _sub(va, vb)
|
|
164
|
+
if fd:
|
|
165
|
+
out[f] = fd
|
|
166
|
+
return out
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _pair_by_content(a: DatasetView, b: DatasetView) -> list[tuple[int, int]]:
|
|
170
|
+
"""Pair A and B episodes: same index + same content first, then same
|
|
171
|
+
content at a different index (moved), then leftovers sharing an index
|
|
172
|
+
(edited in place). Unpaired episodes are added / removed."""
|
|
173
|
+
a_left, b_left = set(a.episodes), set(b.episodes)
|
|
174
|
+
pairs = []
|
|
175
|
+
for idx in sorted(a_left & b_left):
|
|
176
|
+
h = a.episodes[idx].content_hash
|
|
177
|
+
if h is not None and h == b.episodes[idx].content_hash:
|
|
178
|
+
pairs.append((idx, idx))
|
|
179
|
+
a_left -= {p[0] for p in pairs}
|
|
180
|
+
b_left -= {p[1] for p in pairs}
|
|
181
|
+
|
|
182
|
+
by_hash: dict[str, list[int]] = {}
|
|
183
|
+
for ia in sorted(a_left):
|
|
184
|
+
h = a.episodes[ia].content_hash
|
|
185
|
+
if h is not None:
|
|
186
|
+
by_hash.setdefault(h, []).append(ia)
|
|
187
|
+
for ib in sorted(b_left):
|
|
188
|
+
candidates = by_hash.get(b.episodes[ib].content_hash)
|
|
189
|
+
if candidates:
|
|
190
|
+
ia = candidates.pop(0)
|
|
191
|
+
pairs.append((ia, ib))
|
|
192
|
+
a_left.discard(ia)
|
|
193
|
+
b_left.discard(ib)
|
|
194
|
+
|
|
195
|
+
pairs.extend((idx, idx) for idx in sorted(a_left & b_left))
|
|
196
|
+
return sorted(pairs, key=lambda p: p[1])
|
|
55
197
|
|
|
56
198
|
|
|
57
199
|
def diff_datasets(a: DatasetView, b: DatasetView,
|
|
58
|
-
checkpoint: str | None = None
|
|
200
|
+
checkpoint: str | None = None,
|
|
201
|
+
match_by: str = "index") -> DiffResult:
|
|
202
|
+
"""Diff two dataset views.
|
|
203
|
+
|
|
204
|
+
match_by="index" (default) pairs episodes by episode_index.
|
|
205
|
+
match_by="content" pairs them by a hash of their action/state frames, so
|
|
206
|
+
reordered episodes are reported as moved instead of added + removed;
|
|
207
|
+
both views must be loaded with load_dataset(..., content_hash=True).
|
|
208
|
+
"""
|
|
209
|
+
if match_by not in ("index", "content"):
|
|
210
|
+
raise ValueError(f"match_by must be 'index' or 'content', not {match_by!r}")
|
|
211
|
+
if match_by == "content":
|
|
212
|
+
for ds in (a, b):
|
|
213
|
+
if any(e.content_hash is None for e in ds.episodes.values()):
|
|
214
|
+
raise ValueError(f"match_by='content' needs {ds.name} loaded with "
|
|
215
|
+
"content_hash=True (and its data/ files present)")
|
|
59
216
|
res = DiffResult(
|
|
60
217
|
a_name=a.name, b_name=b.name,
|
|
61
218
|
a_version=a.version, b_version=b.version,
|
|
62
219
|
a_fps=a.fps, b_fps=b.fps,
|
|
63
220
|
a_episodes=len(a.episodes), b_episodes=len(b.episodes),
|
|
64
|
-
|
|
221
|
+
a_robot_type=a.robot_type, b_robot_type=b.robot_type,
|
|
222
|
+
a_stats_verified=a.stats_verified, b_stats_verified=b.stats_verified,
|
|
223
|
+
a_stats_mismatches=list(a.stats_mismatches),
|
|
224
|
+
b_stats_mismatches=list(b.stats_mismatches),
|
|
225
|
+
checkpoint=checkpoint, match_by=match_by,
|
|
65
226
|
)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
227
|
+
for name, va, vb in (("fps", a.fps, b.fps),
|
|
228
|
+
("robot_type", a.robot_type, b.robot_type),
|
|
229
|
+
("codebase_version", a.version, b.version)):
|
|
230
|
+
if va != vb:
|
|
231
|
+
res.dataset_changes.append((name, va, vb))
|
|
232
|
+
if match_by == "content":
|
|
233
|
+
pairs = _pair_by_content(a, b)
|
|
234
|
+
else:
|
|
235
|
+
pairs = [(i, i) for i in sorted(set(a.episodes) & set(b.episodes))]
|
|
236
|
+
res.added = sorted(set(b.episodes) - {p[1] for p in pairs})
|
|
237
|
+
res.removed = sorted(set(a.episodes) - {p[0] for p in pairs})
|
|
238
|
+
res.moved = [(ia, ib) for ia, ib in pairs if ia != ib]
|
|
239
|
+
|
|
240
|
+
# only compare per-feature stats that both revisions have
|
|
241
|
+
common = sorted(_stats_features(a) & _stats_features(b))
|
|
242
|
+
|
|
243
|
+
for ia, ib in pairs:
|
|
244
|
+
ea, eb = a.episodes[ia], b.episodes[ib]
|
|
245
|
+
if fingerprint(ea, common) == fingerprint(eb, common):
|
|
246
|
+
if ia == ib:
|
|
247
|
+
res.unchanged += 1
|
|
74
248
|
continue
|
|
75
249
|
res.changed.append(EpisodeDelta(
|
|
76
|
-
index=
|
|
250
|
+
index=ib, old_index=ia,
|
|
77
251
|
old_length=ea.length, new_length=eb.length,
|
|
78
252
|
d_length=eb.length - ea.length,
|
|
79
253
|
tasks_changed=sorted(ea.tasks) != sorted(eb.tasks),
|
|
80
254
|
old_tasks=ea.tasks, new_tasks=eb.tasks,
|
|
255
|
+
feature_deltas=_feature_deltas(ea, eb, common),
|
|
81
256
|
d_action_min=_sub(ea.action_min, eb.action_min),
|
|
82
257
|
d_action_max=_sub(ea.action_max, eb.action_max),
|
|
83
258
|
d_action_mean=_sub(ea.action_mean, eb.action_mean),
|
|
@@ -109,6 +284,7 @@ def diff_datasets(a: DatasetView, b: DatasetView,
|
|
|
109
284
|
if a.features[f] != b.features[f]:
|
|
110
285
|
res.features_changed.append(
|
|
111
286
|
f"{f}: {a.features[f]} -> {b.features[f]}")
|
|
287
|
+
res.features_changed_specs.append((f, a.features[f], b.features[f]))
|
|
112
288
|
|
|
113
289
|
# quality summary
|
|
114
290
|
def qmean(ds: DatasetView):
|
|
@@ -121,4 +297,6 @@ def diff_datasets(a: DatasetView, b: DatasetView,
|
|
|
121
297
|
|
|
122
298
|
res.mean_idle_a, res.spikes_a = qmean(a)
|
|
123
299
|
res.mean_idle_b, res.spikes_b = qmean(b)
|
|
300
|
+
res.a_lengths = [e.length for e in a.episodes.values()]
|
|
301
|
+
res.b_lengths = [e.length for e in b.episodes.values()]
|
|
124
302
|
return res
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""--episodes / --task: restrict a diff to a subset of episodes."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import replace
|
|
5
|
+
|
|
6
|
+
from .loader import DatasetView
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def parse_episodes(spec: str) -> set[int]:
|
|
10
|
+
"""'0-50,99' -> {0, ..., 50, 99}; ranges are inclusive."""
|
|
11
|
+
out: set[int] = set()
|
|
12
|
+
for part in spec.split(","):
|
|
13
|
+
part = part.strip()
|
|
14
|
+
try:
|
|
15
|
+
if "-" in part:
|
|
16
|
+
lo, hi = (int(x) for x in part.split("-", 1))
|
|
17
|
+
if lo > hi:
|
|
18
|
+
raise ValueError
|
|
19
|
+
out.update(range(lo, hi + 1))
|
|
20
|
+
else:
|
|
21
|
+
out.add(int(part))
|
|
22
|
+
except ValueError:
|
|
23
|
+
raise ValueError(f"bad --episodes item {part!r} "
|
|
24
|
+
"(expected e.g. '0-50,99')") from None
|
|
25
|
+
return out
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def select_episodes(a: DatasetView, b: DatasetView, episodes: set[int] | None = None,
|
|
29
|
+
tasks: list[str] | None = None) -> tuple[DatasetView, DatasetView]:
|
|
30
|
+
"""Keep only matching episode indices, on both sides.
|
|
31
|
+
|
|
32
|
+
tasks: case-insensitive substrings; an index is kept when its episode in
|
|
33
|
+
A *or* B has a matching task, so an episode whose task was renamed away
|
|
34
|
+
from the filter still shows up as changed rather than removed.
|
|
35
|
+
"""
|
|
36
|
+
keep = set(a.episodes) | set(b.episodes)
|
|
37
|
+
if episodes is not None:
|
|
38
|
+
keep &= episodes
|
|
39
|
+
if tasks:
|
|
40
|
+
needles = [t.lower() for t in tasks]
|
|
41
|
+
|
|
42
|
+
def matches(ep) -> bool:
|
|
43
|
+
return any(n in t.lower() for n in needles for t in ep.tasks)
|
|
44
|
+
|
|
45
|
+
keep &= {i for ds in (a, b) for i, ep in ds.episodes.items() if matches(ep)}
|
|
46
|
+
|
|
47
|
+
def restrict(ds: DatasetView) -> DatasetView:
|
|
48
|
+
return replace(
|
|
49
|
+
ds,
|
|
50
|
+
episodes={i: ep for i, ep in ds.episodes.items() if i in keep},
|
|
51
|
+
stats_mismatches=[m for m in ds.stats_mismatches if m["episode"] in keep],
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
return restrict(a), restrict(b)
|
robot_data_diff/gate.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""--fail-on: turn a DiffResult into a CI pass/fail decision.
|
|
2
|
+
|
|
3
|
+
A spec is a comma-separated list of rules. Each rule is a name, optionally
|
|
4
|
+
followed by `>N`, `>=N` or (for episode counts) `>N%` of A's episodes:
|
|
5
|
+
|
|
6
|
+
schema, fps, robot_type, codebase_version, dataset, tasks, stale
|
|
7
|
+
fail if that kind of change is present (no threshold)
|
|
8
|
+
added, removed, changed, moved
|
|
9
|
+
fail if the count is > 0, or exceeds the threshold
|
|
10
|
+
idle-increase, spikes-increase
|
|
11
|
+
fail if the mean idle ratio / total spike count rose by more than
|
|
12
|
+
the threshold (default 0)
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import operator
|
|
17
|
+
import re
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
|
|
20
|
+
from .differ import DiffResult
|
|
21
|
+
|
|
22
|
+
FLAG_RULES = ("schema", "fps", "robot_type", "codebase_version", "dataset", "tasks",
|
|
23
|
+
"stale")
|
|
24
|
+
COUNT_RULES = ("added", "removed", "changed", "moved")
|
|
25
|
+
QUALITY_RULES = ("idle-increase", "spikes-increase")
|
|
26
|
+
|
|
27
|
+
_RULE = re.compile(r"^([a-z_-]+)\s*(?:(>=|>)\s*(\d+(?:\.\d+)?|\.\d+)\s*(%)?)?$")
|
|
28
|
+
_OPS = {">": operator.gt, ">=": operator.ge}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class Rule:
|
|
33
|
+
name: str
|
|
34
|
+
op: str = ">"
|
|
35
|
+
threshold: float = 0.0
|
|
36
|
+
percent: bool = False
|
|
37
|
+
text: str = ""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def parse_fail_on(spec: str) -> list[Rule]:
|
|
41
|
+
"""Parse 'schema,fps,changed>5%' into rules; ValueError on a bad spec."""
|
|
42
|
+
rules = []
|
|
43
|
+
for raw in spec.split(","):
|
|
44
|
+
text = raw.strip()
|
|
45
|
+
m = _RULE.match(text)
|
|
46
|
+
if not m:
|
|
47
|
+
raise ValueError(f"bad --fail-on rule {text!r} "
|
|
48
|
+
"(expected e.g. 'schema', 'removed', 'changed>5%')")
|
|
49
|
+
name, op, num, pct = m.groups()
|
|
50
|
+
if name not in FLAG_RULES + COUNT_RULES + QUALITY_RULES:
|
|
51
|
+
known = ", ".join(FLAG_RULES + COUNT_RULES + QUALITY_RULES)
|
|
52
|
+
raise ValueError(f"unknown --fail-on rule {name!r} (known: {known})")
|
|
53
|
+
if name in FLAG_RULES and op:
|
|
54
|
+
raise ValueError(f"--fail-on rule {name!r} takes no threshold")
|
|
55
|
+
if pct and name not in COUNT_RULES:
|
|
56
|
+
raise ValueError(f"--fail-on rule {name!r} can't use a percentage")
|
|
57
|
+
rules.append(Rule(name, op or ">", float(num) if num else 0.0, bool(pct), text))
|
|
58
|
+
return rules
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _flag(r: DiffResult, name: str) -> str | None:
|
|
62
|
+
changes = {n: (va, vb) for n, va, vb in r.dataset_changes}
|
|
63
|
+
if name == "schema":
|
|
64
|
+
n = len(r.features_added) + len(r.features_removed) + len(r.features_changed)
|
|
65
|
+
return f"{n} feature(s) added/removed/changed" if n else None
|
|
66
|
+
if name in ("fps", "robot_type", "codebase_version"):
|
|
67
|
+
if name in changes:
|
|
68
|
+
va, vb = changes[name]
|
|
69
|
+
return f"{va} -> {vb}"
|
|
70
|
+
return None
|
|
71
|
+
if name == "dataset":
|
|
72
|
+
return ", ".join(f"{n} {va} -> {vb}" for n, (va, vb) in changes.items()) or None
|
|
73
|
+
if name == "tasks":
|
|
74
|
+
diff = [t for t, (ca, cb) in r.task_counts.items() if ca != cb]
|
|
75
|
+
return f"episode counts changed for {len(diff)} task(s)" if diff else None
|
|
76
|
+
if name == "stale":
|
|
77
|
+
if not (r.a_stats_verified or r.b_stats_verified):
|
|
78
|
+
raise ValueError("--fail-on stale needs --verify-stats")
|
|
79
|
+
n = len(r.a_stats_mismatches) + len(r.b_stats_mismatches)
|
|
80
|
+
return f"{n} metadata stat(s) disagree with data/" if n else None
|
|
81
|
+
raise AssertionError(name)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def evaluate(rules: list[Rule], r: DiffResult) -> list[str]:
|
|
85
|
+
"""Messages for the rules that fail; empty list means the gate passes."""
|
|
86
|
+
hits = []
|
|
87
|
+
for rule in rules:
|
|
88
|
+
if rule.name in FLAG_RULES:
|
|
89
|
+
msg = _flag(r, rule.name)
|
|
90
|
+
if msg:
|
|
91
|
+
hits.append(f"{rule.text}: {msg}")
|
|
92
|
+
continue
|
|
93
|
+
if rule.name in COUNT_RULES:
|
|
94
|
+
count = len(getattr(r, rule.name))
|
|
95
|
+
if rule.percent:
|
|
96
|
+
value = 100.0 * count / r.a_episodes if r.a_episodes else 0.0
|
|
97
|
+
detail = (f"{count} {rule.name} = {value:.1f}% of "
|
|
98
|
+
f"{r.a_episodes} episodes in A")
|
|
99
|
+
else:
|
|
100
|
+
value, detail = count, f"{count} {rule.name}"
|
|
101
|
+
else:
|
|
102
|
+
if r.mean_idle_a is None or r.mean_idle_b is None:
|
|
103
|
+
raise ValueError(f"--fail-on {rule.name} needs quality stats "
|
|
104
|
+
"(drop --no-quality)")
|
|
105
|
+
if rule.name == "idle-increase":
|
|
106
|
+
value = r.mean_idle_b - r.mean_idle_a
|
|
107
|
+
detail = (f"mean idle ratio {r.mean_idle_a:.4f} -> "
|
|
108
|
+
f"{r.mean_idle_b:.4f} ({value:+.4f})")
|
|
109
|
+
else:
|
|
110
|
+
value = r.spikes_b - r.spikes_a
|
|
111
|
+
detail = f"total spikes {r.spikes_a} -> {r.spikes_b} ({value:+d})"
|
|
112
|
+
if _OPS[rule.op](value, rule.threshold):
|
|
113
|
+
hits.append(f"{rule.text}: {detail}")
|
|
114
|
+
return hits
|