robolint 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
robolint/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """robolint: quality checks for robot learning datasets (LeRobot format)."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from .api import Result, lint # noqa: E402
6
+ from .checks import Config # noqa: E402
7
+ from .findings import Finding, Severity # noqa: E402
8
+
9
+ __all__ = ["Config", "Finding", "Result", "Severity", "__version__", "lint"]
robolint/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
robolint/api.py ADDED
@@ -0,0 +1,108 @@
1
+ """Programmatic entry point: lint a dataset and get a structured result."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+ from collections import Counter
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from . import __version__
12
+ from .checks import Config, all_checks
13
+ from .dataset import Dataset, hub_blob_dirs, load_dataset, resolve_source
14
+ from .findings import Finding, Scores, Severity, score
15
+
16
+
17
+ @dataclass
18
+ class Result:
19
+ source: str
20
+ dataset: Dataset
21
+ findings: list[Finding]
22
+ scores: Scores
23
+ videos_checked: bool
24
+ seconds: float
25
+ extra: dict[str, Any] = field(default_factory=dict)
26
+
27
+ @property
28
+ def dataset_findings(self) -> list[Finding]:
29
+ return [f for f in self.findings if f.episode is None]
30
+
31
+ def episode_findings(self, ep: int) -> list[Finding]:
32
+ return [f for f in self.findings if f.episode == ep]
33
+
34
+ def code_counts(self) -> list[tuple[str, Severity, int]]:
35
+ """(code, worst severity, number of episodes or 1 for dataset-level) sorted by impact."""
36
+ eps: dict[str, set] = {}
37
+ sev: dict[str, Severity] = {}
38
+ for f in self.findings:
39
+ eps.setdefault(f.code, set()).add(f.episode)
40
+ sev[f.code] = max(sev.get(f.code, f.severity), f.severity)
41
+ rows = [(c, sev[c], len(e)) for c, e in eps.items()]
42
+ return sorted(rows, key=lambda r: (-int(r[1]), -r[2], r[0]))
43
+
44
+ def bad_episodes(self, min_severity: Severity = Severity.ERROR) -> list[int]:
45
+ return sorted({f.episode for f in self.findings if f.episode is not None and f.severity >= min_severity})
46
+
47
+ def to_dict(self) -> dict[str, Any]:
48
+ ds = self.dataset
49
+ sev = Counter(f.severity.label for f in self.findings)
50
+ return {
51
+ "tool": {"name": "robolint", "version": __version__},
52
+ "source": self.source,
53
+ "dataset": {
54
+ "codebase_version": ds.version,
55
+ "robot_type": ds.info.get("robot_type"),
56
+ "fps": ds.fps,
57
+ "episodes": len(ds.episodes),
58
+ "frames": ds.total_frames,
59
+ "cameras": ds.video_keys,
60
+ "tasks": len(ds.tasks),
61
+ },
62
+ "score": self.scores.dataset_score,
63
+ "grade": self.scores.grade,
64
+ "episodes": {
65
+ "clean": self.scores.clean_episodes,
66
+ "with_warnings": self.scores.episodes_with_warnings,
67
+ "with_errors": self.scores.episodes_with_errors,
68
+ },
69
+ "videos_checked": self.videos_checked,
70
+ "finding_counts": dict(sev),
71
+ "issues": [{"code": c, "severity": s.label, "affected": n} for c, s, n in self.code_counts()],
72
+ "episode_scores": {str(k): v for k, v in sorted(self.scores.episode_scores.items())},
73
+ "findings": [f.to_dict() for f in self.findings],
74
+ "seconds": round(self.seconds, 2),
75
+ }
76
+
77
+
78
+ def lint(
79
+ source: str | Path,
80
+ *,
81
+ videos: bool = False,
82
+ revision: str | None = None,
83
+ cache_dir: str | None = None,
84
+ max_episodes: int | None = None,
85
+ config: Config | None = None,
86
+ ) -> Result:
87
+ t0 = time.perf_counter()
88
+ cfg = config or Config()
89
+ # Only a Hub download gets the relaxed symlink rule; a local folder keeps the strict one.
90
+ from_hub = not Path(str(source)).expanduser().exists()
91
+ root = resolve_source(
92
+ str(source), include_videos=videos, revision=revision, cache_dir=cache_dir, max_episodes=max_episodes
93
+ )
94
+ hub_blobs = hub_blob_dirs(root) if from_hub else None
95
+ ds = load_dataset(root, source=str(source), max_episodes=max_episodes, hub_blobs=hub_blobs)
96
+ findings: list[Finding] = []
97
+ for check in all_checks(include_video=videos):
98
+ findings.extend(check(ds, cfg))
99
+ findings.sort(key=lambda f: (f.episode if f.episode is not None else -1, -int(f.severity), f.code))
100
+ scores = score(findings, [ep.index for ep in ds.episodes])
101
+ return Result(
102
+ source=str(source),
103
+ dataset=ds,
104
+ findings=findings,
105
+ scores=scores,
106
+ videos_checked=videos,
107
+ seconds=time.perf_counter() - t0,
108
+ )
@@ -0,0 +1,53 @@
1
+ """Check registry and shared configuration."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable, Iterable
6
+ from dataclasses import dataclass
7
+
8
+ from ..dataset import Dataset
9
+ from ..findings import Finding
10
+
11
+
12
+ @dataclass
13
+ class Config:
14
+ # Timing
15
+ gap_factor: float = 1.5 # dt above this many frame periods counts as dropped frames
16
+ jitter_ratio: float = 0.10 # std(dt) / period above this is jittery
17
+ drift_frames: float = 2.0 # end-to-end drift tolerance, in frame periods
18
+ # Signals (fractions of each dimension's dataset-wide range)
19
+ move_tol: float = 0.005 # change smaller than this counts as "not moving"
20
+ dead_tol: float = 0.01 # every dimension moves less than this -> dead episode
21
+ gripper_tol: float = 0.05 # gripper travel smaller than this -> never actuated
22
+ spike_quantile: float = 0.995 # robust scale for per-step action changes
23
+ spike_multiplier: float = 6.0
24
+ spike_min_range: float = 0.15 # a spike must also be at least this fraction of the range
25
+ idle_warn: float = 0.30 # idle lead-in + tail above this fraction of the episode
26
+ idle_info: float = 0.15
27
+ limit_seconds: float = 2.0 # pinned at a joint's extreme for this long
28
+ # Episodes
29
+ min_seconds: float = 1.0
30
+ short_ratio: float = 0.25 # shorter than this fraction of the median length
31
+ long_ratio: float = 3.0
32
+ duplicate_tol: float = 0.002 # mean abs difference (fraction of range) for duplicates
33
+ # Video
34
+ video_samples: int = 8
35
+ black_level: float = 8.0
36
+ white_level: float = 247.0
37
+ blank_std: float = 2.0
38
+ frozen_diff: float = 1.0 # minimum mean pixel change (0-255) before a feed counts as live
39
+ frozen_min_seconds: float = 2.0 # shorter episodes are too brief to judge
40
+
41
+
42
+ CheckFn = Callable[[Dataset, Config], Iterable[Finding]]
43
+
44
+
45
+ def all_checks(include_video: bool) -> list[CheckFn]:
46
+ from . import episodes, meta, signals, timing
47
+
48
+ checks: list[CheckFn] = [meta.check, timing.check, signals.check, episodes.check]
49
+ if include_video:
50
+ from . import video
51
+
52
+ checks.append(video.check)
53
+ return checks
@@ -0,0 +1,126 @@
1
+ """Episode-level checks: lengths, tasks, and duplicate demonstrations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterator
6
+
7
+ import numpy as np
8
+
9
+ from ..dataset import Dataset
10
+ from ..findings import Finding, Severity
11
+ from . import Config
12
+ from .signals import dataset_stats
13
+
14
+ E, W, I = Severity.ERROR, Severity.WARNING, Severity.INFO
15
+ RESAMPLE = 64
16
+
17
+
18
+ def check(ds: Dataset, cfg: Config) -> Iterator[Finding]:
19
+ if not ds.episodes:
20
+ yield Finding("NO_EPISODES", E, "dataset contains no episodes")
21
+ return
22
+ fps = ds.fps or 30.0
23
+ lengths = np.array([ep.length for ep in ds.episodes])
24
+ median = float(np.median(lengths))
25
+
26
+ for ep in ds.episodes:
27
+ n = ep.length
28
+ if ep.length_meta is not None and ep.length_meta != n:
29
+ yield Finding(
30
+ "EPISODE_LENGTH_MISMATCH",
31
+ E,
32
+ f"metadata says {ep.length_meta} frames, data has {n}",
33
+ ep.index,
34
+ {"metadata": ep.length_meta, "data": n},
35
+ )
36
+ if 0 < n < cfg.min_seconds * fps:
37
+ yield Finding("EPISODE_TOO_SHORT", E, f"only {n} frames ({n / fps:.2f}s): likely a false start", ep.index)
38
+ elif len(lengths) >= 5 and n < cfg.short_ratio * median:
39
+ yield Finding(
40
+ "EPISODE_SHORT",
41
+ W,
42
+ f"{n} frames, under {cfg.short_ratio:.0%} of the median ({median:.0f}): possibly truncated",
43
+ ep.index,
44
+ )
45
+ if len(lengths) >= 5 and n > cfg.long_ratio * median and n - median > 5 * fps:
46
+ yield Finding(
47
+ "EPISODE_LONG",
48
+ I,
49
+ f"{n} frames, {n / median:.1f}x the median: recording may not have been stopped",
50
+ ep.index,
51
+ )
52
+
53
+ ti = ep.frames.get("task_index")
54
+ if ti is not None and ds.tasks:
55
+ unknown = sorted({int(x) for x in np.unique(ti)} - set(ds.tasks))
56
+ if unknown:
57
+ yield Finding(
58
+ "TASK_INDEX_UNKNOWN", E, f"task_index values {unknown[:5]} have no task description", ep.index
59
+ )
60
+ if ep.tasks and not any(t.strip() for t in ep.tasks):
61
+ yield Finding("EPISODE_TASK_EMPTY", W, "episode's task description is empty", ep.index)
62
+
63
+ yield from _duplicates(ds, cfg)
64
+
65
+
66
+ def _signature(a: np.ndarray, lo: np.ndarray, rng: np.ndarray) -> np.ndarray:
67
+ a = a.reshape(-1, 1) if a.ndim == 1 else a
68
+ t_old = np.linspace(0.0, 1.0, len(a))
69
+ t_new = np.linspace(0.0, 1.0, RESAMPLE)
70
+ res = np.stack([np.interp(t_new, t_old, a[:, j]) for j in range(a.shape[1])], axis=1)
71
+ return ((res - lo) / rng).ravel()
72
+
73
+
74
+ def _duplicates(ds: Dataset, cfg: Config) -> Iterator[Finding]:
75
+ st = dataset_stats(ds, "action", cfg)
76
+ if st is None:
77
+ return
78
+ act = st.active
79
+ if not act.any():
80
+ return
81
+ rng = np.where(act, st.rng, 1.0)
82
+ eps, sigs, lens = [], [], []
83
+ for ep in ds.episodes:
84
+ a = ep.frames.get("action")
85
+ if a is None or ep.length < 4:
86
+ continue
87
+ a2 = a.reshape(-1, 1) if a.ndim == 1 else a
88
+ if a2.shape[1] != len(rng) or not np.all(np.isfinite(a2)):
89
+ continue
90
+ if np.all((a2.max(0) - a2.min(0))[act] < cfg.dead_tol * st.rng[act]):
91
+ continue # dead episodes are reported elsewhere and would all look alike
92
+ eps.append(ep.index)
93
+ sigs.append(_signature(a2[:, act].astype(np.float64), st.lo[act], rng[act]))
94
+ lens.append(ep.length)
95
+ if len(sigs) < 2:
96
+ return
97
+ S = np.stack(sigs)
98
+ L = np.array(lens, dtype=float)
99
+ # mean|Si - Sj| >= |mean(Si) - mean(Sj)|, so only pairs whose signature means are within
100
+ # the tolerance can be duplicates. Sorting by mean makes this exact pruning, which keeps
101
+ # 100k-episode datasets fast.
102
+ M = S.mean(axis=1)
103
+ order = np.argsort(M, kind="stable")
104
+ M_sorted = M[order]
105
+ seen: set[int] = set()
106
+ for i in range(len(S)):
107
+ if eps[i] in seen:
108
+ continue
109
+ lo_pos = np.searchsorted(M_sorted, M[i] - cfg.duplicate_tol, side="left")
110
+ hi_pos = np.searchsorted(M_sorted, M[i] + cfg.duplicate_tol, side="right")
111
+ cand = order[lo_pos:hi_pos]
112
+ cand = np.sort(cand[cand > i])
113
+ cand = cand[np.abs(L[cand] - L[i]) <= 0.05 * L[i] + 1]
114
+ if not len(cand):
115
+ continue
116
+ dist = np.mean(np.abs(S[cand] - S[i]), axis=1)
117
+ for j in cand[dist < cfg.duplicate_tol]:
118
+ seen.add(eps[j])
119
+ yield Finding(
120
+ "DUPLICATE_EPISODE",
121
+ W,
122
+ f"actions are nearly identical to episode {eps[i]}: a duplicated recording, or a scripted demo "
123
+ "repeated with the same setup",
124
+ eps[j],
125
+ {"duplicate_of": eps[i], "mean_abs_diff": round(float(dist[list(cand).index(j)]), 5)},
126
+ )
@@ -0,0 +1,122 @@
1
+ """Dataset-level structure and metadata consistency."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterator
6
+
7
+ import numpy as np
8
+
9
+ from ..dataset import Dataset
10
+ from ..findings import Finding, Severity
11
+ from . import Config
12
+
13
+ E, W, I = Severity.ERROR, Severity.WARNING, Severity.INFO
14
+
15
+
16
+ def check(ds: Dataset, cfg: Config) -> Iterator[Finding]:
17
+ info = ds.info
18
+ for msg in ds.load_warnings:
19
+ yield Finding("METADATA_INCOMPLETE", W, msg)
20
+
21
+ if ds.fps is None:
22
+ yield Finding("FPS_MISSING", E, "meta/info.json has no valid fps; timing cannot be verified")
23
+
24
+ major = ds.major_version
25
+ if major and major < 2:
26
+ yield Finding("OLD_FORMAT", W, f"codebase_version {ds.version} predates LeRobot v2; results may be incomplete")
27
+
28
+ if ds.partial:
29
+ yield Finding(
30
+ "PARTIAL_CHECK",
31
+ I,
32
+ f"quick mode: only the first {len(ds.episodes)} of {info.get('total_episodes', '?')} episodes were checked",
33
+ )
34
+ # Declared vs found counts (meaningless when only part of the dataset was loaded)
35
+ found_eps = len(ds.episodes)
36
+ declared_eps = info.get("total_episodes")
37
+ if ds.partial:
38
+ declared_eps = None
39
+ if isinstance(declared_eps, int) and declared_eps != found_eps:
40
+ yield Finding(
41
+ "EPISODE_COUNT_MISMATCH",
42
+ E,
43
+ f"info.json declares {declared_eps} episodes but the data contains {found_eps}",
44
+ details={"declared": declared_eps, "found": found_eps},
45
+ )
46
+ declared_frames = None if ds.partial else info.get("total_frames")
47
+ if isinstance(declared_frames, int) and declared_frames != ds.total_frames:
48
+ yield Finding(
49
+ "FRAME_COUNT_MISMATCH",
50
+ E,
51
+ f"info.json declares {declared_frames} frames but the data contains {ds.total_frames}",
52
+ details={"declared": declared_frames, "found": ds.total_frames},
53
+ )
54
+
55
+ # Episode indices should be 0..N-1
56
+ idx = sorted(ep.index for ep in ds.episodes)
57
+ if idx and idx != list(range(idx[0], idx[0] + len(idx))):
58
+ missing = sorted(set(range(idx[0], idx[-1] + 1)) - set(idx))
59
+ yield Finding(
60
+ "EPISODE_INDEX_GAPS",
61
+ W,
62
+ f"{len(missing)} episode indices missing between {idx[0]} and {idx[-1]}",
63
+ details={"missing": missing[:50]},
64
+ )
65
+
66
+ # Global "index" column should be a contiguous range across the dataset
67
+ globals_ = [ep.frames["index"] for ep in ds.episodes if "index" in ep.frames]
68
+ if globals_:
69
+ g = np.sort(np.concatenate(globals_).astype(np.int64))
70
+ dupes = int(np.sum(np.diff(g) == 0))
71
+ if dupes:
72
+ yield Finding("GLOBAL_INDEX_DUPLICATES", E, f"{dupes} duplicate values in the global 'index' column")
73
+ elif len(g) and (g[-1] - g[0] + 1) != len(g):
74
+ yield Finding("GLOBAL_INDEX_GAPS", W, "global 'index' column is not contiguous")
75
+
76
+ # Declared features must exist in the data
77
+ for key, spec in ds.features.items():
78
+ if not isinstance(spec, dict) or spec.get("dtype") in ("video", "image"):
79
+ continue
80
+ if key not in ds.data_columns:
81
+ yield Finding("FEATURE_MISSING", E, f"feature '{key}' is declared in info.json but absent from the data")
82
+
83
+ if "action" not in ds.data_columns:
84
+ yield Finding("NO_ACTIONS", E, "dataset has no 'action' column; it cannot be used for policy training")
85
+ if "timestamp" not in ds.data_columns:
86
+ yield Finding("NO_TIMESTAMPS", E, "dataset has no 'timestamp' column")
87
+
88
+ # Declared shapes should match
89
+ for key in ("action", "observation.state"):
90
+ spec = ds.features.get(key)
91
+ if not isinstance(spec, dict) or not ds.episodes or key not in ds.episodes[0].frames:
92
+ continue
93
+ shape = spec.get("shape")
94
+ arr = ds.episodes[0].frames[key]
95
+ d = arr.shape[1] if arr.ndim == 2 else 1
96
+ if isinstance(shape, list) and shape and isinstance(shape[0], int) and shape[0] != d:
97
+ yield Finding(
98
+ "SHAPE_MISMATCH",
99
+ E,
100
+ f"'{key}' declared with {shape[0]} dimensions but data has {d}",
101
+ )
102
+
103
+ # Tasks
104
+ if not ds.tasks:
105
+ yield Finding("TASKS_MISSING", W, "no task descriptions found (meta/tasks.parquet or tasks.jsonl)")
106
+ else:
107
+ empty = [i for i, t in ds.tasks.items() if not t.strip()]
108
+ if empty:
109
+ yield Finding(
110
+ "TASK_EMPTY", W, f"{len(empty)} task descriptions are empty", details={"task_indices": empty[:20]}
111
+ )
112
+ vague = [t for t in ds.tasks.values() if t.strip() and (len(t.strip()) < 8 or t.strip().lower() in VAGUE_TASKS)]
113
+ if vague:
114
+ yield Finding(
115
+ "TASK_VAGUE",
116
+ I,
117
+ f"{len(vague)} task descriptions are too vague to condition a language policy",
118
+ details={"examples": vague[:5]},
119
+ )
120
+
121
+
122
+ VAGUE_TASKS = {"task", "test", "demo", "default", "debug", "none", "null", "untitled", "do the task", "pick", "place"}