robolint 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ robolint-audit/
10
+ *.html
11
+ !src/**/*.html
12
+ !space_static/index.html
robolint-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Max McCutcheon
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,150 @@
1
+ Metadata-Version: 2.5
2
+ Name: robolint
3
+ Version: 0.1.0
4
+ Summary: Quality checks for robot learning datasets (LeRobot format): dropped frames, dead episodes, teleop glitches, frozen cameras, duplicates.
5
+ Project-URL: Homepage, https://github.com/maxmccutcheon59/robolint
6
+ Project-URL: Repository, https://github.com/maxmccutcheon59/robolint
7
+ Project-URL: Issues, https://github.com/maxmccutcheon59/robolint/issues
8
+ Author-email: Max McCutcheon <MaxMcCutcheon1@outlook.com>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: data-quality,dataset,imitation-learning,lerobot,robot-learning,robotics
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Software Development :: Quality Assurance
25
+ Requires-Python: >=3.10
26
+ Requires-Dist: numpy>=1.24
27
+ Requires-Dist: pyarrow>=14
28
+ Provides-Extra: all
29
+ Requires-Dist: av>=12; extra == 'all'
30
+ Requires-Dist: huggingface-hub>=0.23; extra == 'all'
31
+ Provides-Extra: dev
32
+ Requires-Dist: av>=12; extra == 'dev'
33
+ Requires-Dist: huggingface-hub>=0.23; extra == 'dev'
34
+ Requires-Dist: pytest>=8; extra == 'dev'
35
+ Requires-Dist: ruff>=0.5; extra == 'dev'
36
+ Provides-Extra: hub
37
+ Requires-Dist: huggingface-hub>=0.23; extra == 'hub'
38
+ Provides-Extra: video
39
+ Requires-Dist: av>=12; extra == 'video'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # robolint
43
+
44
+ [![CI](https://github.com/maxmccutcheon59/robolint/actions/workflows/ci.yml/badge.svg)](https://github.com/maxmccutcheon59/robolint/actions/workflows/ci.yml)
45
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/maxmccutcheon59/robolint/blob/main/LICENSE)
46
+
47
+ **Quality checks for robot learning datasets.** Point it at any [LeRobot](https://github.com/huggingface/lerobot) dataset, local or on the Hugging Face Hub, and find out which episodes will quietly hurt your policy before you spend GPU hours training on them.
48
+
49
+ ```bash
50
+ pip install "robolint[all]"
51
+ robolint check lerobot/svla_so101_pickplace --videos --html report.html
52
+ ```
53
+
54
+ ```text
55
+ robolint report: demo_ds
56
+ format v3.0 · robot so101_follower · 40 episodes · 7,032 frames · 30.0 fps · cameras: observation.images.front
57
+
58
+ Score 85.2/100 (grade B)
59
+ clean episodes 30 (75%) · with warnings 5 (12%) · with errors 5 (12%)
60
+
61
+ Issues
62
+ error DEAD_EPISODE 1 episode
63
+ Commanded actions are constant. Drop the episode; it teaches the policy to do nothing.
64
+ error STATE_FROZEN 1 episode
65
+ State never updates while actions change. Check the follower arm connection; drop the episode.
66
+ error VIDEO_BLACK 1 episode
67
+ Camera shows black frames: lens cap, disconnect, or no light.
68
+ warning DUPLICATE_EPISODE 1 episode
69
+ Nearly identical to another episode: duplicated upload or replay. ...
70
+ ```
71
+
72
+ *(Output from a test dataset with deliberately injected faults.)*
73
+
74
+ **New: [What's broken in public robot learning data](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)**, an audit of 18 public LeRobot datasets.
75
+
76
+ [![Data quality scores for 18 public LeRobot datasets](https://raw.githubusercontent.com/maxmccutcheon59/robolint/main/docs/audit-scores.png)](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)
77
+
78
+ **Try it in your browser:** the [robolint Space](https://huggingface.co/spaces/maxmccutcheon59/robolint) checks any public LeRobot dataset without installing anything (experimental; no camera checks).
79
+
80
+ ## Why
81
+
82
+ Robot data is expensive to collect and easy to get subtly wrong. A follower arm unplugs mid-session and you record minutes of actions with no motion. USB bandwidth drops camera frames and video drifts out of sync with actions. A teleop glitch makes a joint jump 150° in one step. An episode is uploaded twice. None of these crash training. They just make the policy worse, and nobody notices.
83
+
84
+ robolint finds them in seconds and tells you exactly which episodes to drop or fix.
85
+
86
+ ## What it checks
87
+
88
+ | Area | Checks |
89
+ |---|---|
90
+ | **Timing** | dropped frames (timestamp gaps), timestamps going backwards, jitter, drift from declared fps, frame_index gaps and duplicates |
91
+ | **Actions & state** | NaN/inf values, dead episodes (robot never moves), frozen state (follower disconnected), teleop glitches (action spikes), idle lead-in/tail padding, gripper never actuating, joints pinned at limits |
92
+ | **Episodes** | false starts, truncated and runaway episodes, metadata/data length mismatches, near-duplicate episodes, missing or unknown tasks |
93
+ | **Video** (`--videos`) | missing or unreadable files, video/action length mismatches, black, blown-out, blank, and frozen camera feeds, fps and resolution mismatches |
94
+ | **Metadata** | declared vs actual episode/frame counts, missing features, shape mismatches, vague task descriptions that can't condition a VLA policy |
95
+
96
+
97
+
98
+ Every episode gets a 0–100 score and the dataset gets a score and letter grade (A ≥ 90, B ≥ 80, C ≥ 70, D ≥ 60). An **error** makes an episode unusable for training, so it scores 0; each distinct **warning** costs 15 points; **info** costs 2. The dataset score is the average episode score, minus small penalties for dataset-level problems. Roughly: the share of your data you can actually train on.
99
+
100
+ Supports LeRobot **v2.x** (per-episode files) and **v3.x** (chunked files) layouts.
101
+
102
+ ## Usage
103
+
104
+ ```bash
105
+ # Local folder or Hub repo id
106
+ robolint check ./my_dataset
107
+ robolint check your-org/your-dataset --videos
108
+
109
+ # Shareable reports
110
+ robolint check your-org/your-dataset --json report.json --html report.html
111
+
112
+ # List bad episodes so you can drop them before training
113
+ robolint check your-org/your-dataset --exclude-out bad_episodes.txt
114
+ robolint check your-org/your-dataset --exclude-out flagged.txt --exclude-level warning
115
+
116
+ # Gate a data pipeline in CI
117
+ robolint check ./my_dataset --fail-under 85
118
+
119
+ # Spot-check a huge dataset: downloads only the files for the first N episodes
120
+ robolint check cadene/droid_1.0.1 --max-episodes 200
121
+
122
+ # Audit many datasets at once
123
+ robolint audit lerobot/pusht lerobot/svla_so101_pickplace --out audit/
124
+ robolint audit --from-file datasets.txt --out audit/
125
+ ```
126
+
127
+ From Python:
128
+
129
+ ```python
130
+ from robolint import lint
131
+
132
+ result = lint("lerobot/pusht")
133
+ print(result.scores.dataset_score, result.scores.grade)
134
+ for f in result.findings:
135
+ print(f.episode, f.severity.label, f.code, f.message)
136
+ ```
137
+
138
+ Install options: `robolint` (local datasets), `robolint[hub]` (Hub downloads), `robolint[video]` (camera checks), `robolint[all]`.
139
+
140
+ ## Exit codes
141
+
142
+ `0` success · `1` score below `--fail-under` · `2` dataset could not be loaded.
143
+
144
+ ## Security
145
+
146
+ Datasets are treated as untrusted input: no pickle, no code execution, file paths from metadata are confined to the dataset folder, and every string in HTML reports is escaped. Report vulnerabilities to MaxMcCutcheon1@outlook.com.
147
+
148
+ ## License
149
+
150
+ MIT © Max McCutcheon
@@ -0,0 +1,109 @@
1
+ # robolint
2
+
3
+ [![CI](https://github.com/maxmccutcheon59/robolint/actions/workflows/ci.yml/badge.svg)](https://github.com/maxmccutcheon59/robolint/actions/workflows/ci.yml)
4
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/maxmccutcheon59/robolint/blob/main/LICENSE)
5
+
6
+ **Quality checks for robot learning datasets.** Point it at any [LeRobot](https://github.com/huggingface/lerobot) dataset, local or on the Hugging Face Hub, and find out which episodes will quietly hurt your policy before you spend GPU hours training on them.
7
+
8
+ ```bash
9
+ pip install "robolint[all]"
10
+ robolint check lerobot/svla_so101_pickplace --videos --html report.html
11
+ ```
12
+
13
+ ```text
14
+ robolint report: demo_ds
15
+ format v3.0 · robot so101_follower · 40 episodes · 7,032 frames · 30.0 fps · cameras: observation.images.front
16
+
17
+ Score 85.2/100 (grade B)
18
+ clean episodes 30 (75%) · with warnings 5 (12%) · with errors 5 (12%)
19
+
20
+ Issues
21
+ error DEAD_EPISODE 1 episode
22
+ Commanded actions are constant. Drop the episode; it teaches the policy to do nothing.
23
+ error STATE_FROZEN 1 episode
24
+ State never updates while actions change. Check the follower arm connection; drop the episode.
25
+ error VIDEO_BLACK 1 episode
26
+ Camera shows black frames: lens cap, disconnect, or no light.
27
+ warning DUPLICATE_EPISODE 1 episode
28
+ Nearly identical to another episode: duplicated upload or replay. ...
29
+ ```
30
+
31
+ *(Output from a test dataset with deliberately injected faults.)*
32
+
33
+ **New: [What's broken in public robot learning data](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)**, an audit of 18 public LeRobot datasets.
34
+
35
+ [![Data quality scores for 18 public LeRobot datasets](https://raw.githubusercontent.com/maxmccutcheon59/robolint/main/docs/audit-scores.png)](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)
36
+
37
+ **Try it in your browser:** the [robolint Space](https://huggingface.co/spaces/maxmccutcheon59/robolint) checks any public LeRobot dataset without installing anything (experimental; no camera checks).
38
+
39
+ ## Why
40
+
41
+ Robot data is expensive to collect and easy to get subtly wrong. A follower arm unplugs mid-session and you record minutes of actions with no motion. USB bandwidth drops camera frames and video drifts out of sync with actions. A teleop glitch makes a joint jump 150° in one step. An episode is uploaded twice. None of these crash training. They just make the policy worse, and nobody notices.
42
+
43
+ robolint finds them in seconds and tells you exactly which episodes to drop or fix.
44
+
45
+ ## What it checks
46
+
47
+ | Area | Checks |
48
+ |---|---|
49
+ | **Timing** | dropped frames (timestamp gaps), timestamps going backwards, jitter, drift from declared fps, frame_index gaps and duplicates |
50
+ | **Actions & state** | NaN/inf values, dead episodes (robot never moves), frozen state (follower disconnected), teleop glitches (action spikes), idle lead-in/tail padding, gripper never actuating, joints pinned at limits |
51
+ | **Episodes** | false starts, truncated and runaway episodes, metadata/data length mismatches, near-duplicate episodes, missing or unknown tasks |
52
+ | **Video** (`--videos`) | missing or unreadable files, video/action length mismatches, black, blown-out, blank, and frozen camera feeds, fps and resolution mismatches |
53
+ | **Metadata** | declared vs actual episode/frame counts, missing features, shape mismatches, vague task descriptions that can't condition a VLA policy |
54
+
55
+
56
+
57
+ Every episode gets a 0–100 score and the dataset gets a score and letter grade (A ≥ 90, B ≥ 80, C ≥ 70, D ≥ 60). An **error** makes an episode unusable for training, so it scores 0; each distinct **warning** costs 15 points; **info** costs 2. The dataset score is the average episode score, minus small penalties for dataset-level problems. Roughly: the share of your data you can actually train on.
58
+
59
+ Supports LeRobot **v2.x** (per-episode files) and **v3.x** (chunked files) layouts.
60
+
61
+ ## Usage
62
+
63
+ ```bash
64
+ # Local folder or Hub repo id
65
+ robolint check ./my_dataset
66
+ robolint check your-org/your-dataset --videos
67
+
68
+ # Shareable reports
69
+ robolint check your-org/your-dataset --json report.json --html report.html
70
+
71
+ # List bad episodes so you can drop them before training
72
+ robolint check your-org/your-dataset --exclude-out bad_episodes.txt
73
+ robolint check your-org/your-dataset --exclude-out flagged.txt --exclude-level warning
74
+
75
+ # Gate a data pipeline in CI
76
+ robolint check ./my_dataset --fail-under 85
77
+
78
+ # Spot-check a huge dataset: downloads only the files for the first N episodes
79
+ robolint check cadene/droid_1.0.1 --max-episodes 200
80
+
81
+ # Audit many datasets at once
82
+ robolint audit lerobot/pusht lerobot/svla_so101_pickplace --out audit/
83
+ robolint audit --from-file datasets.txt --out audit/
84
+ ```
85
+
86
+ From Python:
87
+
88
+ ```python
89
+ from robolint import lint
90
+
91
+ result = lint("lerobot/pusht")
92
+ print(result.scores.dataset_score, result.scores.grade)
93
+ for f in result.findings:
94
+ print(f.episode, f.severity.label, f.code, f.message)
95
+ ```
96
+
97
+ Install options: `robolint` (local datasets), `robolint[hub]` (Hub downloads), `robolint[video]` (camera checks), `robolint[all]`.
98
+
99
+ ## Exit codes
100
+
101
+ `0` success · `1` score below `--fail-under` · `2` dataset could not be loaded.
102
+
103
+ ## Security
104
+
105
+ Datasets are treated as untrusted input: no pickle, no code execution, file paths from metadata are confined to the dataset folder, and every string in HTML reports is escaped. Report vulnerabilities to MaxMcCutcheon1@outlook.com.
106
+
107
+ ## License
108
+
109
+ MIT © Max McCutcheon
@@ -0,0 +1,65 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.27"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "robolint"
7
+ dynamic = ["version"]
8
+ description = "Quality checks for robot learning datasets (LeRobot format): dropped frames, dead episodes, teleop glitches, frozen cameras, duplicates."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{ name = "Max McCutcheon", email = "MaxMcCutcheon1@outlook.com" }]
13
+ requires-python = ">=3.10"
14
+ dependencies = ["numpy>=1.24", "pyarrow>=14"]
15
+ keywords = ["robotics", "lerobot", "dataset", "data-quality", "imitation-learning", "robot-learning"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Environment :: Console",
19
+ "Intended Audience :: Developers",
20
+ "Intended Audience :: Science/Research",
21
+ "Operating System :: OS Independent",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3 :: Only",
24
+ "Programming Language :: Python :: 3.10",
25
+ "Programming Language :: Python :: 3.11",
26
+ "Programming Language :: Python :: 3.12",
27
+ "Programming Language :: Python :: 3.13",
28
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
29
+ "Topic :: Software Development :: Quality Assurance",
30
+ ]
31
+
32
+ [project.optional-dependencies]
33
+ hub = ["huggingface_hub>=0.23"]
34
+ video = ["av>=12"]
35
+ all = ["huggingface_hub>=0.23", "av>=12"]
36
+ dev = ["pytest>=8", "ruff>=0.5", "huggingface_hub>=0.23", "av>=12"]
37
+
38
+ [project.scripts]
39
+ robolint = "robolint.cli:main"
40
+
41
+ [project.urls]
42
+ Homepage = "https://github.com/maxmccutcheon59/robolint"
43
+ Repository = "https://github.com/maxmccutcheon59/robolint"
44
+ Issues = "https://github.com/maxmccutcheon59/robolint/issues"
45
+
46
+ [tool.hatch.version]
47
+ path = "src/robolint/__init__.py"
48
+
49
+ [tool.hatch.build.targets.wheel]
50
+ packages = ["src/robolint"]
51
+
52
+ # Keep the sdist to the package, tests, and docs it needs (no Space app or audit lists).
53
+ [tool.hatch.build.targets.sdist]
54
+ include = ["/src/robolint", "/tests", "/README.md", "/LICENSE", "/pyproject.toml"]
55
+
56
+ [tool.ruff]
57
+ line-length = 120
58
+ target-version = "py310"
59
+
60
+ [tool.ruff.lint]
61
+ select = ["E", "F", "W", "I", "B", "UP"]
62
+ ignore = ["E741", "E501"]
63
+
64
+ [tool.pytest.ini_options]
65
+ testpaths = ["tests"]
@@ -0,0 +1,9 @@
1
+ """robolint: quality checks for robot learning datasets (LeRobot format)."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from .api import Result, lint # noqa: E402
6
+ from .checks import Config # noqa: E402
7
+ from .findings import Finding, Severity # noqa: E402
8
+
9
+ __all__ = ["Config", "Finding", "Result", "Severity", "__version__", "lint"]
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
@@ -0,0 +1,108 @@
1
+ """Programmatic entry point: lint a dataset and get a structured result."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+ from collections import Counter
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from . import __version__
12
+ from .checks import Config, all_checks
13
+ from .dataset import Dataset, hub_blob_dirs, load_dataset, resolve_source
14
+ from .findings import Finding, Scores, Severity, score
15
+
16
+
17
+ @dataclass
18
+ class Result:
19
+ source: str
20
+ dataset: Dataset
21
+ findings: list[Finding]
22
+ scores: Scores
23
+ videos_checked: bool
24
+ seconds: float
25
+ extra: dict[str, Any] = field(default_factory=dict)
26
+
27
+ @property
28
+ def dataset_findings(self) -> list[Finding]:
29
+ return [f for f in self.findings if f.episode is None]
30
+
31
+ def episode_findings(self, ep: int) -> list[Finding]:
32
+ return [f for f in self.findings if f.episode == ep]
33
+
34
+ def code_counts(self) -> list[tuple[str, Severity, int]]:
35
+ """(code, worst severity, number of episodes or 1 for dataset-level) sorted by impact."""
36
+ eps: dict[str, set] = {}
37
+ sev: dict[str, Severity] = {}
38
+ for f in self.findings:
39
+ eps.setdefault(f.code, set()).add(f.episode)
40
+ sev[f.code] = max(sev.get(f.code, f.severity), f.severity)
41
+ rows = [(c, sev[c], len(e)) for c, e in eps.items()]
42
+ return sorted(rows, key=lambda r: (-int(r[1]), -r[2], r[0]))
43
+
44
+ def bad_episodes(self, min_severity: Severity = Severity.ERROR) -> list[int]:
45
+ return sorted({f.episode for f in self.findings if f.episode is not None and f.severity >= min_severity})
46
+
47
+ def to_dict(self) -> dict[str, Any]:
48
+ ds = self.dataset
49
+ sev = Counter(f.severity.label for f in self.findings)
50
+ return {
51
+ "tool": {"name": "robolint", "version": __version__},
52
+ "source": self.source,
53
+ "dataset": {
54
+ "codebase_version": ds.version,
55
+ "robot_type": ds.info.get("robot_type"),
56
+ "fps": ds.fps,
57
+ "episodes": len(ds.episodes),
58
+ "frames": ds.total_frames,
59
+ "cameras": ds.video_keys,
60
+ "tasks": len(ds.tasks),
61
+ },
62
+ "score": self.scores.dataset_score,
63
+ "grade": self.scores.grade,
64
+ "episodes": {
65
+ "clean": self.scores.clean_episodes,
66
+ "with_warnings": self.scores.episodes_with_warnings,
67
+ "with_errors": self.scores.episodes_with_errors,
68
+ },
69
+ "videos_checked": self.videos_checked,
70
+ "finding_counts": dict(sev),
71
+ "issues": [{"code": c, "severity": s.label, "affected": n} for c, s, n in self.code_counts()],
72
+ "episode_scores": {str(k): v for k, v in sorted(self.scores.episode_scores.items())},
73
+ "findings": [f.to_dict() for f in self.findings],
74
+ "seconds": round(self.seconds, 2),
75
+ }
76
+
77
+
78
+ def lint(
79
+ source: str | Path,
80
+ *,
81
+ videos: bool = False,
82
+ revision: str | None = None,
83
+ cache_dir: str | None = None,
84
+ max_episodes: int | None = None,
85
+ config: Config | None = None,
86
+ ) -> Result:
87
+ t0 = time.perf_counter()
88
+ cfg = config or Config()
89
+ # Only a Hub download gets the relaxed symlink rule; a local folder keeps the strict one.
90
+ from_hub = not Path(str(source)).expanduser().exists()
91
+ root = resolve_source(
92
+ str(source), include_videos=videos, revision=revision, cache_dir=cache_dir, max_episodes=max_episodes
93
+ )
94
+ hub_blobs = hub_blob_dirs(root) if from_hub else None
95
+ ds = load_dataset(root, source=str(source), max_episodes=max_episodes, hub_blobs=hub_blobs)
96
+ findings: list[Finding] = []
97
+ for check in all_checks(include_video=videos):
98
+ findings.extend(check(ds, cfg))
99
+ findings.sort(key=lambda f: (f.episode if f.episode is not None else -1, -int(f.severity), f.code))
100
+ scores = score(findings, [ep.index for ep in ds.episodes])
101
+ return Result(
102
+ source=str(source),
103
+ dataset=ds,
104
+ findings=findings,
105
+ scores=scores,
106
+ videos_checked=videos,
107
+ seconds=time.perf_counter() - t0,
108
+ )
@@ -0,0 +1,53 @@
1
+ """Check registry and shared configuration."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable, Iterable
6
+ from dataclasses import dataclass
7
+
8
+ from ..dataset import Dataset
9
+ from ..findings import Finding
10
+
11
+
12
+ @dataclass
13
+ class Config:
14
+ # Timing
15
+ gap_factor: float = 1.5 # dt above this many frame periods counts as dropped frames
16
+ jitter_ratio: float = 0.10 # std(dt) / period above this is jittery
17
+ drift_frames: float = 2.0 # end-to-end drift tolerance, in frame periods
18
+ # Signals (fractions of each dimension's dataset-wide range)
19
+ move_tol: float = 0.005 # change smaller than this counts as "not moving"
20
+ dead_tol: float = 0.01 # every dimension moves less than this -> dead episode
21
+ gripper_tol: float = 0.05 # gripper travel smaller than this -> never actuated
22
+ spike_quantile: float = 0.995 # robust scale for per-step action changes
23
+ spike_multiplier: float = 6.0
24
+ spike_min_range: float = 0.15 # a spike must also be at least this fraction of the range
25
+ idle_warn: float = 0.30 # idle lead-in + tail above this fraction of the episode
26
+ idle_info: float = 0.15
27
+ limit_seconds: float = 2.0 # pinned at a joint's extreme for this long
28
+ # Episodes
29
+ min_seconds: float = 1.0
30
+ short_ratio: float = 0.25 # shorter than this fraction of the median length
31
+ long_ratio: float = 3.0
32
+ duplicate_tol: float = 0.002 # mean abs difference (fraction of range) for duplicates
33
+ # Video
34
+ video_samples: int = 8
35
+ black_level: float = 8.0
36
+ white_level: float = 247.0
37
+ blank_std: float = 2.0
38
+ frozen_diff: float = 1.0 # minimum mean pixel change (0-255) before a feed counts as live
39
+ frozen_min_seconds: float = 2.0 # shorter episodes are too brief to judge
40
+
41
+
42
+ CheckFn = Callable[[Dataset, Config], Iterable[Finding]]
43
+
44
+
45
+ def all_checks(include_video: bool) -> list[CheckFn]:
46
+ from . import episodes, meta, signals, timing
47
+
48
+ checks: list[CheckFn] = [meta.check, timing.check, signals.check, episodes.check]
49
+ if include_video:
50
+ from . import video
51
+
52
+ checks.append(video.check)
53
+ return checks
@@ -0,0 +1,126 @@
1
+ """Episode-level checks: lengths, tasks, and duplicate demonstrations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterator
6
+
7
+ import numpy as np
8
+
9
+ from ..dataset import Dataset
10
+ from ..findings import Finding, Severity
11
+ from . import Config
12
+ from .signals import dataset_stats
13
+
14
+ E, W, I = Severity.ERROR, Severity.WARNING, Severity.INFO
15
+ RESAMPLE = 64
16
+
17
+
18
+ def check(ds: Dataset, cfg: Config) -> Iterator[Finding]:
19
+ if not ds.episodes:
20
+ yield Finding("NO_EPISODES", E, "dataset contains no episodes")
21
+ return
22
+ fps = ds.fps or 30.0
23
+ lengths = np.array([ep.length for ep in ds.episodes])
24
+ median = float(np.median(lengths))
25
+
26
+ for ep in ds.episodes:
27
+ n = ep.length
28
+ if ep.length_meta is not None and ep.length_meta != n:
29
+ yield Finding(
30
+ "EPISODE_LENGTH_MISMATCH",
31
+ E,
32
+ f"metadata says {ep.length_meta} frames, data has {n}",
33
+ ep.index,
34
+ {"metadata": ep.length_meta, "data": n},
35
+ )
36
+ if 0 < n < cfg.min_seconds * fps:
37
+ yield Finding("EPISODE_TOO_SHORT", E, f"only {n} frames ({n / fps:.2f}s): likely a false start", ep.index)
38
+ elif len(lengths) >= 5 and n < cfg.short_ratio * median:
39
+ yield Finding(
40
+ "EPISODE_SHORT",
41
+ W,
42
+ f"{n} frames, under {cfg.short_ratio:.0%} of the median ({median:.0f}): possibly truncated",
43
+ ep.index,
44
+ )
45
+ if len(lengths) >= 5 and n > cfg.long_ratio * median and n - median > 5 * fps:
46
+ yield Finding(
47
+ "EPISODE_LONG",
48
+ I,
49
+ f"{n} frames, {n / median:.1f}x the median: recording may not have been stopped",
50
+ ep.index,
51
+ )
52
+
53
+ ti = ep.frames.get("task_index")
54
+ if ti is not None and ds.tasks:
55
+ unknown = sorted({int(x) for x in np.unique(ti)} - set(ds.tasks))
56
+ if unknown:
57
+ yield Finding(
58
+ "TASK_INDEX_UNKNOWN", E, f"task_index values {unknown[:5]} have no task description", ep.index
59
+ )
60
+ if ep.tasks and not any(t.strip() for t in ep.tasks):
61
+ yield Finding("EPISODE_TASK_EMPTY", W, "episode's task description is empty", ep.index)
62
+
63
+ yield from _duplicates(ds, cfg)
64
+
65
+
66
+ def _signature(a: np.ndarray, lo: np.ndarray, rng: np.ndarray) -> np.ndarray:
67
+ a = a.reshape(-1, 1) if a.ndim == 1 else a
68
+ t_old = np.linspace(0.0, 1.0, len(a))
69
+ t_new = np.linspace(0.0, 1.0, RESAMPLE)
70
+ res = np.stack([np.interp(t_new, t_old, a[:, j]) for j in range(a.shape[1])], axis=1)
71
+ return ((res - lo) / rng).ravel()
72
+
73
+
74
+ def _duplicates(ds: Dataset, cfg: Config) -> Iterator[Finding]:
75
+ st = dataset_stats(ds, "action", cfg)
76
+ if st is None:
77
+ return
78
+ act = st.active
79
+ if not act.any():
80
+ return
81
+ rng = np.where(act, st.rng, 1.0)
82
+ eps, sigs, lens = [], [], []
83
+ for ep in ds.episodes:
84
+ a = ep.frames.get("action")
85
+ if a is None or ep.length < 4:
86
+ continue
87
+ a2 = a.reshape(-1, 1) if a.ndim == 1 else a
88
+ if a2.shape[1] != len(rng) or not np.all(np.isfinite(a2)):
89
+ continue
90
+ if np.all((a2.max(0) - a2.min(0))[act] < cfg.dead_tol * st.rng[act]):
91
+ continue # dead episodes are reported elsewhere and would all look alike
92
+ eps.append(ep.index)
93
+ sigs.append(_signature(a2[:, act].astype(np.float64), st.lo[act], rng[act]))
94
+ lens.append(ep.length)
95
+ if len(sigs) < 2:
96
+ return
97
+ S = np.stack(sigs)
98
+ L = np.array(lens, dtype=float)
99
+ # mean|Si - Sj| >= |mean(Si) - mean(Sj)|, so only pairs whose signature means are within
100
+ # the tolerance can be duplicates. Sorting by mean makes this exact pruning, which keeps
101
+ # 100k-episode datasets fast.
102
+ M = S.mean(axis=1)
103
+ order = np.argsort(M, kind="stable")
104
+ M_sorted = M[order]
105
+ seen: set[int] = set()
106
+ for i in range(len(S)):
107
+ if eps[i] in seen:
108
+ continue
109
+ lo_pos = np.searchsorted(M_sorted, M[i] - cfg.duplicate_tol, side="left")
110
+ hi_pos = np.searchsorted(M_sorted, M[i] + cfg.duplicate_tol, side="right")
111
+ cand = order[lo_pos:hi_pos]
112
+ cand = np.sort(cand[cand > i])
113
+ cand = cand[np.abs(L[cand] - L[i]) <= 0.05 * L[i] + 1]
114
+ if not len(cand):
115
+ continue
116
+ dist = np.mean(np.abs(S[cand] - S[i]), axis=1)
117
+ for j in cand[dist < cfg.duplicate_tol]:
118
+ seen.add(eps[j])
119
+ yield Finding(
120
+ "DUPLICATE_EPISODE",
121
+ W,
122
+ f"actions are nearly identical to episode {eps[i]}: a duplicated recording, or a scripted demo "
123
+ "repeated with the same setup",
124
+ eps[j],
125
+ {"duplicate_of": eps[i], "mean_abs_diff": round(float(dist[list(cand).index(j)]), 5)},
126
+ )