robolint 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robolint-0.1.0/.gitignore +12 -0
- robolint-0.1.0/LICENSE +21 -0
- robolint-0.1.0/PKG-INFO +150 -0
- robolint-0.1.0/README.md +109 -0
- robolint-0.1.0/pyproject.toml +65 -0
- robolint-0.1.0/src/robolint/__init__.py +9 -0
- robolint-0.1.0/src/robolint/__main__.py +5 -0
- robolint-0.1.0/src/robolint/api.py +108 -0
- robolint-0.1.0/src/robolint/checks/__init__.py +53 -0
- robolint-0.1.0/src/robolint/checks/episodes.py +126 -0
- robolint-0.1.0/src/robolint/checks/meta.py +122 -0
- robolint-0.1.0/src/robolint/checks/signals.py +278 -0
- robolint-0.1.0/src/robolint/checks/timing.py +104 -0
- robolint-0.1.0/src/robolint/checks/video.py +211 -0
- robolint-0.1.0/src/robolint/cli.py +136 -0
- robolint-0.1.0/src/robolint/dataset.py +516 -0
- robolint-0.1.0/src/robolint/findings.py +90 -0
- robolint-0.1.0/src/robolint/report.py +262 -0
- robolint-0.1.0/tests/__init__.py +0 -0
- robolint-0.1.0/tests/synth.py +249 -0
- robolint-0.1.0/tests/test_checks.py +288 -0
- robolint-0.1.0/tests/test_hub_cache_symlinks.py +194 -0
robolint-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Max McCutcheon
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
robolint-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: robolint
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Quality checks for robot learning datasets (LeRobot format): dropped frames, dead episodes, teleop glitches, frozen cameras, duplicates.
|
|
5
|
+
Project-URL: Homepage, https://github.com/maxmccutcheon59/robolint
|
|
6
|
+
Project-URL: Repository, https://github.com/maxmccutcheon59/robolint
|
|
7
|
+
Project-URL: Issues, https://github.com/maxmccutcheon59/robolint/issues
|
|
8
|
+
Author-email: Max McCutcheon <MaxMcCutcheon1@outlook.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: data-quality,dataset,imitation-learning,lerobot,robot-learning,robotics
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: numpy>=1.24
|
|
27
|
+
Requires-Dist: pyarrow>=14
|
|
28
|
+
Provides-Extra: all
|
|
29
|
+
Requires-Dist: av>=12; extra == 'all'
|
|
30
|
+
Requires-Dist: huggingface-hub>=0.23; extra == 'all'
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: av>=12; extra == 'dev'
|
|
33
|
+
Requires-Dist: huggingface-hub>=0.23; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
35
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
36
|
+
Provides-Extra: hub
|
|
37
|
+
Requires-Dist: huggingface-hub>=0.23; extra == 'hub'
|
|
38
|
+
Provides-Extra: video
|
|
39
|
+
Requires-Dist: av>=12; extra == 'video'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# robolint
|
|
43
|
+
|
|
44
|
+
[](https://github.com/maxmccutcheon59/robolint/actions/workflows/ci.yml)
|
|
45
|
+
[](https://github.com/maxmccutcheon59/robolint/blob/main/LICENSE)
|
|
46
|
+
|
|
47
|
+
**Quality checks for robot learning datasets.** Point it at any [LeRobot](https://github.com/huggingface/lerobot) dataset, local or on the Hugging Face Hub, and find out which episodes will quietly hurt your policy before you spend GPU hours training on them.
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install "robolint[all]"
|
|
51
|
+
robolint check lerobot/svla_so101_pickplace --videos --html report.html
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
```text
|
|
55
|
+
robolint report: demo_ds
|
|
56
|
+
format v3.0 · robot so101_follower · 40 episodes · 7,032 frames · 30.0 fps · cameras: observation.images.front
|
|
57
|
+
|
|
58
|
+
Score 85.2/100 (grade B)
|
|
59
|
+
clean episodes 30 (75%) · with warnings 5 (12%) · with errors 5 (12%)
|
|
60
|
+
|
|
61
|
+
Issues
|
|
62
|
+
error DEAD_EPISODE 1 episode
|
|
63
|
+
Commanded actions are constant. Drop the episode; it teaches the policy to do nothing.
|
|
64
|
+
error STATE_FROZEN 1 episode
|
|
65
|
+
State never updates while actions change. Check the follower arm connection; drop the episode.
|
|
66
|
+
error VIDEO_BLACK 1 episode
|
|
67
|
+
Camera shows black frames: lens cap, disconnect, or no light.
|
|
68
|
+
warning DUPLICATE_EPISODE 1 episode
|
|
69
|
+
Nearly identical to another episode: duplicated upload or replay. ...
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
*(Output from a test dataset with deliberately injected faults.)*
|
|
73
|
+
|
|
74
|
+
**New: [What's broken in public robot learning data](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)**, an audit of 18 public LeRobot datasets.
|
|
75
|
+
|
|
76
|
+
[](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)
|
|
77
|
+
|
|
78
|
+
**Try it in your browser:** the [robolint Space](https://huggingface.co/spaces/maxmccutcheon59/robolint) checks any public LeRobot dataset without installing anything (experimental; no camera checks).
|
|
79
|
+
|
|
80
|
+
## Why
|
|
81
|
+
|
|
82
|
+
Robot data is expensive to collect and easy to get subtly wrong. A follower arm unplugs mid-session and you record minutes of actions with no motion. USB bandwidth drops camera frames and video drifts out of sync with actions. A teleop glitch makes a joint jump 150° in one step. An episode is uploaded twice. None of these crash training. They just make the policy worse, and nobody notices.
|
|
83
|
+
|
|
84
|
+
robolint finds them in seconds and tells you exactly which episodes to drop or fix.
|
|
85
|
+
|
|
86
|
+
## What it checks
|
|
87
|
+
|
|
88
|
+
| Area | Checks |
|
|
89
|
+
|---|---|
|
|
90
|
+
| **Timing** | dropped frames (timestamp gaps), timestamps going backwards, jitter, drift from declared fps, frame_index gaps and duplicates |
|
|
91
|
+
| **Actions & state** | NaN/inf values, dead episodes (robot never moves), frozen state (follower disconnected), teleop glitches (action spikes), idle lead-in/tail padding, gripper never actuating, joints pinned at limits |
|
|
92
|
+
| **Episodes** | false starts, truncated and runaway episodes, metadata/data length mismatches, near-duplicate episodes, missing or unknown tasks |
|
|
93
|
+
| **Video** (`--videos`) | missing or unreadable files, video/action length mismatches, black, blown-out, blank, and frozen camera feeds, fps and resolution mismatches |
|
|
94
|
+
| **Metadata** | declared vs actual episode/frame counts, missing features, shape mismatches, vague task descriptions that can't condition a VLA policy |
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
Every episode gets a 0–100 score and the dataset gets a score and letter grade (A ≥ 90, B ≥ 80, C ≥ 70, D ≥ 60). An **error** makes an episode unusable for training, so it scores 0; each distinct **warning** costs 15 points; **info** costs 2. The dataset score is the average episode score, minus small penalties for dataset-level problems. Roughly: the share of your data you can actually train on.
|
|
99
|
+
|
|
100
|
+
Supports LeRobot **v2.x** (per-episode files) and **v3.x** (chunked files) layouts.
|
|
101
|
+
|
|
102
|
+
## Usage
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
# Local folder or Hub repo id
|
|
106
|
+
robolint check ./my_dataset
|
|
107
|
+
robolint check your-org/your-dataset --videos
|
|
108
|
+
|
|
109
|
+
# Shareable reports
|
|
110
|
+
robolint check your-org/your-dataset --json report.json --html report.html
|
|
111
|
+
|
|
112
|
+
# List bad episodes so you can drop them before training
|
|
113
|
+
robolint check your-org/your-dataset --exclude-out bad_episodes.txt
|
|
114
|
+
robolint check your-org/your-dataset --exclude-out flagged.txt --exclude-level warning
|
|
115
|
+
|
|
116
|
+
# Gate a data pipeline in CI
|
|
117
|
+
robolint check ./my_dataset --fail-under 85
|
|
118
|
+
|
|
119
|
+
# Spot-check a huge dataset: downloads only the files for the first N episodes
|
|
120
|
+
robolint check cadene/droid_1.0.1 --max-episodes 200
|
|
121
|
+
|
|
122
|
+
# Audit many datasets at once
|
|
123
|
+
robolint audit lerobot/pusht lerobot/svla_so101_pickplace --out audit/
|
|
124
|
+
robolint audit --from-file datasets.txt --out audit/
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
From Python:
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from robolint import lint
|
|
131
|
+
|
|
132
|
+
result = lint("lerobot/pusht")
|
|
133
|
+
print(result.scores.dataset_score, result.scores.grade)
|
|
134
|
+
for f in result.findings:
|
|
135
|
+
print(f.episode, f.severity.label, f.code, f.message)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Install options: `robolint` (local datasets), `robolint[hub]` (Hub downloads), `robolint[video]` (camera checks), `robolint[all]`.
|
|
139
|
+
|
|
140
|
+
## Exit codes
|
|
141
|
+
|
|
142
|
+
`0` success · `1` score below `--fail-under` · `2` dataset could not be loaded.
|
|
143
|
+
|
|
144
|
+
## Security
|
|
145
|
+
|
|
146
|
+
Datasets are treated as untrusted input: no pickle, no code execution, file paths from metadata are confined to the dataset folder, and every string in HTML reports is escaped. Report vulnerabilities to MaxMcCutcheon1@outlook.com.
|
|
147
|
+
|
|
148
|
+
## License
|
|
149
|
+
|
|
150
|
+
MIT © Max McCutcheon
|
robolint-0.1.0/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# robolint
|
|
2
|
+
|
|
3
|
+
[](https://github.com/maxmccutcheon59/robolint/actions/workflows/ci.yml)
|
|
4
|
+
[](https://github.com/maxmccutcheon59/robolint/blob/main/LICENSE)
|
|
5
|
+
|
|
6
|
+
**Quality checks for robot learning datasets.** Point it at any [LeRobot](https://github.com/huggingface/lerobot) dataset, local or on the Hugging Face Hub, and find out which episodes will quietly hurt your policy before you spend GPU hours training on them.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install "robolint[all]"
|
|
10
|
+
robolint check lerobot/svla_so101_pickplace --videos --html report.html
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```text
|
|
14
|
+
robolint report: demo_ds
|
|
15
|
+
format v3.0 · robot so101_follower · 40 episodes · 7,032 frames · 30.0 fps · cameras: observation.images.front
|
|
16
|
+
|
|
17
|
+
Score 85.2/100 (grade B)
|
|
18
|
+
clean episodes 30 (75%) · with warnings 5 (12%) · with errors 5 (12%)
|
|
19
|
+
|
|
20
|
+
Issues
|
|
21
|
+
error DEAD_EPISODE 1 episode
|
|
22
|
+
Commanded actions are constant. Drop the episode; it teaches the policy to do nothing.
|
|
23
|
+
error STATE_FROZEN 1 episode
|
|
24
|
+
State never updates while actions change. Check the follower arm connection; drop the episode.
|
|
25
|
+
error VIDEO_BLACK 1 episode
|
|
26
|
+
Camera shows black frames: lens cap, disconnect, or no light.
|
|
27
|
+
warning DUPLICATE_EPISODE 1 episode
|
|
28
|
+
Nearly identical to another episode: duplicated upload or replay. ...
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
*(Output from a test dataset with deliberately injected faults.)*
|
|
32
|
+
|
|
33
|
+
**New: [What's broken in public robot learning data](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)**, an audit of 18 public LeRobot datasets.
|
|
34
|
+
|
|
35
|
+
[](https://github.com/maxmccutcheon59/robolint/blob/main/AUDIT.md)
|
|
36
|
+
|
|
37
|
+
**Try it in your browser:** the [robolint Space](https://huggingface.co/spaces/maxmccutcheon59/robolint) checks any public LeRobot dataset without installing anything (experimental; no camera checks).
|
|
38
|
+
|
|
39
|
+
## Why
|
|
40
|
+
|
|
41
|
+
Robot data is expensive to collect and easy to get subtly wrong. A follower arm unplugs mid-session and you record minutes of actions with no motion. USB bandwidth drops camera frames and video drifts out of sync with actions. A teleop glitch makes a joint jump 150° in one step. An episode is uploaded twice. None of these crash training. They just make the policy worse, and nobody notices.
|
|
42
|
+
|
|
43
|
+
robolint finds them in seconds and tells you exactly which episodes to drop or fix.
|
|
44
|
+
|
|
45
|
+
## What it checks
|
|
46
|
+
|
|
47
|
+
| Area | Checks |
|
|
48
|
+
|---|---|
|
|
49
|
+
| **Timing** | dropped frames (timestamp gaps), timestamps going backwards, jitter, drift from declared fps, frame_index gaps and duplicates |
|
|
50
|
+
| **Actions & state** | NaN/inf values, dead episodes (robot never moves), frozen state (follower disconnected), teleop glitches (action spikes), idle lead-in/tail padding, gripper never actuating, joints pinned at limits |
|
|
51
|
+
| **Episodes** | false starts, truncated and runaway episodes, metadata/data length mismatches, near-duplicate episodes, missing or unknown tasks |
|
|
52
|
+
| **Video** (`--videos`) | missing or unreadable files, video/action length mismatches, black, blown-out, blank, and frozen camera feeds, fps and resolution mismatches |
|
|
53
|
+
| **Metadata** | declared vs actual episode/frame counts, missing features, shape mismatches, vague task descriptions that can't condition a VLA policy |
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
Every episode gets a 0–100 score and the dataset gets a score and letter grade (A ≥ 90, B ≥ 80, C ≥ 70, D ≥ 60). An **error** makes an episode unusable for training, so it scores 0; each distinct **warning** costs 15 points; **info** costs 2. The dataset score is the average episode score, minus small penalties for dataset-level problems. Roughly: the share of your data you can actually train on.
|
|
58
|
+
|
|
59
|
+
Supports LeRobot **v2.x** (per-episode files) and **v3.x** (chunked files) layouts.
|
|
60
|
+
|
|
61
|
+
## Usage
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# Local folder or Hub repo id
|
|
65
|
+
robolint check ./my_dataset
|
|
66
|
+
robolint check your-org/your-dataset --videos
|
|
67
|
+
|
|
68
|
+
# Shareable reports
|
|
69
|
+
robolint check your-org/your-dataset --json report.json --html report.html
|
|
70
|
+
|
|
71
|
+
# List bad episodes so you can drop them before training
|
|
72
|
+
robolint check your-org/your-dataset --exclude-out bad_episodes.txt
|
|
73
|
+
robolint check your-org/your-dataset --exclude-out flagged.txt --exclude-level warning
|
|
74
|
+
|
|
75
|
+
# Gate a data pipeline in CI
|
|
76
|
+
robolint check ./my_dataset --fail-under 85
|
|
77
|
+
|
|
78
|
+
# Spot-check a huge dataset: downloads only the files for the first N episodes
|
|
79
|
+
robolint check cadene/droid_1.0.1 --max-episodes 200
|
|
80
|
+
|
|
81
|
+
# Audit many datasets at once
|
|
82
|
+
robolint audit lerobot/pusht lerobot/svla_so101_pickplace --out audit/
|
|
83
|
+
robolint audit --from-file datasets.txt --out audit/
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
From Python:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from robolint import lint
|
|
90
|
+
|
|
91
|
+
result = lint("lerobot/pusht")
|
|
92
|
+
print(result.scores.dataset_score, result.scores.grade)
|
|
93
|
+
for f in result.findings:
|
|
94
|
+
print(f.episode, f.severity.label, f.code, f.message)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Install options: `robolint` (local datasets), `robolint[hub]` (Hub downloads), `robolint[video]` (camera checks), `robolint[all]`.
|
|
98
|
+
|
|
99
|
+
## Exit codes
|
|
100
|
+
|
|
101
|
+
`0` success · `1` score below `--fail-under` · `2` dataset could not be loaded.
|
|
102
|
+
|
|
103
|
+
## Security
|
|
104
|
+
|
|
105
|
+
Datasets are treated as untrusted input: no pickle, no code execution, file paths from metadata are confined to the dataset folder, and every string in HTML reports is escaped. Report vulnerabilities to MaxMcCutcheon1@outlook.com.
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT © Max McCutcheon
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "robolint"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Quality checks for robot learning datasets (LeRobot format): dropped frames, dead episodes, teleop glitches, frozen cameras, duplicates."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Max McCutcheon", email = "MaxMcCutcheon1@outlook.com" }]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
dependencies = ["numpy>=1.24", "pyarrow>=14"]
|
|
15
|
+
keywords = ["robotics", "lerobot", "dataset", "data-quality", "imitation-learning", "robot-learning"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
29
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
hub = ["huggingface_hub>=0.23"]
|
|
34
|
+
video = ["av>=12"]
|
|
35
|
+
all = ["huggingface_hub>=0.23", "av>=12"]
|
|
36
|
+
dev = ["pytest>=8", "ruff>=0.5", "huggingface_hub>=0.23", "av>=12"]
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
robolint = "robolint.cli:main"
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/maxmccutcheon59/robolint"
|
|
43
|
+
Repository = "https://github.com/maxmccutcheon59/robolint"
|
|
44
|
+
Issues = "https://github.com/maxmccutcheon59/robolint/issues"
|
|
45
|
+
|
|
46
|
+
[tool.hatch.version]
|
|
47
|
+
path = "src/robolint/__init__.py"
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.wheel]
|
|
50
|
+
packages = ["src/robolint"]
|
|
51
|
+
|
|
52
|
+
# Keep the sdist to the package, tests, and docs it needs (no Space app or audit lists).
|
|
53
|
+
[tool.hatch.build.targets.sdist]
|
|
54
|
+
include = ["/src/robolint", "/tests", "/README.md", "/LICENSE", "/pyproject.toml"]
|
|
55
|
+
|
|
56
|
+
[tool.ruff]
|
|
57
|
+
line-length = 120
|
|
58
|
+
target-version = "py310"
|
|
59
|
+
|
|
60
|
+
[tool.ruff.lint]
|
|
61
|
+
select = ["E", "F", "W", "I", "B", "UP"]
|
|
62
|
+
ignore = ["E741", "E501"]
|
|
63
|
+
|
|
64
|
+
[tool.pytest.ini_options]
|
|
65
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""robolint: quality checks for robot learning datasets (LeRobot format)."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
from .api import Result, lint # noqa: E402
|
|
6
|
+
from .checks import Config # noqa: E402
|
|
7
|
+
from .findings import Finding, Severity # noqa: E402
|
|
8
|
+
|
|
9
|
+
__all__ = ["Config", "Finding", "Result", "Severity", "__version__", "lint"]
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Programmatic entry point: lint a dataset and get a structured result."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .checks import Config, all_checks
|
|
13
|
+
from .dataset import Dataset, hub_blob_dirs, load_dataset, resolve_source
|
|
14
|
+
from .findings import Finding, Scores, Severity, score
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class Result:
|
|
19
|
+
source: str
|
|
20
|
+
dataset: Dataset
|
|
21
|
+
findings: list[Finding]
|
|
22
|
+
scores: Scores
|
|
23
|
+
videos_checked: bool
|
|
24
|
+
seconds: float
|
|
25
|
+
extra: dict[str, Any] = field(default_factory=dict)
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def dataset_findings(self) -> list[Finding]:
|
|
29
|
+
return [f for f in self.findings if f.episode is None]
|
|
30
|
+
|
|
31
|
+
def episode_findings(self, ep: int) -> list[Finding]:
|
|
32
|
+
return [f for f in self.findings if f.episode == ep]
|
|
33
|
+
|
|
34
|
+
def code_counts(self) -> list[tuple[str, Severity, int]]:
|
|
35
|
+
"""(code, worst severity, number of episodes or 1 for dataset-level) sorted by impact."""
|
|
36
|
+
eps: dict[str, set] = {}
|
|
37
|
+
sev: dict[str, Severity] = {}
|
|
38
|
+
for f in self.findings:
|
|
39
|
+
eps.setdefault(f.code, set()).add(f.episode)
|
|
40
|
+
sev[f.code] = max(sev.get(f.code, f.severity), f.severity)
|
|
41
|
+
rows = [(c, sev[c], len(e)) for c, e in eps.items()]
|
|
42
|
+
return sorted(rows, key=lambda r: (-int(r[1]), -r[2], r[0]))
|
|
43
|
+
|
|
44
|
+
def bad_episodes(self, min_severity: Severity = Severity.ERROR) -> list[int]:
|
|
45
|
+
return sorted({f.episode for f in self.findings if f.episode is not None and f.severity >= min_severity})
|
|
46
|
+
|
|
47
|
+
def to_dict(self) -> dict[str, Any]:
|
|
48
|
+
ds = self.dataset
|
|
49
|
+
sev = Counter(f.severity.label for f in self.findings)
|
|
50
|
+
return {
|
|
51
|
+
"tool": {"name": "robolint", "version": __version__},
|
|
52
|
+
"source": self.source,
|
|
53
|
+
"dataset": {
|
|
54
|
+
"codebase_version": ds.version,
|
|
55
|
+
"robot_type": ds.info.get("robot_type"),
|
|
56
|
+
"fps": ds.fps,
|
|
57
|
+
"episodes": len(ds.episodes),
|
|
58
|
+
"frames": ds.total_frames,
|
|
59
|
+
"cameras": ds.video_keys,
|
|
60
|
+
"tasks": len(ds.tasks),
|
|
61
|
+
},
|
|
62
|
+
"score": self.scores.dataset_score,
|
|
63
|
+
"grade": self.scores.grade,
|
|
64
|
+
"episodes": {
|
|
65
|
+
"clean": self.scores.clean_episodes,
|
|
66
|
+
"with_warnings": self.scores.episodes_with_warnings,
|
|
67
|
+
"with_errors": self.scores.episodes_with_errors,
|
|
68
|
+
},
|
|
69
|
+
"videos_checked": self.videos_checked,
|
|
70
|
+
"finding_counts": dict(sev),
|
|
71
|
+
"issues": [{"code": c, "severity": s.label, "affected": n} for c, s, n in self.code_counts()],
|
|
72
|
+
"episode_scores": {str(k): v for k, v in sorted(self.scores.episode_scores.items())},
|
|
73
|
+
"findings": [f.to_dict() for f in self.findings],
|
|
74
|
+
"seconds": round(self.seconds, 2),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def lint(
|
|
79
|
+
source: str | Path,
|
|
80
|
+
*,
|
|
81
|
+
videos: bool = False,
|
|
82
|
+
revision: str | None = None,
|
|
83
|
+
cache_dir: str | None = None,
|
|
84
|
+
max_episodes: int | None = None,
|
|
85
|
+
config: Config | None = None,
|
|
86
|
+
) -> Result:
|
|
87
|
+
t0 = time.perf_counter()
|
|
88
|
+
cfg = config or Config()
|
|
89
|
+
# Only a Hub download gets the relaxed symlink rule; a local folder keeps the strict one.
|
|
90
|
+
from_hub = not Path(str(source)).expanduser().exists()
|
|
91
|
+
root = resolve_source(
|
|
92
|
+
str(source), include_videos=videos, revision=revision, cache_dir=cache_dir, max_episodes=max_episodes
|
|
93
|
+
)
|
|
94
|
+
hub_blobs = hub_blob_dirs(root) if from_hub else None
|
|
95
|
+
ds = load_dataset(root, source=str(source), max_episodes=max_episodes, hub_blobs=hub_blobs)
|
|
96
|
+
findings: list[Finding] = []
|
|
97
|
+
for check in all_checks(include_video=videos):
|
|
98
|
+
findings.extend(check(ds, cfg))
|
|
99
|
+
findings.sort(key=lambda f: (f.episode if f.episode is not None else -1, -int(f.severity), f.code))
|
|
100
|
+
scores = score(findings, [ep.index for ep in ds.episodes])
|
|
101
|
+
return Result(
|
|
102
|
+
source=str(source),
|
|
103
|
+
dataset=ds,
|
|
104
|
+
findings=findings,
|
|
105
|
+
scores=scores,
|
|
106
|
+
videos_checked=videos,
|
|
107
|
+
seconds=time.perf_counter() - t0,
|
|
108
|
+
)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Check registry and shared configuration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Iterable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
from ..dataset import Dataset
|
|
9
|
+
from ..findings import Finding
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Config:
|
|
14
|
+
# Timing
|
|
15
|
+
gap_factor: float = 1.5 # dt above this many frame periods counts as dropped frames
|
|
16
|
+
jitter_ratio: float = 0.10 # std(dt) / period above this is jittery
|
|
17
|
+
drift_frames: float = 2.0 # end-to-end drift tolerance, in frame periods
|
|
18
|
+
# Signals (fractions of each dimension's dataset-wide range)
|
|
19
|
+
move_tol: float = 0.005 # change smaller than this counts as "not moving"
|
|
20
|
+
dead_tol: float = 0.01 # every dimension moves less than this -> dead episode
|
|
21
|
+
gripper_tol: float = 0.05 # gripper travel smaller than this -> never actuated
|
|
22
|
+
spike_quantile: float = 0.995 # robust scale for per-step action changes
|
|
23
|
+
spike_multiplier: float = 6.0
|
|
24
|
+
spike_min_range: float = 0.15 # a spike must also be at least this fraction of the range
|
|
25
|
+
idle_warn: float = 0.30 # idle lead-in + tail above this fraction of the episode
|
|
26
|
+
idle_info: float = 0.15
|
|
27
|
+
limit_seconds: float = 2.0 # pinned at a joint's extreme for this long
|
|
28
|
+
# Episodes
|
|
29
|
+
min_seconds: float = 1.0
|
|
30
|
+
short_ratio: float = 0.25 # shorter than this fraction of the median length
|
|
31
|
+
long_ratio: float = 3.0
|
|
32
|
+
duplicate_tol: float = 0.002 # mean abs difference (fraction of range) for duplicates
|
|
33
|
+
# Video
|
|
34
|
+
video_samples: int = 8
|
|
35
|
+
black_level: float = 8.0
|
|
36
|
+
white_level: float = 247.0
|
|
37
|
+
blank_std: float = 2.0
|
|
38
|
+
frozen_diff: float = 1.0 # minimum mean pixel change (0-255) before a feed counts as live
|
|
39
|
+
frozen_min_seconds: float = 2.0 # shorter episodes are too brief to judge
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
CheckFn = Callable[[Dataset, Config], Iterable[Finding]]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def all_checks(include_video: bool) -> list[CheckFn]:
|
|
46
|
+
from . import episodes, meta, signals, timing
|
|
47
|
+
|
|
48
|
+
checks: list[CheckFn] = [meta.check, timing.check, signals.check, episodes.check]
|
|
49
|
+
if include_video:
|
|
50
|
+
from . import video
|
|
51
|
+
|
|
52
|
+
checks.append(video.check)
|
|
53
|
+
return checks
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Episode-level checks: lengths, tasks, and duplicate demonstrations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from ..dataset import Dataset
|
|
10
|
+
from ..findings import Finding, Severity
|
|
11
|
+
from . import Config
|
|
12
|
+
from .signals import dataset_stats
|
|
13
|
+
|
|
14
|
+
E, W, I = Severity.ERROR, Severity.WARNING, Severity.INFO
|
|
15
|
+
RESAMPLE = 64
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check(ds: Dataset, cfg: Config) -> Iterator[Finding]:
|
|
19
|
+
if not ds.episodes:
|
|
20
|
+
yield Finding("NO_EPISODES", E, "dataset contains no episodes")
|
|
21
|
+
return
|
|
22
|
+
fps = ds.fps or 30.0
|
|
23
|
+
lengths = np.array([ep.length for ep in ds.episodes])
|
|
24
|
+
median = float(np.median(lengths))
|
|
25
|
+
|
|
26
|
+
for ep in ds.episodes:
|
|
27
|
+
n = ep.length
|
|
28
|
+
if ep.length_meta is not None and ep.length_meta != n:
|
|
29
|
+
yield Finding(
|
|
30
|
+
"EPISODE_LENGTH_MISMATCH",
|
|
31
|
+
E,
|
|
32
|
+
f"metadata says {ep.length_meta} frames, data has {n}",
|
|
33
|
+
ep.index,
|
|
34
|
+
{"metadata": ep.length_meta, "data": n},
|
|
35
|
+
)
|
|
36
|
+
if 0 < n < cfg.min_seconds * fps:
|
|
37
|
+
yield Finding("EPISODE_TOO_SHORT", E, f"only {n} frames ({n / fps:.2f}s): likely a false start", ep.index)
|
|
38
|
+
elif len(lengths) >= 5 and n < cfg.short_ratio * median:
|
|
39
|
+
yield Finding(
|
|
40
|
+
"EPISODE_SHORT",
|
|
41
|
+
W,
|
|
42
|
+
f"{n} frames, under {cfg.short_ratio:.0%} of the median ({median:.0f}): possibly truncated",
|
|
43
|
+
ep.index,
|
|
44
|
+
)
|
|
45
|
+
if len(lengths) >= 5 and n > cfg.long_ratio * median and n - median > 5 * fps:
|
|
46
|
+
yield Finding(
|
|
47
|
+
"EPISODE_LONG",
|
|
48
|
+
I,
|
|
49
|
+
f"{n} frames, {n / median:.1f}x the median: recording may not have been stopped",
|
|
50
|
+
ep.index,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
ti = ep.frames.get("task_index")
|
|
54
|
+
if ti is not None and ds.tasks:
|
|
55
|
+
unknown = sorted({int(x) for x in np.unique(ti)} - set(ds.tasks))
|
|
56
|
+
if unknown:
|
|
57
|
+
yield Finding(
|
|
58
|
+
"TASK_INDEX_UNKNOWN", E, f"task_index values {unknown[:5]} have no task description", ep.index
|
|
59
|
+
)
|
|
60
|
+
if ep.tasks and not any(t.strip() for t in ep.tasks):
|
|
61
|
+
yield Finding("EPISODE_TASK_EMPTY", W, "episode's task description is empty", ep.index)
|
|
62
|
+
|
|
63
|
+
yield from _duplicates(ds, cfg)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _signature(a: np.ndarray, lo: np.ndarray, rng: np.ndarray) -> np.ndarray:
|
|
67
|
+
a = a.reshape(-1, 1) if a.ndim == 1 else a
|
|
68
|
+
t_old = np.linspace(0.0, 1.0, len(a))
|
|
69
|
+
t_new = np.linspace(0.0, 1.0, RESAMPLE)
|
|
70
|
+
res = np.stack([np.interp(t_new, t_old, a[:, j]) for j in range(a.shape[1])], axis=1)
|
|
71
|
+
return ((res - lo) / rng).ravel()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _duplicates(ds: Dataset, cfg: Config) -> Iterator[Finding]:
|
|
75
|
+
st = dataset_stats(ds, "action", cfg)
|
|
76
|
+
if st is None:
|
|
77
|
+
return
|
|
78
|
+
act = st.active
|
|
79
|
+
if not act.any():
|
|
80
|
+
return
|
|
81
|
+
rng = np.where(act, st.rng, 1.0)
|
|
82
|
+
eps, sigs, lens = [], [], []
|
|
83
|
+
for ep in ds.episodes:
|
|
84
|
+
a = ep.frames.get("action")
|
|
85
|
+
if a is None or ep.length < 4:
|
|
86
|
+
continue
|
|
87
|
+
a2 = a.reshape(-1, 1) if a.ndim == 1 else a
|
|
88
|
+
if a2.shape[1] != len(rng) or not np.all(np.isfinite(a2)):
|
|
89
|
+
continue
|
|
90
|
+
if np.all((a2.max(0) - a2.min(0))[act] < cfg.dead_tol * st.rng[act]):
|
|
91
|
+
continue # dead episodes are reported elsewhere and would all look alike
|
|
92
|
+
eps.append(ep.index)
|
|
93
|
+
sigs.append(_signature(a2[:, act].astype(np.float64), st.lo[act], rng[act]))
|
|
94
|
+
lens.append(ep.length)
|
|
95
|
+
if len(sigs) < 2:
|
|
96
|
+
return
|
|
97
|
+
S = np.stack(sigs)
|
|
98
|
+
L = np.array(lens, dtype=float)
|
|
99
|
+
# mean|Si - Sj| >= |mean(Si) - mean(Sj)|, so only pairs whose signature means are within
|
|
100
|
+
# the tolerance can be duplicates. Sorting by mean makes this exact pruning, which keeps
|
|
101
|
+
# 100k-episode datasets fast.
|
|
102
|
+
M = S.mean(axis=1)
|
|
103
|
+
order = np.argsort(M, kind="stable")
|
|
104
|
+
M_sorted = M[order]
|
|
105
|
+
seen: set[int] = set()
|
|
106
|
+
for i in range(len(S)):
|
|
107
|
+
if eps[i] in seen:
|
|
108
|
+
continue
|
|
109
|
+
lo_pos = np.searchsorted(M_sorted, M[i] - cfg.duplicate_tol, side="left")
|
|
110
|
+
hi_pos = np.searchsorted(M_sorted, M[i] + cfg.duplicate_tol, side="right")
|
|
111
|
+
cand = order[lo_pos:hi_pos]
|
|
112
|
+
cand = np.sort(cand[cand > i])
|
|
113
|
+
cand = cand[np.abs(L[cand] - L[i]) <= 0.05 * L[i] + 1]
|
|
114
|
+
if not len(cand):
|
|
115
|
+
continue
|
|
116
|
+
dist = np.mean(np.abs(S[cand] - S[i]), axis=1)
|
|
117
|
+
for j in cand[dist < cfg.duplicate_tol]:
|
|
118
|
+
seen.add(eps[j])
|
|
119
|
+
yield Finding(
|
|
120
|
+
"DUPLICATE_EPISODE",
|
|
121
|
+
W,
|
|
122
|
+
f"actions are nearly identical to episode {eps[i]}: a duplicated recording, or a scripted demo "
|
|
123
|
+
"repeated with the same setup",
|
|
124
|
+
eps[j],
|
|
125
|
+
{"duplicate_of": eps[i], "mean_abs_diff": round(float(dist[list(cand).index(j)]), 5)},
|
|
126
|
+
)
|