kinema 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kinema-0.1.0/LICENSE +196 -0
- kinema-0.1.0/PKG-INFO +264 -0
- kinema-0.1.0/README.md +243 -0
- kinema-0.1.0/pyproject.toml +45 -0
- kinema-0.1.0/setup.cfg +4 -0
- kinema-0.1.0/src/kinema/__init__.py +3 -0
- kinema-0.1.0/src/kinema/causes.json +112 -0
- kinema-0.1.0/src/kinema/checks/__init__.py +21 -0
- kinema-0.1.0/src/kinema/checks/actions.py +155 -0
- kinema-0.1.0/src/kinema/checks/base.py +188 -0
- kinema-0.1.0/src/kinema/checks/calibration.py +364 -0
- kinema-0.1.0/src/kinema/checks/deployment.py +283 -0
- kinema-0.1.0/src/kinema/checks/duplicates.py +276 -0
- kinema-0.1.0/src/kinema/checks/idle.py +351 -0
- kinema-0.1.0/src/kinema/checks/metadata.py +244 -0
- kinema-0.1.0/src/kinema/checks/outliers.py +267 -0
- kinema-0.1.0/src/kinema/checks/stats_drift.py +308 -0
- kinema-0.1.0/src/kinema/checks/timestamps.py +235 -0
- kinema-0.1.0/src/kinema/checks/visual.py +241 -0
- kinema-0.1.0/src/kinema/cli.py +166 -0
- kinema-0.1.0/src/kinema/dataset.py +909 -0
- kinema-0.1.0/src/kinema/diagnose.py +402 -0
- kinema-0.1.0/src/kinema/engine.py +57 -0
- kinema-0.1.0/src/kinema/fixtures/__init__.py +0 -0
- kinema-0.1.0/src/kinema/fixtures/synth.py +239 -0
- kinema-0.1.0/src/kinema/mcp_server.py +762 -0
- kinema-0.1.0/src/kinema/policies.json +46 -0
- kinema-0.1.0/src/kinema/report.py +416 -0
- kinema-0.1.0/src/kinema/similar.py +136 -0
- kinema-0.1.0/src/kinema/stats.py +267 -0
- kinema-0.1.0/src/kinema.egg-info/PKG-INFO +264 -0
- kinema-0.1.0/src/kinema.egg-info/SOURCES.txt +56 -0
- kinema-0.1.0/src/kinema.egg-info/dependency_links.txt +1 -0
- kinema-0.1.0/src/kinema.egg-info/entry_points.txt +2 -0
- kinema-0.1.0/src/kinema.egg-info/requires.txt +12 -0
- kinema-0.1.0/src/kinema.egg-info/top_level.txt +1 -0
- kinema-0.1.0/tests/test_causes_schema.py +81 -0
- kinema-0.1.0/tests/test_checks.py +295 -0
- kinema-0.1.0/tests/test_cli_sample_caution.py +37 -0
- kinema-0.1.0/tests/test_diagnose.py +236 -0
- kinema-0.1.0/tests/test_end_to_end.py +33 -0
- kinema-0.1.0/tests/test_engine.py +20 -0
- kinema-0.1.0/tests/test_false_positives.py +805 -0
- kinema-0.1.0/tests/test_findings_batch3.py +407 -0
- kinema-0.1.0/tests/test_findings_batch4.py +138 -0
- kinema-0.1.0/tests/test_harm_study.py +82 -0
- kinema-0.1.0/tests/test_harm_table.py +455 -0
- kinema-0.1.0/tests/test_leaderboard.py +50 -0
- kinema-0.1.0/tests/test_mcp_protocol.py +575 -0
- kinema-0.1.0/tests/test_netcheck.py +31 -0
- kinema-0.1.0/tests/test_real_datasets.py +61 -0
- kinema-0.1.0/tests/test_revalidate_gripper_prior.py +70 -0
- kinema-0.1.0/tests/test_sampling.py +284 -0
- kinema-0.1.0/tests/test_scale_test.py +26 -0
- kinema-0.1.0/tests/test_scoring.py +451 -0
- kinema-0.1.0/tests/test_sensitivity.py +85 -0
- kinema-0.1.0/tests/test_similar.py +122 -0
- kinema-0.1.0/tests/test_visual.py +118 -0
kinema-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was necessarily
|
|
82
|
+
submitted. If You institute patent litigation against any entity
|
|
83
|
+
(including a cross-claim or counterclaim in a lawsuit) alleging
|
|
84
|
+
that the Work or a Contribution incorporated within the Work
|
|
85
|
+
constitutes direct or contributory patent infringement, then any
|
|
86
|
+
patent licenses granted to You under this License for that Work
|
|
87
|
+
shall terminate as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the source code or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add your own attribution
|
|
118
|
+
notices to any Derivative Works, and to the attribution notices
|
|
119
|
+
within such NOTICE file, and shall include only the information
|
|
120
|
+
supplied by You to the Licensor in or on behalf of the Licensor.
|
|
121
|
+
|
|
122
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
123
|
+
the terms of any separate license agreement you may have executed
|
|
124
|
+
with Licensor regarding such Contributions.
|
|
125
|
+
|
|
126
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
127
|
+
any contribution intentionally submitted for inclusion in the Work
|
|
128
|
+
by You to the Licensor shall automatically be under the terms and
|
|
129
|
+
conditions of this License. You may reproduce and distribute copies
|
|
130
|
+
of the Work, with or without modifications, in Source or Object
|
|
131
|
+
form, and at your option, any such additional terms, as set forth
|
|
132
|
+
in this License.
|
|
133
|
+
|
|
134
|
+
Unless required by applicable law or agreed to in writing, Licensor
|
|
135
|
+
shall have no obligation or grant of any maintenance, support,
|
|
136
|
+
updates, upgrades, or bug fixes for the Work beyond making the
|
|
137
|
+
source code available on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
138
|
+
CONDITIONS OF ANY KIND, either express or implied, including,
|
|
139
|
+
without limitation, the implied warranties or conditions of
|
|
140
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
141
|
+
|
|
142
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
143
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
144
|
+
except as required for reasonable and customary use in describing the
|
|
145
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
146
|
+
|
|
147
|
+
7. Disclaimer of Warranty. Unless required by applicable law or agreed
|
|
148
|
+
to in writing, Licensor provides the Work on an "AS IS" BASIS,
|
|
149
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
150
|
+
implied. See the License for the specific language governing
|
|
151
|
+
permissions and limitations under the License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, will any Contributor be
|
|
157
|
+
liable to You for damages, including any lost profits, lost monies,
|
|
158
|
+
or any direct, indirect, special, incidental, or consequential
|
|
159
|
+
damages of any character arising as a result of this License or out
|
|
160
|
+
of the use or inability to use the Work.
|
|
161
|
+
|
|
162
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
163
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
164
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
165
|
+
or other liability obligations and/or rights consistent with this
|
|
166
|
+
License. However, in accepting such obligations, You may act only
|
|
167
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
168
|
+
of any other Contributor, and only in providing a warranty, if any,
|
|
169
|
+
or additional liability obligations.
|
|
170
|
+
|
|
171
|
+
END OF TERMS AND CONDITIONS
|
|
172
|
+
|
|
173
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
174
|
+
|
|
175
|
+
To apply the Apache License to your work, attach the following
|
|
176
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
177
|
+
replaced with your own identifying information. (Don't include
|
|
178
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
179
|
+
comment syntax for the file format. We also recommend that a
|
|
180
|
+
file or class name and description of purpose be included on the
|
|
181
|
+
same "printed page" as the copyright notice for easier
|
|
182
|
+
identification within third-party archives.
|
|
183
|
+
|
|
184
|
+
Copyright 2026 Ved Patel
|
|
185
|
+
|
|
186
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
187
|
+
you may not use this file except in compliance with the License.
|
|
188
|
+
You may obtain a copy of the License at
|
|
189
|
+
|
|
190
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
191
|
+
|
|
192
|
+
Unless required by applicable law or agreed to in writing, software
|
|
193
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
194
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
195
|
+
See the License for the specific language governing permissions and
|
|
196
|
+
limitations under the License.
|
kinema-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kinema
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Quality checks for LeRobot robot-learning datasets: find the data problems that kill fine-tunes.
|
|
5
|
+
Author: Ved Patel
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pyarrow>=14
|
|
11
|
+
Requires-Dist: numpy>=1.24
|
|
12
|
+
Requires-Dist: huggingface-hub>=0.20
|
|
13
|
+
Requires-Dist: rich>=13
|
|
14
|
+
Provides-Extra: video
|
|
15
|
+
Requires-Dist: av>=11; extra == "video"
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
18
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
19
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# kinema
|
|
23
|
+
|
|
24
|
+
Quality checks for [LeRobot](https://github.com/huggingface/lerobot) robot-learning datasets.
|
|
25
|
+
|
|
26
|
+
Your fine-tune failed and the loss looked fine. The problem is almost always in the data:
|
|
27
|
+
normalization stats that drifted from the actual values, calibration offset between the
|
|
28
|
+
leader and follower arm, episodes that are mostly the robot sitting still, timestamps off
|
|
29
|
+
the sampling grid. kinema finds those before you burn the GPU hours.
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# not on PyPI yet - install from the repo
|
|
33
|
+
pip install "git+https://github.com/vedevpatel/kinema"
|
|
34
|
+
# or from source: git clone https://github.com/vedevpatel/kinema && pip install -e kinema
|
|
35
|
+
|
|
36
|
+
kinema check lerobot/pusht # straight off the Hugging Face Hub
|
|
37
|
+
kinema check ./my-dataset --policy pi0 # local dataset, deployment checks for pi0
|
|
38
|
+
kinema check hf://owner/name --json out.json --markdown report.md
|
|
39
|
+
kinema check owner/name@<commit> --video # pinned revision; + blur/exposure from the MP4s
|
|
40
|
+
kinema check lerobot/droid_1.0.1 --max-episodes 50 # spread sample; downloads only the shards it reads
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
No `lerobot` dependency. Installs in seconds: `pyarrow`, `numpy`, `huggingface-hub`, `rich`.
|
|
44
|
+
Works on LeRobot **v3.0 and v2.1** layouts, local directories, `.zip`s, and Hub repo ids.
|
|
45
|
+
|
|
46
|
+
## What it looks like
|
|
47
|
+
|
|
48
|
+
`kinema check lerobot/pusht --policy smolvla`:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
lerobot/pusht health score 79/100
|
|
52
|
+
format v3.0 | 206 episodes | 25650 frames | fps 10.0
|
|
53
|
+
|
|
54
|
+
check verdict summary
|
|
55
|
+
stats-drift PASS data matches meta/stats.json (worst 0% of the FAIL threshold)
|
|
56
|
+
timestamp-grid PASS evenly spaced at 10 fps and monotonic
|
|
57
|
+
dropped-frames PASS frame_index contiguous; episode lengths and video segments match the data
|
|
58
|
+
calibration-drift PASS 2 joint(s) + 0 gripper(s) agree; worst offset -0.0% of range (joint 1)
|
|
59
|
+
idle-segments PASS no episode starts idle or idles >= 30%
|
|
60
|
+
stuck-actuator PASS every joint moves wherever the dataset uses it
|
|
61
|
+
action-saturation PASS no continuous action dim pinned at a limit above 5% (2 dims checked)
|
|
62
|
+
nan-and-normalization PASS no NaN/Inf; all dims have usable variance
|
|
63
|
+
duplicate-and-leakage PASS no exact, partial or near duplicates
|
|
64
|
+
outlier-episodes WARN 2 episode(s) to review, 23 mild, of 206; worst: [107, 159, 15, 97, ...]
|
|
65
|
+
embedded-images PASS no image features declared
|
|
66
|
+
camera-coverage PASS 1 camera(s) over 206 episodes; 0 missing segments
|
|
67
|
+
task-strings PASS 206 episodes, 1 tasks in meta/tasks; 0 finding(s)
|
|
68
|
+
reward-done WARN done is True before the last frame in 206 episodes - the trajectory continues after termination
|
|
69
|
+
visual-quality SKIP pass --video to run (downloads MP4s; needs the video extra)
|
|
70
|
+
deployment-readiness WARN 0 blocker(s), 1 warning(s) for smolvla
|
|
71
|
+
|
|
72
|
+
verdict: WARN (0 fail, 4 warn, 23 info)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Exit code is `1` on any FAIL (`--fail-on warn` to also gate on warnings), so it drops
|
|
76
|
+
straight into CI as a pre-training gate.
|
|
77
|
+
|
|
78
|
+
## The checks
|
|
79
|
+
|
|
80
|
+
The long form of each check - method, severity logic, config knobs, known limits -
|
|
81
|
+
lives in [docs/checks.md](docs/checks.md).
|
|
82
|
+
|
|
83
|
+
| check | what it catches | why it kills fine-tunes |
|
|
84
|
+
|---|---|---|
|
|
85
|
+
| `stats-drift` | stored stats disagree with the data: mean shift > 0.1 std, std off > 5%, min/max off > 1% of range. Sampled loads are checked against the per-episode stats of exactly the loaded episodes, and `stats.json` against all per-episode stats; v2.1 uses aggregated `episodes_stats.jsonl` like LeRobot does | every run normalizes with those numbers; drift = wrong-scaled inputs at training *and* inference |
|
|
86
|
+
| `timestamp-grid` | timestamps off the 1/fps grid, non-monotonic runs | `delta_timestamps` sampling returns misaligned windows |
|
|
87
|
+
| `dropped-frames` | `frame_index` gaps, metadata length vs data mismatch | corrupt episodes; the metadata lie blocks training outright |
|
|
88
|
+
| `calibration-drift` | mean action-state offset on still frames, per joint, independent of position. Only on dims where action is a position command for the same joint (names agree, action tracks state); grippers excluded | leader/follower drift (the 17-degree case from LeRobot PR #3762) survives recording *and* training |
|
|
89
|
+
| `idle-segments` | episodes that start idle, or idle >30% of the time in runs >2s (command held AND robot still); a still tail after the task is INFO | teaches the policy to do nothing - the classic "robot stays still at inference" postmortem |
|
|
90
|
+
| `stuck-actuator` | episodes where nothing moves; a joint that moves in most episodes but not in a few; gripper never firing on pick/place | the policy learns to ignore that joint |
|
|
91
|
+
| `action-saturation` | >5% of commands during motion pinned at exactly the min/max of a continuous, non-gripper dim | clipped commands the policy will imitate |
|
|
92
|
+
| `nan-and-normalization` | NaN/Inf, zero-variance dims, std < 1e-6 | one NaN poisons the loss for the whole batch |
|
|
93
|
+
| `duplicate-and-leakage` | exact-copy episodes, partial copies (an opening only two episodes share is a copied beginning, not a scripted start), near-duplicates (exact scan up to 100k episodes with memory-bounded blocks, an even sample with a note above that) | silent metric inflation |
|
|
94
|
+
| `outlier-episodes` | IQR outliers on the harmful side only: duration and end pose within a task, jerk/velocity per dim in std units without grippers or velocity commands (recognised by name or by tracking the state's delta, not its level - a spike beyond a velocity dim's own step scale still counts). WARN when two independent signals agree or one is far out (Tukey k=3, >=3x median, and for end pose >= half the end-pose range) | a handful of broken episodes dominate the gradient |
|
|
95
|
+
| `embedded-images` | image features stored inside the parquet shards instead of MP4 video chunks | bloated shards; every frame decodes on read |
|
|
96
|
+
| `camera-coverage` | episodes with no video for a declared camera (v3 segment metadata, or the v2.1 files with `--video`) | a modality silently drops out for those episodes |
|
|
97
|
+
| `task-strings` | empty task strings, frames pointing at a `task_index` the task table does not define, episode tasks with no table entry | language conditioning trains on `""` or an unrecoverable task |
|
|
98
|
+
| `reward-done` | constant or NaN reward, `done` never True, `done` True mid-episode with frames after it | RL / reward-conditioned training gets labels with no signal |
|
|
99
|
+
| `visual-quality` | blur (variance of Laplacian, frames decimated to ~VGA first so one threshold works at any resolution), underexposure (95th-percentile brightness) and overexposure (pixels clipped at the codec's brightest level), per camera, sampled inside each episode's segment (v2.1 per-episode and v3 shared MP4s) | bad visuals cost ~30% of training loss ([HF GIGO](https://huggingface.co/blog/robotdata/gigo)). Needs `--video` and the `video` extra |
|
|
100
|
+
| `deployment-readiness` | episode length vs the policy's action chunk, stats coverage, quantile-norm support, declared shapes, and what the policy conditions on (camera input, task language, typical resolution - declared per policy in `src/kinema/policies.json`) | the openpi #906 failure mode: low loss, failed robot |
|
|
101
|
+
|
|
102
|
+
## Deployment readiness
|
|
103
|
+
|
|
104
|
+
Point it at the policy you're about to train:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
kinema check ./so100-pickplace --policy pi0
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
It verifies the dataset can actually serve that policy: episodes long enough to fill an
|
|
111
|
+
action chunk, stats.json covering what the normalizer consumes, `q01/q99` quantiles when
|
|
112
|
+
the policy normalizes that way (openpi does), that declared shapes and timestamps
|
|
113
|
+
match what the policy's data loader will assume, and that the dataset carries what the
|
|
114
|
+
policy conditions on - camera input for vision policies, task strings for
|
|
115
|
+
language-conditioned ones, and a note when a camera is smaller than the resolution the
|
|
116
|
+
policy is usually trained at.
|
|
117
|
+
|
|
118
|
+
What each policy expects is declared in `src/kinema/policies.json` (chunk size,
|
|
119
|
+
normalization, camera/language needs, typical resolution; aliases like `pi0.5`
|
|
120
|
+
resolve to their family entry). Adding or updating a policy is a JSON edit, no
|
|
121
|
+
code change; an unknown `--policy` still gets the policy-agnostic checks.
|
|
122
|
+
|
|
123
|
+
## The fine-tune copilot (MCP)
|
|
124
|
+
|
|
125
|
+
`kinema mcp` starts an MCP server over stdio - the "why did my fine-tune fail"
|
|
126
|
+
layer on top of the same checks. Point your MCP client at it:
|
|
127
|
+
|
|
128
|
+
```json
|
|
129
|
+
{
|
|
130
|
+
"mcpServers": {
|
|
131
|
+
"kinema": {
|
|
132
|
+
"command": "uvx",
|
|
133
|
+
"args": ["--from", "git+https://github.com/vedevpatel/kinema", "kinema", "mcp"]
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
(The plain `uvx kinema mcp` form works once the package is on PyPI - a planned
|
|
140
|
+
milestone, not done yet. Until then a `pip install -e .` checkout works too:
|
|
141
|
+
`"command": "kinema", "args": ["mcp"]`.)
|
|
142
|
+
|
|
143
|
+
Three tools:
|
|
144
|
+
|
|
145
|
+
- `diagnose_finetune(dataset, policy, symptom, policy_overrides?, max_episodes?)` -
|
|
146
|
+
ranks the data causes that explain a failed run, each with check evidence, a
|
|
147
|
+
docs citation and a next step. Every check runs whatever the symptom says;
|
|
148
|
+
the symptom only decides which causes rank first. Scope is deliberately narrow: the pi0 family
|
|
149
|
+
(`pi0`, `pi05`/`pi0.5`, `pi0_fast`) and two failure-mode clusters -
|
|
150
|
+
normalization/stats mismatch and idle/static segments. Training with a custom
|
|
151
|
+
chunk size or mean/std normalization? Pass `policy_overrides`
|
|
152
|
+
(`{"chunk_size": 10, "quantile_norm": false}`) and the diagnosis computes
|
|
153
|
+
against *your* setup, not the declared defaults.
|
|
154
|
+
- `run_checks(dataset, checks?, ...)` - the raw engine over MCP: a capped summary
|
|
155
|
+
(verdict, score, top findings) sized for client output limits, with the full
|
|
156
|
+
JSON report written to a file (`<dataset>.kinema-report-<timestamp>.json` next
|
|
157
|
+
to a local dataset, else `~/.cache/kinema/reports/`; an explicit `report_path`
|
|
158
|
+
must be absolute and only replaces an earlier kinema report).
|
|
159
|
+
- `list_checks()` - the check catalog from the registry, with docs links.
|
|
160
|
+
|
|
161
|
+
`dataset` is an absolute local path (directory or `.zip`) or a Hub repo id
|
|
162
|
+
(`owner/name`). Relative paths are rejected: the server runs in whatever
|
|
163
|
+
directory your client launched it from, not yours.
|
|
164
|
+
|
|
165
|
+
The answers show their work instead of claiming your data is fine: every answer
|
|
166
|
+
enumerates which checks ran, which skipped and why, whether the load was sampled,
|
|
167
|
+
which action dims sat out, any findings outside the v0 clusters (verbatim), and
|
|
168
|
+
the blind spots no check covers (unit conventions, joint order, delta-vs-absolute
|
|
169
|
+
at inference, control-loop fps). Cause entries live in `src/kinema/causes.json` -
|
|
170
|
+
adding a failure-mode cluster or a policy is a JSON edit, and a schema test keeps
|
|
171
|
+
the map honest against the engine.
|
|
172
|
+
|
|
173
|
+
## Tuning the thresholds
|
|
174
|
+
|
|
175
|
+
Every threshold lives in `CheckConfig` with a conservative default. Override any
|
|
176
|
+
of them without editing code - a JSON file of field names to values:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
kinema check ./my-dataset --config tuning.json
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
```json
|
|
183
|
+
{
|
|
184
|
+
"stats_std_fail": 0.4,
|
|
185
|
+
"idle_min_seconds": 5,
|
|
186
|
+
"calib_fail_range_frac": 0.2
|
|
187
|
+
}
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Unknown keys and non-numeric values are rejected, so a typo fails loudly instead
|
|
191
|
+
of silently tuning nothing. `kinema check --help` lists the rest of the flags.
|
|
192
|
+
|
|
193
|
+
Resident frame memory is capped separately: `KINEMA_FRAME_BUDGET_BYTES` (default
|
|
194
|
+
1 GiB). Above it the frame arrays are memory-mapped to disk instead of held in
|
|
195
|
+
RAM, so a full load of a DROID-sized dataset cannot OOM a smaller machine.
|
|
196
|
+
|
|
197
|
+
## The leaderboard
|
|
198
|
+
|
|
199
|
+
The most-downloaded LeRobot datasets on Hugging Face, audited nightly:
|
|
200
|
+
**[vedevpatel.github.io/kinema](https://vedevpatel.github.io/kinema/)**
|
|
201
|
+
|
|
202
|
+
Board audits a sample of 50 episodes per dataset (`--max-episodes 50`), spread
|
|
203
|
+
evenly across the dataset, to keep a nightly run of 200 datasets feasible; a
|
|
204
|
+
local `kinema check` scans the full dataset.
|
|
205
|
+
Sampled and full scans can score differently - for example pusht scores 89 on the
|
|
206
|
+
board and 79 on a full local scan, because a 50-episode sample's recomputed stats
|
|
207
|
+
are compared against stats.json values computed over all 206 episodes.
|
|
208
|
+
|
|
209
|
+
## Try it on the demo dataset
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
kinema synth /tmp/demo --defects stats-drift,idle-segments,calibration-drift
|
|
213
|
+
kinema check /tmp/demo
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
## Roadmap
|
|
217
|
+
|
|
218
|
+
- Auto-fix suggestions with safe rewrites (stats recompute, idle trimming).
|
|
219
|
+
- Video checks on the leaderboard: v3 shared-MP4 attribution works locally (`--video`); the nightly board skips it because the MP4s run to hundreds of MB per camera.
|
|
220
|
+
|
|
221
|
+
## Development
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
pip install -e .[dev]
|
|
225
|
+
python -m pytest # unit tests (synthetic defects + false-positive regressions)
|
|
226
|
+
ruff check src scripts tests # lint
|
|
227
|
+
mypy src/kinema # type check
|
|
228
|
+
KINEMA_E2E=1 python -m pytest # + real Hub datasets (pinned revisions) and the sensitivity guard
|
|
229
|
+
python scripts/calibrate.py --quick # how big a defect must be before each check catches it
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
### The CI guards
|
|
233
|
+
|
|
234
|
+
The nightly e2e job runs on pinned real datasets: known-healthy data must not
|
|
235
|
+
FAIL, known defects must still be caught, and the sensitivity guard
|
|
236
|
+
(`tests/test_sensitivity.py`) injects defects of known size. Its **floors** are
|
|
237
|
+
detection rates a little under the measured ones - a drop means a threshold
|
|
238
|
+
lost its teeth. Its **ceilings** are the measured baseline: how many untouched
|
|
239
|
+
datasets each check already flags.
|
|
240
|
+
|
|
241
|
+
The pinned calibration list splits into disjoint `tune` and `eval` halves
|
|
242
|
+
(`scripts/calibration_datasets.txt`): thresholds, floors and ceilings are derived
|
|
243
|
+
from the tune half, and the sensitivity guard runs on the eval half - the numbers
|
|
244
|
+
that protect the checks were never tuned on the data that guards them.
|
|
245
|
+
|
|
246
|
+
These suites need the Hub. If huggingface.co itself is unreachable they skip with a
|
|
247
|
+
visible reason (`tests/netcheck.py`) instead of failing CI on someone else's outage.
|
|
248
|
+
|
|
249
|
+
A ceiling failure is not automatically a false positive: it can also be a real
|
|
250
|
+
issue a change newly surfaces (a new check firing, a sharper threshold). When
|
|
251
|
+
one fails:
|
|
252
|
+
|
|
253
|
+
1. Reproduce locally: `KINEMA_E2E=1 python -m pytest tests/test_sensitivity.py -q`.
|
|
254
|
+
2. Read the new finding on the flagged dataset. If it is a false positive, fix
|
|
255
|
+
the check - do not raise the ceiling to make it pass.
|
|
256
|
+
3. If the finding is real and the new count is the intended behavior, raise the
|
|
257
|
+
number in `CEILINGS` in the same commit, with the message saying what the
|
|
258
|
+
finding was and why it stays. The same rule applies in reverse to floors:
|
|
259
|
+
lower one only with a reason in the commit message.
|
|
260
|
+
|
|
261
|
+
The full pinned list (~700 MB) runs weekly in the `calibration` workflow and on
|
|
262
|
+
demand from the Actions tab; the report lands as a run artifact.
|
|
263
|
+
|
|
264
|
+
Apache-2.0. Named for *kinema*, "motion" - the thing your dataset is supposed to contain.
|