datahive-tools 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datahive_tools-0.1.0/LICENSE +21 -0
- datahive_tools-0.1.0/PKG-INFO +117 -0
- datahive_tools-0.1.0/README.md +92 -0
- datahive_tools-0.1.0/pyproject.toml +35 -0
- datahive_tools-0.1.0/setup.cfg +4 -0
- datahive_tools-0.1.0/src/datahive/__init__.py +31 -0
- datahive_tools-0.1.0/src/datahive/annotate.py +123 -0
- datahive_tools-0.1.0/src/datahive/attachments.py +71 -0
- datahive_tools-0.1.0/src/datahive/attachments.yaml +124 -0
- datahive_tools-0.1.0/src/datahive/check.py +192 -0
- datahive_tools-0.1.0/src/datahive/cli.py +567 -0
- datahive_tools-0.1.0/src/datahive/collect.py +158 -0
- datahive_tools-0.1.0/src/datahive/config.py +200 -0
- datahive_tools-0.1.0/src/datahive/consistency.py +313 -0
- datahive_tools-0.1.0/src/datahive/episode.py +378 -0
- datahive_tools-0.1.0/src/datahive/errors.py +65 -0
- datahive_tools-0.1.0/src/datahive/hf_limits.py +48 -0
- datahive_tools-0.1.0/src/datahive/hub.py +118 -0
- datahive_tools-0.1.0/src/datahive/index.py +236 -0
- datahive_tools-0.1.0/src/datahive/interface/__init__.py +0 -0
- datahive_tools-0.1.0/src/datahive/interface/api.py +748 -0
- datahive_tools-0.1.0/src/datahive/interface/app.py +32 -0
- datahive_tools-0.1.0/src/datahive/interface/static/app.js +2678 -0
- datahive_tools-0.1.0/src/datahive/interface/static/automatic.js +242 -0
- datahive_tools-0.1.0/src/datahive/interface/static/favicon.svg +9 -0
- datahive_tools-0.1.0/src/datahive/interface/static/guide.js +116 -0
- datahive_tools-0.1.0/src/datahive/interface/static/hiveboard-logo.svg +43 -0
- datahive_tools-0.1.0/src/datahive/interface/static/index.html +350 -0
- datahive_tools-0.1.0/src/datahive/interface/static/nav.js +34 -0
- datahive_tools-0.1.0/src/datahive/interface/static/runner.js +1137 -0
- datahive_tools-0.1.0/src/datahive/interface/static/style.css +1828 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/big_valve_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/box_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/button_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/key_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/lamp_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/m30_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/m8_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/peg_and_hole_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/small_valve_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/spring_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/switch_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/interface/static/tasks/torque_valve_noFriction_3d.png +0 -0
- datahive_tools-0.1.0/src/datahive/ops.py +384 -0
- datahive_tools-0.1.0/src/datahive/paths.py +143 -0
- datahive_tools-0.1.0/src/datahive/preflight.py +111 -0
- datahive_tools-0.1.0/src/datahive/profile.py +530 -0
- datahive_tools-0.1.0/src/datahive/runner.py +485 -0
- datahive_tools-0.1.0/src/datahive/schema.py +251 -0
- datahive_tools-0.1.0/src/datahive/skill_install.py +45 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-auto-collect/SKILL.md +108 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-auto-collect/reference/adapting.md +152 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-auto-collect/reference/protocol.md +73 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-data-prep/SKILL.md +108 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-data-prep/reference/annotation.md +84 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-data-prep/reference/layout-and-format.md +144 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-data-prep/reference/robot-profile.md +92 -0
- datahive_tools-0.1.0/src/datahive/skills/datahive-data-prep/reference/troubleshooting.md +68 -0
- datahive_tools-0.1.0/src/datahive/trials.py +120 -0
- datahive_tools-0.1.0/src/datahive/validate.py +125 -0
- datahive_tools-0.1.0/src/datahive_tools.egg-info/PKG-INFO +117 -0
- datahive_tools-0.1.0/src/datahive_tools.egg-info/SOURCES.txt +76 -0
- datahive_tools-0.1.0/src/datahive_tools.egg-info/dependency_links.txt +1 -0
- datahive_tools-0.1.0/src/datahive_tools.egg-info/entry_points.txt +2 -0
- datahive_tools-0.1.0/src/datahive_tools.egg-info/requires.txt +17 -0
- datahive_tools-0.1.0/src/datahive_tools.egg-info/top_level.txt +1 -0
- datahive_tools-0.1.0/tests/test_api.py +966 -0
- datahive_tools-0.1.0/tests/test_check.py +158 -0
- datahive_tools-0.1.0/tests/test_config_security.py +180 -0
- datahive_tools-0.1.0/tests/test_consistency.py +180 -0
- datahive_tools-0.1.0/tests/test_end_to_end.py +188 -0
- datahive_tools-0.1.0/tests/test_index_sync.py +97 -0
- datahive_tools-0.1.0/tests/test_profile.py +555 -0
- datahive_tools-0.1.0/tests/test_runner_collect.py +913 -0
- datahive_tools-0.1.0/tests/test_schema.py +150 -0
- datahive_tools-0.1.0/tests/test_skills.py +24 -0
- datahive_tools-0.1.0/tests/test_trials.py +53 -0
- datahive_tools-0.1.0/tests/test_upload_safety.py +119 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 HiveBoard
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datahive-tools
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Client tooling to collect, validate, annotate and upload HiveBoard manipulation episodes.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: huggingface_hub>=0.24
|
|
10
|
+
Requires-Dist: h5py>=3.10
|
|
11
|
+
Requires-Dist: pydantic>=2.6
|
|
12
|
+
Requires-Dist: pyyaml>=6.0
|
|
13
|
+
Requires-Dist: numpy>=1.24
|
|
14
|
+
Requires-Dist: fastapi>=0.110
|
|
15
|
+
Requires-Dist: uvicorn>=0.29
|
|
16
|
+
Requires-Dist: typer>=0.12
|
|
17
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
18
|
+
Provides-Extra: video
|
|
19
|
+
Requires-Dist: opencv-python-headless>=4.8; extra == "video"
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
22
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
23
|
+
Requires-Dist: opencv-python-headless>=4.8; extra == "dev"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# DataHive
|
|
27
|
+
|
|
28
|
+
`datahive-tools`: the client-side package a lab installs to collect,
|
|
29
|
+
validate, annotate, and upload HiveBoard manipulation episodes to its
|
|
30
|
+
private Hugging Face dataset repo.
|
|
31
|
+
|
|
32
|
+
> [!IMPORTANT]
|
|
33
|
+
> **Credits & Attribution**: All pipeline architecture, workflows, and core design in DataHive are directly adapted from and credited to [**Oopsie Data**](https://github.com/oopsie-data) (`oopsie-data` / `oopsie_data_tools`). Everything in DataHive — from the CLI workflow, robot profiles, and dataset indexing to the annotation GUI and Hugging Face Hub synchronization pipeline — was built upon the pipeline and concepts developed by the Oopsie Data team.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
pip install -e .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Quickstart
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
datahive init # one-time: writes ~/.datahive/config.yaml (0600)
|
|
45
|
+
datahive new-profile # writes samples/robot_profile.yaml -- fill in ONCE
|
|
46
|
+
# ... record episodes into samples/{session}/episodes/{episode}.h5 + .mp4 ...
|
|
47
|
+
datahive check <episode_id> # pre-annotation health check (frames, Hz, MP4s)
|
|
48
|
+
datahive annotate <episode_id>
|
|
49
|
+
datahive validate <episode_id>
|
|
50
|
+
datahive upload <episode_id>
|
|
51
|
+
datahive sync # reconcile all of samples/ with the Hub
|
|
52
|
+
datahive list
|
|
53
|
+
datahive delete <episode_id>
|
|
54
|
+
datahive interface --port 8000 # local web GUI, localhost only (alias: serve)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
See `docs/MANUAL_SMOKE_TEST.md` for the manual (non-automated) end-to-end
|
|
58
|
+
check against a real, disposable Hugging Face repo.
|
|
59
|
+
|
|
60
|
+
## The sections of `datahive serve`
|
|
61
|
+
|
|
62
|
+
Opening the interface shows a start screen with two sections, plus an About and FAQ area. Both sections share the same `samples/` tree.
|
|
63
|
+
|
|
64
|
+
- **Runner** – runs HiveBoard trials. Create a session (operator and date; the lab ID comes from `datahive init` and the platform ID from the robot profile), generate the trial plan (5 trials for each chosen task, shuffled or ordered), then run each trial in one of two modes, switchable at any time:
|
|
65
|
+
- **Manual**: 5 s countdown, stopwatch, sounds and automatic timeout in the browser (Space starts and stops), then record the outcome and **upload the HDF5 file and one video per camera in the robot profile**. The trial counts as valid only when all of them are uploaded and the episode passes validation.
|
|
66
|
+
- **Automatic**: send the trial to your robot script (below). It records the episode, and you annotate it in the Annotate section, which opens on that episode.
|
|
67
|
+
- **Annotate** – review recorded episodes, annotate them, validate the data, and upload it to the Hub.
|
|
68
|
+
|
|
69
|
+
Robot side of automatic mode:
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from datahive.collect import CollectClient
|
|
73
|
+
|
|
74
|
+
robot = CollectClient("samples", base_url="http://127.0.0.1:8000")
|
|
75
|
+
while True:
|
|
76
|
+
task = robot.wait_for_task() # blocks until you press "Send to robot"
|
|
77
|
+
with robot.new_writer(task) as writer: # EpisodeWriter for that trial
|
|
78
|
+
... # append_proprioception / append_command / attach_video
|
|
79
|
+
robot.finish(writer) # hands the episode over, waits for the annotation
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Trials are saved as rows of `samples/<session>/trials.csv`; all times are stored in UTC.
|
|
83
|
+
|
|
84
|
+
## AI assistant skills
|
|
85
|
+
|
|
86
|
+
Two skills for AI coding assistants (e.g. Claude Code) live in `src/datahive/skills/`:
|
|
87
|
+
|
|
88
|
+
- `datahive-data-prep` – how to lay out, format, annotate and check data under `samples/` so it validates.
|
|
89
|
+
- `datahive-auto-collect` – how automatic collection works and how to adapt the robot script to your setup.
|
|
90
|
+
|
|
91
|
+
Install them for Claude Code with:
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
datahive install-skill # into ./.claude/skills (this project)
|
|
95
|
+
datahive install-skill --user # into ~/.claude/skills (all projects)
|
|
96
|
+
datahive install-skill --dir DIR --force
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Tests
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
pip install -e ".[dev]"
|
|
103
|
+
pytest
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
All tests run with no real network access and no real Hugging Face
|
|
107
|
+
credentials.
|
|
108
|
+
|
|
109
|
+
## Credits & Acknowledgments
|
|
110
|
+
|
|
111
|
+
All credit for the underlying pipeline architecture goes to the **Oopsie Data** (`oopsie-data` / `oopsie_data_tools`) project. DataHive was built on top of their pipeline patterns, including:
|
|
112
|
+
|
|
113
|
+
- **Interactive Annotation & GUI**: The web-based annotator interface, video playback controls, task picker, episode navigation, and failure classification.
|
|
114
|
+
- **Data Lifecycle**: The full `init` -> `new-profile` -> `annotate` -> `validate` -> `upload` -> `sync` pipeline.
|
|
115
|
+
- **Dataset Structure & Profiling**: Hardware platform specification snapshots, HDF5 episode storage, and Hub reconciliation.
|
|
116
|
+
|
|
117
|
+
We are deeply grateful to the Oopsie Data contributors for their pioneering work in robotic data pipelines.
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# DataHive
|
|
2
|
+
|
|
3
|
+
`datahive-tools`: the client-side package a lab installs to collect,
|
|
4
|
+
validate, annotate, and upload HiveBoard manipulation episodes to its
|
|
5
|
+
private Hugging Face dataset repo.
|
|
6
|
+
|
|
7
|
+
> [!IMPORTANT]
|
|
8
|
+
> **Credits & Attribution**: All pipeline architecture, workflows, and core design in DataHive are directly adapted from and credited to [**Oopsie Data**](https://github.com/oopsie-data) (`oopsie-data` / `oopsie_data_tools`). Everything in DataHive — from the CLI workflow, robot profiles, and dataset indexing to the annotation GUI and Hugging Face Hub synchronization pipeline — was built upon the pipeline and concepts developed by the Oopsie Data team.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
pip install -e .
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Quickstart
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
datahive init # one-time: writes ~/.datahive/config.yaml (0600)
|
|
20
|
+
datahive new-profile # writes samples/robot_profile.yaml -- fill in ONCE
|
|
21
|
+
# ... record episodes into samples/{session}/episodes/{episode}.h5 + .mp4 ...
|
|
22
|
+
datahive check <episode_id> # pre-annotation health check (frames, Hz, MP4s)
|
|
23
|
+
datahive annotate <episode_id>
|
|
24
|
+
datahive validate <episode_id>
|
|
25
|
+
datahive upload <episode_id>
|
|
26
|
+
datahive sync # reconcile all of samples/ with the Hub
|
|
27
|
+
datahive list
|
|
28
|
+
datahive delete <episode_id>
|
|
29
|
+
datahive interface --port 8000 # local web GUI, localhost only (alias: serve)
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
See `docs/MANUAL_SMOKE_TEST.md` for the manual (non-automated) end-to-end
|
|
33
|
+
check against a real, disposable Hugging Face repo.
|
|
34
|
+
|
|
35
|
+
## The sections of `datahive serve`
|
|
36
|
+
|
|
37
|
+
Opening the interface shows a start screen with two sections, plus an About and FAQ area. Both sections share the same `samples/` tree.
|
|
38
|
+
|
|
39
|
+
- **Runner** – runs HiveBoard trials. Create a session (operator and date; the lab ID comes from `datahive init` and the platform ID from the robot profile), generate the trial plan (5 trials for each chosen task, shuffled or ordered), then run each trial in one of two modes, switchable at any time:
|
|
40
|
+
- **Manual**: 5 s countdown, stopwatch, sounds and automatic timeout in the browser (Space starts and stops), then record the outcome and **upload the HDF5 file and one video per camera in the robot profile**. The trial counts as valid only when all of them are uploaded and the episode passes validation.
|
|
41
|
+
- **Automatic**: send the trial to your robot script (below). It records the episode, and you annotate it in the Annotate section, which opens on that episode.
|
|
42
|
+
- **Annotate** – review recorded episodes, annotate them, validate the data, and upload it to the Hub.
|
|
43
|
+
|
|
44
|
+
Robot side of automatic mode:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from datahive.collect import CollectClient
|
|
48
|
+
|
|
49
|
+
robot = CollectClient("samples", base_url="http://127.0.0.1:8000")
|
|
50
|
+
while True:
|
|
51
|
+
task = robot.wait_for_task() # blocks until you press "Send to robot"
|
|
52
|
+
with robot.new_writer(task) as writer: # EpisodeWriter for that trial
|
|
53
|
+
... # append_proprioception / append_command / attach_video
|
|
54
|
+
robot.finish(writer) # hands the episode over, waits for the annotation
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Trials are saved as rows of `samples/<session>/trials.csv`; all times are stored in UTC.
|
|
58
|
+
|
|
59
|
+
## AI assistant skills
|
|
60
|
+
|
|
61
|
+
Two skills for AI coding assistants (e.g. Claude Code) live in `src/datahive/skills/`:
|
|
62
|
+
|
|
63
|
+
- `datahive-data-prep` – how to lay out, format, annotate and check data under `samples/` so it validates.
|
|
64
|
+
- `datahive-auto-collect` – how automatic collection works and how to adapt the robot script to your setup.
|
|
65
|
+
|
|
66
|
+
Install them for Claude Code with:
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
datahive install-skill # into ./.claude/skills (this project)
|
|
70
|
+
datahive install-skill --user # into ~/.claude/skills (all projects)
|
|
71
|
+
datahive install-skill --dir DIR --force
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Tests
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
pip install -e ".[dev]"
|
|
78
|
+
pytest
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
All tests run with no real network access and no real Hugging Face
|
|
82
|
+
credentials.
|
|
83
|
+
|
|
84
|
+
## Credits & Acknowledgments
|
|
85
|
+
|
|
86
|
+
All credit for the underlying pipeline architecture goes to the **Oopsie Data** (`oopsie-data` / `oopsie_data_tools`) project. DataHive was built on top of their pipeline patterns, including:
|
|
87
|
+
|
|
88
|
+
- **Interactive Annotation & GUI**: The web-based annotator interface, video playback controls, task picker, episode navigation, and failure classification.
|
|
89
|
+
- **Data Lifecycle**: The full `init` -> `new-profile` -> `annotate` -> `validate` -> `upload` -> `sync` pipeline.
|
|
90
|
+
- **Dataset Structure & Profiling**: Hardware platform specification snapshots, HDF5 episode storage, and Hub reconciliation.
|
|
91
|
+
|
|
92
|
+
We are deeply grateful to the Oopsie Data contributors for their pioneering work in robotic data pipelines.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "datahive-tools"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Client tooling to collect, validate, annotate and upload HiveBoard manipulation episodes."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
dependencies = [
|
|
13
|
+
"huggingface_hub>=0.24",
|
|
14
|
+
"h5py>=3.10",
|
|
15
|
+
"pydantic>=2.6",
|
|
16
|
+
"pyyaml>=6.0",
|
|
17
|
+
"numpy>=1.24",
|
|
18
|
+
"fastapi>=0.110",
|
|
19
|
+
"uvicorn>=0.29",
|
|
20
|
+
"typer>=0.12",
|
|
21
|
+
"python-multipart>=0.0.9",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
video = ["opencv-python-headless>=4.8"]
|
|
26
|
+
dev = ["pytest>=8.0", "httpx>=0.27", "opencv-python-headless>=4.8"]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
datahive = "datahive.cli:main"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
|
|
34
|
+
[tool.setuptools.package-data]
|
|
35
|
+
"datahive" = ["attachments.yaml", "interface/static/*", "interface/static/tasks/*", "skills/*/SKILL.md", "skills/*/reference/*.md"]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""datahive-tools: client package for collecting, validating, annotating and
|
|
2
|
+
uploading HiveBoard manipulation episodes.
|
|
3
|
+
|
|
4
|
+
Public library API (also used internally by the CLI and the local GUI, so
|
|
5
|
+
all three surfaces stay in sync):
|
|
6
|
+
|
|
7
|
+
from datahive import EpisodeWriter, RobotProfile, TrialAnnotation
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from datahive.episode import EpisodeWriter, read_header, read_trajectory, resolve_episode
|
|
11
|
+
from datahive.errors import DatahiveError
|
|
12
|
+
from datahive.profile import RobotProfile, load_profile, write_profile_skeleton
|
|
13
|
+
from datahive.schema import EpisodeHeader, FailureCause, Outcome, Strategy, TrialAnnotation
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"EpisodeWriter",
|
|
17
|
+
"read_header",
|
|
18
|
+
"read_trajectory",
|
|
19
|
+
"resolve_episode",
|
|
20
|
+
"DatahiveError",
|
|
21
|
+
"RobotProfile",
|
|
22
|
+
"load_profile",
|
|
23
|
+
"write_profile_skeleton",
|
|
24
|
+
"EpisodeHeader",
|
|
25
|
+
"FailureCause",
|
|
26
|
+
"Outcome",
|
|
27
|
+
"Strategy",
|
|
28
|
+
"TrialAnnotation",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""`datahive annotate`: fills outcome/failure_cause/etc. for an episode's
|
|
2
|
+
trial, shared by the CLI's interactive prompts and the GUI's validation
|
|
3
|
+
form (both end up calling `annotate_episode`)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from datahive.attachments import is_composed_assembly
|
|
11
|
+
from datahive.episode import read_header
|
|
12
|
+
from datahive.errors import AnnotationError
|
|
13
|
+
from datahive.paths import resolve_episode_paths
|
|
14
|
+
from datahive.schema import ANNOTATION_SCHEMA_CURRENT, TrialAnnotation
|
|
15
|
+
from datahive.trials import upsert_row
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def annotate_episode(
|
|
19
|
+
samples_root: Path, episode_id: str, fields: dict[str, Any], *, validate_after: bool = True
|
|
20
|
+
) -> TrialAnnotation:
|
|
21
|
+
"""`fields` is a dict of TrialAnnotation column values (trial_id/lab_id/
|
|
22
|
+
platform_id/attachment_id/date are filled in automatically from the
|
|
23
|
+
episode header + config when omitted). Writes the row to trials.csv and,
|
|
24
|
+
by default, runs full validation afterward so the index status reflects
|
|
25
|
+
whether the episode is now `validated`."""
|
|
26
|
+
paths = resolve_episode_paths(samples_root, episode_id)
|
|
27
|
+
header = read_header(paths.h5)
|
|
28
|
+
|
|
29
|
+
data = dict(fields)
|
|
30
|
+
data.setdefault("trial_id", header.trial_id)
|
|
31
|
+
data.setdefault("lab_id", header.lab_id)
|
|
32
|
+
data.setdefault("platform_id", header.platform_id)
|
|
33
|
+
if "date" not in data or data["date"] is None:
|
|
34
|
+
from datetime import datetime, timezone
|
|
35
|
+
|
|
36
|
+
data["date"] = datetime.now(timezone.utc).date().isoformat()
|
|
37
|
+
for name_field in ("operator_name", "annotator_name", "failure_cause_detail"):
|
|
38
|
+
if data.get(name_field) is None:
|
|
39
|
+
data[name_field] = ""
|
|
40
|
+
|
|
41
|
+
if data.get("outcome") == "success" and data.get("completion_time_s") is None:
|
|
42
|
+
from datahive.episode import episode_stats
|
|
43
|
+
|
|
44
|
+
duration = episode_stats(paths.h5).get("duration_s")
|
|
45
|
+
if duration is not None:
|
|
46
|
+
data["completion_time_s"] = round(duration, 3)
|
|
47
|
+
data.setdefault("completion_source", "hdf5")
|
|
48
|
+
|
|
49
|
+
from datetime import datetime, timezone
|
|
50
|
+
|
|
51
|
+
data["annotated_at"] = datetime.now(timezone.utc).isoformat()
|
|
52
|
+
data["schema_version"] = ANNOTATION_SCHEMA_CURRENT
|
|
53
|
+
|
|
54
|
+
attachment_id = data.get("attachment_id")
|
|
55
|
+
composed = is_composed_assembly(attachment_id or "", samples_root)
|
|
56
|
+
context = {} if composed is None else {"composed_assembly": composed}
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
annotation = TrialAnnotation.model_validate(data, context=context)
|
|
60
|
+
except Exception as e:
|
|
61
|
+
raise AnnotationError(f"Invalid annotation: {e}") from e
|
|
62
|
+
|
|
63
|
+
upsert_row(paths.trials_csv, annotation)
|
|
64
|
+
write_h5_annotation(paths.h5, annotation)
|
|
65
|
+
|
|
66
|
+
if validate_after:
|
|
67
|
+
from datahive.validate import validate_episode
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
validate_episode(samples_root, episode_id)
|
|
71
|
+
except Exception:
|
|
72
|
+
pass
|
|
73
|
+
|
|
74
|
+
return annotation
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _annotator_key(name: str) -> str:
|
|
78
|
+
return (name or "unknown").strip().replace("/", "_") or "unknown"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def write_h5_annotation(h5_path: Path, annotation: TrialAnnotation) -> None:
|
|
82
|
+
"""Stores the annotation inside the episode as
|
|
83
|
+
episode_annotations/<annotator>/ (attrs), stamped with schema_version, so
|
|
84
|
+
the file stays self-describing if trials.csv is lost or the episode moved."""
|
|
85
|
+
import json
|
|
86
|
+
|
|
87
|
+
import h5py
|
|
88
|
+
|
|
89
|
+
row = annotation.to_csv_row()
|
|
90
|
+
key = _annotator_key(annotation.annotator_name)
|
|
91
|
+
with h5py.File(h5_path, "r+") as f:
|
|
92
|
+
root = f.require_group("episode_annotations")
|
|
93
|
+
if key in root:
|
|
94
|
+
del root[key]
|
|
95
|
+
grp = root.create_group(key)
|
|
96
|
+
grp.attrs["schema_version"] = annotation.schema_version
|
|
97
|
+
grp.attrs["source"] = "human"
|
|
98
|
+
grp.attrs["timestamp"] = row["annotated_at"]
|
|
99
|
+
grp.attrs["annotation"] = json.dumps(row, sort_keys=True)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def read_h5_annotations(h5_path: Path) -> dict[str, dict]:
|
|
103
|
+
"""{annotator: row-dict} from the episode file. Groups without a
|
|
104
|
+
schema_version are upcast to the legacy version; nothing is rewritten."""
|
|
105
|
+
import json
|
|
106
|
+
|
|
107
|
+
import h5py
|
|
108
|
+
|
|
109
|
+
from datahive.schema import ANNOTATION_SCHEMA_LEGACY
|
|
110
|
+
|
|
111
|
+
out: dict[str, dict] = {}
|
|
112
|
+
with h5py.File(h5_path, "r") as f:
|
|
113
|
+
root = f.get("episode_annotations")
|
|
114
|
+
if root is None:
|
|
115
|
+
return out
|
|
116
|
+
for name, grp in root.items():
|
|
117
|
+
try:
|
|
118
|
+
row = json.loads(grp.attrs.get("annotation", "{}"))
|
|
119
|
+
except (TypeError, ValueError):
|
|
120
|
+
row = {}
|
|
121
|
+
row["schema_version"] = grp.attrs.get("schema_version") or row.get("schema_version") or ANNOTATION_SCHEMA_LEGACY
|
|
122
|
+
out[name] = row
|
|
123
|
+
return out
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Attachment registry: which attachment_ids are "composed-assembly"
|
|
2
|
+
(multi-stage) attachments, for which stage_reached is meaningful.
|
|
3
|
+
|
|
4
|
+
Bundled defaults live in datahive/attachments.yaml; a lab can add or
|
|
5
|
+
override entries with samples/attachments.yaml.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from functools import lru_cache
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from datahive.paths import attachments_override_path
|
|
17
|
+
|
|
18
|
+
_BUNDLED_PATH = Path(__file__).parent / "attachments.yaml"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class AttachmentInfo:
|
|
23
|
+
attachment_id: str
|
|
24
|
+
name: str
|
|
25
|
+
composed_assembly: bool
|
|
26
|
+
n_stages: int | None = None
|
|
27
|
+
family: str | None = None
|
|
28
|
+
timeout: int | None = None
|
|
29
|
+
success: str | None = None
|
|
30
|
+
reset: str | None = None
|
|
31
|
+
stages: list[str] | None = None
|
|
32
|
+
image: str | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@lru_cache(maxsize=1)
|
|
36
|
+
def _bundled() -> dict:
|
|
37
|
+
return yaml.safe_load(_BUNDLED_PATH.read_text(encoding="utf-8")) or {}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_registry(samples_root: Path | None = None) -> dict[str, AttachmentInfo]:
|
|
41
|
+
raw: dict = dict(_bundled())
|
|
42
|
+
if samples_root is not None:
|
|
43
|
+
override_path = attachments_override_path(samples_root)
|
|
44
|
+
if override_path.is_file():
|
|
45
|
+
override = yaml.safe_load(override_path.read_text(encoding="utf-8")) or {}
|
|
46
|
+
raw.update(override)
|
|
47
|
+
return {
|
|
48
|
+
aid: AttachmentInfo(
|
|
49
|
+
attachment_id=aid,
|
|
50
|
+
name=info.get("name", aid),
|
|
51
|
+
composed_assembly=bool(info.get("composed_assembly", False)),
|
|
52
|
+
n_stages=info.get("n_stages"),
|
|
53
|
+
family=info.get("family"),
|
|
54
|
+
timeout=info.get("timeout"),
|
|
55
|
+
success=info.get("success"),
|
|
56
|
+
reset=info.get("reset"),
|
|
57
|
+
stages=info.get("stages"),
|
|
58
|
+
image=info.get("image"),
|
|
59
|
+
)
|
|
60
|
+
for aid, info in raw.items()
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def is_composed_assembly(attachment_id: str, samples_root: Path | None = None) -> bool | None:
|
|
65
|
+
"""Returns True/False if the attachment is known, or None if unknown
|
|
66
|
+
(caller should treat this as 'no constraint, but warn')."""
|
|
67
|
+
registry = load_registry(samples_root)
|
|
68
|
+
info = registry.get(attachment_id)
|
|
69
|
+
if info is None:
|
|
70
|
+
return None
|
|
71
|
+
return info.composed_assembly
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Bundled HiveBoard attachment (task) registry.
|
|
2
|
+
#
|
|
3
|
+
# This mirrors the 13 official HiveBoard evaluation conditions from
|
|
4
|
+
# HiveBoard's Evaluation Runner
|
|
5
|
+
# (https://hiveboard-bench.github.io/hivedocs/benchmark/evaluation-runner),
|
|
6
|
+
# so `attachment_id` here means the same thing as `attachment_id` in a
|
|
7
|
+
# HiveBoard submission. `composed_assembly: true` marks attachments made
|
|
8
|
+
# of multiple stages that must be completed in sequence (stage_reached is
|
|
9
|
+
# meaningful for these and required in the trial annotation).
|
|
10
|
+
#
|
|
11
|
+
# Labs can override/extend this list by placing an attachments.yaml with
|
|
12
|
+
# the same shape at the root of their samples/ directory.
|
|
13
|
+
valve_ball:
|
|
14
|
+
name: Ball valve
|
|
15
|
+
family: Torque
|
|
16
|
+
timeout: 60
|
|
17
|
+
composed_assembly: false
|
|
18
|
+
success: Rotate the handle 90° from the closed state to the open state.
|
|
19
|
+
reset: Return the handle to the closed position and confirm that the attachment is fully seated.
|
|
20
|
+
image: torque_valve_noFriction_3d.png
|
|
21
|
+
valve_ball_ring:
|
|
22
|
+
name: Ball valve with friction ring
|
|
23
|
+
family: Torque
|
|
24
|
+
timeout: 90
|
|
25
|
+
composed_assembly: false
|
|
26
|
+
success: Rotate the handle 90° from closed to open with the friction ring fitted.
|
|
27
|
+
reset: Fit the friction ring, return the handle to closed, and confirm that the attachment is seated.
|
|
28
|
+
image: torque_valve_noFriction_3d.png
|
|
29
|
+
valve_gate_small:
|
|
30
|
+
name: Small gate valve
|
|
31
|
+
family: Torque
|
|
32
|
+
timeout: 90
|
|
33
|
+
composed_assembly: false
|
|
34
|
+
success: Complete one full turn of the valve stem.
|
|
35
|
+
reset: Return the stem to the marked initial orientation without changing the board position.
|
|
36
|
+
image: small_valve_3d.png
|
|
37
|
+
valve_gate_large:
|
|
38
|
+
name: Large gate valve
|
|
39
|
+
family: Torque
|
|
40
|
+
timeout: 120
|
|
41
|
+
composed_assembly: false
|
|
42
|
+
success: Complete one full turn of the valve stem.
|
|
43
|
+
reset: Return the stem to the marked initial orientation without changing the board position.
|
|
44
|
+
image: big_valve_3d.png
|
|
45
|
+
circuit_breaker:
|
|
46
|
+
name: Circuit breaker
|
|
47
|
+
family: Torque
|
|
48
|
+
timeout: 60
|
|
49
|
+
composed_assembly: false
|
|
50
|
+
success: Move the toggle to the opposite state and hold it there.
|
|
51
|
+
reset: Return the toggle to its initial state and confirm that it moves freely.
|
|
52
|
+
image: switch_3d.png
|
|
53
|
+
light_bulb:
|
|
54
|
+
name: Light bulb and socket
|
|
55
|
+
family: Precision
|
|
56
|
+
timeout: 120
|
|
57
|
+
composed_assembly: false
|
|
58
|
+
success: Thread the bulb into the socket until it is seated.
|
|
59
|
+
reset: Remove the bulb, restore the documented starting pose, and inspect the thread.
|
|
60
|
+
image: lamp_3d.png
|
|
61
|
+
thread_m8:
|
|
62
|
+
name: M8 threaded fastener
|
|
63
|
+
family: Precision
|
|
64
|
+
timeout: 120
|
|
65
|
+
composed_assembly: false
|
|
66
|
+
success: Thread the bolt along the available length.
|
|
67
|
+
reset: Return the bolt to the documented initial engagement and check that the thread is clear.
|
|
68
|
+
image: m8_3d.png
|
|
69
|
+
thread_m30:
|
|
70
|
+
name: M30 threaded fastener
|
|
71
|
+
family: Precision
|
|
72
|
+
timeout: 120
|
|
73
|
+
composed_assembly: false
|
|
74
|
+
success: Thread the bolt along the available length.
|
|
75
|
+
reset: Return the bolt to the documented initial engagement and check that the thread is clear.
|
|
76
|
+
image: m30_3d.png
|
|
77
|
+
peg_insertion:
|
|
78
|
+
name: Threaded peg insertion
|
|
79
|
+
family: Precision
|
|
80
|
+
timeout: 120
|
|
81
|
+
composed_assembly: false
|
|
82
|
+
success: Thread the free 8 mm peg into the empty socket until it is seated.
|
|
83
|
+
reset: Remove the peg and return it to the initial pose next to the empty socket.
|
|
84
|
+
image: peg_and_hole_3d.png
|
|
85
|
+
button:
|
|
86
|
+
name: Covered button
|
|
87
|
+
family: Composed assembly
|
|
88
|
+
timeout: 60
|
|
89
|
+
composed_assembly: true
|
|
90
|
+
n_stages: 2
|
|
91
|
+
stages: [Open cover, Press button]
|
|
92
|
+
success: Open the cover and press the button.
|
|
93
|
+
reset: Close the cover and confirm that the button has returned.
|
|
94
|
+
image: button_3d.png
|
|
95
|
+
lock:
|
|
96
|
+
name: Lock and key
|
|
97
|
+
family: Composed assembly
|
|
98
|
+
timeout: 180
|
|
99
|
+
composed_assembly: true
|
|
100
|
+
n_stages: 3
|
|
101
|
+
stages: [Grasp key, Insert key vertically, Rotate to unlock]
|
|
102
|
+
success: Grasp the key, insert it vertically, and rotate it to unlock.
|
|
103
|
+
reset: Remove the key, return the lock to its initial state, and restore the key pose.
|
|
104
|
+
image: key_3d.png
|
|
105
|
+
drawer:
|
|
106
|
+
name: Sliding drawer
|
|
107
|
+
family: Composed assembly
|
|
108
|
+
timeout: 120
|
|
109
|
+
composed_assembly: true
|
|
110
|
+
n_stages: 3
|
|
111
|
+
stages: [Grasp handle, Pull open, Push closed]
|
|
112
|
+
success: Grasp the handle, pull the drawer open, and push it closed.
|
|
113
|
+
reset: Return the drawer to the fully closed initial position.
|
|
114
|
+
image: box_3d.png
|
|
115
|
+
shock_absorber:
|
|
116
|
+
name: Shock absorber
|
|
117
|
+
family: Composed assembly
|
|
118
|
+
timeout: 180
|
|
119
|
+
composed_assembly: true
|
|
120
|
+
n_stages: 3
|
|
121
|
+
stages: [Grasp pin, Align with hole, Insert fully]
|
|
122
|
+
success: Grasp the pin, align it with the hole, and insert it fully.
|
|
123
|
+
reset: Remove the pin, restore its starting pose, and check both occupied board cells.
|
|
124
|
+
image: spring_3d.png
|