oopsie-data-tools 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.gitignore +2 -1
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/PKG-INFO +6 -3
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/README.md +5 -2
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/cli.py +204 -36
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/SKILL.md +53 -2
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/reference/robot-profile.md +4 -1
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/reference/setup.md +4 -2
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/reference/troubleshooting.md +7 -1
- oopsie_data_tools-0.2.2/oopsie_data_tools/test/test_cli_json.py +200 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_install_skill.py +91 -17
- oopsie_data_tools-0.2.2/oopsie_data_tools/test/test_skill_matches_cli.py +97 -0
- oopsie_data_tools-0.2.2/oopsie_data_tools/utils/claude_skill.py +198 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/h5_inspect.py +74 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/validation_utils.py +35 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/pyproject.toml +1 -1
- oopsie_data_tools-0.2.1/oopsie_data_tools/utils/claude_skill.py +0 -128
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.github/ISSUE_TEMPLATE/help-request--data-conversion.md +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.github/ISSUE_TEMPLATE/help-request--robot-profile.md +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.github/ISSUE_TEMPLATE/improvement-suggestion.md +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.github/workflows/ci.yml +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.github/workflows/publish.yml +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/.python-version +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/LICENSE +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/configs/robot_profiles/act_plus_plus_robot_profile.yaml +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/configs/robot_profiles/openpi_example_robot_profile.yaml +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/configs/robot_profiles/template.yaml +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/conversion_script_examples/_common.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/conversion_script_examples/convert_ar_aloha.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/conversion_script_examples/convert_rlds.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/conversion_script_examples/convert_soar.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/inference_examples/act_plus_plus/run_policy.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/inference_examples/act_plus_plus/run_policy_browser.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/inference_examples/act_plus_plus/run_policy_bulk.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/inference_examples/act_plus_plus/run_policy_inloop.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/inference_examples/openpi/run_policy_bulk.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/examples/inference_examples/openpi/run_policy_inloop.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/annotation_tool/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/annotation_tool/annotation_schema.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/annotation_tool/annotator_server.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/annotation_tool/episode_recorder.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/annotation_tool/rollout_annotator.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/annotation_tool/ui/annotator.html +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/init_wizard.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/reference/conversion.md +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/reference/format.md +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/conftest.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/fixtures/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/fixtures/make_invalid.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/fixtures/make_valid.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_annotation_completeness.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_annotation_schema.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_annotator_server.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_annotator_server_guards.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_bulk_inference_end_to_end.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_cli_config.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_cli_tools.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_contributor_config.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_conversion_utils_annotations.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_conversion_utils_writers.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_credentials_location.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_diversity.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_episode_recorder.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_episode_video_paths.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_hf_upload.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_init_wizard.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_migrate_taxonomy_v2.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_new_profile.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_paths.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_python38_compat.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_record_step_purity.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_robot_setup.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_rollout_annotator.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_rotation_utils.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/test/test_validate.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/contributor_config.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/conversion_utils.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/h5.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/hf_limits.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/hf_upload.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/log.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/migrate_taxonomy_v2.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/paths.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/restructure.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/robot_profile/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/robot_profile/robot_profile.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/robot_profile/rotation_utils.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/robot_profile/template.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/__init__.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/annotation_completeness.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/diversity.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/episode_data.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/episode_loader.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/episode_validator.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/utils/validation/errors.py +0 -0
- {oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oopsie-data-tools
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Collect, annotate, validate and upload robotic manipulation rollout data.
|
|
5
5
|
License-File: LICENSE
|
|
6
6
|
Requires-Python: >=3.8
|
|
@@ -24,18 +24,21 @@ Description-Content-Type: text/markdown
|
|
|
24
24
|
|
|
25
25
|
Tools for collecting, annotating, inspecting, and converting robotic manipulation rollout data.
|
|
26
26
|
|
|
27
|
-
This repository currently provides
|
|
27
|
+
This repository currently provides tools for:
|
|
28
28
|
|
|
29
29
|
- HDF5 episode recording (`EpisodeRecorder`)
|
|
30
30
|
- Web annotation workflows
|
|
31
31
|
- In-the-loop annotation during policy rollout
|
|
32
32
|
|
|
33
|
+
as well as all the necessary utilities to validate, inspect, and upload Oopsie-Data to the official repositories.
|
|
34
|
+
|
|
35
|
+
[Sign up today](https://forms.gle/9arwZHAvRjvbozoT7) and start contributing!
|
|
36
|
+
|
|
33
37
|
---
|
|
34
38
|
|
|
35
39
|
For detailed explanations on how to use our tooling and contribute to the project, please visit [our website](https://oopsie-data.com/).
|
|
36
40
|
|
|
37
41
|
For an overview of the steps necessary to integrate the tooling into your workflow and to contribute data to the official Oopsie Data repositories, check out [our quickstart guide](https://oopsie-data.com/quickstart).
|
|
38
|
-
You can also use the information in AI_CONTEXT.md to guide a coding agent through the setup.
|
|
39
42
|
|
|
40
43
|
## Repository structure
|
|
41
44
|
|
|
@@ -2,18 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
Tools for collecting, annotating, inspecting, and converting robotic manipulation rollout data.
|
|
4
4
|
|
|
5
|
-
This repository currently provides
|
|
5
|
+
This repository currently provides tools for:
|
|
6
6
|
|
|
7
7
|
- HDF5 episode recording (`EpisodeRecorder`)
|
|
8
8
|
- Web annotation workflows
|
|
9
9
|
- In-the-loop annotation during policy rollout
|
|
10
10
|
|
|
11
|
+
as well as all the necessary utilities to validate, inspect, and upload Oopsie-Data to the official repositories.
|
|
12
|
+
|
|
13
|
+
[Sign up today](https://forms.gle/9arwZHAvRjvbozoT7) and start contributing!
|
|
14
|
+
|
|
11
15
|
---
|
|
12
16
|
|
|
13
17
|
For detailed explanations on how to use our tooling and contribute to the project, please visit [our website](https://oopsie-data.com/).
|
|
14
18
|
|
|
15
19
|
For an overview of the steps necessary to integrate the tooling into your workflow and to contribute data to the official Oopsie Data repositories, check out [our quickstart guide](https://oopsie-data.com/quickstart).
|
|
16
|
-
You can also use the information in AI_CONTEXT.md to guide a coding agent through the setup.
|
|
17
20
|
|
|
18
21
|
## Repository structure
|
|
19
22
|
|
|
@@ -15,6 +15,7 @@ notes and reStructuredText markup cannot leak into a terminal.
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
|
+
import contextlib
|
|
18
19
|
import difflib
|
|
19
20
|
import logging
|
|
20
21
|
import os
|
|
@@ -23,6 +24,10 @@ import sys
|
|
|
23
24
|
import textwrap
|
|
24
25
|
from pathlib import Path
|
|
25
26
|
|
|
27
|
+
# Imported eagerly, unlike the heavier command modules below: the parser needs the agent
|
|
28
|
+
# names to build --agent's choices, and this module costs nothing but the standard library.
|
|
29
|
+
from oopsie_data_tools.utils.claude_skill import AGENT_SKILL_DIRS as _AGENT_CHOICES
|
|
30
|
+
from oopsie_data_tools.utils.claude_skill import DEFAULT_AGENT as _DEFAULT_AGENT
|
|
26
31
|
from oopsie_data_tools.utils.hf_limits import BATCH_SIZE, FILE_LIMIT
|
|
27
32
|
from oopsie_data_tools.utils.paths import ENV_CONFIG_DIR
|
|
28
33
|
|
|
@@ -43,6 +48,36 @@ def _version() -> str:
|
|
|
43
48
|
return "unknown (not installed)"
|
|
44
49
|
|
|
45
50
|
|
|
51
|
+
# ── machine-readable output ───────────────────────────────────────────────────
|
|
52
|
+
|
|
53
|
+
#: Commands whose result is a structure rather than a narrative accept ``--json``, for
|
|
54
|
+
#: callers — scripts, CI, coding agents — that would otherwise have to parse prose. The
|
|
55
|
+
#: prose output is unchanged and stays the default; exit codes mean the same thing either
|
|
56
|
+
#: way. JSON goes to stdout on its own, so logs must not be interleaved with it.
|
|
57
|
+
def _emit_json(payload: dict) -> None:
|
|
58
|
+
import json
|
|
59
|
+
|
|
60
|
+
print(json.dumps(payload, indent=2, default=str))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@contextlib.contextmanager
|
|
64
|
+
def _quiet_logging(enabled: bool):
|
|
65
|
+
"""Silence log output so --json emits a parseable document and nothing else.
|
|
66
|
+
|
|
67
|
+
Restores the previous level on the way out: ``main`` is called more than once in a
|
|
68
|
+
single process by the test suite and by anything embedding the CLI, and a root logger
|
|
69
|
+
left at CRITICAL would silence every later command.
|
|
70
|
+
"""
|
|
71
|
+
root = logging.getLogger()
|
|
72
|
+
previous = root.level
|
|
73
|
+
if enabled:
|
|
74
|
+
root.setLevel(logging.CRITICAL)
|
|
75
|
+
try:
|
|
76
|
+
yield
|
|
77
|
+
finally:
|
|
78
|
+
root.setLevel(previous)
|
|
79
|
+
|
|
80
|
+
|
|
46
81
|
# ── show-config ───────────────────────────────────────────────────────────────
|
|
47
82
|
|
|
48
83
|
|
|
@@ -87,13 +122,77 @@ def _print_wrapped_field(label: str, value: str) -> None:
|
|
|
87
122
|
print(line)
|
|
88
123
|
|
|
89
124
|
|
|
90
|
-
def
|
|
91
|
-
"""
|
|
125
|
+
def _read_config_yaml(config_path: Path) -> tuple[dict, str | None]:
|
|
126
|
+
"""The contributor config as a plain dict, plus a parse error if there was one.
|
|
127
|
+
|
|
128
|
+
Read directly rather than through ``read_contributor_config``, which raises on a
|
|
129
|
+
missing or placeholder lab_id — this command must still report what it found.
|
|
130
|
+
"""
|
|
92
131
|
import yaml
|
|
93
132
|
|
|
133
|
+
if not config_path.is_file():
|
|
134
|
+
return {}, None
|
|
135
|
+
try:
|
|
136
|
+
loaded = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
|
137
|
+
return (loaded if isinstance(loaded, dict) else {}), None
|
|
138
|
+
except yaml.YAMLError as e:
|
|
139
|
+
return {}, str(e)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _show_config_json(args: argparse.Namespace) -> int:
|
|
143
|
+
from oopsie_data_tools.init_wizard import _mask
|
|
144
|
+
from oopsie_data_tools.utils import paths
|
|
145
|
+
|
|
146
|
+
config_path = paths.contributor_config_path()
|
|
147
|
+
profiles_dir = paths.robot_profiles_dir()
|
|
148
|
+
data, parse_error = _read_config_yaml(config_path)
|
|
149
|
+
|
|
150
|
+
lab_id = str(data.get("lab_id") or "").strip()
|
|
151
|
+
config_token = str(data.get("huggingface_token") or "").strip()
|
|
152
|
+
env_token = os.environ.get("HF_TOKEN", "").strip()
|
|
153
|
+
token = env_token or config_token
|
|
154
|
+
|
|
155
|
+
_emit_json(
|
|
156
|
+
{
|
|
157
|
+
"credentials": {
|
|
158
|
+
"env_var": paths.ENV_CONFIG_DIR,
|
|
159
|
+
"env_value": os.environ.get(paths.ENV_CONFIG_DIR),
|
|
160
|
+
"search_path": [str(p) for p in paths.config_search_dirs()],
|
|
161
|
+
"active": str(config_path) if config_path.is_file() else None,
|
|
162
|
+
"parse_error": parse_error,
|
|
163
|
+
"lab_id": lab_id or None,
|
|
164
|
+
# Masked unless asked, so --json is safe to paste into a bug report.
|
|
165
|
+
"hf_token": (token if args.show_token else _mask(token)) if token else None,
|
|
166
|
+
"hf_token_source": (
|
|
167
|
+
None if not token else ("HF_TOKEN" if env_token else "contributor_config.yaml")
|
|
168
|
+
),
|
|
169
|
+
"uploads_to": f"OopsieData-Submissions/{lab_id}" if lab_id else None,
|
|
170
|
+
},
|
|
171
|
+
"robot_profiles": {
|
|
172
|
+
"env_var": paths.ENV_PROFILES_DIR,
|
|
173
|
+
"env_value": os.environ.get(paths.ENV_PROFILES_DIR),
|
|
174
|
+
"search_path": [str(p) for p in paths.profiles_search_dirs()],
|
|
175
|
+
"active": str(profiles_dir) if profiles_dir.is_dir() else None,
|
|
176
|
+
"profiles": (
|
|
177
|
+
sorted(p.name for p in profiles_dir.glob("*.yaml"))
|
|
178
|
+
if profiles_dir.is_dir()
|
|
179
|
+
else []
|
|
180
|
+
),
|
|
181
|
+
"write_target": str(paths.write_profiles_dir()),
|
|
182
|
+
},
|
|
183
|
+
}
|
|
184
|
+
)
|
|
185
|
+
return 0
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def cmd_show_config(args: argparse.Namespace) -> int:
|
|
189
|
+
"""Show where every config is read from right now, and what is in it. Read-only."""
|
|
94
190
|
from oopsie_data_tools.init_wizard import _mask
|
|
95
191
|
from oopsie_data_tools.utils import paths
|
|
96
192
|
|
|
193
|
+
if args.json:
|
|
194
|
+
return _show_config_json(args)
|
|
195
|
+
|
|
97
196
|
config_path = paths.contributor_config_path()
|
|
98
197
|
profiles_dir = paths.robot_profiles_dir()
|
|
99
198
|
|
|
@@ -110,15 +209,9 @@ def cmd_show_config(args: argparse.Namespace) -> int:
|
|
|
110
209
|
else:
|
|
111
210
|
print(f" No config found. 'oopsie-data init' would write {config_path}")
|
|
112
211
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if config_path.is_file():
|
|
117
|
-
try:
|
|
118
|
-
loaded = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
|
119
|
-
data = loaded if isinstance(loaded, dict) else {}
|
|
120
|
-
except yaml.YAMLError as e:
|
|
121
|
-
print(f" Could not parse the file: {e}")
|
|
212
|
+
data, parse_error = _read_config_yaml(config_path)
|
|
213
|
+
if parse_error:
|
|
214
|
+
print(f" Could not parse the file: {parse_error}")
|
|
122
215
|
|
|
123
216
|
lab_id = str(data.get("lab_id") or "").strip()
|
|
124
217
|
config_token = str(data.get("huggingface_token") or "").strip()
|
|
@@ -249,10 +342,34 @@ def cmd_validate(args: argparse.Namespace) -> int:
|
|
|
249
342
|
from oopsie_data_tools.utils.hf_upload import run_validation
|
|
250
343
|
|
|
251
344
|
target = os.path.abspath(os.path.normpath(args.path))
|
|
345
|
+
if args.episode_id:
|
|
346
|
+
target = os.path.join(target, f"{args.episode_id}.h5")
|
|
252
347
|
if not os.path.exists(target):
|
|
348
|
+
if args.json:
|
|
349
|
+
_emit_json({"path": target, "error": "path does not exist", "episodes": []})
|
|
350
|
+
return 1
|
|
253
351
|
logger.error("Path does not exist: %s", target)
|
|
254
352
|
return 1
|
|
255
|
-
|
|
353
|
+
|
|
354
|
+
if not args.json:
|
|
355
|
+
return run_validation(args.path, args.episode_id, args.log_path)
|
|
356
|
+
|
|
357
|
+
from oopsie_data_tools.utils.validation.validation_utils import collect_validation_results
|
|
358
|
+
|
|
359
|
+
episodes = collect_validation_results(target)
|
|
360
|
+
failed = [e for e in episodes if not e["passed"]]
|
|
361
|
+
_emit_json(
|
|
362
|
+
{
|
|
363
|
+
"path": target,
|
|
364
|
+
"episodes": episodes,
|
|
365
|
+
"total": len(episodes),
|
|
366
|
+
"passed": len(episodes) - len(failed),
|
|
367
|
+
"failed": len(failed),
|
|
368
|
+
}
|
|
369
|
+
)
|
|
370
|
+
# An empty directory is a failure in the prose path too — nothing was checked, so
|
|
371
|
+
# nothing can be said to have passed.
|
|
372
|
+
return 1 if failed or not episodes else 0
|
|
256
373
|
|
|
257
374
|
|
|
258
375
|
# ── upload ────────────────────────────────────────────────────────────────────
|
|
@@ -383,12 +500,19 @@ def cmd_submissions(args: argparse.Namespace) -> int:
|
|
|
383
500
|
|
|
384
501
|
|
|
385
502
|
def cmd_inspect(args: argparse.Namespace) -> int:
|
|
386
|
-
from oopsie_data_tools.utils.h5_inspect import inspect_h5
|
|
503
|
+
from oopsie_data_tools.utils.h5_inspect import inspect_h5, inspect_h5_structure
|
|
387
504
|
|
|
388
505
|
if not os.path.isfile(args.path):
|
|
506
|
+
if args.json:
|
|
507
|
+
_emit_json({"path": args.path, "error": "not a file"})
|
|
508
|
+
return 1
|
|
389
509
|
logger.error("Not a file: %s", args.path)
|
|
390
510
|
return 1
|
|
391
|
-
|
|
511
|
+
|
|
512
|
+
if args.json:
|
|
513
|
+
_emit_json(inspect_h5_structure(args.path))
|
|
514
|
+
else:
|
|
515
|
+
inspect_h5(args.path)
|
|
392
516
|
return 0
|
|
393
517
|
|
|
394
518
|
|
|
@@ -405,9 +529,11 @@ def cmd_restructure(args: argparse.Namespace) -> int:
|
|
|
405
529
|
|
|
406
530
|
|
|
407
531
|
def cmd_install_skill(args: argparse.Namespace) -> int:
|
|
408
|
-
from oopsie_data_tools.utils.claude_skill import install_skill
|
|
532
|
+
from oopsie_data_tools.utils.claude_skill import check_installations, install_skill
|
|
409
533
|
|
|
410
|
-
|
|
534
|
+
if args.check:
|
|
535
|
+
return check_installations()
|
|
536
|
+
return install_skill(user=args.user, force=args.force, agent=args.agent)
|
|
411
537
|
|
|
412
538
|
|
|
413
539
|
# ── parser ────────────────────────────────────────────────────────────────────
|
|
@@ -473,6 +599,15 @@ class _ArgumentParser(argparse.ArgumentParser):
|
|
|
473
599
|
super().error(message)
|
|
474
600
|
|
|
475
601
|
|
|
602
|
+
def _add_json_flag(parser: argparse.ArgumentParser, what: str) -> None:
|
|
603
|
+
"""Give a command a ``--json`` mode. Exit codes are unchanged; only the format differs."""
|
|
604
|
+
parser.add_argument(
|
|
605
|
+
"--json",
|
|
606
|
+
action="store_true",
|
|
607
|
+
help=f"Print {what} as JSON on stdout instead of prose, for scripts and agents",
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
|
|
476
611
|
def _add_command(
|
|
477
612
|
sub, name: str, *, summary: str, description: str, examples: str = ""
|
|
478
613
|
) -> argparse.ArgumentParser:
|
|
@@ -576,6 +711,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
576
711
|
p_config.add_argument(
|
|
577
712
|
"--show-token", action="store_true", help="Print the HuggingFace token in full"
|
|
578
713
|
)
|
|
714
|
+
_add_json_flag(p_config, "the resolved config, both search paths, and what is in effect")
|
|
579
715
|
p_config.set_defaults(func=cmd_show_config)
|
|
580
716
|
|
|
581
717
|
# new-profile
|
|
@@ -688,6 +824,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
688
824
|
p_validate.add_argument(
|
|
689
825
|
"--log-path", "-l", default=None, metavar="FILE", help="Also write logs to this file"
|
|
690
826
|
)
|
|
827
|
+
_add_json_flag(p_validate, "one record per episode, with the error that rejected it")
|
|
691
828
|
p_validate.set_defaults(func=cmd_validate)
|
|
692
829
|
|
|
693
830
|
# upload
|
|
@@ -795,6 +932,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
795
932
|
),
|
|
796
933
|
)
|
|
797
934
|
p_inspect.add_argument("path", metavar="FILE", help="Path to a .h5 / .hdf5 file")
|
|
935
|
+
_add_json_flag(p_inspect, "the full group/dataset/attribute tree")
|
|
798
936
|
p_inspect.set_defaults(func=cmd_inspect)
|
|
799
937
|
|
|
800
938
|
# restructure
|
|
@@ -840,29 +978,49 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
840
978
|
"install-skill",
|
|
841
979
|
summary="Install the bundled agent skill for oopsie-data",
|
|
842
980
|
description=(
|
|
843
|
-
"Copy the skill that ships with this package into
|
|
844
|
-
"knows how to drive the contributor workflow. SKILL.md is a shared
|
|
845
|
-
"same payload works for Claude Code, Cursor, Codex and anything
|
|
846
|
-
"
|
|
847
|
-
"
|
|
848
|
-
"
|
|
849
|
-
"
|
|
850
|
-
"
|
|
851
|
-
"Code and Cursor both read in every project."
|
|
981
|
+
"Copy the skill that ships with this package into a directory your coding agent "
|
|
982
|
+
"scans, so it knows how to drive the contributor workflow. SKILL.md is a shared "
|
|
983
|
+
"format, so the same payload works for Claude Code, Cursor, Codex and anything "
|
|
984
|
+
"else that reads it — --agent only chooses the destination. Defaults to "
|
|
985
|
+
"./.claude/skills/oopsie-data/, which Claude Code and Cursor both read; --user "
|
|
986
|
+
"installs into your home directory instead, making it available in every project. "
|
|
987
|
+
"Entirely optional: nothing else in oopsie-data needs an agent, and no files are "
|
|
988
|
+
"written anywhere unless you run this command."
|
|
852
989
|
),
|
|
853
990
|
examples=(
|
|
854
991
|
" oopsie-data install-skill\n"
|
|
855
|
-
" oopsie-data install-skill --user
|
|
992
|
+
" oopsie-data install-skill --user # available in every project\n"
|
|
993
|
+
" oopsie-data install-skill --agent cursor\n"
|
|
994
|
+
" oopsie-data install-skill --agent none # a plain ./skills/ to copy onward\n"
|
|
995
|
+
" oopsie-data install-skill --check # is an installed copy out of date?\n"
|
|
996
|
+
),
|
|
997
|
+
)
|
|
998
|
+
p_skill.add_argument(
|
|
999
|
+
"--agent",
|
|
1000
|
+
choices=list(_AGENT_CHOICES),
|
|
1001
|
+
default=_DEFAULT_AGENT,
|
|
1002
|
+
help=(
|
|
1003
|
+
f"Which agent's skills directory to install into (default: {_DEFAULT_AGENT}). "
|
|
1004
|
+
"'agents' is the vendor-neutral .agents/skills/; 'none' writes a plain ./skills/ "
|
|
1005
|
+
"that no agent scans, for copying onward yourself"
|
|
856
1006
|
),
|
|
857
1007
|
)
|
|
858
1008
|
p_skill.add_argument(
|
|
859
1009
|
"--user",
|
|
860
1010
|
action="store_true",
|
|
861
|
-
help="Install
|
|
1011
|
+
help="Install under your home directory instead of the working directory",
|
|
862
1012
|
)
|
|
863
1013
|
p_skill.add_argument(
|
|
864
1014
|
"--force", action="store_true", help="Overwrite an existing installation of the skill"
|
|
865
1015
|
)
|
|
1016
|
+
p_skill.add_argument(
|
|
1017
|
+
"--check",
|
|
1018
|
+
action="store_true",
|
|
1019
|
+
help=(
|
|
1020
|
+
"Report installed copies and whether they came from this version of the package, "
|
|
1021
|
+
"instead of installing anything. Exits 1 if any copy is stale or missing"
|
|
1022
|
+
),
|
|
1023
|
+
)
|
|
866
1024
|
p_skill.set_defaults(func=cmd_install_skill)
|
|
867
1025
|
|
|
868
1026
|
return parser
|
|
@@ -882,15 +1040,25 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
882
1040
|
# Export before any command runs, so library code resolving config paths sees it.
|
|
883
1041
|
if args.config_dir is not None:
|
|
884
1042
|
os.environ[ENV_CONFIG_DIR] = str(Path(args.config_dir).expanduser().resolve())
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1043
|
+
|
|
1044
|
+
# --json owns stdout: a log line interleaved with the document would make it
|
|
1045
|
+
# unparseable, which is the whole point of the flag.
|
|
1046
|
+
as_json = getattr(args, "json", False)
|
|
1047
|
+
with _quiet_logging(as_json):
|
|
1048
|
+
try:
|
|
1049
|
+
return args.func(args)
|
|
1050
|
+
except KeyboardInterrupt:
|
|
1051
|
+
logger.info("Interrupted.")
|
|
1052
|
+
return 130
|
|
1053
|
+
except RuntimeError as e:
|
|
1054
|
+
# Config/auth problems already carry an actionable message; don't dump a
|
|
1055
|
+
# traceback. Under --json logging is silenced, so the message has to go into
|
|
1056
|
+
# the document or the caller gets an empty stdout and a bare exit code.
|
|
1057
|
+
if as_json:
|
|
1058
|
+
_emit_json({"error": str(e)})
|
|
1059
|
+
else:
|
|
1060
|
+
logger.error("%s", e)
|
|
1061
|
+
return 1
|
|
894
1062
|
|
|
895
1063
|
|
|
896
1064
|
if __name__ == "__main__":
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: oopsie-data
|
|
3
|
-
description: Use when working with Oopsie Data robotic manipulation failure datasets — recording rollouts, annotating episodes, validating
|
|
3
|
+
description: Use when working with Oopsie Data robotic manipulation failure datasets or the oopsie-data / oopsie-data-tools CLI — recording rollouts with EpisodeRecorder, writing or debugging a robot profile, annotating episodes and the failure taxonomy, reading or validating oopsiedata_format_v1 HDF5 (.h5) episode files, converting an existing robot dataset into the format, or uploading a lab's submission to HuggingFace.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Oopsie Data
|
|
7
7
|
|
|
8
|
+
This skill is for using the toolkit: recording, annotating, validating and uploading a
|
|
9
|
+
contributor's data.
|
|
10
|
+
|
|
8
11
|
Oopsie Data is a community dataset of **robotic manipulation failures**. Contributors record
|
|
9
12
|
rollouts, annotate what went wrong, and upload them to a shared HuggingFace repo. The
|
|
10
13
|
`oopsie-data` CLI is the single entry point for all of it — there are no side scripts.
|
|
@@ -36,11 +39,59 @@ Supporting commands: `show-config` (which config files are in effect), `submissi
|
|
|
36
39
|
lab has already uploaded), `inspect <file.h5>` (dump one episode's structure — a debugging aid
|
|
37
40
|
that works even on files `validate` rejects; the path is positional, there is no `--path`),
|
|
38
41
|
`restructure` (split a directory HuggingFace would reject), `install-skill` (copy this skill
|
|
39
|
-
into
|
|
42
|
+
into an agent's skills directory).
|
|
40
43
|
|
|
41
44
|
Inside a git checkout that was set up with `uv sync`, prefix every command with `uv run`. A
|
|
42
45
|
pip- or uv-installed package puts `oopsie-data` on `PATH` directly.
|
|
43
46
|
|
|
47
|
+
## Machine-readable output
|
|
48
|
+
|
|
49
|
+
`validate`, `show-config` and `inspect` each take `--json`, which prints a structure on stdout
|
|
50
|
+
instead of prose and logs nothing else. Prefer it: it is the difference between reading a
|
|
51
|
+
result and parsing one. Exit codes are identical either way.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
oopsie-data validate --path ./samples --json # one record per episode: passed, error
|
|
55
|
+
oopsie-data show-config --json # both search chains and what won
|
|
56
|
+
oopsie-data inspect ./samples/000000.h5 --json # the full group/dataset/attr tree
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`validate --json` gives `episodes[]` with `episode`, `path`, `passed`, and on failure `error`
|
|
60
|
+
and an `error_type` of `validation` (the episode is bad — the user's to fix) or `unexpected`
|
|
61
|
+
(the validator broke — report it, do not work around it), plus `total`/`passed`/`failed`
|
|
62
|
+
counts. That is what to use when triaging a directory rather than checking a single file.
|
|
63
|
+
`show-config --json` masks the token unless `--show-token` is given, so it is safe to quote
|
|
64
|
+
back to the user.
|
|
65
|
+
|
|
66
|
+
## Commands that block or prompt
|
|
67
|
+
|
|
68
|
+
Three commands are the human's to run, and running them in the foreground yourself stalls the
|
|
69
|
+
session or fails for a reason that has nothing to do with the data.
|
|
70
|
+
|
|
71
|
+
**`annotate` never returns.** It serves the UI until Ctrl-C, by design. Start it in the
|
|
72
|
+
background and give the user the URL, or hand them the command to run — do not wait on it.
|
|
73
|
+
Annotating is a human judgement about what happened in a rollout, so there is nothing for you
|
|
74
|
+
to do while it runs anyway.
|
|
75
|
+
|
|
76
|
+
**`restructure` prompts for confirmation** and has no terminal check, so without `--yes` it
|
|
77
|
+
either hangs waiting for input or dies on `EOFError`. It copies the whole dataset, which is
|
|
78
|
+
why the prompt exists — get the user's agreement, then pass `--yes`. `upload
|
|
79
|
+
--with-restructure` treats the flag itself as that agreement and does not prompt.
|
|
80
|
+
|
|
81
|
+
**`init` is interactive**, but it checks for a terminal and exits 1 with an explanation
|
|
82
|
+
rather than hanging. It runs unattended only when `--lab-id` and `--hf-token` are both given —
|
|
83
|
+
and a token passed as a flag lands in the user's shell history, so prefer letting them run
|
|
84
|
+
`init` themselves. The same applies to `annotate --annotator-name`: omitted, it prompts;
|
|
85
|
+
without a terminal it errors out.
|
|
86
|
+
You can run it, but ask the user to provide the values and pass them as flags rather than typing them yourself.
|
|
87
|
+
|
|
88
|
+
**`upload`**: out of principle, you should not run it for the user. It publishes their data to the submission repository, so
|
|
89
|
+
it is best practice to let the user run it themselves. If you do run it, confirm with them first, and pass `--skip-upload` to do a dry run that validates everything without publishing. Only run it
|
|
90
|
+
in full if the user explicitly asks and insists.
|
|
91
|
+
|
|
92
|
+
Everything else — `validate`, `show-config`, `inspect`, `submissions`, `new-profile`,
|
|
93
|
+
`install-skill` — runs to completion on its own and is safe to invoke directly.
|
|
94
|
+
|
|
44
95
|
## Rules
|
|
45
96
|
|
|
46
97
|
**Never invent identity or annotation content.** `lab_id` comes from the registration form,
|
|
@@ -9,6 +9,9 @@ Start from `oopsie-data new-profile --name <name>`, which writes a commented ske
|
|
|
9
9
|
fills it in. Ask the user the questions below; where the choice is constrained to a set, list the
|
|
10
10
|
options and explain them if asked.
|
|
11
11
|
|
|
12
|
+
Be careful to ask the user these questions step by step and explain their choices. Providing
|
|
13
|
+
a full list can easily overwhelm.
|
|
14
|
+
|
|
12
15
|
## Required fields
|
|
13
16
|
|
|
14
17
|
All nine must be present or `load_robot_profile` raises `Robot profile missing keys: [...]`.
|
|
@@ -101,4 +104,4 @@ Stored for reproducibility, never validated:
|
|
|
101
104
|
the underscored and the spaced spelling (`intrinsic calibration matrix`) are read; the
|
|
102
105
|
underscored one is canonical.
|
|
103
106
|
|
|
104
|
-
Please ask explicitly if the user wants to provide the optional inputs!
|
|
107
|
+
Please ask explicitly if the user wants to provide the optional inputs!
|
{oopsie_data_tools-0.2.1 → oopsie_data_tools-0.2.2}/oopsie_data_tools/skill/reference/setup.md
RENAMED
|
@@ -78,7 +78,8 @@ upload --path ...`. Placing it after the subcommand is an argparse error.
|
|
|
78
78
|
|
|
79
79
|
When unsure what is in effect, run `oopsie-data show-config`: it prints both chains, the
|
|
80
80
|
location that wins, and the lab id and token in use (`--show-token` prints the token unmasked).
|
|
81
|
-
`$HF_TOKEN` in the environment overrides the stored token.
|
|
81
|
+
`$HF_TOKEN` in the environment overrides the stored token. Add `--json` to get the same thing
|
|
82
|
+
as a structure — the token stays masked unless `--show-token` is also given.
|
|
82
83
|
|
|
83
84
|
Below, `<config-dir>` means the resolved credential location and `<profiles-dir>` the resolved
|
|
84
85
|
profile location.
|
|
@@ -129,7 +130,8 @@ python -c "from oopsie_data_tools.utils.robot_profile.robot_profile import load_
|
|
|
129
130
|
|
|
130
131
|
`oopsie-data show-config` is what verifies a contributor config; loading the profile as above is
|
|
131
132
|
what verifies a profile. Real confirmation comes from `oopsie-data validate --path ./samples`
|
|
132
|
-
once episodes exist.
|
|
133
|
+
once episodes exist. Add `--json` to any of the read-only commands (`validate`, `show-config`,
|
|
134
|
+
`inspect`) to read the result as a structure instead of parsing prose — exit codes are the same.
|
|
133
135
|
|
|
134
136
|
Running `pytest oopsie_data_tools/test/` checks the toolkit, not the user's setup — the suite
|
|
135
137
|
deliberately isolates `HOME`, `XDG_CONFIG_HOME`, both `OOPSIE_*` variables and the working
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Troubleshooting
|
|
2
2
|
|
|
3
|
+
`oopsie-data validate --path <dir> --json` is the fastest way to see which episodes failed and
|
|
4
|
+
on what: it reports one record per episode with `passed`, `error`, and an `error_type` of
|
|
5
|
+
`validation` (the episode is wrong) or `unexpected` (the validator is wrong — report it rather
|
|
6
|
+
than working around it). `oopsie-data inspect <file.h5> --json` does the same for one file's
|
|
7
|
+
structure.
|
|
8
|
+
|
|
3
9
|
## Errors you will actually see
|
|
4
10
|
|
|
5
11
|
**`lab_id` unset, blank (`lab_id:`), or still `your_lab_id`.** A `RuntimeError` pointing at the
|
|
@@ -66,7 +72,7 @@ silence. The check runs even under `--skip-validate`.
|
|
|
66
72
|
**`--config-dir` rejected.** It is a flag on `oopsie-data` itself and must come *before* the
|
|
67
73
|
subcommand.
|
|
68
74
|
|
|
69
|
-
|
|
75
|
+
**`--path` passed to `inspect`.** It takes a positional path — `oopsie-data inspect <file.h5>`.
|
|
70
76
|
|
|
71
77
|
**`--robot-profile` missing** when running an `examples/inference_examples/` script. It is
|
|
72
78
|
required and deliberately has no default. Those examples also need `uv sync --extra droid`.
|