oopsie-data-tools 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. oopsie_data_tools-0.2.0/.github/ISSUE_TEMPLATE/bug_report.md +33 -0
  2. oopsie_data_tools-0.2.0/.github/ISSUE_TEMPLATE/help-request--data-conversion.md +14 -0
  3. oopsie_data_tools-0.2.0/.github/ISSUE_TEMPLATE/help-request--robot-profile.md +14 -0
  4. oopsie_data_tools-0.2.0/.github/ISSUE_TEMPLATE/improvement-suggestion.md +20 -0
  5. oopsie_data_tools-0.2.0/.github/workflows/ci.yml +95 -0
  6. oopsie_data_tools-0.2.0/.gitignore +238 -0
  7. oopsie_data_tools-0.2.0/.python-version +1 -0
  8. oopsie_data_tools-0.2.0/AI_CONTEXT.md +269 -0
  9. oopsie_data_tools-0.2.0/LICENSE +21 -0
  10. oopsie_data_tools-0.2.0/PKG-INFO +131 -0
  11. oopsie_data_tools-0.2.0/README.md +109 -0
  12. oopsie_data_tools-0.2.0/configs/robot_profiles/act_plus_plus_robot_profile.yaml +66 -0
  13. oopsie_data_tools-0.2.0/configs/robot_profiles/openpi_example_robot_profile.yaml +60 -0
  14. oopsie_data_tools-0.2.0/configs/robot_profiles/template.yaml +95 -0
  15. oopsie_data_tools-0.2.0/examples/inference_examples/act_plus_plus/run_policy.py +803 -0
  16. oopsie_data_tools-0.2.0/examples/inference_examples/act_plus_plus/run_policy_browser.py +664 -0
  17. oopsie_data_tools-0.2.0/examples/inference_examples/act_plus_plus/run_policy_bulk.py +470 -0
  18. oopsie_data_tools-0.2.0/examples/inference_examples/act_plus_plus/run_policy_inloop.py +463 -0
  19. oopsie_data_tools-0.2.0/examples/inference_examples/openpi/run_policy_bulk.py +331 -0
  20. oopsie_data_tools-0.2.0/examples/inference_examples/openpi/run_policy_inloop.py +342 -0
  21. oopsie_data_tools-0.2.0/oopsie_data_tools/__init__.py +7 -0
  22. oopsie_data_tools-0.2.0/oopsie_data_tools/annotation_tool/__init__.py +5 -0
  23. oopsie_data_tools-0.2.0/oopsie_data_tools/annotation_tool/annotation_schema.py +266 -0
  24. oopsie_data_tools-0.2.0/oopsie_data_tools/annotation_tool/annotator_server.py +850 -0
  25. oopsie_data_tools-0.2.0/oopsie_data_tools/annotation_tool/episode_recorder.py +648 -0
  26. oopsie_data_tools-0.2.0/oopsie_data_tools/annotation_tool/rollout_annotator.py +333 -0
  27. oopsie_data_tools-0.2.0/oopsie_data_tools/annotation_tool/ui/annotator.html +2240 -0
  28. oopsie_data_tools-0.2.0/oopsie_data_tools/cli.py +894 -0
  29. oopsie_data_tools-0.2.0/oopsie_data_tools/init_wizard.py +329 -0
  30. oopsie_data_tools-0.2.0/oopsie_data_tools/skill/SKILL.md +98 -0
  31. oopsie_data_tools-0.2.0/oopsie_data_tools/skill/reference/conversion.md +257 -0
  32. oopsie_data_tools-0.2.0/oopsie_data_tools/skill/reference/format.md +112 -0
  33. oopsie_data_tools-0.2.0/oopsie_data_tools/skill/reference/robot-profile.md +104 -0
  34. oopsie_data_tools-0.2.0/oopsie_data_tools/skill/reference/setup.md +216 -0
  35. oopsie_data_tools-0.2.0/oopsie_data_tools/skill/reference/troubleshooting.md +97 -0
  36. oopsie_data_tools-0.2.0/oopsie_data_tools/test/__init__.py +1 -0
  37. oopsie_data_tools-0.2.0/oopsie_data_tools/test/conftest.py +174 -0
  38. oopsie_data_tools-0.2.0/oopsie_data_tools/test/fixtures/__init__.py +0 -0
  39. oopsie_data_tools-0.2.0/oopsie_data_tools/test/fixtures/make_invalid.py +633 -0
  40. oopsie_data_tools-0.2.0/oopsie_data_tools/test/fixtures/make_valid.py +406 -0
  41. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_annotation_completeness.py +154 -0
  42. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_annotation_schema.py +190 -0
  43. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_annotator_server.py +240 -0
  44. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_annotator_server_guards.py +146 -0
  45. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_bulk_inference_end_to_end.py +112 -0
  46. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_cli_config.py +111 -0
  47. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_cli_tools.py +438 -0
  48. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_contributor_config.py +38 -0
  49. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_conversion_utils_annotations.py +75 -0
  50. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_credentials_location.py +106 -0
  51. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_diversity.py +33 -0
  52. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_episode_recorder.py +335 -0
  53. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_episode_video_paths.py +173 -0
  54. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_hf_upload.py +234 -0
  55. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_init_wizard.py +193 -0
  56. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_install_skill.py +120 -0
  57. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_migrate_taxonomy_v2.py +255 -0
  58. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_new_profile.py +84 -0
  59. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_paths.py +137 -0
  60. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_python38_compat.py +79 -0
  61. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_record_step_purity.py +127 -0
  62. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_robot_setup.py +244 -0
  63. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_rollout_annotator.py +134 -0
  64. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_rotation_utils.py +126 -0
  65. oopsie_data_tools-0.2.0/oopsie_data_tools/test/test_validate.py +494 -0
  66. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/__init__.py +1 -0
  67. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/claude_skill.py +96 -0
  68. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/contributor_config.py +91 -0
  69. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/conversion_utils.py +220 -0
  70. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/h5.py +60 -0
  71. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/h5_inspect.py +167 -0
  72. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/hf_limits.py +22 -0
  73. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/hf_upload.py +235 -0
  74. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/log.py +35 -0
  75. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/migrate_taxonomy_v2.py +215 -0
  76. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/paths.py +162 -0
  77. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/restructure.py +402 -0
  78. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/robot_profile/__init__.py +1 -0
  79. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/robot_profile/robot_profile.py +240 -0
  80. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/robot_profile/rotation_utils.py +119 -0
  81. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/robot_profile/template.py +108 -0
  82. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/__init__.py +5 -0
  83. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/annotation_completeness.py +67 -0
  84. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/diversity.py +93 -0
  85. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/episode_data.py +54 -0
  86. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/episode_loader.py +201 -0
  87. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/episode_validator.py +315 -0
  88. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/errors.py +17 -0
  89. oopsie_data_tools-0.2.0/oopsie_data_tools/utils/validation/validation_utils.py +88 -0
  90. oopsie_data_tools-0.2.0/pyproject.toml +74 -0
  91. oopsie_data_tools-0.2.0/uv.lock +2724 -0
@@ -0,0 +1,33 @@
1
+ ---
2
+ name: Bug report
3
+ about: Create a report to help us improve
4
+ title: ''
5
+ labels: bug
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Describe the bug**
11
+ A clear and concise description of what the bug is.
12
+
13
+ **To Reproduce**
14
+ Steps to reproduce the behavior:
15
+ 1. Script details or minimal code example
16
+ 2. Stack trace
17
+
18
+ **Python environment**
19
+ Provide details about your local python environment
20
+
21
+ **Expected behavior**
22
+ A clear and concise description of what you expected to happen.
23
+
24
+ **Screenshots**
25
+ If applicable, add screenshots to help explain your problem.
26
+
27
+ **System (please complete the following information):**
28
+ - OS: [e.g. iOS]
29
+ - Browser (if using web tool) [e.g. chrome, safari]
30
+ - Version [e.g. 22]
31
+
32
+ **Additional context**
33
+ Add any other context about the problem here.
@@ -0,0 +1,14 @@
1
+ ---
2
+ name: 'Help Request: Data Conversion'
3
+ about: Use this template to request help with custom data conversion
4
+ title: ''
5
+ labels: data conversion
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Please describe the current format of your data**
11
+ Please validate that the minimum data required for the Oopsie Dataset is available in your data.
12
+
13
+ **Download link**
14
+ Please either provide a minimal downloadable of your dataset, or send it to us via mail.
@@ -0,0 +1,14 @@
1
+ ---
2
+ name: 'Help Request: Robot Profile'
3
+ about: Please use this template to request help with setting up a robot profile
4
+ title: ''
5
+ labels: ''
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Robot information**
11
+ Please provide as much information as possible about your current robot setup
12
+
13
+ **Issues**
14
+ Please describe which parts of the robot profile cause issue or are incompatible with your setup.
@@ -0,0 +1,20 @@
1
+ ---
2
+ name: Improvement Suggestion
3
+ about: Suggest an improvement to this project
4
+ title: "[Suggestion]"
5
+ labels: ''
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Is your feature request related to a problem? Please describe.**
11
+ A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12
+
13
+ **Describe the solution you'd like**
14
+ A clear and concise description of what you want to happen.
15
+
16
+ **Describe what part of the tooling the improvement touches.**
17
+ Clarify whether the improvement is a suggestion for the robot profile and recording utilities, the data annotation webpage, or the validation and upload scripts, or if you are proposing an extension of the tooling.
18
+
19
+ **Additional context**
20
+ Add any other context or screenshots about the feature request here.
@@ -0,0 +1,95 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ concurrency:
9
+ group: ${{ github.workflow }}-${{ github.ref }}
10
+ cancel-in-progress: true
11
+
12
+ jobs:
13
+ lint:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: astral-sh/setup-uv@v5
18
+ with:
19
+ enable-cache: true
20
+ - run: uv run ruff check oopsie_data_tools/ examples/
21
+ # The PEP 604 / 3.8 guard that used to run here is now part of the test suite
22
+ # (test_python38_compat.py), so the matrix below covers it.
23
+
24
+ test:
25
+ strategy:
26
+ fail-fast: false
27
+ matrix:
28
+ include:
29
+ # 3.8 is the declared floor (requires-python = ">=3.8") and what .python-version
30
+ # pins locally. It needs ubuntu-22.04: setup-python cannot install 3.8 on 24.04,
31
+ # which is what ubuntu-latest now resolves to.
32
+ - python: "3.8"
33
+ os: ubuntu-22.04
34
+ - python: "3.10"
35
+ os: ubuntu-latest
36
+ - python: "3.12"
37
+ os: ubuntu-latest
38
+ runs-on: ${{ matrix.os }}
39
+ steps:
40
+ - uses: actions/checkout@v4
41
+ - uses: astral-sh/setup-uv@v5
42
+ with:
43
+ enable-cache: true
44
+ - run: uv sync --python ${{ matrix.python }}
45
+ - run: uv run --python ${{ matrix.python }} pytest -q
46
+
47
+ # `python -O` strips assert statements. Validation used to be built entirely out of
48
+ # them, so the schema check silently evaporated under an optimised interpreter — for the
49
+ # gate in front of a public dataset. It raises EpisodeValidationError now; this keeps it
50
+ # that way. The -W filter silences pytest's own notice that it cannot rewrite assertions.
51
+ optimised-interpreter:
52
+ runs-on: ubuntu-latest
53
+ steps:
54
+ - uses: actions/checkout@v4
55
+ - uses: astral-sh/setup-uv@v5
56
+ with:
57
+ enable-cache: true
58
+ - run: uv sync
59
+ - run: uv run python -O -m pytest -q -W ignore::pytest.PytestConfigWarning
60
+
61
+ # Ruff's target-version only tunes autofixes; it does not reject newer syntax, and it
62
+ # cannot see runtime-only breakage at all (a PEP 604 annotation resolved by
63
+ # typing.get_type_hints, a 3.9+ stdlib call). Importing every module on the floor
64
+ # version is the only check that actually enforces it.
65
+ import-on-floor:
66
+ runs-on: ubuntu-22.04
67
+ steps:
68
+ - uses: actions/checkout@v4
69
+ - uses: astral-sh/setup-uv@v5
70
+ with:
71
+ enable-cache: true
72
+ - run: uv sync --python 3.8
73
+ - name: Import every module under oopsie_data_tools/
74
+ run: |
75
+ uv run --python 3.8 python - <<'PY'
76
+ import importlib, pathlib, sys
77
+
78
+ failures = []
79
+ for path in sorted(pathlib.Path("oopsie_data_tools").rglob("*.py")):
80
+ if "__pycache__" in path.parts:
81
+ continue
82
+ module = ".".join(path.with_suffix("").parts)
83
+ if module.endswith(".__init__"):
84
+ module = module[: -len(".__init__")]
85
+ try:
86
+ importlib.import_module(module)
87
+ except Exception as e:
88
+ failures.append("{}: {}: {}".format(module, type(e).__name__, e))
89
+
90
+ for line in failures:
91
+ print("FAIL", line)
92
+ print("{} modules imported, {} failed".format(
93
+ len(list(pathlib.Path("oopsie_data_tools").rglob("*.py"))), len(failures)))
94
+ sys.exit(1 if failures else 0)
95
+ PY
@@ -0,0 +1,238 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ #uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+ #poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ #pdm.lock
116
+ #pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ #pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # SageMath parsed files
135
+ *.sage.py
136
+
137
+ # Environments
138
+ .env
139
+ .envrc
140
+ .venv
141
+ env/
142
+ venv/
143
+ ENV/
144
+ env.bak/
145
+ venv.bak/
146
+
147
+ # Local contributor credentials
148
+ configs/contributor_config.yaml
149
+
150
+ # Spyder project settings
151
+ .spyderproject
152
+ .spyproject
153
+
154
+ # Rope project settings
155
+ .ropeproject
156
+
157
+ # mkdocs documentation
158
+ /site
159
+
160
+ # mypy
161
+ .mypy_cache/
162
+ .dmypy.json
163
+ dmypy.json
164
+
165
+ # Pyre type checker
166
+ .pyre/
167
+
168
+ # pytype static type analyzer
169
+ .pytype/
170
+
171
+ # Cython debug symbols
172
+ cython_debug/
173
+
174
+ # PyCharm
175
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
176
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
177
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
178
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
179
+ #.idea/
180
+
181
+ # Abstra
182
+ # Abstra is an AI-powered process automation framework.
183
+ # Ignore directories containing user credentials, local state, and settings.
184
+ # Learn more at https://abstra.io/docs
185
+ .abstra/
186
+
187
+ # Visual Studio Code
188
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
189
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
190
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
191
+ # you could uncomment the following to ignore the entire vscode folder
192
+ .vscode/
193
+
194
+ # Ruff stuff:
195
+ .ruff_cache/
196
+
197
+ # PyPI configuration file
198
+ .pypirc
199
+
200
+ # Cursor
201
+ # Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
202
+ # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
203
+ # refer to https://docs.cursor.com/context/ignore-files
204
+ .cursorignore
205
+ .cursorindexingignore
206
+
207
+ # Marimo
208
+ marimo/_static/
209
+ marimo/_lsp/
210
+ __marimo__/
211
+
212
+ data/
213
+ .secrets/
214
+
215
+ # Exported samples (large video files)
216
+ samples/
217
+ stalling_samples/
218
+ trajectory_*/
219
+ *.zip
220
+
221
+ # Claude Code memory file
222
+ CLAUDE.md
223
+ .claude
224
+
225
+ # Jekyll build output
226
+ docs/_site/
227
+ docs/.jekyll-cache/
228
+ docs/.jekyll-metadata
229
+ docs/.sass-cache/
230
+ docs/apidocs/
231
+
232
+ # Bundler
233
+ .bundle/
234
+ .bundle-user/
235
+ vendor/
236
+
237
+ # OS/editor noise
238
+ .DS_Store
@@ -0,0 +1 @@
1
+ 3.8
@@ -0,0 +1,269 @@
1
+ # AI_CONTEXT.md — oopsie-data-tools setup guide for AI assistants
2
+
3
+ This file is a step-by-step setup skill for AI tools helping a user integrate `oopsie-data-tools` into their robot codebase. Work through the sections in order, asking the user the listed questions before writing any config files.
4
+
5
+ ---
6
+
7
+ ## 1. Verify prerequisites
8
+
9
+ Ask the user to confirm:
10
+ - [ ] Python 3.8 or newer is installed. 3.8 is the supported floor and is tested in CI alongside 3.10 and 3.12; note 3.8 itself has been end-of-life since October 2024.
11
+ - [ ] `uv` (preferred) or `pip` is available.
12
+ - [ ] They have completed the registration form at https://forms.gle/9arwZHAvRjvbozoT7 and received a **lab ID** and **HuggingFace token**. If not, send them there first — nothing else works without these.
13
+
14
+ ---
15
+
16
+ ## 2. Installation
17
+
18
+ ```bash
19
+ git clone https://github.com/oopsie-data/oopsie-data-tools
20
+ cd oopsie-data-tools
21
+ uv sync # or: pip install -e .
22
+ ```
23
+
24
+ Confirm the install succeeded before proceeding.
25
+
26
+ ---
27
+
28
+ ## 3. Where configs live
29
+
30
+ The two configs are looked up through **separate chains**, because they belong to different
31
+ things. In each chain the first location that exists wins.
32
+
33
+ **Credentials** (`contributor_config.yaml`) belong to the user and are shared by every project:
34
+
35
+ 1. `$OOPSIE_CONFIG_DIR` — explicit override
36
+ 2. `~/.config/oopsie-data` (or `$XDG_CONFIG_HOME/oopsie-data`)
37
+ 3. the repository's `configs/` directory — only when working from a clone
38
+
39
+ **Robot profiles** belong to the robot code that loads them, and are *never* looked up in the
40
+ user config directory:
41
+
42
+ 1. `$OOPSIE_ROBOT_PROFILES_DIR` — explicit override
43
+ 2. `./robot_profiles` or `./configs/robot_profiles`, relative to the working directory
44
+ 3. the repository's `configs/robot_profiles` — only when working from a clone
45
+
46
+ So: write the contributor config to `configs/` when working from a clone, otherwise to
47
+ `~/.config/oopsie-data/`. Put the robot profile next to the user's robot code (a
48
+ `robot_profiles/` directory beside their eval script is the normal choice) and load it by
49
+ explicit path. To keep either somewhere custom, have them persist it in their shell rc file:
50
+
51
+ ```bash
52
+ echo 'export OOPSIE_CONFIG_DIR=/path/to/oopsie-config' >> ~/.bashrc # or ~/.zshrc
53
+ echo 'export OOPSIE_ROBOT_PROFILES_DIR=/path/to/profiles' >> ~/.bashrc
54
+ ```
55
+
56
+ `--config-dir <dir>` overrides the credential location for one run. It is a flag on
57
+ `oopsie-data` itself, so it goes **before** the subcommand — `oopsie-data --config-dir <dir>
58
+ upload --path ...`. Placing it after the subcommand is rejected. When unsure what is actually
59
+ in use, run `oopsie-data show-config`: it prints both chains, the location that wins, and the
60
+ lab id and token in effect.
61
+
62
+ Below, `<config-dir>` means the resolved credential location and `<profiles-dir>` the resolved
63
+ profile location.
64
+
65
+ ---
66
+
67
+ ## 3b. Fast path for section 4: `oopsie-data init`
68
+
69
+ If the user is at a terminal, have them run:
70
+
71
+ ```bash
72
+ oopsie-data init
73
+ ```
74
+
75
+ It asks which config directory to use — defaulting to the per-user one, never the checkout,
76
+ because the file holds a token and a token inside a git working tree can be committed — then
77
+ for the lab id and HuggingFace token, rejects the `your_lab_id` placeholder, and writes
78
+ `contributor_config.yaml` (mode 0600). `--lab-id`, `--hf-token`, `--no-verify-token` and
79
+ `--force` skip the corresponding prompts, so it also runs unattended.
80
+
81
+ The token check is **advisory**: a token that fails to verify is reported as a warning and
82
+ saved anyway, and `init` still exits 0. Only the lab id is actually rejected. You find out a
83
+ token is wrong at `oopsie-data upload`.
84
+
85
+ It does **not** create robot profiles. Run `oopsie-data new-profile` to write a commented
86
+ skeleton into `./robot_profiles/`, then fill it in — see section 5. The skeleton deliberately
87
+ does not load until you do, so a half-edited profile cannot stamp placeholder metadata into
88
+ recorded episodes.
89
+
90
+ ---
91
+
92
+ ## 4. Contributor config (`<config-dir>/contributor_config.yaml`)
93
+
94
+ Ask the user:
95
+ 1. **What is your lab ID?** (exact string provided at registration — capitalization matters; a wrong value will block access to the lab-specific HuggingFace repo)
96
+ 2. **What is your HuggingFace token?**
97
+
98
+ Then write/update `<config-dir>/contributor_config.yaml`:
99
+
100
+ ```yaml
101
+ lab_id: <EXACT_LAB_ID>
102
+ huggingface_token: <HF_TOKEN>
103
+ ```
104
+
105
+ ---
106
+
107
+ ## 5. Robot profile (`<profiles-dir>/<name>.yaml`)
108
+
109
+ A robot profile captures hardware and policy metadata. Start from a skeleton — `oopsie-data new-profile --name <name>` writes one into `./robot_profiles/` — then ask the user the questions below and fill it in. Do not start from one of the bundled example profiles: they describe someone else's robot, and an unnoticed leftover field is recorded into every episode's HDF5 attrs and uploaded.
110
+
111
+ Make sure to list available options to the user where the choice is constrained to a set of options, and explain them if the user asks for additional detail.
112
+
113
+ ### 5a. Robot & policy identity
114
+ | Question | YAML key | Example |
115
+ |---|---|---|
116
+ | What is the policy name? | `policy_name` | `pi0.5`, `act_plus_plus` |
117
+ | What is the robot name? | `robot_name` | `franka_droid`, `aloha` |
118
+ | What is the gripper name? | `gripper_name` | `robotiq_2f_85`, `aloha_gripper` |
119
+ | Is this a bimanual (dual-arm) setup? | `is_biarm` | `true` / `false` |
120
+ | Does the robot use a mobile base? | `uses_mobile_base` | `true` / `false` |
121
+ | What is the control frequency (Hz)? | `control_freq` | `10`, `50` |
122
+ | What are the camera names? (list) | `camera_names` | `[left, right, wrist]` |
123
+
124
+ ### 5b. Observation space
125
+ | Question | YAML key | Options |
126
+ |---|---|---|
127
+ | Which robot state keys are recorded? | `robot_state_keys` | **`gripper_position` is always mandatory.** The rest follow from `action_space`: a joint-space action (`joint_position`, `joint_velocity`) requires `joint_position`, a Cartesian action (`cartesian_position`, `cartesian_velocity`) requires `cartesian_position` — the state must observe whatever space the action controls. Note velocity control requires *position* state. An action space mixing joint and Cartesian requires both. `base_position` is an optional addition. |
128
+ | What are the joint names (in order)? | `robot_state_joint_names` | **Required — and only required — when `joint_position` is in `robot_state_keys`.** E.g. `joint_1 … joint_7`. A pure-Cartesian profile omits it entirely. |
129
+ | If `cartesian_position` is included: what orientation representation does the robot state use? | `robot_state_orientation_representation` | `euler_xyz`, `quat`, `matrix`, `rot6d`, `rotvec` |
130
+
131
+ ### 5c. Action space
132
+ | Question | YAML key | Options |
133
+ |---|---|---|
134
+ | What action types does the policy output? | `action_space` | Not a free choice — **at least one arm action** from `joint_position`, `joint_velocity`, `cartesian_position`, `cartesian_velocity`; **at least one gripper action** from `gripper_position`, `gripper_velocity`, `gripper_binary`; **at most one base action** from `base_velocity`, `base_position`; no other keys. A profile declaring `uses_mobile_base: true` must include a base action. |
135
+ | What are the joint names for arm actions? | `action_joint_names` | same order as the action vector. **Required** whenever `joint_position` or `joint_velocity` is in the action space — not optional. |
136
+ | If `cartesian_position` is in the action space: what orientation representation? | `orientation_representation` | `euler_xyz`, `quat`, `matrix`, `rot6d`, `rotvec`. Applies to `cartesian_position` only — `cartesian_velocity` is recorded exactly as given, with no conversion and no shape check. |
137
+
138
+ ### 5d. Optional keys
139
+ These are not required but can be stored for reproducibility:
140
+ - `controller` — e.g. `OSC`, `joint_position`, `joint_velocity`
141
+ - `gains` — controller gain parameters (see template)
142
+ - Camera intrinsic / extrinsic calibration matrices
143
+
144
+ ---
145
+
146
+ ## 6. Validate the config
147
+
148
+ Run the test suite to catch config errors early:
149
+
150
+ ```bash
151
+ pytest oopsie_data_tools/test/
152
+ ```
153
+
154
+ DO not modify the project as this can cause issues later on. Instead, ask the user to manually check issues and to contact the project team if necessary. It is vital that you do not change the code in the oopsie_data_tools directory, only templates and configs, without the user's expressed permission.
155
+
156
+ ---
157
+
158
+ ## 7. Choose a data collection workflow
159
+
160
+ Ask the user which workflow they need:
161
+
162
+ **A. In-the-loop** — annotate each episode right after it is collected (requires the annotation server to be running during robot operation).
163
+
164
+ **B. Bulk collection** — collect all episodes first, annotate later using the standalone annotation server.
165
+
166
+ For **A**, the annotation server will be launched as part of the robot script.
167
+
168
+ For **B**, run this command needs to be run after.
169
+ ```bash
170
+ oopsie-data annotate \
171
+ --samples-dir ./samples \
172
+ --annotator-name <YOUR_NAME> \
173
+ --port 5001
174
+ ```
175
+
176
+ Omitting `--annotator-name` makes the command prompt for it. The equivalent long form is
177
+ `python -m oopsie_data_tools.annotation_tool.annotator_server --samples-dir ./samples
178
+ --annotator-name <YOUR_NAME> --port 5001`.
179
+
180
+ ---
181
+
182
+ ## 8. Integrate `EpisodeRecorder` into the robot script
183
+
184
+ Ask the user:
185
+ - Where is their robot control loop? (file path)
186
+ - What variable holds the robot observation dict? (must have `robot_state` and `image_observation` keys)
187
+ - What variable holds the action dict? (keys must match `action_space` in the robot profile)
188
+ - Where should episode HDF5 files and videos be saved? (`data_root_dir`)
189
+ - Who is running the evaluation? (`operator_name`, stamped into every episode)
190
+
191
+ Minimal integration pattern:
192
+ ```python
193
+ from oopsie_data_tools.annotation_tool.episode_recorder import EpisodeRecorder
194
+ from oopsie_data_tools.utils.robot_profile.robot_profile import load_robot_profile
195
+
196
+ profile = load_robot_profile("<profiles-dir>/<your_profile>.yaml")
197
+ recorder = EpisodeRecorder(
198
+ robot_profile=profile,
199
+ data_root_dir="./samples",
200
+ operator_name="<operator>",
201
+ # resume_session_name="20260101_120000", # optional: append to an existing session
202
+ )
203
+
204
+ for _ in range(num_episodes):
205
+ recorder.reset_episode_recorder() # clears the buffers between episodes
206
+
207
+ # Inside the control loop:
208
+ recorder.record_step(observation=obs, action=action)
209
+
210
+ # After the rollout ends. `instruction` is required; `success` is optional
211
+ # (omit it to leave the episode unannotated for the web annotator).
212
+ recorder.finish_rollout(instruction="pick up the red block", success=success)
213
+ ```
214
+
215
+ Verify that the keys are consistent between the robot profile and the ones passed for recording. The data validation will fail otherwise.
216
+
217
+ ---
218
+
219
+ ## 9. Upload data
220
+
221
+ After annotation is complete:
222
+ ```bash
223
+ oopsie-data upload --path ./samples
224
+ ```
225
+
226
+ This validates and pushes episodes to the lab-specific HuggingFace repository. To check the
227
+ data without uploading, run `oopsie-data validate --path ./samples` (or
228
+ `oopsie-data upload --path ./samples --skip-upload` for the full pre-upload check).
229
+
230
+ If upload refuses because a directory holds more than 10,000 files (a HuggingFace limit),
231
+ add `--with-restructure` to do it in one step:
232
+
233
+ ```bash
234
+ oopsie-data upload --path ./samples --with-restructure
235
+ ```
236
+
237
+ That writes a restructured copy to `./samples_restructured` and uploads it. Every directory
238
+ over the limit — at any depth — is split into numbered subfolders of 500 episodes; the rest
239
+ of the tree is copied through unchanged. It copies rather than moves, so you need room for a
240
+ second copy and the original is left untouched until you delete it yourself. Run
241
+ `oopsie-data restructure --source ./samples --output <dir>` separately if you need the copy
242
+ somewhere else.
243
+
244
+ ---
245
+
246
+ ## Common mistakes to catch
247
+
248
+ - `lab_id` unset, blank (`lab_id:`), or still the placeholder in `contributor_config.yaml` → a clear `RuntimeError` (pointing to the registration form) at `EpisodeRecorder.__init__` and when running `oopsie-data upload`. Capitalisation must match exactly the value you were given.
249
+ - Config edited in the wrong place — e.g. editing the clone's `configs/` while `$OOPSIE_CONFIG_DIR` or `~/.config/oopsie-data` also exists, which take precedence. The error message names the file that was actually read.
250
+ - After uploading, run `oopsie-data submissions` to confirm your episodes landed in the lab HuggingFace repo.
251
+ - Action dict keys not matching `action_space` in the robot profile → validation error at `record_step`.
252
+ - Joint-space actions require `joint_position` in `robot_state_keys`; Cartesian actions require `cartesian_position`.
253
+ - If `joint_position` is in `robot_state_keys`, `robot_state_joint_names` must be a non-empty list.
254
+ - `robot_state_joint_names` length not matching the `joint_position` array length → HDF5 schema error.
255
+ - Running `uv sync` without `--extra tfds` or `--extra droid` when those features are needed (note: those two extras conflict with each other).
256
+ - Passing an action chunk instead of per-step actions → caught only for `cartesian_position`, which is shape-checked to `(7,)` or `(14,)`. For joint action spaces a `(T, chunk, dof)` array passes both the DOF and trajectory-length checks and is recorded silently, so check this yourself.
257
+ - `cartesian_position` in state/action but `orientation_representation` not set → the value is recorded unconverted and then rejected unless it is already `[x, y, z, qx, qy, qz, qw]` with a unit quaternion. A representation that is set but does not match what the policy emits is reported by width, e.g. "QUAT orientation expects 4 value(s), got 3".
258
+ - `robot_state_joint_names` length not matching the `joint_position` array length → `EpisodeValidationError` raised inside `finish_rollout`, before any HDF5 file is written. (`EpisodeValidationError` subclasses `AssertionError`, so `except AssertionError` still catches it.)
259
+ - Running `uv sync` without the `droid` extra when using `examples/inference_examples/`. It is the only extra; everything the CLI does works off a bare `uv sync`.
260
+ - Running one of the `examples/inference_examples/` scripts without `--robot-profile`. It is required, and deliberately has no default.
261
+
262
+ ## Important mistakes that will not raise an error
263
+
264
+ These need to be verified manually by the user.
265
+
266
+ - Action space not in absolute, but in delta coordinates
267
+ - Delta coordinates cannot easily be processed by downstream applications as the base offset is not recorded
268
+ - Quaternion representation is in wrong order if passed explicitly
269
+ - The framework provides a best effort test, but it cannot catch all edge-cases