oopsie-data-tools 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. oopsie_data_tools/__init__.py +7 -0
  2. oopsie_data_tools/annotation_tool/__init__.py +5 -0
  3. oopsie_data_tools/annotation_tool/annotation_schema.py +266 -0
  4. oopsie_data_tools/annotation_tool/annotator_server.py +850 -0
  5. oopsie_data_tools/annotation_tool/episode_recorder.py +648 -0
  6. oopsie_data_tools/annotation_tool/rollout_annotator.py +333 -0
  7. oopsie_data_tools/annotation_tool/ui/annotator.html +2240 -0
  8. oopsie_data_tools/cli.py +894 -0
  9. oopsie_data_tools/init_wizard.py +329 -0
  10. oopsie_data_tools/skill/SKILL.md +98 -0
  11. oopsie_data_tools/skill/reference/conversion.md +257 -0
  12. oopsie_data_tools/skill/reference/format.md +112 -0
  13. oopsie_data_tools/skill/reference/robot-profile.md +104 -0
  14. oopsie_data_tools/skill/reference/setup.md +216 -0
  15. oopsie_data_tools/skill/reference/troubleshooting.md +97 -0
  16. oopsie_data_tools/test/__init__.py +1 -0
  17. oopsie_data_tools/test/conftest.py +174 -0
  18. oopsie_data_tools/test/fixtures/__init__.py +0 -0
  19. oopsie_data_tools/test/fixtures/make_invalid.py +633 -0
  20. oopsie_data_tools/test/fixtures/make_valid.py +406 -0
  21. oopsie_data_tools/test/test_annotation_completeness.py +154 -0
  22. oopsie_data_tools/test/test_annotation_schema.py +190 -0
  23. oopsie_data_tools/test/test_annotator_server.py +240 -0
  24. oopsie_data_tools/test/test_annotator_server_guards.py +146 -0
  25. oopsie_data_tools/test/test_bulk_inference_end_to_end.py +112 -0
  26. oopsie_data_tools/test/test_cli_config.py +111 -0
  27. oopsie_data_tools/test/test_cli_tools.py +438 -0
  28. oopsie_data_tools/test/test_contributor_config.py +38 -0
  29. oopsie_data_tools/test/test_conversion_utils_annotations.py +75 -0
  30. oopsie_data_tools/test/test_credentials_location.py +106 -0
  31. oopsie_data_tools/test/test_diversity.py +33 -0
  32. oopsie_data_tools/test/test_episode_recorder.py +335 -0
  33. oopsie_data_tools/test/test_episode_video_paths.py +173 -0
  34. oopsie_data_tools/test/test_hf_upload.py +234 -0
  35. oopsie_data_tools/test/test_init_wizard.py +193 -0
  36. oopsie_data_tools/test/test_install_skill.py +120 -0
  37. oopsie_data_tools/test/test_migrate_taxonomy_v2.py +255 -0
  38. oopsie_data_tools/test/test_new_profile.py +84 -0
  39. oopsie_data_tools/test/test_paths.py +137 -0
  40. oopsie_data_tools/test/test_python38_compat.py +79 -0
  41. oopsie_data_tools/test/test_record_step_purity.py +127 -0
  42. oopsie_data_tools/test/test_robot_setup.py +244 -0
  43. oopsie_data_tools/test/test_rollout_annotator.py +134 -0
  44. oopsie_data_tools/test/test_rotation_utils.py +126 -0
  45. oopsie_data_tools/test/test_validate.py +494 -0
  46. oopsie_data_tools/utils/__init__.py +1 -0
  47. oopsie_data_tools/utils/claude_skill.py +96 -0
  48. oopsie_data_tools/utils/contributor_config.py +91 -0
  49. oopsie_data_tools/utils/conversion_utils.py +220 -0
  50. oopsie_data_tools/utils/h5.py +60 -0
  51. oopsie_data_tools/utils/h5_inspect.py +167 -0
  52. oopsie_data_tools/utils/hf_limits.py +22 -0
  53. oopsie_data_tools/utils/hf_upload.py +235 -0
  54. oopsie_data_tools/utils/log.py +35 -0
  55. oopsie_data_tools/utils/migrate_taxonomy_v2.py +215 -0
  56. oopsie_data_tools/utils/paths.py +162 -0
  57. oopsie_data_tools/utils/restructure.py +402 -0
  58. oopsie_data_tools/utils/robot_profile/__init__.py +1 -0
  59. oopsie_data_tools/utils/robot_profile/robot_profile.py +240 -0
  60. oopsie_data_tools/utils/robot_profile/rotation_utils.py +119 -0
  61. oopsie_data_tools/utils/robot_profile/template.py +108 -0
  62. oopsie_data_tools/utils/validation/__init__.py +5 -0
  63. oopsie_data_tools/utils/validation/annotation_completeness.py +67 -0
  64. oopsie_data_tools/utils/validation/diversity.py +93 -0
  65. oopsie_data_tools/utils/validation/episode_data.py +54 -0
  66. oopsie_data_tools/utils/validation/episode_loader.py +201 -0
  67. oopsie_data_tools/utils/validation/episode_validator.py +315 -0
  68. oopsie_data_tools/utils/validation/errors.py +17 -0
  69. oopsie_data_tools/utils/validation/validation_utils.py +88 -0
  70. oopsie_data_tools-0.2.0.dist-info/METADATA +131 -0
  71. oopsie_data_tools-0.2.0.dist-info/RECORD +74 -0
  72. oopsie_data_tools-0.2.0.dist-info/WHEEL +4 -0
  73. oopsie_data_tools-0.2.0.dist-info/entry_points.txt +2 -0
  74. oopsie_data_tools-0.2.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,111 @@
1
+ """Tests for ``oopsie-data show-config`` (``cli.cmd_show_config``).
2
+
3
+ The command is purely diagnostic: it prints the lookup chains, marks the entry that wins,
4
+ and shows the credentials in effect. It must never raise on a missing or half-filled
5
+ config — reporting that state is exactly what it is for.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import pytest
11
+
12
+ from oopsie_data_tools import cli
13
+ from oopsie_data_tools.utils import paths
14
+
15
+ TOKEN = "hf_secrettokenvalue"
16
+
17
+
18
+ @pytest.fixture
19
+ def env(tmp_path, monkeypatch):
20
+ """Isolate both chains and the working directory; returns the credential dir."""
21
+ config_dir = tmp_path / "cfg"
22
+ config_dir.mkdir()
23
+ monkeypatch.setenv(paths.ENV_CONFIG_DIR, str(config_dir))
24
+ monkeypatch.delenv(paths.ENV_PROFILES_DIR, raising=False)
25
+ monkeypatch.delenv("HF_TOKEN", raising=False)
26
+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg"))
27
+ monkeypatch.chdir(tmp_path)
28
+ return config_dir
29
+
30
+
31
+ def _write_config(config_dir, lab_id="MyLab", token=TOKEN):
32
+ (config_dir / "contributor_config.yaml").write_text(
33
+ f"lab_id: {lab_id}\nhuggingface_token: {token}\n"
34
+ )
35
+
36
+
37
+ def test_reports_active_config_and_masks_the_token(env, capsys):
38
+ _write_config(env)
39
+
40
+ assert cli.main(["show-config"]) == 0
41
+
42
+ out = capsys.readouterr().out
43
+ assert f"Reading: {env / 'contributor_config.yaml'}" in out
44
+ assert "lab_id: MyLab" in out
45
+ assert "OopsieData-Submissions/MyLab" in out
46
+ assert TOKEN not in out, "the token must not be printed in full by default"
47
+
48
+
49
+ def test_show_token_reveals_it(env, capsys):
50
+ _write_config(env)
51
+
52
+ assert cli.main(["show-config", "--show-token"]) == 0
53
+ assert TOKEN in capsys.readouterr().out
54
+
55
+
56
+ def test_env_token_is_reported_as_the_override(env, capsys, monkeypatch):
57
+ _write_config(env)
58
+ monkeypatch.setenv("HF_TOKEN", "hf_from_the_environment")
59
+
60
+ assert cli.main(["show-config", "--show-token"]) == 0
61
+
62
+ out = capsys.readouterr().out
63
+ assert "hf_from_the_environment" in out
64
+ assert "overrides the config" in out
65
+
66
+
67
+ def test_missing_config_says_where_init_would_write(env, capsys, monkeypatch):
68
+ # Hide the checkout so the chain has no fallback, as for a pip install without a clone.
69
+ monkeypatch.setattr(paths, "_REPO_CONFIG_DIR", env / "no_such_checkout")
70
+
71
+ assert cli.main(["show-config"]) == 0
72
+
73
+ out = capsys.readouterr().out
74
+ assert "No config found" in out
75
+ assert str(env / "contributor_config.yaml") in out
76
+ assert "lab_id: (not set)" in out
77
+
78
+
79
+ def test_placeholder_config_is_reported_not_raised(env, capsys):
80
+ """read_contributor_config raises on this; the diagnostic command must not."""
81
+ _write_config(env, lab_id="your_lab_id")
82
+
83
+ assert cli.main(["show-config"]) == 0
84
+ assert "your_lab_id" in capsys.readouterr().out
85
+
86
+
87
+ def test_project_local_profiles_are_listed(env, tmp_path, capsys):
88
+ profiles = tmp_path / "robot_profiles"
89
+ profiles.mkdir()
90
+ (profiles / "my_robot.yaml").write_text("policy_name: p\n")
91
+ _write_config(env)
92
+
93
+ assert cli.main(["show-config"]) == 0
94
+
95
+ out = capsys.readouterr().out
96
+ assert f"Reading: {profiles}" in out
97
+ assert "profiles: my_robot.yaml" in out
98
+ assert "1 profile\n" in out, "singular when there is exactly one"
99
+
100
+
101
+ def test_chain_lists_each_directory_once(env, monkeypatch, tmp_path, capsys):
102
+ """From a checkout root, ./configs/robot_profiles and the repo's dir are the same path."""
103
+ checkout = tmp_path / "checkout"
104
+ (checkout / "configs" / paths.PROFILES_DIR_NAME).mkdir(parents=True)
105
+ monkeypatch.setattr(paths, "_REPO_CONFIG_DIR", checkout / "configs")
106
+ monkeypatch.chdir(checkout)
107
+
108
+ assert cli.main(["show-config"]) == 0
109
+
110
+ listed = capsys.readouterr().out.count(str(checkout / "configs" / paths.PROFILES_DIR_NAME))
111
+ assert listed == 2, f"expected one chain entry plus the 'Reading:' line, got {listed}"
@@ -0,0 +1,438 @@
1
+ """Tests for the three subcommands folded in from the old ``scripts/`` directory.
2
+
3
+ ``inspect``, ``restructure`` and ``submissions`` used to be standalone scripts that nothing
4
+ tested and that a pip-installed user could not reach at all. They are subcommands now, so
5
+ they get the same coverage as the rest of the CLI. Nothing here touches the network.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from pathlib import Path
12
+
13
+ import h5py
14
+ import pytest
15
+
16
+ from oopsie_data_tools.cli import main
17
+ from oopsie_data_tools.test.fixtures.make_valid import write_valid_episode
18
+ from oopsie_data_tools.utils import hf_upload, restructure
19
+
20
+
21
+ @pytest.fixture
22
+ def info_log(caplog):
23
+ """caplog at INFO.
24
+
25
+ ``cli.main`` calls ``logging.basicConfig``, which is a no-op once pytest's logging
26
+ plugin has attached its handler, so the root logger stays at WARNING and every
27
+ ``logger.info`` line these commands emit as their *result* would be dropped.
28
+ """
29
+ caplog.set_level(logging.INFO)
30
+ return caplog
31
+
32
+
33
+ # ── inspect ───────────────────────────────────────────────────────────────────
34
+
35
+
36
+ def test_inspect_dumps_groups_datasets_and_attrs(tmp_path, capsys):
37
+ episode = write_valid_episode(tmp_path, "ep_a")
38
+
39
+ assert main(["inspect", str(episode)]) == 0
40
+
41
+ out = capsys.readouterr().out
42
+ assert "[group] /" in out
43
+ assert "observations/robot_states" in out
44
+ assert "[dataset]" in out
45
+ assert "'schema'" in out, "root attrs must be shown"
46
+
47
+
48
+ def test_inspect_works_on_a_file_validate_would_reject(tmp_path, capsys):
49
+ """It is a structure dump, not a validator — that is the whole point of having it."""
50
+ broken = tmp_path / "broken.h5"
51
+ with h5py.File(broken, "w") as f:
52
+ f.attrs["schema"] = "something_else_entirely"
53
+ f.create_dataset("stray", data=[1, 2, 3])
54
+
55
+ assert main(["inspect", str(broken)]) == 0
56
+ assert "stray" in capsys.readouterr().out
57
+
58
+
59
+ def test_inspect_reports_a_missing_file_rather_than_raising(tmp_path, caplog):
60
+ assert main(["inspect", str(tmp_path / "nope.h5")]) == 1
61
+ assert "Not a file" in caplog.text
62
+
63
+
64
+ # ── restructure ───────────────────────────────────────────────────────────────
65
+
66
+
67
+ def test_restructure_does_nothing_when_the_folder_is_small_enough(tmp_path, info_log):
68
+ write_valid_episode(tmp_path, "ep_a")
69
+
70
+ assert main(["restructure", "--source", str(tmp_path), "--yes"]) == 0
71
+
72
+ assert "No restructuring needed" in info_log.text
73
+ assert not (tmp_path.parent / f"{tmp_path.name}_restructured").exists()
74
+
75
+
76
+ def test_restructure_copies_episodes_and_rewrites_video_paths(tmp_path, monkeypatch):
77
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 1)
78
+ source = tmp_path / "session"
79
+ source.mkdir()
80
+ write_valid_episode(source, "ep_a")
81
+ write_valid_episode(source, "ep_b")
82
+ output = tmp_path / "out"
83
+
84
+ assert main(
85
+ ["restructure", "--source", str(source), "--output", str(output), "--yes"]
86
+ ) == 0
87
+
88
+ # Source is untouched — this command must never be the thing that loses data.
89
+ assert (source / "ep_a.h5").exists()
90
+ assert (source / "ep_a_front.mp4").exists()
91
+
92
+ copies = sorted(output.rglob("*.h5"))
93
+ assert [p.name for p in copies] == ["ep_a.h5", "ep_b.h5"]
94
+ for h5_path in copies:
95
+ with h5py.File(h5_path, "r") as f:
96
+ stored = f["observations/video_paths/front"][()]
97
+ rel = stored.decode() if isinstance(stored, bytes) else stored
98
+ assert "/" not in rel, "videos are copied flat next to their episode"
99
+ assert (h5_path.parent / rel).is_file(), "the rewritten path must resolve"
100
+
101
+
102
+ def test_unique_video_dest_separates_two_videos_that_want_the_same_name(tmp_path):
103
+ """Two distinct videos claiming one name must not collapse onto it.
104
+
105
+ This used to return the colliding name regardless, so the copy silently overwrote the
106
+ first episode's video. Exercised directly: ``restructure()`` collects HDF5 files from
107
+ the source root only, where stems are unique by construction, so this branch is
108
+ unreachable through the command and would otherwise never be tested.
109
+ """
110
+ used: dict[str, Path] = {}
111
+ first = tmp_path / "one" / "ep_front.mp4"
112
+ second = tmp_path / "two" / "ep_front.mp4"
113
+
114
+ a = restructure._unique_video_dest(first, "front", "ep", used)
115
+ b = restructure._unique_video_dest(second, "front", "ep", used)
116
+
117
+ assert a == "ep_front.mp4"
118
+ assert b != a, "a second, different video must not take the first one's name"
119
+ # Idempotent: re-asking for a video already registered returns the same name, so a
120
+ # caller may ask more than once per file without inventing new copies.
121
+ assert restructure._unique_video_dest(first, "front", "ep", used) == a
122
+
123
+
124
+ def test_restructure_copies_a_shared_video_once(tmp_path, monkeypatch):
125
+ """Two episodes referencing one video share the copy rather than duplicating it."""
126
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 1)
127
+ source = tmp_path / "session"
128
+ source.mkdir()
129
+ write_valid_episode(source, "ep_a")
130
+ write_valid_episode(source, "ep_b")
131
+ # Point ep_b at ep_a's video, the way a re-annotated episode would.
132
+ with h5py.File(source / "ep_b.h5", "r+") as f:
133
+ del f["observations/video_paths/front"]
134
+ f["observations/video_paths"].create_dataset(
135
+ "front", data="ep_a_front.mp4", dtype=h5py.string_dtype(encoding="utf-8")
136
+ )
137
+ output = tmp_path / "out"
138
+
139
+ assert main(
140
+ ["restructure", "--source", str(source), "--output", str(output), "--yes"]
141
+ ) == 0
142
+
143
+ for h5_path in sorted(output.rglob("*.h5")):
144
+ with h5py.File(h5_path, "r") as f:
145
+ stored = f["observations/video_paths/front"][()]
146
+ rel = stored.decode() if isinstance(stored, bytes) else stored
147
+ assert (h5_path.parent / rel).is_file(), "every rewritten path must resolve"
148
+
149
+
150
+ def test_restructure_splits_a_nested_oversized_directory(tmp_path, monkeypatch, info_log):
151
+ """Detection is recursive, so repair must be too.
152
+
153
+ This used to detect the nested directory, restructure the *root* instead, and report
154
+ success — leaving the upload just as blocked as before.
155
+ """
156
+ # Limit 2 puts the root exactly at the limit (one .h5 + one .mp4) while the nested
157
+ # directory is over it. This is the shape that used to misfire: the old code saw an
158
+ # oversized directory, then acted on the root because that is where it looked for
159
+ # episodes.
160
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 2)
161
+ source = tmp_path / "session"
162
+ nested = source / "day_two"
163
+ nested.mkdir(parents=True)
164
+ write_valid_episode(source, "ep_root")
165
+ write_valid_episode(nested, "ep_nested_a")
166
+ write_valid_episode(nested, "ep_nested_b")
167
+ output = tmp_path / "out"
168
+
169
+ assert restructure.oversized_dirs(source) == [(nested, 4)], "only the nested dir is over"
170
+
171
+ assert main(
172
+ ["restructure", "--source", str(source), "--output", str(output), "--yes"]
173
+ ) == 0
174
+
175
+ # The nested directory keeps its position in the tree and gains numbered subfolders.
176
+ nested_out = output / "day_two"
177
+ assert nested_out.is_dir()
178
+ assert sorted(p.name for p in nested_out.rglob("*.h5")) == [
179
+ "ep_nested_a.h5",
180
+ "ep_nested_b.h5",
181
+ ]
182
+ assert [p for p in nested_out.iterdir() if p.is_dir()], "the oversized dir is batched"
183
+
184
+ # The root was within the limit, so it is copied through rather than reorganised.
185
+ assert (output / "ep_root.h5").is_file(), "an in-limit root must not be restructured"
186
+
187
+ # Every episode in the output still resolves its own video.
188
+ for h5_path in output.rglob("*.h5"):
189
+ with h5py.File(h5_path, "r") as f:
190
+ stored = f["observations/video_paths/front"][()]
191
+ rel = stored.decode() if isinstance(stored, bytes) else stored
192
+ assert (h5_path.parent / rel).is_file(), f"{h5_path} lost its video"
193
+
194
+ # Nothing is left behind: the root episode is in the output exactly once.
195
+ assert len(list(output.rglob("ep_root.h5"))) == 1
196
+
197
+
198
+ def test_restructure_copies_untouched_directories_through(tmp_path, monkeypatch):
199
+ """The output is a full copy, not just the episodes that needed splitting."""
200
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 2)
201
+ source = tmp_path / "session"
202
+ big = source / "big"
203
+ big.mkdir(parents=True)
204
+ write_valid_episode(big, "ep_a")
205
+ write_valid_episode(big, "ep_b") # 4 files > limit of 2
206
+ small = source / "small"
207
+ small.mkdir()
208
+ write_valid_episode(small, "ep_small")
209
+ (source / "NOTES.md").write_text("keep me", encoding="utf-8")
210
+ output = tmp_path / "out"
211
+
212
+ assert main(
213
+ ["restructure", "--source", str(source), "--output", str(output), "--yes"]
214
+ ) == 0
215
+
216
+ assert (output / "NOTES.md").read_text(encoding="utf-8") == "keep me"
217
+ assert (output / "small" / "ep_small.h5").is_file(), "small dirs pass through as-is"
218
+ assert (output / "small" / "ep_small_front.mp4").is_file()
219
+ assert not list((output / "small").glob("0*")), "an in-limit dir must not be batched"
220
+ assert list((output / "big").glob("*/ep_a.h5")), "the oversized dir is batched"
221
+
222
+
223
+ def test_restructure_refuses_an_output_inside_the_source(tmp_path, monkeypatch, caplog):
224
+ """Otherwise the pass-through copy would walk into its own output."""
225
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 1)
226
+ source = tmp_path / "session"
227
+ source.mkdir()
228
+ write_valid_episode(source, "ep_a")
229
+
230
+ assert main(
231
+ ["restructure", "--source", str(source), "--output", str(source / "out"), "--yes"]
232
+ ) == 1
233
+ assert "must not be inside the source" in caplog.text
234
+
235
+
236
+ def test_restructure_reports_an_oversized_dir_it_cannot_split(tmp_path, monkeypatch, caplog):
237
+ """A videos-only folder cannot be batched by episode; say so rather than claiming success."""
238
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 1)
239
+ source = tmp_path / "session"
240
+ source.mkdir()
241
+ for i in range(3):
242
+ (source / f"clip{i}.mp4").write_bytes(b"x")
243
+
244
+ assert main(["restructure", "--source", str(source), "--yes"]) == 1
245
+ assert "nothing to split" in caplog.text
246
+
247
+
248
+ def test_restructure_rejects_a_source_that_is_not_a_directory(tmp_path, caplog):
249
+ episode = write_valid_episode(tmp_path, "ep_a")
250
+ assert main(["restructure", "--source", str(episode), "--yes"]) == 1
251
+ assert "not a directory" in caplog.text.lower()
252
+
253
+
254
+ def test_restructure_without_yes_aborts_on_anything_but_yes(tmp_path, monkeypatch, info_log):
255
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 0)
256
+ write_valid_episode(tmp_path, "ep_a")
257
+ monkeypatch.setattr("builtins.input", lambda _prompt: "y")
258
+
259
+ assert main(["restructure", "--source", str(tmp_path)]) == 0
260
+
261
+ assert "Aborted" in info_log.text
262
+ assert not (tmp_path.parent / f"{tmp_path.name}_restructured").exists()
263
+
264
+
265
+ # ── upload --with-restructure ─────────────────────────────────────────────────
266
+
267
+
268
+ def _oversized_session(tmp_path, monkeypatch, n=3):
269
+ """A session directory that trips the folder-size precheck.
270
+
271
+ BATCH_SIZE is scaled down alongside FILE_LIMIT. One batch holds BATCH_SIZE episodes
272
+ plus their videos, so a batch only ends up under the limit if the two are set in a
273
+ realistic ratio — at the shipped 500 / 10,000 it takes about twenty cameras per episode
274
+ to overflow a batch.
275
+ """
276
+ monkeypatch.setattr(restructure, "FILE_LIMIT", 2)
277
+ monkeypatch.setattr(restructure, "BATCH_SIZE", 1)
278
+ monkeypatch.setattr(hf_upload, "FILE_LIMIT", 2)
279
+ source = tmp_path / "samples"
280
+ source.mkdir()
281
+ for i in range(n):
282
+ write_valid_episode(source, f"ep_{i}")
283
+ return source
284
+
285
+
286
+ def test_upload_aborts_on_an_oversized_folder_without_the_flag(tmp_path, monkeypatch, caplog):
287
+ source = _oversized_session(tmp_path, monkeypatch)
288
+
289
+ # No --skip-upload: the precheck only runs when something would actually be uploaded.
290
+ # It returns non-zero before any network call is reached.
291
+ assert main(["upload", "--path", str(source)]) == 1
292
+
293
+ assert "exceed" in caplog.text
294
+ assert "--with-restructure" in caplog.text, "point at the one-step fix"
295
+ assert not (tmp_path / "samples_restructured").exists()
296
+
297
+
298
+ def test_upload_with_restructure_does_not_tell_you_to_restructure_by_hand(
299
+ tmp_path, monkeypatch, caplog
300
+ ):
301
+ """The manual instruction contradicts what the next line does."""
302
+ source = _oversized_session(tmp_path, monkeypatch)
303
+
304
+ assert main(["upload", "--path", str(source), "--with-restructure", "--skip-upload"]) == 0
305
+
306
+ assert "exceed" in caplog.text, "still say which directories were over"
307
+ assert "oopsie-data restructure --source" not in caplog.text
308
+
309
+
310
+ def test_upload_with_restructure_uploads_the_restructured_copy(tmp_path, monkeypatch, info_log):
311
+ source = _oversized_session(tmp_path, monkeypatch)
312
+ uploaded = {}
313
+ monkeypatch.setattr(
314
+ "oopsie_data_tools.utils.hf_upload.resolve_hf_target", lambda: ("tok", "repo/x")
315
+ )
316
+ monkeypatch.setattr("oopsie_data_tools.utils.hf_upload.hf_login", lambda token: "someone")
317
+ monkeypatch.setattr("oopsie_data_tools.utils.hf_upload.ensure_repo", lambda api, repo: None)
318
+ monkeypatch.setattr(
319
+ "oopsie_data_tools.utils.hf_upload.upload_dataset",
320
+ lambda api, repo, d: uploaded.update(dir=d),
321
+ )
322
+ monkeypatch.setattr("huggingface_hub.HfApi", lambda token=None: object())
323
+
324
+ assert main(["upload", "--path", str(source), "--with-restructure"]) == 0
325
+
326
+ output = tmp_path / "samples_restructured"
327
+ assert uploaded["dir"] == str(output), "the copy is uploaded, not the original"
328
+ # The original is untouched, which is the whole contract of restructure.
329
+ assert sorted(p.name for p in source.glob("*.h5")) == ["ep_0.h5", "ep_1.h5", "ep_2.h5"]
330
+ assert list(output.glob("*/ep_0.h5")), "the copy is batched into subfolders"
331
+
332
+
333
+ def test_upload_with_restructure_validates_the_copy_not_the_original(tmp_path, monkeypatch):
334
+ """Video paths are rewritten by the split, so validation must run on the copy."""
335
+ source = _oversized_session(tmp_path, monkeypatch)
336
+ validated = {}
337
+
338
+ def fake_validation(path, episode_id, log_path):
339
+ validated["path"] = path
340
+ return 0
341
+
342
+ monkeypatch.setattr("oopsie_data_tools.utils.hf_upload.run_validation", fake_validation)
343
+
344
+ assert main(["upload", "--path", str(source), "--with-restructure", "--skip-upload"]) == 0
345
+ assert validated["path"] == str(tmp_path / "samples_restructured")
346
+
347
+
348
+ def test_upload_with_restructure_is_a_no_op_when_the_folder_is_fine(tmp_path, monkeypatch):
349
+ source = tmp_path / "samples"
350
+ source.mkdir()
351
+ write_valid_episode(source, "ep_a")
352
+
353
+ assert main(["upload", "--path", str(source), "--with-restructure", "--skip-upload"]) == 0
354
+ assert not (tmp_path / "samples_restructured").exists()
355
+
356
+
357
+ def test_upload_with_restructure_aborts_if_the_layout_is_still_oversized(
358
+ tmp_path, monkeypatch, caplog
359
+ ):
360
+ """A videos-only directory cannot be split by episode; do not upload it anyway."""
361
+ source = _oversized_session(tmp_path, monkeypatch)
362
+ clips = source / "clips"
363
+ clips.mkdir()
364
+ for i in range(5):
365
+ (clips / f"c{i}.mp4").write_bytes(b"x")
366
+
367
+ assert main(["upload", "--path", str(source), "--with-restructure", "--skip-upload"]) == 1
368
+ assert "still over the file limit" in caplog.text
369
+
370
+
371
+ # ── submissions ───────────────────────────────────────────────────────────────
372
+
373
+
374
+ class _FakeApi:
375
+ def __init__(self, files=None):
376
+ self.files = files
377
+ self.token = None
378
+
379
+ def repo_info(self, repo_id, repo_type):
380
+ if self.files is None:
381
+ raise RuntimeError("404")
382
+ return {"id": repo_id}
383
+
384
+ def list_repo_files(self, repo_id, repo_type):
385
+ return self.files
386
+
387
+
388
+ @pytest.fixture
389
+ def fake_hf(monkeypatch):
390
+ """Patch HfApi at its import site inside query_submissions."""
391
+
392
+ def install(files):
393
+ api = _FakeApi(files)
394
+ monkeypatch.setattr("huggingface_hub.HfApi", lambda token=None: api)
395
+ return api
396
+
397
+ return install
398
+
399
+
400
+ def test_submissions_counts_episodes_videos_and_folders(fake_hf, info_log):
401
+ fake_hf(["000/a.h5", "000/a_front.mp4", "500/b.hdf5", "README.md"])
402
+
403
+ assert main(["submissions"]) == 0
404
+
405
+ assert "Episodes (.h5): 2" in info_log.text
406
+ assert "Videos (.mp4): 1" in info_log.text
407
+ assert "Total files: 4" in info_log.text
408
+ assert "000" in info_log.text and "500" in info_log.text
409
+
410
+
411
+ def test_submissions_treats_a_missing_repo_as_success(fake_hf, info_log):
412
+ """The repo is created on first upload, so 'not there yet' is not a failure."""
413
+ fake_hf(None)
414
+
415
+ assert main(["submissions"]) == 0
416
+ assert "created automatically on your first successful upload" in info_log.text
417
+
418
+
419
+ def test_submissions_lab_id_flag_overrides_the_config(fake_hf, info_log):
420
+ fake_hf([])
421
+
422
+ assert main(["submissions", "--lab-id", "SomeOtherLab"]) == 0
423
+ assert "OopsieData-Submissions/SomeOtherLab" in info_log.text
424
+
425
+
426
+ def test_submissions_prefers_the_env_token(monkeypatch):
427
+ """$HF_TOKEN overrides the config token, matching upload's precedence."""
428
+ seen = {}
429
+
430
+ def fake_api(token=None):
431
+ seen["token"] = token
432
+ return _FakeApi([])
433
+
434
+ monkeypatch.setenv("HF_TOKEN", "env-token")
435
+ monkeypatch.setattr("huggingface_hub.HfApi", fake_api)
436
+
437
+ assert hf_upload.query_submissions() == 0
438
+ assert seen["token"] == "env-token"
@@ -0,0 +1,38 @@
1
+ """Tests for the shared contributor-config reader (issue #20)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+ import pytest
8
+
9
+ from oopsie_data_tools.utils.contributor_config import read_contributor_config
10
+
11
+
12
+ def _write(tmp_path: Path, content: str) -> Path:
13
+ path = tmp_path / "contributor_config.yaml"
14
+ path.write_text(content)
15
+ return path
16
+
17
+
18
+ def test_valid_config_returns_lab_and_token(tmp_path: Path) -> None:
19
+ path = _write(tmp_path, "lab_id: MyLab\nhuggingface_token: hf_abc\n")
20
+ assert read_contributor_config(path) == ("MyLab", "hf_abc")
21
+
22
+
23
+ def test_blank_lab_id_gives_clear_error(tmp_path: Path) -> None:
24
+ # `lab_id:` parses to None; must not crash with `None.strip()` (the original #20 bug).
25
+ path = _write(tmp_path, "lab_id:\nhuggingface_token:\n")
26
+ with pytest.raises(RuntimeError, match="lab_id is not set"):
27
+ read_contributor_config(path)
28
+
29
+
30
+ def test_placeholder_lab_id_rejected(tmp_path: Path) -> None:
31
+ path = _write(tmp_path, "lab_id: your_lab_id\n")
32
+ with pytest.raises(RuntimeError, match="placeholder"):
33
+ read_contributor_config(path)
34
+
35
+
36
+ def test_missing_file_gives_clear_error(tmp_path: Path) -> None:
37
+ with pytest.raises(RuntimeError, match="not found"):
38
+ read_contributor_config(tmp_path / "does_not_exist.yaml")
@@ -0,0 +1,75 @@
1
+ """``write_episode_annotations`` — the public API third-party converters write against.
2
+
3
+ It is the one annotation writer that is not driven by the annotation tool, so it has its
4
+ own argument shape and its own consistency rules. The rules changed in v2: the failure
5
+ trio is no longer all-or-nothing, and ``outcome`` joined the numeric ``success``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+
12
+ import h5py
13
+ import pytest
14
+
15
+ from oopsie_data_tools.utils.conversion_utils import write_episode_annotations
16
+
17
+
18
+ def _write(tmp_path, **kwargs) -> dict:
19
+ path = tmp_path / "ep.h5"
20
+ with h5py.File(path, "w") as f:
21
+ write_episode_annotations(f, annotator_name="conv", **kwargs)
22
+ with h5py.File(path, "r") as f:
23
+ return dict(f["episode_annotations"]["conv"].attrs)
24
+
25
+
26
+ def test_a_partial_taxonomy_is_accepted(tmp_path):
27
+ """v1 raised here; v2 lets a converter record whatever it actually knows."""
28
+ attrs = _write(tmp_path, success=0.0, severity="medium")
29
+
30
+ assert json.loads(attrs["taxonomy"])["severity"] == "medium"
31
+ assert attrs["episode_description"] == ""
32
+
33
+
34
+ def test_outcome_defaults_to_the_coarse_reading_of_success(tmp_path):
35
+ assert json.loads(_write(tmp_path, success=1.0)["taxonomy"])["outcome"] == "success"
36
+ assert json.loads(_write(tmp_path, success=0.0)["taxonomy"])["outcome"] == "failure"
37
+
38
+
39
+ def test_a_qualified_success_needs_an_explicit_outcome(tmp_path):
40
+ """The float alone cannot express the two qualified successes."""
41
+ attrs = _write(
42
+ tmp_path,
43
+ success=1.0,
44
+ outcome="success_side_effect",
45
+ side_effect_category=["collision"],
46
+ severity="low",
47
+ )
48
+ taxonomy = json.loads(attrs["taxonomy"])
49
+
50
+ assert taxonomy["outcome"] == "success_side_effect"
51
+ assert taxonomy["side_effect_category"] == ["collision"]
52
+ assert float(attrs["success"]) == 1.0
53
+
54
+
55
+ def test_a_fractional_success_survives(tmp_path):
56
+ """The exact float is authoritative for magnitude, the slug only for the branch."""
57
+ attrs = _write(tmp_path, success=0.75, outcome="success_suboptimal")
58
+
59
+ assert float(attrs["success"]) == 0.75
60
+ assert json.loads(attrs["taxonomy"])["outcome"] == "success_suboptimal"
61
+
62
+
63
+ def test_an_outcome_contradicting_the_success_float_is_rejected(tmp_path):
64
+ with pytest.raises(ValueError, match="disagrees with success"):
65
+ _write(tmp_path, success=1.0, outcome="failure")
66
+
67
+
68
+ def test_an_unrecognized_outcome_is_rejected(tmp_path):
69
+ with pytest.raises(ValueError, match="outcome must be one of"):
70
+ _write(tmp_path, success=0.0, outcome="sort_of_worked")
71
+
72
+
73
+ def test_success_out_of_range_is_rejected(tmp_path):
74
+ with pytest.raises(ValueError, match=r"success must be in \[0.0, 1.0\]"):
75
+ _write(tmp_path, success=1.5)