kleinkram 0.48.0.dev20250723090520__py3-none-any.whl → 0.58.0.dev20260121112512__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kleinkram/api/client.py +6 -18
- kleinkram/api/deser.py +152 -1
- kleinkram/api/file_transfer.py +57 -87
- kleinkram/api/pagination.py +11 -2
- kleinkram/api/query.py +10 -10
- kleinkram/api/routes.py +192 -59
- kleinkram/auth.py +108 -7
- kleinkram/cli/_action.py +131 -0
- kleinkram/cli/_download.py +6 -18
- kleinkram/cli/_endpoint.py +2 -4
- kleinkram/cli/_file.py +6 -18
- kleinkram/cli/_file_validator.py +125 -0
- kleinkram/cli/_list.py +5 -15
- kleinkram/cli/_mission.py +24 -28
- kleinkram/cli/_project.py +10 -26
- kleinkram/cli/_run.py +220 -0
- kleinkram/cli/_upload.py +58 -26
- kleinkram/cli/_verify.py +48 -15
- kleinkram/cli/app.py +56 -17
- kleinkram/cli/error_handling.py +1 -3
- kleinkram/config.py +6 -21
- kleinkram/core.py +19 -36
- kleinkram/errors.py +12 -0
- kleinkram/models.py +49 -0
- kleinkram/printing.py +225 -15
- kleinkram/utils.py +8 -22
- kleinkram/wrappers.py +13 -34
- {kleinkram-0.48.0.dev20250723090520.dist-info → kleinkram-0.58.0.dev20260121112512.dist-info}/METADATA +6 -5
- kleinkram-0.58.0.dev20260121112512.dist-info/RECORD +53 -0
- {kleinkram-0.48.0.dev20250723090520.dist-info → kleinkram-0.58.0.dev20260121112512.dist-info}/WHEEL +1 -1
- {kleinkram-0.48.0.dev20250723090520.dist-info → kleinkram-0.58.0.dev20260121112512.dist-info}/top_level.txt +0 -1
- {testing → tests}/backend_fixtures.py +27 -3
- tests/conftest.py +1 -1
- tests/generate_test_data.py +322 -0
- tests/test_config.py +2 -6
- tests/test_core.py +11 -31
- tests/test_end_to_end.py +3 -5
- tests/test_fixtures.py +3 -5
- tests/test_printing.py +1 -3
- tests/test_utils.py +1 -3
- tests/test_wrappers.py +9 -27
- kleinkram-0.48.0.dev20250723090520.dist-info/RECORD +0 -50
- testing/__init__.py +0 -0
- {kleinkram-0.48.0.dev20250723090520.dist-info → kleinkram-0.58.0.dev20260121112512.dist-info}/entry_points.txt +0 -0
kleinkram/wrappers.py
CHANGED
|
@@ -9,7 +9,8 @@ conversion to the internal representation
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
from pathlib import Path
|
|
12
|
-
from typing import
|
|
12
|
+
from typing import Any
|
|
13
|
+
from typing import Collection
|
|
13
14
|
from typing import Dict
|
|
14
15
|
from typing import List
|
|
15
16
|
from typing import Literal
|
|
@@ -66,9 +67,7 @@ def _args_to_mission_query(
|
|
|
66
67
|
return MissionQuery(
|
|
67
68
|
ids=[parse_uuid_like(_id) for _id in mission_ids or []],
|
|
68
69
|
patterns=list(mission_names or []),
|
|
69
|
-
project_query=_args_to_project_query(
|
|
70
|
-
project_names=project_names, project_ids=project_ids
|
|
71
|
-
),
|
|
70
|
+
project_query=_args_to_project_query(project_names=project_names, project_ids=project_ids),
|
|
72
71
|
)
|
|
73
72
|
|
|
74
73
|
|
|
@@ -85,9 +84,7 @@ def _verify_string_sequence(arg_name: str, arg_value: Optional[Sequence[Any]]) -
|
|
|
85
84
|
if not isinstance(arg_value, Sequence):
|
|
86
85
|
raise TypeError(f"{arg_name} must be a Sequence, None, or empty array.")
|
|
87
86
|
if isinstance(arg_value, str):
|
|
88
|
-
raise TypeError(
|
|
89
|
-
f"{arg_name} cannot be a string, but a sequence of strings."
|
|
90
|
-
)
|
|
87
|
+
raise TypeError(f"{arg_name} cannot be a string, but a sequence of strings.")
|
|
91
88
|
for item in arg_value:
|
|
92
89
|
if not isinstance(item, str):
|
|
93
90
|
raise TypeError(f"{arg_name} must contain strings only.")
|
|
@@ -358,15 +355,11 @@ def create_mission(
|
|
|
358
355
|
|
|
359
356
|
|
|
360
357
|
def create_project(project_name: str, description: str) -> None:
|
|
361
|
-
kleinkram.api.routes._create_project(
|
|
362
|
-
AuthenticatedClient(), project_name, description
|
|
363
|
-
)
|
|
358
|
+
kleinkram.api.routes._create_project(AuthenticatedClient(), project_name, description)
|
|
364
359
|
|
|
365
360
|
|
|
366
361
|
def update_file(file_id: IdLike) -> None:
|
|
367
|
-
kleinkram.core.update_file(
|
|
368
|
-
client=AuthenticatedClient(), file_id=parse_uuid_like(file_id)
|
|
369
|
-
)
|
|
362
|
+
kleinkram.core.update_file(client=AuthenticatedClient(), file_id=parse_uuid_like(file_id))
|
|
370
363
|
|
|
371
364
|
|
|
372
365
|
def update_mission(mission_id: IdLike, metadata: Dict[str, str]) -> None:
|
|
@@ -399,48 +392,34 @@ def delete_file(file_id: IdLike) -> None:
|
|
|
399
392
|
"""\
|
|
400
393
|
delete a single file by id
|
|
401
394
|
"""
|
|
402
|
-
file = kleinkram.api.routes.get_file(
|
|
403
|
-
|
|
404
|
-
)
|
|
405
|
-
kleinkram.api.routes._delete_files(
|
|
406
|
-
AuthenticatedClient(), file_ids=[file.id], mission_id=file.mission_id
|
|
407
|
-
)
|
|
395
|
+
file = kleinkram.api.routes.get_file(AuthenticatedClient(), FileQuery(ids=[parse_uuid_like(file_id)]))
|
|
396
|
+
kleinkram.api.routes._delete_files(AuthenticatedClient(), file_ids=[file.id], mission_id=file.mission_id)
|
|
408
397
|
|
|
409
398
|
|
|
410
399
|
def delete_mission(mission_id: IdLike) -> None:
|
|
411
|
-
kleinkram.core.delete_mission(
|
|
412
|
-
client=AuthenticatedClient(), mission_id=parse_uuid_like(mission_id)
|
|
413
|
-
)
|
|
400
|
+
kleinkram.core.delete_mission(client=AuthenticatedClient(), mission_id=parse_uuid_like(mission_id))
|
|
414
401
|
|
|
415
402
|
|
|
416
403
|
def delete_project(project_id: IdLike) -> None:
|
|
417
|
-
kleinkram.core.delete_project(
|
|
418
|
-
client=AuthenticatedClient(), project_id=parse_uuid_like(project_id)
|
|
419
|
-
)
|
|
404
|
+
kleinkram.core.delete_project(client=AuthenticatedClient(), project_id=parse_uuid_like(project_id))
|
|
420
405
|
|
|
421
406
|
|
|
422
407
|
def get_file(file_id: IdLike) -> File:
|
|
423
408
|
"""\
|
|
424
409
|
get a file by its id
|
|
425
410
|
"""
|
|
426
|
-
return kleinkram.api.routes.get_file(
|
|
427
|
-
AuthenticatedClient(), FileQuery(ids=[parse_uuid_like(file_id)])
|
|
428
|
-
)
|
|
411
|
+
return kleinkram.api.routes.get_file(AuthenticatedClient(), FileQuery(ids=[parse_uuid_like(file_id)]))
|
|
429
412
|
|
|
430
413
|
|
|
431
414
|
def get_mission(mission_id: IdLike) -> Mission:
|
|
432
415
|
"""\
|
|
433
416
|
get a mission by its id
|
|
434
417
|
"""
|
|
435
|
-
return kleinkram.api.routes.get_mission(
|
|
436
|
-
AuthenticatedClient(), MissionQuery(ids=[parse_uuid_like(mission_id)])
|
|
437
|
-
)
|
|
418
|
+
return kleinkram.api.routes.get_mission(AuthenticatedClient(), MissionQuery(ids=[parse_uuid_like(mission_id)]))
|
|
438
419
|
|
|
439
420
|
|
|
440
421
|
def get_project(project_id: IdLike) -> Project:
|
|
441
422
|
"""\
|
|
442
423
|
get a project by its id
|
|
443
424
|
"""
|
|
444
|
-
return kleinkram.api.routes.get_project(
|
|
445
|
-
AuthenticatedClient(), ProjectQuery(ids=[parse_uuid_like(project_id)])
|
|
446
|
-
)
|
|
425
|
+
return kleinkram.api.routes.get_project(AuthenticatedClient(), ProjectQuery(ids=[parse_uuid_like(project_id)]))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kleinkram
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.58.0.dev20260121112512
|
|
4
4
|
Summary: give me your bags
|
|
5
5
|
Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
|
|
6
6
|
Author-email: pucyril@ethz.ch, dgarmier@ethz.ch, jschwab@ethz.ch
|
|
@@ -16,13 +16,14 @@ Requires-Python: >=3.8
|
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
Requires-Dist: boto3
|
|
18
18
|
Requires-Dist: botocore
|
|
19
|
+
Requires-Dist: click
|
|
19
20
|
Requires-Dist: httpx
|
|
20
21
|
Requires-Dist: python-dateutil
|
|
21
22
|
Requires-Dist: pyyaml
|
|
23
|
+
Requires-Dist: requests
|
|
22
24
|
Requires-Dist: rich
|
|
23
25
|
Requires-Dist: tqdm
|
|
24
26
|
Requires-Dist: typer
|
|
25
|
-
Requires-Dist: click
|
|
26
27
|
|
|
27
28
|
# Kleinkram: CLI
|
|
28
29
|
|
|
@@ -76,7 +77,7 @@ Instead of downloading files from a specified mission you can download arbitrary
|
|
|
76
77
|
klein download --dest out *id1* *id2* *id3*
|
|
77
78
|
```
|
|
78
79
|
|
|
79
|
-
For more information consult the [documentation](https://docs.datasets.leggedrobotics.com
|
|
80
|
+
For more information consult the [documentation](https://docs.datasets.leggedrobotics.com//usage/python/setup).
|
|
80
81
|
|
|
81
82
|
## Development
|
|
82
83
|
|
|
@@ -118,7 +119,7 @@ pytest
|
|
|
118
119
|
```
|
|
119
120
|
For the latter you need to have an instance of the backend running locally.
|
|
120
121
|
See instructions in the root of the repository for this.
|
|
121
|
-
On top of that these tests require particular files to be present in the `cli/data
|
|
122
|
-
|
|
122
|
+
On top of that these tests require particular files to be present in the `cli/tests/data` directory.
|
|
123
|
+
These files are automatically generated by the `cli/tests/generate_test_data.py` script.
|
|
123
124
|
|
|
124
125
|
You also need to make sure to be logged in with the cli with `klein login`.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
kleinkram/__init__.py,sha256=xIJqTJw2kbCGryGlCeAdpmtR1FTxmrW1MklUNQEaj74,1061
|
|
2
|
+
kleinkram/__main__.py,sha256=B9RiZxfO4jpCmWPUHyKJ7_EoZlEG4sPpH-nz7T_YhhQ,125
|
|
3
|
+
kleinkram/_version.py,sha256=QYJyRTcqFcJj4qWYpqs7WcoOP6jxDMqyvxLY-cD6KcE,129
|
|
4
|
+
kleinkram/auth.py,sha256=zbjSIPBS5xzLzo75Ou2dgUluXLwBRxKxBdSwtIQdA5g,6835
|
|
5
|
+
kleinkram/config.py,sha256=fCRyDdBKXbH7HQ_Kc3YGTamqGL9zkKLRIIQTQDql0Ec,7273
|
|
6
|
+
kleinkram/core.py,sha256=AhFIxm2jjzRUlGXrOJksebYHOyuioAcbpCx0WmbIpbs,9790
|
|
7
|
+
kleinkram/errors.py,sha256=Bevo8LNPlM46C9JY_uj1Z3FSvPJ051Vyd0M9IYENLtc,1140
|
|
8
|
+
kleinkram/main.py,sha256=BTE0mZN__xd46wBhFi6iBlK9eGGQvJ1LdUMsbnysLi0,172
|
|
9
|
+
kleinkram/models.py,sha256=YB3pcZGJ5nXeZ9gLTbpaaWsUCnvaDIcnvpXUnyWZAaY,2824
|
|
10
|
+
kleinkram/printing.py,sha256=U8I43Ju3orA_jKqF5f-vmgl3cC8Ffjo5m2fgChmY6ik,19163
|
|
11
|
+
kleinkram/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
kleinkram/types.py,sha256=nfDjj8TB1Jn5vqO0Xg6qhLOuKom9DDhe62BrngqnVGM,185
|
|
13
|
+
kleinkram/utils.py,sha256=iPZXUbiG4cYfkL3CG-Rs-dwVovNYkgybfAAvD0oamBY,6725
|
|
14
|
+
kleinkram/wrappers.py,sha256=TprQWvEw-8QsgcDNsQxjBkcEwVFQyfmclTpt8wcZ4tA,12967
|
|
15
|
+
kleinkram/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
+
kleinkram/api/client.py,sha256=j-ZWBsHI-PQOk86RtDfqunaSVyKqCnKdqPMy0tCbdNI,5840
|
|
17
|
+
kleinkram/api/deser.py,sha256=94O1ZeP0kW-LaKaZeP4o1PQLeHbxxZ_C7nU2FzunntY,10077
|
|
18
|
+
kleinkram/api/file_transfer.py,sha256=j1iINYFV4iXMjAz59eM6y8G35Uq4GpPgqf1T0z73Jzs,19710
|
|
19
|
+
kleinkram/api/pagination.py,sha256=NRVGorgNthKi4t_H9lOn87rDebx33WVOvTadA6-4QXw,1693
|
|
20
|
+
kleinkram/api/query.py,sha256=WR_j0WRt9LDW9uyqc8pEdjmgN5eOLVQi1y3LSOh2Y9A,3498
|
|
21
|
+
kleinkram/api/routes.py,sha256=zKgjEiuFP-v06pBnN87yLcdYyuAV4Jx-hbEf44qu_e4,17461
|
|
22
|
+
kleinkram/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
+
kleinkram/cli/_action.py,sha256=Gb3kIs4jcutINX4pmrnOjhetcEpeAGVs3llX0GR9gz0,4911
|
|
24
|
+
kleinkram/cli/_download.py,sha256=n4d9y8ViTLnzrfn8MLaM_6Jlsuh9_Y_F0I-dfAhZEqI,2318
|
|
25
|
+
kleinkram/cli/_endpoint.py,sha256=dFLTaARXP96ide4J4cOHx5qHc3_iFYGJmhz-fKLGbP8,1785
|
|
26
|
+
kleinkram/cli/_file.py,sha256=CiTmy3Di7lVBH7mfsNbWpiwyDl8WGw0xi8MV0-EmAV4,2897
|
|
27
|
+
kleinkram/cli/_file_validator.py,sha256=6SkQGNysq1ydZimfv7MmAqyRP7xiKwaWcoK7R-tL_M4,4552
|
|
28
|
+
kleinkram/cli/_list.py,sha256=jDZlG76b0KWr8imBQdA--V2DC6O3j6jrMd6qyJm6cTw,3028
|
|
29
|
+
kleinkram/cli/_mission.py,sha256=nZjk9gHsTCMkmwbtNMIb9fxfySyggc3nGJdTC0t45G4,5759
|
|
30
|
+
kleinkram/cli/_project.py,sha256=C6mnNJM4oloFto9wDY4zgwdmXNsqR9ErbgZeFdhCENk,3338
|
|
31
|
+
kleinkram/cli/_run.py,sha256=TRx08DPB4PRGIqZl2CjHEDODgQac3Jr7iNQqBN9A7nE,8008
|
|
32
|
+
kleinkram/cli/_upload.py,sha256=PkD9O4ApBo5VUxdt70pu0XdJqZ88FrY_vZWQzTtN0pg,4030
|
|
33
|
+
kleinkram/cli/_verify.py,sha256=j-bRmFqW2j88ZwVOSFEy0Dgok-JY0nAPkcTAOwgxQyY,3211
|
|
34
|
+
kleinkram/cli/app.py,sha256=fpme23Y6pd2LeTadcFYBNfyjDn4AEGltSJVg79GCDAI,8940
|
|
35
|
+
kleinkram/cli/error_handling.py,sha256=ZFcJhPXWITG1cGpnsiuDVVehjIqO5pbz47zWdL1QcQs,1907
|
|
36
|
+
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
37
|
+
tests/backend_fixtures.py,sha256=Kjr1W9NYQ1yCOqLheAIILYeTo9B59WW1M0FMr2MPKew,2308
|
|
38
|
+
tests/conftest.py,sha256=0kbRbC4AJhwjOlZYdwkzQHLBmcrU5pB3o76PImunP4c,102
|
|
39
|
+
tests/generate_test_data.py,sha256=ujp_CoSUr_v9ZrTrYDIjlsOcxqpsc2pzE307z_pS1qY,12116
|
|
40
|
+
tests/test_config.py,sha256=RUIUEqTnAKg8EyqXVGohwfm9jtuaWo1hWqVxZMV6_bU,5843
|
|
41
|
+
tests/test_core.py,sha256=qIPYuMpI3yh2G6OEqqqjRCWlvaM9bd9VC0bB5kI7PJs,5473
|
|
42
|
+
tests/test_end_to_end.py,sha256=mySbCPZPmrszo2AjD18TDitrPAJJ73oz7DaBpTXIkiI,3337
|
|
43
|
+
tests/test_error_handling.py,sha256=qPSMKF1qsAHyUME0-krxbIrk38iGKkhAyAah-KwN4NE,1300
|
|
44
|
+
tests/test_fixtures.py,sha256=SaOe7naSqGH3nivdbwyNo1tcYOg_XU-NJy1cFv3__2o,1053
|
|
45
|
+
tests/test_printing.py,sha256=Kfg0XjGV7ymvI4RgHtNc8wiwpqB3VG0wc1XYWO1ld_k,2217
|
|
46
|
+
tests/test_query.py,sha256=fExmCKXLA7-9j2S2sF_sbvRX_2s6Cp3a7OTcqE25q9g,3864
|
|
47
|
+
tests/test_utils.py,sha256=_DGxsQ60YEKeLNmrwLdJbevSFhWgIK9f38fH02VA8MI,4762
|
|
48
|
+
tests/test_wrappers.py,sha256=Jr7qFbkEBbs21S-SkwF8A5cH-OcFsVz0Y0kBc6gGBuY,2547
|
|
49
|
+
kleinkram-0.58.0.dev20260121112512.dist-info/METADATA,sha256=uZsYFjw4P-9121f4ImhNHVhjqWaAJQIR_OWNu2uNk0U,2862
|
|
50
|
+
kleinkram-0.58.0.dev20260121112512.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
|
|
51
|
+
kleinkram-0.58.0.dev20260121112512.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
|
|
52
|
+
kleinkram-0.58.0.dev20260121112512.dist-info/top_level.txt,sha256=G1Lj9vHAtZn402Ukkrfll-6BCmnDNy_HVtWeNvXzdDA,16
|
|
53
|
+
kleinkram-0.58.0.dev20260121112512.dist-info/RECORD,,
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
3
5
|
import time
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
from secrets import token_hex
|
|
@@ -14,7 +16,7 @@ from kleinkram import list_projects
|
|
|
14
16
|
from kleinkram import upload
|
|
15
17
|
|
|
16
18
|
# we expect the mission files to be in this folder that is not commited to the repo
|
|
17
|
-
DATA_PATH = Path(__file__).parent
|
|
19
|
+
DATA_PATH = Path(__file__).parent / "data"
|
|
18
20
|
DATA_FILES = [
|
|
19
21
|
DATA_PATH / "10_KB.bag",
|
|
20
22
|
DATA_PATH / "50_KB.bag",
|
|
@@ -26,7 +28,7 @@ DATA_FILES = [
|
|
|
26
28
|
PROJECT_DESCRIPTION = "This is a test project"
|
|
27
29
|
|
|
28
30
|
|
|
29
|
-
|
|
31
|
+
WAIT_BEFORE_DELETION = 5
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
@pytest.fixture(scope="session")
|
|
@@ -38,7 +40,7 @@ def project():
|
|
|
38
40
|
|
|
39
41
|
yield project
|
|
40
42
|
|
|
41
|
-
time.sleep(
|
|
43
|
+
time.sleep(WAIT_BEFORE_DELETION)
|
|
42
44
|
delete_project(project.id)
|
|
43
45
|
|
|
44
46
|
|
|
@@ -63,3 +65,25 @@ def empty_mission(project):
|
|
|
63
65
|
mission = list_missions(project_ids=[project.id], mission_names=[mission_name])[0]
|
|
64
66
|
|
|
65
67
|
yield mission
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@pytest.fixture(scope="session", autouse=True)
|
|
71
|
+
def auto_login():
|
|
72
|
+
"""
|
|
73
|
+
Automatically logs in using the CLI before running any tests.
|
|
74
|
+
"""
|
|
75
|
+
try:
|
|
76
|
+
# Run: python3 -m kleinkram login --user 1
|
|
77
|
+
result = subprocess.run(
|
|
78
|
+
[sys.executable, "-m", "kleinkram", "login", "--user", "1"],
|
|
79
|
+
cwd=str(Path(__file__).parent.parent), # Run from cli root
|
|
80
|
+
capture_output=True,
|
|
81
|
+
text=True,
|
|
82
|
+
)
|
|
83
|
+
if result.returncode != 0:
|
|
84
|
+
pytest.fail(
|
|
85
|
+
f"Failed to auto-login. Return code: {result.returncode}\nStdout: {result.stdout}\nStderr: {result.stderr}"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
except Exception as e:
|
|
89
|
+
pytest.fail(f"Failed to auto-login: {e}")
|
tests/conftest.py
CHANGED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import struct
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from rosbags.rosbag1 import Writer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def serialize_string(data):
|
|
11
|
+
"""Serialize a string for ROS1 (std_msgs/String)."""
|
|
12
|
+
encoded = data.encode("utf-8")
|
|
13
|
+
return struct.pack("<I", len(encoded)) + encoded
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def serialize_time(secs, nsecs):
|
|
17
|
+
return struct.pack("<II", secs, nsecs)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def serialize_header(seq, secs, nsecs, frame_id):
|
|
21
|
+
return struct.pack("<I", seq) + serialize_time(secs, nsecs) + serialize_string(frame_id)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def serialize_log(seq, secs, nsecs, frame_id, level, name, msg, file, function, line, topics):
|
|
25
|
+
# rosgraph_msgs/Log
|
|
26
|
+
# Header header
|
|
27
|
+
# byte level
|
|
28
|
+
# string name
|
|
29
|
+
# string msg
|
|
30
|
+
# string file
|
|
31
|
+
# string function
|
|
32
|
+
# uint32 line
|
|
33
|
+
# string[] topics
|
|
34
|
+
data = serialize_header(seq, secs, nsecs, frame_id)
|
|
35
|
+
data += struct.pack("<b", level)
|
|
36
|
+
data += serialize_string(name)
|
|
37
|
+
data += serialize_string(msg)
|
|
38
|
+
data += serialize_string(file)
|
|
39
|
+
data += serialize_string(function)
|
|
40
|
+
data += struct.pack("<I", line)
|
|
41
|
+
data += struct.pack("<I", len(topics))
|
|
42
|
+
for t in topics:
|
|
43
|
+
data += serialize_string(t)
|
|
44
|
+
return data
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def serialize_temperature(seq, secs, nsecs, frame_id, temp, variance):
|
|
48
|
+
# sensor_msgs/Temperature
|
|
49
|
+
# Header header
|
|
50
|
+
# float64 temperature
|
|
51
|
+
# float64 variance
|
|
52
|
+
data = serialize_header(seq, secs, nsecs, frame_id)
|
|
53
|
+
data += struct.pack("<dd", temp, variance)
|
|
54
|
+
return data
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def serialize_time_reference(seq, secs, nsecs, frame_id, ref_secs, ref_nsecs, source):
|
|
58
|
+
# sensor_msgs/TimeReference
|
|
59
|
+
# Header header
|
|
60
|
+
# time time_ref
|
|
61
|
+
# string source
|
|
62
|
+
data = serialize_header(seq, secs, nsecs, frame_id)
|
|
63
|
+
data += serialize_time(ref_secs, ref_nsecs)
|
|
64
|
+
data += serialize_string(source)
|
|
65
|
+
return data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def serialize_twist_stamped(seq, secs, nsecs, frame_id, linear, angular):
|
|
69
|
+
# geometry_msgs/TwistStamped
|
|
70
|
+
# Header header
|
|
71
|
+
# Twist twist (Vector3 linear, Vector3 angular)
|
|
72
|
+
data = serialize_header(seq, secs, nsecs, frame_id)
|
|
73
|
+
data += struct.pack("<ddd", linear[0], linear[1], linear[2])
|
|
74
|
+
data += struct.pack("<ddd", angular[0], angular[1], angular[2])
|
|
75
|
+
return data
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def serialize_tf_message(transforms):
|
|
79
|
+
# tf2_msgs/TFMessage
|
|
80
|
+
# geometry_msgs/TransformStamped[] transforms
|
|
81
|
+
data = struct.pack("<I", len(transforms))
|
|
82
|
+
for t in transforms:
|
|
83
|
+
# TransformStamped
|
|
84
|
+
# Header header
|
|
85
|
+
# string child_frame_id
|
|
86
|
+
# Transform transform (Vector3 translation, Quaternion rotation)
|
|
87
|
+
data += serialize_header(t["seq"], t["secs"], t["nsecs"], t["frame_id"])
|
|
88
|
+
data += serialize_string(t["child_frame_id"])
|
|
89
|
+
data += struct.pack("<ddd", t["tx"], t["ty"], t["tz"])
|
|
90
|
+
data += struct.pack("<dddd", t["rx"], t["ry"], t["rz"], t["rw"])
|
|
91
|
+
return data
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def generate_bag(filename, target_size):
|
|
95
|
+
# Adjust payload to be smaller for small files
|
|
96
|
+
payload_size = 1024
|
|
97
|
+
if target_size > 1024 * 1024:
|
|
98
|
+
payload_size = 1024 * 1024
|
|
99
|
+
|
|
100
|
+
if target_size < 2000: # Very small files
|
|
101
|
+
payload_size = 100
|
|
102
|
+
|
|
103
|
+
payload = "x" * payload_size
|
|
104
|
+
serialized_msg = serialize_string(payload)
|
|
105
|
+
|
|
106
|
+
# Calculate number of messages needed
|
|
107
|
+
# Approximate overhead per message in bag file is ~30-50 bytes (record header) + connection ref
|
|
108
|
+
# We'll assume overhead is small compared to payload for large files, but significant for small ones.
|
|
109
|
+
# We'll just write until we think we are close.
|
|
110
|
+
|
|
111
|
+
msg_size = len(serialized_msg) + 50 # rough estimate including record headers
|
|
112
|
+
num_msgs = int(target_size / msg_size) + 1
|
|
113
|
+
if num_msgs < 1:
|
|
114
|
+
num_msgs = 1
|
|
115
|
+
|
|
116
|
+
print(f"Generating {filename} (~{target_size} bytes) with {num_msgs} messages of payload {payload_size}...")
|
|
117
|
+
|
|
118
|
+
if os.path.exists(filename):
|
|
119
|
+
os.remove(filename)
|
|
120
|
+
|
|
121
|
+
with Writer(filename) as writer:
|
|
122
|
+
# Add a connection
|
|
123
|
+
# msg_def for std_msgs/String is just "string data"
|
|
124
|
+
conn = writer.add_connection(
|
|
125
|
+
topic="/test_topic",
|
|
126
|
+
msgtype="std_msgs/msg/String",
|
|
127
|
+
msgdef="string data",
|
|
128
|
+
md5sum="992ce8a1687cec8c8bd883ec73ca41d1",
|
|
129
|
+
)
|
|
130
|
+
timestamp = 1000
|
|
131
|
+
for i in range(num_msgs):
|
|
132
|
+
writer.write(conn, timestamp + i, serialized_msg)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def generate_frontend_bag(filename):
|
|
136
|
+
print(f"Generating frontend test bag: {filename}")
|
|
137
|
+
if os.path.exists(filename):
|
|
138
|
+
os.remove(filename)
|
|
139
|
+
|
|
140
|
+
with Writer(filename) as writer:
|
|
141
|
+
# 1. rosgraph_msgs/Log
|
|
142
|
+
conn_log = writer.add_connection(
|
|
143
|
+
topic="/rosout",
|
|
144
|
+
msgtype="rosgraph_msgs/msg/Log",
|
|
145
|
+
msgdef="Header header\nbyte level\nstring name\nstring msg\nstring file\nstring function\nuint32 "
|
|
146
|
+
"line\nstring[] topics\n================================================================================\n"
|
|
147
|
+
"MSG: std_msgs/Header\nuint32 seq\ntime stamp\nstring frame_id",
|
|
148
|
+
md5sum="acffd30cd6b6de30f120938c17c593fb",
|
|
149
|
+
)
|
|
150
|
+
# 2. sensor_msgs/Temperature
|
|
151
|
+
conn_temp = writer.add_connection(
|
|
152
|
+
topic="/sensors/temperature",
|
|
153
|
+
msgtype="sensor_msgs/msg/Temperature",
|
|
154
|
+
msgdef="Header header\nfloat64 temperature\nfloat64 variance"
|
|
155
|
+
"\n================================================================================\n"
|
|
156
|
+
"MSG: std_msgs/Header\nuint32 seq\ntime stamp\nstring frame_id",
|
|
157
|
+
md5sum="ff71b307acdbe7c871a5a6d7edce2f6e",
|
|
158
|
+
)
|
|
159
|
+
# 3. sensor_msgs/TimeReference
|
|
160
|
+
conn_time = writer.add_connection(
|
|
161
|
+
topic="/time_ref",
|
|
162
|
+
msgtype="sensor_msgs/msg/TimeReference",
|
|
163
|
+
msgdef="Header header\ntime time_ref\nstring source\n"
|
|
164
|
+
"================================================================================\n"
|
|
165
|
+
"MSG: std_msgs/Header\nuint32 seq\ntime stamp\nstring frame_id",
|
|
166
|
+
md5sum="fded64a0265108ba86c3d38fb11c0c16",
|
|
167
|
+
)
|
|
168
|
+
# 4. geometry_msgs/TwistStamped
|
|
169
|
+
conn_twist = writer.add_connection(
|
|
170
|
+
topic="/cmd_vel",
|
|
171
|
+
msgtype="geometry_msgs/msg/TwistStamped",
|
|
172
|
+
msgdef="Header header\ngeometry_msgs/Twist twist\n"
|
|
173
|
+
"================================================================================\n"
|
|
174
|
+
"MSG: std_msgs/Header\nuint32 seq\ntime stamp\nstring frame_id\n"
|
|
175
|
+
"================================================================================\n"
|
|
176
|
+
"MSG: geometry_msgs/Twist\nVector3 linear\nVector3 angular\n"
|
|
177
|
+
"================================================================================\n"
|
|
178
|
+
"MSG: geometry_msgs/Vector3\nfloat64 x\nfloat64 y\nfloat64 z",
|
|
179
|
+
md5sum="98d34b0043a2093cf9d9345ab6eef12e",
|
|
180
|
+
)
|
|
181
|
+
# 5. tf2_msgs/TFMessage
|
|
182
|
+
conn_tf = writer.add_connection(
|
|
183
|
+
topic="/tf",
|
|
184
|
+
msgtype="tf2_msgs/msg/TFMessage",
|
|
185
|
+
msgdef="geometry_msgs/TransformStamped[] transforms\n"
|
|
186
|
+
"================================================================================\n"
|
|
187
|
+
"MSG: geometry_msgs/TransformStamped\nHeader header\nstring child_frame_id\n"
|
|
188
|
+
"geometry_msgs/Transform transform\n"
|
|
189
|
+
"================================================================================\n"
|
|
190
|
+
"MSG: std_msgs/Header\nuint32 seq\ntime stamp\nstring frame_id\n"
|
|
191
|
+
"================================================================================\n"
|
|
192
|
+
"MSG: geometry_msgs/Transform\ngeometry_msgs/Vector3 translation\n"
|
|
193
|
+
"geometry_msgs/Quaternion rotation\n"
|
|
194
|
+
"================================================================================\n"
|
|
195
|
+
"MSG: geometry_msgs/Vector3\nfloat64 x\nfloat64 y\nfloat64 z\n"
|
|
196
|
+
"================================================================================\n"
|
|
197
|
+
"MSG: geometry_msgs/Quaternion\nfloat64 x\nfloat64 y\nfloat64 z\nfloat64 w",
|
|
198
|
+
md5sum="94810edda583a504dfda3829e70d7eec",
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# Write messages
|
|
202
|
+
for i in range(100):
|
|
203
|
+
timestamp = 1000 + i * 100000000 # 100ms steps
|
|
204
|
+
secs = int(timestamp / 1000000000)
|
|
205
|
+
nsecs = timestamp % 1000000000
|
|
206
|
+
|
|
207
|
+
# Log
|
|
208
|
+
writer.write(
|
|
209
|
+
conn_log,
|
|
210
|
+
timestamp,
|
|
211
|
+
serialize_log(
|
|
212
|
+
i,
|
|
213
|
+
secs,
|
|
214
|
+
nsecs,
|
|
215
|
+
"",
|
|
216
|
+
2,
|
|
217
|
+
"test_node",
|
|
218
|
+
f"Log message {i}",
|
|
219
|
+
"test.cpp",
|
|
220
|
+
"main",
|
|
221
|
+
i,
|
|
222
|
+
["/rosout"],
|
|
223
|
+
),
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# Temperature (sine wave)
|
|
227
|
+
import math
|
|
228
|
+
|
|
229
|
+
temp = 20.0 + 5.0 * math.sin(i * 0.1)
|
|
230
|
+
writer.write(
|
|
231
|
+
conn_temp,
|
|
232
|
+
timestamp,
|
|
233
|
+
serialize_temperature(i, secs, nsecs, "sensor_frame", temp, 0.1),
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
# TimeReference
|
|
237
|
+
writer.write(
|
|
238
|
+
conn_time,
|
|
239
|
+
timestamp,
|
|
240
|
+
serialize_time_reference(i, secs, nsecs, "time_frame", secs, nsecs, "GPS"),
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
# TwistStamped (circle)
|
|
244
|
+
writer.write(
|
|
245
|
+
conn_twist,
|
|
246
|
+
timestamp,
|
|
247
|
+
serialize_twist_stamped(i, secs, nsecs, "base_link", [1.0, 0.0, 0.0], [0.0, 0.0, 0.5]),
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
# TF
|
|
251
|
+
writer.write(
|
|
252
|
+
conn_tf,
|
|
253
|
+
timestamp,
|
|
254
|
+
serialize_tf_message(
|
|
255
|
+
[
|
|
256
|
+
{
|
|
257
|
+
"seq": i,
|
|
258
|
+
"secs": secs,
|
|
259
|
+
"nsecs": nsecs,
|
|
260
|
+
"frame_id": "map",
|
|
261
|
+
"child_frame_id": "base_link",
|
|
262
|
+
"tx": i * 0.1,
|
|
263
|
+
"ty": 0.0,
|
|
264
|
+
"tz": 0.0,
|
|
265
|
+
"rx": 0.0,
|
|
266
|
+
"ry": 0.0,
|
|
267
|
+
"rz": 0.0,
|
|
268
|
+
"rw": 1.0,
|
|
269
|
+
}
|
|
270
|
+
]
|
|
271
|
+
),
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def main():
|
|
276
|
+
data_dir = os.path.join(os.path.dirname(__file__), "data")
|
|
277
|
+
os.makedirs(data_dir, exist_ok=True)
|
|
278
|
+
|
|
279
|
+
files = {
|
|
280
|
+
"10_KB.bag": 10 * 1024,
|
|
281
|
+
"50_KB.bag": 50 * 1024,
|
|
282
|
+
"1_MB.bag": 1 * 1024 * 1024,
|
|
283
|
+
"17_MB.bag": 17 * 1024 * 1024,
|
|
284
|
+
"125_MB.bag": 125 * 1024 * 1024,
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
for filename, size in files.items():
|
|
288
|
+
filepath = os.path.join(data_dir, filename)
|
|
289
|
+
generate_bag(filepath, size)
|
|
290
|
+
|
|
291
|
+
# Generate backend fixtures
|
|
292
|
+
backend_fixtures_dir = os.path.join(os.path.dirname(__file__), "../../backend/tests/fixtures")
|
|
293
|
+
os.makedirs(backend_fixtures_dir, exist_ok=True)
|
|
294
|
+
generate_bag(os.path.join(backend_fixtures_dir, "test.bag"), 10 * 1024)
|
|
295
|
+
generate_bag(os.path.join(backend_fixtures_dir, "to_delete.bag"), 10 * 1024)
|
|
296
|
+
generate_bag(os.path.join(backend_fixtures_dir, "file1.bag"), 10 * 1024)
|
|
297
|
+
generate_bag(os.path.join(backend_fixtures_dir, "file2.bag"), 10 * 1024)
|
|
298
|
+
generate_bag(os.path.join(backend_fixtures_dir, "move_me.bag"), 10 * 1024)
|
|
299
|
+
generate_bag(os.path.join(backend_fixtures_dir, "state_test.bag"), 10 * 1024)
|
|
300
|
+
|
|
301
|
+
# Generate backend dummy MCAP and YAML
|
|
302
|
+
with open(os.path.join(backend_fixtures_dir, "config.yaml"), "w") as f:
|
|
303
|
+
f.write("test: true\nvalue: 123\n")
|
|
304
|
+
with open(os.path.join(backend_fixtures_dir, "config.yml"), "w") as f:
|
|
305
|
+
f.write("test: true\nvalue: 123\n")
|
|
306
|
+
with open(os.path.join(backend_fixtures_dir, "test.mcap"), "wb") as f:
|
|
307
|
+
f.write(b"\x89MCAP\x30\r\n")
|
|
308
|
+
|
|
309
|
+
generate_frontend_bag(os.path.join(data_dir, "frontend_test.bag"))
|
|
310
|
+
|
|
311
|
+
# Generate dummy MCAP and YAML
|
|
312
|
+
with open(os.path.join(data_dir, "test.yaml"), "w") as f:
|
|
313
|
+
f.write("test: true\nvalue: 123\n")
|
|
314
|
+
|
|
315
|
+
with open(os.path.join(data_dir, "test.mcap"), "wb") as f:
|
|
316
|
+
f.write(b"\x89MCAP\x30\r\n") # Minimal magic bytes
|
|
317
|
+
|
|
318
|
+
print("Done.")
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
if __name__ == "__main__":
|
|
322
|
+
main()
|
tests/test_config.py
CHANGED
|
@@ -72,9 +72,7 @@ def test_load_config_default(config_path):
|
|
|
72
72
|
assert config.selected_endpoint == get_env().value
|
|
73
73
|
|
|
74
74
|
|
|
75
|
-
def test_load_default_config_with_env_var_api_key_specified(
|
|
76
|
-
config_path, set_api_key_env
|
|
77
|
-
):
|
|
75
|
+
def test_load_default_config_with_env_var_api_key_specified(config_path, set_api_key_env):
|
|
78
76
|
assert set_api_key_env is None
|
|
79
77
|
|
|
80
78
|
config = _load_config(path=config_path)
|
|
@@ -87,9 +85,7 @@ def test_load_default_config_with_env_var_api_key_specified(
|
|
|
87
85
|
assert not config_path.exists()
|
|
88
86
|
|
|
89
87
|
|
|
90
|
-
def test_load_default_config_with_env_var_endpoints_specified(
|
|
91
|
-
config_path, set_endpoint_env
|
|
92
|
-
):
|
|
88
|
+
def test_load_default_config_with_env_var_endpoints_specified(config_path, set_endpoint_env):
|
|
93
89
|
assert set_endpoint_env is None
|
|
94
90
|
config = _load_config(path=config_path)
|
|
95
91
|
|