kleinkram 0.38.1.dev20250207122632__py3-none-any.whl → 0.38.1.dev20250218095010__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kleinkram might be problematic. Click here for more details.
- kleinkram/api/client.py +54 -7
- kleinkram/api/file_transfer.py +3 -3
- kleinkram/api/routes.py +10 -10
- kleinkram/auth.py +1 -1
- kleinkram/cli/_file.py +94 -0
- kleinkram/cli/app.py +6 -2
- kleinkram/config.py +64 -14
- kleinkram/printing.py +36 -0
- {kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/METADATA +10 -5
- {kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/RECORD +16 -15
- testing/backend_fixtures.py +0 -3
- tests/test_config.py +66 -2
- tests/test_end_to_end.py +6 -0
- {kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/WHEEL +0 -0
- {kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/entry_points.txt +0 -0
- {kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/top_level.txt +0 -0
kleinkram/api/client.py
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
from collections import abc
|
|
4
5
|
from threading import Lock
|
|
5
6
|
from typing import Any
|
|
7
|
+
from typing import List
|
|
8
|
+
from typing import Mapping
|
|
9
|
+
from typing import Sequence
|
|
10
|
+
from typing import Tuple
|
|
11
|
+
from typing import Union
|
|
6
12
|
|
|
7
13
|
import httpx
|
|
14
|
+
from httpx._types import PrimitiveData
|
|
8
15
|
|
|
9
16
|
from kleinkram.config import Config
|
|
10
17
|
from kleinkram.config import Credentials
|
|
@@ -17,10 +24,41 @@ logger = logging.getLogger(__name__)
|
|
|
17
24
|
|
|
18
25
|
COOKIE_AUTH_TOKEN = "authtoken"
|
|
19
26
|
COOKIE_REFRESH_TOKEN = "refreshtoken"
|
|
20
|
-
|
|
27
|
+
COOKIE_API_KEY = "clikey"
|
|
21
28
|
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
Data = Union[PrimitiveData, Any]
|
|
31
|
+
NestedData = Mapping[str, Data]
|
|
32
|
+
ListData = Sequence[Data]
|
|
33
|
+
QueryParams = Mapping[str, Union[Data, NestedData, ListData]]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _convert_nested_data_query_params_values(
|
|
37
|
+
key: str, values: NestedData
|
|
38
|
+
) -> List[Tuple[str, Data]]:
|
|
39
|
+
return [(f"{key}[{k}]", v) for k, v in values.items()]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _convert_list_data_query_params_values(
|
|
43
|
+
key: str, values: ListData
|
|
44
|
+
) -> List[Tuple[str, Data]]:
|
|
45
|
+
return [(key, value) for value in values]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _convert_query_params_to_httpx_format(
|
|
49
|
+
params: QueryParams,
|
|
50
|
+
) -> List[Tuple[str, Data]]:
|
|
51
|
+
ret: List[Tuple[str, Data]] = []
|
|
52
|
+
for key, value in params.items():
|
|
53
|
+
if isinstance(value, (str, int, float, bool)) or value is None:
|
|
54
|
+
ret.append((key, value))
|
|
55
|
+
elif isinstance(value, abc.Mapping):
|
|
56
|
+
ret.extend(_convert_nested_data_query_params_values(key, value))
|
|
57
|
+
elif isinstance(value, abc.Sequence):
|
|
58
|
+
ret.extend(_convert_list_data_query_params_values(key, value))
|
|
59
|
+
else: # TODO: handle this better
|
|
60
|
+
ret.append((key, str(value)))
|
|
61
|
+
return ret
|
|
24
62
|
|
|
25
63
|
|
|
26
64
|
class AuthenticatedClient(httpx.Client):
|
|
@@ -36,9 +74,9 @@ class AuthenticatedClient(httpx.Client):
|
|
|
36
74
|
if self._config.credentials is None:
|
|
37
75
|
logger.info("not authenticated...")
|
|
38
76
|
raise NotAuthenticated
|
|
39
|
-
elif (
|
|
77
|
+
elif (api_key := self._config.credentials.api_key) is not None:
|
|
40
78
|
logger.info("using cli key...")
|
|
41
|
-
self.cookies.set(
|
|
79
|
+
self.cookies.set(COOKIE_API_KEY, api_key)
|
|
42
80
|
else:
|
|
43
81
|
logger.info("using refresh token...")
|
|
44
82
|
assert self._config.credentials.auth_token is not None, "unreachable"
|
|
@@ -48,7 +86,7 @@ class AuthenticatedClient(httpx.Client):
|
|
|
48
86
|
if self._config.credentials is None:
|
|
49
87
|
raise NotAuthenticated
|
|
50
88
|
|
|
51
|
-
if self._config.credentials.
|
|
89
|
+
if self._config.credentials.api_key is not None:
|
|
52
90
|
raise RuntimeError("cannot refresh token when using cli key auth")
|
|
53
91
|
|
|
54
92
|
refresh_token = self._config.credentials.refresh_token
|
|
@@ -73,7 +111,12 @@ class AuthenticatedClient(httpx.Client):
|
|
|
73
111
|
self.cookies.set(COOKIE_AUTH_TOKEN, new_access_token)
|
|
74
112
|
|
|
75
113
|
def request(
|
|
76
|
-
self,
|
|
114
|
+
self,
|
|
115
|
+
method: str,
|
|
116
|
+
url: str | httpx.URL,
|
|
117
|
+
params: QueryParams | None = None,
|
|
118
|
+
*args: Any,
|
|
119
|
+
**kwargs: Any,
|
|
77
120
|
) -> httpx.Response:
|
|
78
121
|
if isinstance(url, httpx.URL):
|
|
79
122
|
raise NotImplementedError(f"`httpx.URL` is not supported {url!r}")
|
|
@@ -83,7 +126,11 @@ class AuthenticatedClient(httpx.Client):
|
|
|
83
126
|
# try to do a request
|
|
84
127
|
full_url = f"{self._config.endpoint.api}{url}"
|
|
85
128
|
logger.info(f"requesting {method} {full_url}")
|
|
86
|
-
|
|
129
|
+
|
|
130
|
+
httpx_params = _convert_query_params_to_httpx_format(params or {})
|
|
131
|
+
response = super().request(
|
|
132
|
+
method, full_url, params=httpx_params, *args, **kwargs
|
|
133
|
+
)
|
|
87
134
|
|
|
88
135
|
logger.info(f"got response {response}")
|
|
89
136
|
|
kleinkram/api/file_transfer.py
CHANGED
|
@@ -32,12 +32,12 @@ from kleinkram.utils import styled_string
|
|
|
32
32
|
|
|
33
33
|
logger = logging.getLogger(__name__)
|
|
34
34
|
|
|
35
|
-
UPLOAD_CREDS = "/
|
|
35
|
+
UPLOAD_CREDS = "/files/temporaryAccess"
|
|
36
36
|
UPLOAD_CONFIRM = "/queue/confirmUpload"
|
|
37
|
-
UPLOAD_CANCEL = "/
|
|
37
|
+
UPLOAD_CANCEL = "/files/cancelUpload"
|
|
38
38
|
|
|
39
39
|
DOWNLOAD_CHUNK_SIZE = 1024 * 1024 * 16
|
|
40
|
-
DOWNLOAD_URL = "/
|
|
40
|
+
DOWNLOAD_URL = "/files/download"
|
|
41
41
|
|
|
42
42
|
S3_MAX_RETRIES = 60 # same as frontend
|
|
43
43
|
S3_READ_TIMEOUT = 60 * 5 # 5 minutes
|
kleinkram/api/routes.py
CHANGED
|
@@ -65,15 +65,15 @@ __all__ = [
|
|
|
65
65
|
CLAIM_ADMIN = "/user/claimAdmin"
|
|
66
66
|
GET_STATUS = "/user/me"
|
|
67
67
|
|
|
68
|
-
UPDATE_PROJECT = "/
|
|
69
|
-
UPDATE_MISSION = "/
|
|
70
|
-
CREATE_MISSION = "/
|
|
71
|
-
CREATE_PROJECT = "/
|
|
68
|
+
UPDATE_PROJECT = "/projects"
|
|
69
|
+
UPDATE_MISSION = "/missions/tags" # TODO: just metadata for now
|
|
70
|
+
CREATE_MISSION = "/missions/create"
|
|
71
|
+
CREATE_PROJECT = "/projects"
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
FILE_ENDPOINT = "/
|
|
75
|
-
MISSION_ENDPOINT = "/
|
|
76
|
-
PROJECT_ENDPOINT = "/
|
|
74
|
+
FILE_ENDPOINT = "/files"
|
|
75
|
+
MISSION_ENDPOINT = "/missions"
|
|
76
|
+
PROJECT_ENDPOINT = "/projects"
|
|
77
77
|
|
|
78
78
|
TAG_TYPE_BY_NAME = "/tag/filtered"
|
|
79
79
|
|
|
@@ -370,7 +370,7 @@ def _claim_admin(client: AuthenticatedClient) -> None:
|
|
|
370
370
|
return
|
|
371
371
|
|
|
372
372
|
|
|
373
|
-
FILE_DELETE_MANY = "/
|
|
373
|
+
FILE_DELETE_MANY = "/files/deleteMultiple"
|
|
374
374
|
|
|
375
375
|
|
|
376
376
|
def _delete_files(
|
|
@@ -384,7 +384,7 @@ def _delete_files(
|
|
|
384
384
|
resp.raise_for_status()
|
|
385
385
|
|
|
386
386
|
|
|
387
|
-
MISSION_DELETE_ONE = "/
|
|
387
|
+
MISSION_DELETE_ONE = "/missions/{}"
|
|
388
388
|
|
|
389
389
|
|
|
390
390
|
def _delete_mission(client: AuthenticatedClient, mission_id: UUID) -> None:
|
|
@@ -396,7 +396,7 @@ def _delete_mission(client: AuthenticatedClient, mission_id: UUID) -> None:
|
|
|
396
396
|
resp.raise_for_status()
|
|
397
397
|
|
|
398
398
|
|
|
399
|
-
PROJECT_DELETE_ONE = "/
|
|
399
|
+
PROJECT_DELETE_ONE = "/projects/{}"
|
|
400
400
|
|
|
401
401
|
|
|
402
402
|
def _delete_project(client: AuthenticatedClient, project_id: UUID) -> None:
|
kleinkram/auth.py
CHANGED
|
@@ -84,7 +84,7 @@ def login_flow(*, key: Optional[str] = None, headless: bool = False) -> None:
|
|
|
84
84
|
config = get_config()
|
|
85
85
|
# use cli key login
|
|
86
86
|
if key is not None:
|
|
87
|
-
config.credentials = Credentials(
|
|
87
|
+
config.credentials = Credentials(api_key=key)
|
|
88
88
|
save_config(config)
|
|
89
89
|
return
|
|
90
90
|
|
kleinkram/cli/_file.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
import kleinkram.api.routes
|
|
8
|
+
import kleinkram.core
|
|
9
|
+
from kleinkram.api.client import AuthenticatedClient
|
|
10
|
+
from kleinkram.api.query import FileQuery
|
|
11
|
+
from kleinkram.api.query import MissionQuery
|
|
12
|
+
from kleinkram.api.query import ProjectQuery
|
|
13
|
+
from kleinkram.api.routes import get_file
|
|
14
|
+
from kleinkram.config import get_shared_state
|
|
15
|
+
from kleinkram.printing import print_file_info
|
|
16
|
+
from kleinkram.utils import split_args
|
|
17
|
+
|
|
18
|
+
INFO_HELP = "get information about a file"
|
|
19
|
+
DELETE_HELP = "delete a file"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
file_typer = typer.Typer(
|
|
23
|
+
no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]}
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@file_typer.command(help=INFO_HELP)
|
|
28
|
+
def info(
|
|
29
|
+
project: Optional[str] = typer.Option(
|
|
30
|
+
None, "--project", "-p", help="project id or name"
|
|
31
|
+
),
|
|
32
|
+
mission: Optional[str] = typer.Option(
|
|
33
|
+
None, "--mission", "-m", help="mission id or name"
|
|
34
|
+
),
|
|
35
|
+
file: str = typer.Option(..., "--file", "-f", help="file id or name"),
|
|
36
|
+
) -> None:
|
|
37
|
+
project_ids, project_patterns = split_args([project] if project else [])
|
|
38
|
+
project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
|
|
39
|
+
|
|
40
|
+
mission_ids, mission_patterns = split_args([mission] if mission else [])
|
|
41
|
+
mission_query = MissionQuery(
|
|
42
|
+
ids=mission_ids,
|
|
43
|
+
patterns=mission_patterns,
|
|
44
|
+
project_query=project_query,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
file_ids, file_patterns = split_args([file])
|
|
48
|
+
file_query = FileQuery(
|
|
49
|
+
ids=file_ids,
|
|
50
|
+
patterns=file_patterns,
|
|
51
|
+
mission_query=mission_query,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
client = AuthenticatedClient()
|
|
55
|
+
file_parsed = get_file(client, file_query)
|
|
56
|
+
print_file_info(file_parsed, pprint=get_shared_state().verbose)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@file_typer.command(help=DELETE_HELP)
|
|
60
|
+
def delete(
|
|
61
|
+
project: Optional[str] = typer.Option(
|
|
62
|
+
None, "--project", "-p", help="project id or name"
|
|
63
|
+
),
|
|
64
|
+
mission: Optional[str] = typer.Option(
|
|
65
|
+
None, "--mission", "-m", help="mission id or name"
|
|
66
|
+
),
|
|
67
|
+
file: str = typer.Option(..., "--file", "-f", help="file id or name"),
|
|
68
|
+
confirm: bool = typer.Option(
|
|
69
|
+
False, "--confirm", "-y", "--yes", help="confirm deletion"
|
|
70
|
+
),
|
|
71
|
+
) -> None:
|
|
72
|
+
if not confirm:
|
|
73
|
+
typer.confirm(f"delete {project} {mission}", abort=True)
|
|
74
|
+
|
|
75
|
+
project_ids, project_patterns = split_args([project] if project else [])
|
|
76
|
+
project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
|
|
77
|
+
|
|
78
|
+
mission_ids, mission_patterns = split_args([mission] if mission else [])
|
|
79
|
+
mission_query = MissionQuery(
|
|
80
|
+
ids=mission_ids,
|
|
81
|
+
patterns=mission_patterns,
|
|
82
|
+
project_query=project_query,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
file_ids, file_patterns = split_args([file])
|
|
86
|
+
file_query = FileQuery(
|
|
87
|
+
ids=file_ids,
|
|
88
|
+
patterns=file_patterns,
|
|
89
|
+
mission_query=mission_query,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
client = AuthenticatedClient()
|
|
93
|
+
file_parsed = get_file(client, file_query)
|
|
94
|
+
kleinkram.core.delete_files(client=client, file_ids=[file_parsed.id])
|
kleinkram/cli/app.py
CHANGED
|
@@ -21,6 +21,7 @@ from kleinkram.api.routes import _get_api_version
|
|
|
21
21
|
from kleinkram.auth import login_flow
|
|
22
22
|
from kleinkram.cli._download import download_typer
|
|
23
23
|
from kleinkram.cli._endpoint import endpoint_typer
|
|
24
|
+
from kleinkram.cli._file import file_typer
|
|
24
25
|
from kleinkram.cli._list import list_typer
|
|
25
26
|
from kleinkram.cli._mission import mission_typer
|
|
26
27
|
from kleinkram.cli._project import project_typer
|
|
@@ -57,7 +58,7 @@ Kleinkram CLI
|
|
|
57
58
|
|
|
58
59
|
The Kleinkram CLI is a command line interface for Kleinkram.
|
|
59
60
|
For a list of available commands, run `klein --help` or visit \
|
|
60
|
-
https://docs.datasets.leggedrobotics.com/usage/
|
|
61
|
+
https://docs.datasets.leggedrobotics.com/usage/python/getting-started.html \
|
|
61
62
|
for more information.
|
|
62
63
|
"""
|
|
63
64
|
|
|
@@ -98,11 +99,14 @@ app = ErrorHandledTyper(
|
|
|
98
99
|
no_args_is_help=True,
|
|
99
100
|
)
|
|
100
101
|
|
|
102
|
+
app.add_typer(endpoint_typer, name="endpoint", rich_help_panel=CommandTypes.AUTH)
|
|
103
|
+
|
|
101
104
|
app.add_typer(download_typer, name="download", rich_help_panel=CommandTypes.CORE)
|
|
102
105
|
app.add_typer(upload_typer, name="upload", rich_help_panel=CommandTypes.CORE)
|
|
103
106
|
app.add_typer(verify_typer, name="verify", rich_help_panel=CommandTypes.CORE)
|
|
104
107
|
app.add_typer(list_typer, name="list", rich_help_panel=CommandTypes.CORE)
|
|
105
|
-
|
|
108
|
+
|
|
109
|
+
app.add_typer(file_typer, name="file", rich_help_panel=CommandTypes.CRUD)
|
|
106
110
|
app.add_typer(mission_typer, name="mission", rich_help_panel=CommandTypes.CRUD)
|
|
107
111
|
app.add_typer(project_typer, name="project", rich_help_panel=CommandTypes.CRUD)
|
|
108
112
|
|
kleinkram/config.py
CHANGED
|
@@ -45,7 +45,7 @@ class Endpoint(NamedTuple):
|
|
|
45
45
|
class Credentials(NamedTuple):
|
|
46
46
|
auth_token: Optional[str] = None
|
|
47
47
|
refresh_token: Optional[str] = None
|
|
48
|
-
|
|
48
|
+
api_key: Optional[str] = None
|
|
49
49
|
|
|
50
50
|
|
|
51
51
|
DEFAULT_LOCAL_API = "http://localhost:3000"
|
|
@@ -73,14 +73,58 @@ def get_env() -> Environment:
|
|
|
73
73
|
return Environment.PROD
|
|
74
74
|
|
|
75
75
|
|
|
76
|
+
ACTION_API_KEY = "KLEINKRAM_API_KEY"
|
|
77
|
+
ACTION_API = "KLEINKRAM_API"
|
|
78
|
+
ACTION_S3 = "KLEINKRAM_S3"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _get_endpoint_from_action_env_vars() -> Optional[Endpoint]:
|
|
82
|
+
api = os.getenv(ACTION_API)
|
|
83
|
+
s3 = os.getenv(ACTION_S3)
|
|
84
|
+
if api is None or s3 is None:
|
|
85
|
+
return None
|
|
86
|
+
return Endpoint("action", api, s3)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _get_api_key_from_action_env_vars() -> Optional[str]:
|
|
90
|
+
return os.getenv(ACTION_API_KEY)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _get_default_selected_endpoint() -> Endpoint:
|
|
94
|
+
env_endpoint = _get_endpoint_from_action_env_vars()
|
|
95
|
+
if env_endpoint is not None:
|
|
96
|
+
return env_endpoint
|
|
97
|
+
return DEFAULT_ENDPOINTS[get_env().value]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _get_default_endpoints() -> Dict[str, Endpoint]:
|
|
101
|
+
env_endpoint = _get_endpoint_from_action_env_vars()
|
|
102
|
+
|
|
103
|
+
default_endpoints = DEFAULT_ENDPOINTS.copy()
|
|
104
|
+
if env_endpoint is not None:
|
|
105
|
+
default_endpoints["action"] = env_endpoint
|
|
106
|
+
return default_endpoints
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _get_default_credentials() -> Dict[str, Credentials]:
|
|
110
|
+
endpoint = _get_default_selected_endpoint()
|
|
111
|
+
|
|
112
|
+
api_key = _get_api_key_from_action_env_vars()
|
|
113
|
+
if api_key is not None:
|
|
114
|
+
return {endpoint.name: Credentials(api_key=api_key)}
|
|
115
|
+
return {}
|
|
116
|
+
|
|
117
|
+
|
|
76
118
|
@dataclass
|
|
77
119
|
class Config:
|
|
78
120
|
version: str = __version__
|
|
79
|
-
selected_endpoint: str = field(
|
|
80
|
-
|
|
81
|
-
|
|
121
|
+
selected_endpoint: str = field(
|
|
122
|
+
default_factory=lambda: _get_default_selected_endpoint().name
|
|
123
|
+
)
|
|
124
|
+
endpoints: Dict[str, Endpoint] = field(default_factory=_get_default_endpoints)
|
|
125
|
+
endpoint_credentials: Dict[str, Credentials] = field(
|
|
126
|
+
default_factory=_get_default_credentials
|
|
82
127
|
)
|
|
83
|
-
endpoint_credentials: Dict[str, Credentials] = field(default_factory=dict)
|
|
84
128
|
|
|
85
129
|
@property
|
|
86
130
|
def endpoint(self) -> Endpoint:
|
|
@@ -129,11 +173,21 @@ def save_config(config: Config, path: Path = CONFIG_PATH) -> None:
|
|
|
129
173
|
os.replace(temp_path, path)
|
|
130
174
|
|
|
131
175
|
|
|
132
|
-
def
|
|
176
|
+
def _load_config_if_compatible(path: Path) -> Optional[Config]:
|
|
133
177
|
if not path.exists():
|
|
134
|
-
return
|
|
178
|
+
return None
|
|
135
179
|
with open(path, "r") as f:
|
|
136
|
-
|
|
180
|
+
try:
|
|
181
|
+
return _config_from_dict(json.load(f))
|
|
182
|
+
except Exception:
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _load_config(*, path: Path = CONFIG_PATH) -> Config:
|
|
187
|
+
config = _load_config_if_compatible(path)
|
|
188
|
+
if config is None:
|
|
189
|
+
return Config()
|
|
190
|
+
return config
|
|
137
191
|
|
|
138
192
|
|
|
139
193
|
LOADED_CONFIGS: Dict[Path, Config] = {}
|
|
@@ -166,12 +220,8 @@ def check_config_compatibility(path: Path = CONFIG_PATH) -> bool:
|
|
|
166
220
|
"""
|
|
167
221
|
if not path.exists():
|
|
168
222
|
return True
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
except Exception as e:
|
|
172
|
-
logger.info(f"Error loading config: {e}")
|
|
173
|
-
return False
|
|
174
|
-
return True
|
|
223
|
+
config = _load_config_if_compatible(path)
|
|
224
|
+
return config is not None
|
|
175
225
|
|
|
176
226
|
|
|
177
227
|
def endpoint_table(config: Config) -> Table:
|
kleinkram/printing.py
CHANGED
|
@@ -191,6 +191,28 @@ def files_to_table(
|
|
|
191
191
|
return table
|
|
192
192
|
|
|
193
193
|
|
|
194
|
+
def file_info_table(file: File) -> Table:
|
|
195
|
+
table = Table("k", "v", title=f"file info: {file.name}", show_header=False)
|
|
196
|
+
|
|
197
|
+
table.add_row("name", file.name)
|
|
198
|
+
table.add_row("id", Text(str(file.id), style="green"))
|
|
199
|
+
table.add_row("project", file.project_name)
|
|
200
|
+
table.add_row("project id", Text(str(file.project_id), style="green"))
|
|
201
|
+
table.add_row("mission", file.mission_name)
|
|
202
|
+
table.add_row("mission id", Text(str(file.mission_id), style="green"))
|
|
203
|
+
table.add_row("created", str(file.created_at))
|
|
204
|
+
table.add_row("updated", str(file.updated_at))
|
|
205
|
+
table.add_row("size", format_bytes(file.size))
|
|
206
|
+
table.add_row("state", file_state_to_text(file.state))
|
|
207
|
+
table.add_row("categories", ", ".join(file.categories))
|
|
208
|
+
table.add_row("topics", ", ".join(file.topics))
|
|
209
|
+
table.add_row("hash", file.hash)
|
|
210
|
+
table.add_row("type", file.type_)
|
|
211
|
+
table.add_row("date", str(file.date))
|
|
212
|
+
|
|
213
|
+
return table
|
|
214
|
+
|
|
215
|
+
|
|
194
216
|
def mission_info_table(
|
|
195
217
|
mission: Mission, print_metadata: bool = False
|
|
196
218
|
) -> Tuple[Table, ...]:
|
|
@@ -297,6 +319,20 @@ def print_projects(projects: Sequence[Project], *, pprint: bool) -> None:
|
|
|
297
319
|
print(project.id)
|
|
298
320
|
|
|
299
321
|
|
|
322
|
+
def print_file_info(file: File, *, pprint: bool) -> None:
|
|
323
|
+
"""\
|
|
324
|
+
prints the file info to stdout
|
|
325
|
+
either using pprint or as a list for piping
|
|
326
|
+
"""
|
|
327
|
+
if pprint:
|
|
328
|
+
Console().print(file_info_table(file))
|
|
329
|
+
else:
|
|
330
|
+
file_dct = asdict(file)
|
|
331
|
+
for key in file_dct:
|
|
332
|
+
file_dct[key] = str(file_dct[key]) # TODO: improve this
|
|
333
|
+
print(json.dumps(file_dct))
|
|
334
|
+
|
|
335
|
+
|
|
300
336
|
def print_mission_info(mission: Mission, *, pprint: bool) -> None:
|
|
301
337
|
"""\
|
|
302
338
|
prints the mission info to stdout
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: kleinkram
|
|
3
|
-
Version: 0.38.1.
|
|
3
|
+
Version: 0.38.1.dev20250218095010
|
|
4
4
|
Summary: give me your bags
|
|
5
5
|
Author: Cyrill Püntener, Dominique Garmier, Johann Schwabe
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -74,7 +74,7 @@ Instead of downloading files from a specified mission you can download arbitrary
|
|
|
74
74
|
klein download --dest out *id1* *id2* *id3*
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
For more information consult the [documentation](https://docs.datasets.leggedrobotics.com/usage/
|
|
77
|
+
For more information consult the [documentation](https://docs.datasets.leggedrobotics.com/usage/python/getting-started.html).
|
|
78
78
|
|
|
79
79
|
## Development
|
|
80
80
|
|
|
@@ -106,10 +106,15 @@ klein --help
|
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
### Run Tests
|
|
109
|
+
to run unit tests:
|
|
109
110
|
```bash
|
|
110
|
-
pytest
|
|
111
|
+
pytest -m "not slow"
|
|
111
112
|
```
|
|
112
|
-
|
|
113
|
+
to run all tests (including e2e and integration tests):
|
|
113
114
|
```bash
|
|
114
|
-
pytest
|
|
115
|
+
pytest
|
|
115
116
|
```
|
|
117
|
+
For the latter you need to have an instance of the backend running locally.
|
|
118
|
+
See instructions in the root of the repository for this.
|
|
119
|
+
On top of that these tests require particular files to be present in the `cli/data/testing` directory.
|
|
120
|
+
To see the exact files that are required, see `cli/testing/backend_fixtures.py`.
|
{kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/RECORD
RENAMED
|
@@ -1,49 +1,50 @@
|
|
|
1
1
|
kleinkram/__init__.py,sha256=xIJqTJw2kbCGryGlCeAdpmtR1FTxmrW1MklUNQEaj74,1061
|
|
2
2
|
kleinkram/__main__.py,sha256=B9RiZxfO4jpCmWPUHyKJ7_EoZlEG4sPpH-nz7T_YhhQ,125
|
|
3
3
|
kleinkram/_version.py,sha256=QYJyRTcqFcJj4qWYpqs7WcoOP6jxDMqyvxLY-cD6KcE,129
|
|
4
|
-
kleinkram/auth.py,sha256=
|
|
5
|
-
kleinkram/config.py,sha256=
|
|
4
|
+
kleinkram/auth.py,sha256=XD_rHOyJmYYfO7QJf3TLYH5qXA22gXGWi7PT3jujlVs,2968
|
|
5
|
+
kleinkram/config.py,sha256=3yCEfoG-me4VKjB5ez5yXc_u2S-391e9-22VL1NMW-E,6819
|
|
6
6
|
kleinkram/core.py,sha256=Q7OYIKPN9K6kxf9Eq7r5XRHPJ3RtT7SBZp_3_CS8yuY,8429
|
|
7
7
|
kleinkram/errors.py,sha256=4mygNxkf6IBgaiRWY95qu0v6z4TAXA3G6CUcXC9FU3s,772
|
|
8
8
|
kleinkram/main.py,sha256=BTE0mZN__xd46wBhFi6iBlK9eGGQvJ1LdUMsbnysLi0,172
|
|
9
9
|
kleinkram/models.py,sha256=8nJlPrKVLSmehspeuQSFV6nUo76JzehUn6KIZYH1xy4,1832
|
|
10
|
-
kleinkram/printing.py,sha256=
|
|
10
|
+
kleinkram/printing.py,sha256=OLApOtXytGZGzdpQ9Kd-swMrPAjdnvCo5jCpLoa2lIE,11238
|
|
11
11
|
kleinkram/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
12
|
kleinkram/types.py,sha256=nfDjj8TB1Jn5vqO0Xg6qhLOuKom9DDhe62BrngqnVGM,185
|
|
13
13
|
kleinkram/utils.py,sha256=6HFqTw7-eqDEjNG_PsVEQNMNK-RWOqPsoiZI5SK8F7Q,6270
|
|
14
14
|
kleinkram/wrappers.py,sha256=4xXU43eNnvMG2sssU330MmTLSSRdurOpnZ-zNGOGmt0,11342
|
|
15
15
|
kleinkram/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
-
kleinkram/api/client.py,sha256=
|
|
16
|
+
kleinkram/api/client.py,sha256=yFanAc8VkpcyO5rVXIPyGaYHmF8_tBeC0ZMRUx0StWU,5061
|
|
17
17
|
kleinkram/api/deser.py,sha256=-eP0haBAFr-dRWJ1v-P5o_rxA8vOBlZMtAGXW8ItIAk,4870
|
|
18
|
-
kleinkram/api/file_transfer.py,sha256=
|
|
18
|
+
kleinkram/api/file_transfer.py,sha256=3wNlVQdjnRtxOzih5HhCTF18xPbYClFIDxCqbwkLl6c,12985
|
|
19
19
|
kleinkram/api/pagination.py,sha256=P_zPsBKlMWkmAv-YfUNHaGW-XLB_4U8BDMrKyiDFIXk,1370
|
|
20
20
|
kleinkram/api/query.py,sha256=gn5yf-eRB_Bcw2diLjt66yQtorrZMKdj5_oNA_oOhvc,3281
|
|
21
|
-
kleinkram/api/routes.py,sha256=
|
|
21
|
+
kleinkram/api/routes.py,sha256=q2MhoeGRXFKcQlIwxk9kzdzSiFB-EWdyeXVCqLp6ydw,12099
|
|
22
22
|
kleinkram/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
23
|
kleinkram/cli/_download.py,sha256=H4YlXJkZE4Md02nzgrO_i8Hsm4ZIejPsxBEKkcn4KHs,2371
|
|
24
24
|
kleinkram/cli/_endpoint.py,sha256=oY0p4bnuHLEDJCXtTmir4AHswcKAygZ8I4IWC3RFcKc,1796
|
|
25
|
+
kleinkram/cli/_file.py,sha256=Q2fLDdUyfHFmdGC6wIxMqgEl0F76qszhzWJrRV5rTBM,2973
|
|
25
26
|
kleinkram/cli/_list.py,sha256=5gI3aIUeKC0_eWPQqdFXSBBFvpkTTJSm31TamHa197c,3090
|
|
26
27
|
kleinkram/cli/_mission.py,sha256=zDFnOozOFckpuREFgIPt1IzG5q3b1bsNxYlWQoHoz5A,5301
|
|
27
28
|
kleinkram/cli/_project.py,sha256=N0C96NC_onCEwTteYp2wgkkwkdJt-1q43LFdqNXfjC8,3398
|
|
28
29
|
kleinkram/cli/_upload.py,sha256=gOhbjbmqhmwW7p6bWlSvI53vLHvBFO9QqD1kdU92I2k,2813
|
|
29
30
|
kleinkram/cli/_verify.py,sha256=0ABVa4U_WzaV36ClR8NsOIG7KAMRlnFmsbtnHhbWVj4,1742
|
|
30
|
-
kleinkram/cli/app.py,sha256=
|
|
31
|
+
kleinkram/cli/app.py,sha256=pQwyX6Bilg3-GJwkfs5K49KtwnTnl8pzsa7urbR3TMg,6657
|
|
31
32
|
kleinkram/cli/error_handling.py,sha256=wK3tzeKVSrZm-xmiyzGLnGT2E4TRpyxhaak6GWGP7P8,1921
|
|
32
33
|
testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
|
-
testing/backend_fixtures.py,sha256=
|
|
34
|
+
testing/backend_fixtures.py,sha256=4m4J51KLpuJsNanAKdivQWNflbmWGfODYJUnH-lBTIs,1570
|
|
34
35
|
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
35
36
|
tests/conftest.py,sha256=5MLYQOtQoXWl0TRkYntYKNdqpd4hl9m0XTRi5OXanYI,104
|
|
36
|
-
tests/test_config.py,sha256=
|
|
37
|
+
tests/test_config.py,sha256=2hyk-CY4D_Q1bLDwcpdEs2tpYeR8zdWxW7VvrckoEVk,5225
|
|
37
38
|
tests/test_core.py,sha256=JbzB05LWmaaP77uXeTOQtCJD2AJT0zO9zhDfcZ3GNH8,5139
|
|
38
|
-
tests/test_end_to_end.py,sha256=
|
|
39
|
+
tests/test_end_to_end.py,sha256=0W5pUES5hek-pXq4NZtpPZqKTORkGCRsDv5_D3rDMjY,3372
|
|
39
40
|
tests/test_error_handling.py,sha256=qPSMKF1qsAHyUME0-krxbIrk38iGKkhAyAah-KwN4NE,1300
|
|
40
41
|
tests/test_fixtures.py,sha256=UlPmGbEsGvrDPsaStGMRjNvrVPGjCqOB0RMfLJq2VRA,1071
|
|
41
42
|
tests/test_printing.py,sha256=qCr04OJVl5ouht9FoeWGKOi8MZXevVV1EDghzV1JaMc,1903
|
|
42
43
|
tests/test_query.py,sha256=fExmCKXLA7-9j2S2sF_sbvRX_2s6Cp3a7OTcqE25q9g,3864
|
|
43
44
|
tests/test_utils.py,sha256=eUBYrn3xrcgcaxm1X4fqZaX4tRvkbI6rh6BUbNbu9T0,4784
|
|
44
45
|
tests/test_wrappers.py,sha256=TbcTyO2L7fslbzgfDdcVZkencxNQ8cGPZm_iB6c9d6Q,2673
|
|
45
|
-
kleinkram-0.38.1.
|
|
46
|
-
kleinkram-0.38.1.
|
|
47
|
-
kleinkram-0.38.1.
|
|
48
|
-
kleinkram-0.38.1.
|
|
49
|
-
kleinkram-0.38.1.
|
|
46
|
+
kleinkram-0.38.1.dev20250218095010.dist-info/METADATA,sha256=6tpuu8kbIDQtsEvjnHu6GvJzm_0n9_T2imocskUJzq0,2683
|
|
47
|
+
kleinkram-0.38.1.dev20250218095010.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
48
|
+
kleinkram-0.38.1.dev20250218095010.dist-info/entry_points.txt,sha256=SaB2l5aqhSr8gmaMw2kvQU90a8Bnl7PedU8cWYxkfYo,46
|
|
49
|
+
kleinkram-0.38.1.dev20250218095010.dist-info/top_level.txt,sha256=N3-sJagEHu1Tk1X6Dx1X1q0pLDNbDZpLzRxVftvepds,24
|
|
50
|
+
kleinkram-0.38.1.dev20250218095010.dist-info/RECORD,,
|
testing/backend_fixtures.py
CHANGED
|
@@ -32,10 +32,7 @@ WAIT_BEOFORE_DELETION = 5
|
|
|
32
32
|
@pytest.fixture(scope="session")
|
|
33
33
|
def project():
|
|
34
34
|
project_name = token_hex(8)
|
|
35
|
-
|
|
36
|
-
print("here")
|
|
37
35
|
create_project(project_name, description="This is a test project")
|
|
38
|
-
|
|
39
36
|
project = list_projects(project_names=[project_name])[0]
|
|
40
37
|
|
|
41
38
|
yield project
|
tests/test_config.py
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import os
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from tempfile import TemporaryDirectory
|
|
6
|
+
from unittest import mock
|
|
5
7
|
|
|
6
8
|
import pytest
|
|
7
9
|
|
|
10
|
+
from kleinkram.config import ACTION_API
|
|
11
|
+
from kleinkram.config import ACTION_API_KEY
|
|
12
|
+
from kleinkram.config import ACTION_S3
|
|
8
13
|
from kleinkram.config import Config
|
|
9
14
|
from kleinkram.config import Endpoint
|
|
10
15
|
from kleinkram.config import _load_config
|
|
16
|
+
from kleinkram.config import _load_config_if_compatible
|
|
11
17
|
from kleinkram.config import add_endpoint
|
|
12
18
|
from kleinkram.config import check_config_compatibility
|
|
13
19
|
from kleinkram.config import endpoint_table
|
|
@@ -17,7 +23,30 @@ from kleinkram.config import get_shared_state
|
|
|
17
23
|
from kleinkram.config import save_config
|
|
18
24
|
from kleinkram.config import select_endpoint
|
|
19
25
|
|
|
20
|
-
CONFIG_FILENAME = "kleinkram.json"
|
|
26
|
+
CONFIG_FILENAME = ".kleinkram.json"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
TEST_API_KEY = "test_key"
|
|
30
|
+
TEST_API = "test_api"
|
|
31
|
+
TEST_S3 = "test_s3"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@pytest.fixture()
|
|
35
|
+
def set_api_key_env(monkeypatch):
|
|
36
|
+
with mock.patch.dict(os.environ, clear=True):
|
|
37
|
+
envvars = {ACTION_API_KEY: TEST_API_KEY}
|
|
38
|
+
for k, v in envvars.items():
|
|
39
|
+
monkeypatch.setenv(k, v)
|
|
40
|
+
yield # This is the magical bit which restore the environment after
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.fixture()
|
|
44
|
+
def set_endpoint_env(monkeypatch):
|
|
45
|
+
with mock.patch.dict(os.environ, clear=True):
|
|
46
|
+
envvars = {ACTION_API: TEST_API, ACTION_S3: TEST_S3}
|
|
47
|
+
for k, v in envvars.items():
|
|
48
|
+
monkeypatch.setenv(k, v)
|
|
49
|
+
yield # This is the magical bit which restore the environment after
|
|
21
50
|
|
|
22
51
|
|
|
23
52
|
@pytest.fixture
|
|
@@ -26,15 +55,50 @@ def config_path():
|
|
|
26
55
|
yield Path(tmpdir) / CONFIG_FILENAME
|
|
27
56
|
|
|
28
57
|
|
|
58
|
+
def test_load_config_if_compatible_with_invalid_config(config_path):
|
|
59
|
+
with open(config_path, "w") as f:
|
|
60
|
+
f.write("this is not a valid config")
|
|
61
|
+
assert _load_config_if_compatible(config_path) is None
|
|
62
|
+
|
|
63
|
+
|
|
29
64
|
def test_load_config_default(config_path):
|
|
30
65
|
config = _load_config(path=config_path)
|
|
31
66
|
|
|
32
67
|
assert not config_path.exists()
|
|
33
68
|
assert Config() == config
|
|
34
69
|
|
|
70
|
+
assert config.endpoint_credentials == {}
|
|
71
|
+
assert config.selected_endpoint == get_env().value
|
|
72
|
+
|
|
35
73
|
|
|
36
|
-
def
|
|
74
|
+
def test_load_default_config_with_env_var_api_key_specified(
|
|
75
|
+
config_path, set_api_key_env
|
|
76
|
+
):
|
|
77
|
+
assert set_api_key_env is None
|
|
37
78
|
|
|
79
|
+
config = _load_config(path=config_path)
|
|
80
|
+
|
|
81
|
+
creds = config.endpoint_credentials[config.selected_endpoint]
|
|
82
|
+
assert creds.auth_token is None
|
|
83
|
+
assert creds.refresh_token is None
|
|
84
|
+
assert creds.api_key == TEST_API_KEY
|
|
85
|
+
|
|
86
|
+
assert not config_path.exists()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_load_default_config_with_env_var_endpoints_specified(
|
|
90
|
+
config_path, set_endpoint_env
|
|
91
|
+
):
|
|
92
|
+
assert set_endpoint_env is None
|
|
93
|
+
config = _load_config(path=config_path)
|
|
94
|
+
|
|
95
|
+
assert config.selected_endpoint == "action"
|
|
96
|
+
assert config.endpoint == Endpoint("action", TEST_API, TEST_S3)
|
|
97
|
+
|
|
98
|
+
assert not config_path.exists()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_save_and_load_config(config_path):
|
|
38
102
|
config = Config(version="foo")
|
|
39
103
|
|
|
40
104
|
assert not config_path.exists()
|
tests/test_end_to_end.py
CHANGED
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import os
|
|
4
4
|
import secrets
|
|
5
5
|
import shutil
|
|
6
|
+
import time
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
|
|
8
9
|
import pytest
|
|
@@ -43,6 +44,8 @@ def run_cmd(command, *, verbose=VERBOSE):
|
|
|
43
44
|
def test_upload_verify_update_download_mission(project, tmp_path, api):
|
|
44
45
|
assert api
|
|
45
46
|
|
|
47
|
+
file_names = list(DATA_PATH.glob("*.bag"))
|
|
48
|
+
|
|
46
49
|
mission_name = secrets.token_hex(8)
|
|
47
50
|
upload = f"{CLI} upload -p {project.name} -m {mission_name} --create {DATA_PATH.absolute()}/*.bag"
|
|
48
51
|
verify = (
|
|
@@ -50,12 +53,15 @@ def test_upload_verify_update_download_mission(project, tmp_path, api):
|
|
|
50
53
|
)
|
|
51
54
|
# update = f"{CLI} mission update -p {project.name} -m {mission_name} --metadata {DATA_PATH.absolute()}/metadata.yaml"
|
|
52
55
|
download = f"{CLI} download -p {project.name} -m {mission_name} --dest {tmp_path.absolute()}"
|
|
56
|
+
delete_file = f"{CLI} file delete -p {project.name} -m {mission_name} -f {file_names[0].name} -y"
|
|
53
57
|
|
|
54
58
|
assert run_cmd(upload) == 0
|
|
55
59
|
assert run_cmd(verify) == 0
|
|
56
60
|
# assert run_cmd(update) == 0
|
|
57
61
|
assert run_cmd(download) == 0
|
|
58
62
|
|
|
63
|
+
assert run_cmd(delete_file) == 0
|
|
64
|
+
|
|
59
65
|
|
|
60
66
|
@pytest.mark.slow
|
|
61
67
|
def test_list_files(project, mission, api):
|
{kleinkram-0.38.1.dev20250207122632.dist-info → kleinkram-0.38.1.dev20250218095010.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|