kleinkram 0.43.2.dev20250331124109__py3-none-any.whl → 0.58.0.dev20260110152317__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kleinkram/api/client.py +6 -18
- kleinkram/api/deser.py +152 -1
- kleinkram/api/file_transfer.py +202 -101
- kleinkram/api/pagination.py +11 -2
- kleinkram/api/query.py +10 -10
- kleinkram/api/routes.py +192 -59
- kleinkram/auth.py +108 -7
- kleinkram/cli/_action.py +131 -0
- kleinkram/cli/_download.py +8 -19
- kleinkram/cli/_endpoint.py +2 -4
- kleinkram/cli/_file.py +6 -18
- kleinkram/cli/_file_validator.py +125 -0
- kleinkram/cli/_list.py +5 -15
- kleinkram/cli/_mission.py +24 -28
- kleinkram/cli/_project.py +10 -26
- kleinkram/cli/_run.py +220 -0
- kleinkram/cli/_upload.py +58 -26
- kleinkram/cli/_verify.py +59 -16
- kleinkram/cli/app.py +56 -17
- kleinkram/cli/error_handling.py +1 -3
- kleinkram/config.py +6 -21
- kleinkram/core.py +53 -43
- kleinkram/errors.py +12 -0
- kleinkram/models.py +51 -1
- kleinkram/printing.py +229 -18
- kleinkram/utils.py +10 -24
- kleinkram/wrappers.py +54 -30
- {kleinkram-0.43.2.dev20250331124109.dist-info → kleinkram-0.58.0.dev20260110152317.dist-info}/METADATA +6 -4
- kleinkram-0.58.0.dev20260110152317.dist-info/RECORD +53 -0
- {kleinkram-0.43.2.dev20250331124109.dist-info → kleinkram-0.58.0.dev20260110152317.dist-info}/WHEEL +1 -1
- {kleinkram-0.43.2.dev20250331124109.dist-info → kleinkram-0.58.0.dev20260110152317.dist-info}/top_level.txt +0 -1
- {testing → tests}/backend_fixtures.py +27 -3
- tests/conftest.py +1 -1
- tests/generate_test_data.py +314 -0
- tests/test_config.py +2 -6
- tests/test_core.py +11 -31
- tests/test_end_to_end.py +3 -5
- tests/test_fixtures.py +3 -5
- tests/test_printing.py +9 -11
- tests/test_utils.py +1 -3
- tests/test_wrappers.py +9 -27
- kleinkram-0.43.2.dev20250331124109.dist-info/RECORD +0 -50
- testing/__init__.py +0 -0
- {kleinkram-0.43.2.dev20250331124109.dist-info → kleinkram-0.58.0.dev20260110152317.dist-info}/entry_points.txt +0 -0
kleinkram/cli/_run.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
import tarfile
|
|
7
|
+
import time
|
|
8
|
+
from typing import List
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
import typer
|
|
13
|
+
|
|
14
|
+
import kleinkram.api.routes
|
|
15
|
+
from kleinkram.api.client import AuthenticatedClient
|
|
16
|
+
from kleinkram.api.query import RunQuery
|
|
17
|
+
from kleinkram.config import get_shared_state
|
|
18
|
+
from kleinkram.models import LogEntry
|
|
19
|
+
from kleinkram.models import Run
|
|
20
|
+
from kleinkram.printing import print_run_info
|
|
21
|
+
from kleinkram.printing import print_run_logs
|
|
22
|
+
from kleinkram.printing import print_runs_table
|
|
23
|
+
from kleinkram.utils import split_args
|
|
24
|
+
|
|
25
|
+
HELP = """\
|
|
26
|
+
Manage and inspect action runs.
|
|
27
|
+
|
|
28
|
+
You can list action runs, get detailed information about specific runs, stream their logs,
|
|
29
|
+
cancel runs in progress, and retry failed runs.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
run_typer = typer.Typer(
|
|
33
|
+
no_args_is_help=True,
|
|
34
|
+
context_settings={"help_option_names": ["-h", "--help"]},
|
|
35
|
+
help=HELP,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
LIST_HELP = "List action runs. Optionally filter by mission or project."
|
|
39
|
+
INFO_HELP = "Get detailed information about a specific action run."
|
|
40
|
+
LOGS_HELP = "Stream the logs for a specific action run."
|
|
41
|
+
CANCEL_HELP = "Cancel an action run that is in progress."
|
|
42
|
+
RETRY_HELP = "Retry a failed action run."
|
|
43
|
+
DOWNLOAD_HELP = "Download artifacts for a specific action run."
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@run_typer.command(help=LIST_HELP, name="list")
|
|
47
|
+
def list_runs(
|
|
48
|
+
mission: Optional[str] = typer.Option(None, "--mission", "-m", help="Mission ID or name to filter by."),
|
|
49
|
+
project: Optional[str] = typer.Option(None, "--project", "-p", help="Project ID or name to filter by."),
|
|
50
|
+
) -> None:
|
|
51
|
+
"""
|
|
52
|
+
List action runs.
|
|
53
|
+
"""
|
|
54
|
+
client = AuthenticatedClient()
|
|
55
|
+
|
|
56
|
+
mission_ids, mission_patterns = split_args([mission] if mission else [])
|
|
57
|
+
project_ids, project_patterns = split_args([project] if project else [])
|
|
58
|
+
|
|
59
|
+
query = RunQuery(
|
|
60
|
+
mission_ids=mission_ids,
|
|
61
|
+
mission_patterns=mission_patterns,
|
|
62
|
+
project_ids=project_ids,
|
|
63
|
+
project_patterns=project_patterns,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
runs = list(kleinkram.api.routes.get_runs(client, query=query))
|
|
67
|
+
print_runs_table(runs, pprint=get_shared_state().verbose)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@run_typer.command(name="info", help=INFO_HELP)
|
|
71
|
+
def get_info(run_id: str = typer.Argument(..., help="The ID of the run to get information for.")) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Get detailed information for a single run.
|
|
74
|
+
"""
|
|
75
|
+
client = AuthenticatedClient()
|
|
76
|
+
run: Run = kleinkram.api.routes.get_run(client, run_id=run_id)
|
|
77
|
+
print_run_info(run, pprint=get_shared_state().verbose)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@run_typer.command(help=LOGS_HELP)
|
|
81
|
+
def logs(
|
|
82
|
+
run_id: str = typer.Argument(..., help="The ID of the run to fetch logs for."),
|
|
83
|
+
follow: bool = typer.Option(False, "--follow", "-f", help="Follow the log output in real-time."),
|
|
84
|
+
) -> None:
|
|
85
|
+
"""
|
|
86
|
+
Fetch and display logs for a specific run.
|
|
87
|
+
"""
|
|
88
|
+
client = AuthenticatedClient()
|
|
89
|
+
|
|
90
|
+
if follow:
|
|
91
|
+
typer.echo(f"Watching logs for run {run_id}. Press Ctrl+C to stop.")
|
|
92
|
+
try:
|
|
93
|
+
|
|
94
|
+
# TODO: fine for now, but ideally we would have a streaming endpoint
|
|
95
|
+
# currently there is no following, thus we just poll every 2 seconds
|
|
96
|
+
# from the get_run endpoint
|
|
97
|
+
last_log_index = 0
|
|
98
|
+
while True:
|
|
99
|
+
run: Run = kleinkram.api.routes.get_run(client, run_id=run_id)
|
|
100
|
+
log_entries: List[LogEntry] = run.logs
|
|
101
|
+
new_log_entries = log_entries[last_log_index:]
|
|
102
|
+
if new_log_entries:
|
|
103
|
+
print_run_logs(new_log_entries, pprint=get_shared_state().verbose)
|
|
104
|
+
last_log_index += len(new_log_entries)
|
|
105
|
+
|
|
106
|
+
time.sleep(2)
|
|
107
|
+
|
|
108
|
+
except KeyboardInterrupt:
|
|
109
|
+
typer.echo("Stopped following logs.")
|
|
110
|
+
sys.exit(0)
|
|
111
|
+
else:
|
|
112
|
+
log_entries = kleinkram.api.routes.get_run(client, run_id=run_id).logs
|
|
113
|
+
print_run_logs(log_entries, pprint=get_shared_state().verbose)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _get_filename_from_cd(cd: str) -> Optional[str]:
|
|
117
|
+
"""Extract filename from Content-Disposition header."""
|
|
118
|
+
if not cd:
|
|
119
|
+
return None
|
|
120
|
+
fname = re.findall("filename=(.+)", cd)
|
|
121
|
+
if len(fname) == 0:
|
|
122
|
+
return None
|
|
123
|
+
return fname[0].strip().strip('"')
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@run_typer.command(name="download", help=DOWNLOAD_HELP)
|
|
127
|
+
def download_artifacts(
|
|
128
|
+
run_id: str = typer.Argument(..., help="The ID of the run to download artifacts for."),
|
|
129
|
+
output: Optional[str] = typer.Option(None, "--output", "-o", help="Path or filename to save the artifacts to."),
|
|
130
|
+
extract: bool = typer.Option(
|
|
131
|
+
False,
|
|
132
|
+
"--extract",
|
|
133
|
+
"-x",
|
|
134
|
+
help="Automatically extract the archive after downloading.",
|
|
135
|
+
),
|
|
136
|
+
) -> None:
|
|
137
|
+
"""
|
|
138
|
+
Download the artifacts (.tar.gz) for a finished run.
|
|
139
|
+
"""
|
|
140
|
+
client = AuthenticatedClient()
|
|
141
|
+
|
|
142
|
+
# Fetch Run Details
|
|
143
|
+
try:
|
|
144
|
+
run: Run = kleinkram.api.routes.get_run(client, run_id=run_id)
|
|
145
|
+
except Exception as e:
|
|
146
|
+
typer.secho(f"Failed to fetch run details: {e}", fg=typer.colors.RED)
|
|
147
|
+
raise typer.Exit(1)
|
|
148
|
+
|
|
149
|
+
if not run.artifact_url:
|
|
150
|
+
typer.secho(
|
|
151
|
+
f"No artifacts found for run {run_id}. The run might not be finished or artifacts expired.",
|
|
152
|
+
fg=typer.colors.YELLOW,
|
|
153
|
+
)
|
|
154
|
+
raise typer.Exit(1)
|
|
155
|
+
|
|
156
|
+
typer.echo(f"Downloading artifacts for run {run_id}...")
|
|
157
|
+
|
|
158
|
+
# Stream Download
|
|
159
|
+
try:
|
|
160
|
+
with requests.get(run.artifact_url, stream=True) as r:
|
|
161
|
+
r.raise_for_status()
|
|
162
|
+
|
|
163
|
+
# Determine Filename
|
|
164
|
+
filename = output
|
|
165
|
+
if not filename:
|
|
166
|
+
filename = _get_filename_from_cd(r.headers.get("content-disposition"))
|
|
167
|
+
|
|
168
|
+
if not filename:
|
|
169
|
+
filename = f"{run_id}.tar.gz"
|
|
170
|
+
|
|
171
|
+
# If output is a directory, join with filename
|
|
172
|
+
if output and os.path.isdir(output):
|
|
173
|
+
filename = os.path.join(
|
|
174
|
+
output,
|
|
175
|
+
_get_filename_from_cd(r.headers.get("content-disposition")) or f"{run_id}.tar.gz",
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
total_length = int(r.headers.get("content-length", 0))
|
|
179
|
+
|
|
180
|
+
# Write to file with Progress Bar
|
|
181
|
+
with open(filename, "wb") as f:
|
|
182
|
+
with typer.progressbar(length=total_length, label=f"Saving to {filename}") as progress:
|
|
183
|
+
for chunk in r.iter_content(chunk_size=8192):
|
|
184
|
+
if chunk:
|
|
185
|
+
f.write(chunk)
|
|
186
|
+
progress.update(len(chunk))
|
|
187
|
+
|
|
188
|
+
typer.secho(f"\nSuccessfully downloaded to {filename}", fg=typer.colors.GREEN)
|
|
189
|
+
|
|
190
|
+
# Extraction Logic
|
|
191
|
+
if extract:
|
|
192
|
+
try:
|
|
193
|
+
# Determine extraction directory (based on filename without extension)
|
|
194
|
+
# e.g., "downloads/my-run.tar" -> "downloads/my-run"
|
|
195
|
+
base_name = os.path.basename(filename)
|
|
196
|
+
folder_name = base_name.split(".")[0]
|
|
197
|
+
|
|
198
|
+
# Get the parent directory of the downloaded file
|
|
199
|
+
parent_dir = os.path.dirname(os.path.abspath(filename))
|
|
200
|
+
extract_path = os.path.join(parent_dir, folder_name)
|
|
201
|
+
|
|
202
|
+
typer.echo(f"Extracting to: {extract_path}...")
|
|
203
|
+
|
|
204
|
+
with tarfile.open(filename, "r:gz") as tar:
|
|
205
|
+
|
|
206
|
+
# Safety check: filter_data prevents extraction outside target dir (CVE-2007-4559)
|
|
207
|
+
# Available in Python 3.12+, for older python use generic extractall
|
|
208
|
+
if hasattr(tarfile, "data_filter"):
|
|
209
|
+
tar.extractall(path=extract_path, filter="data")
|
|
210
|
+
else:
|
|
211
|
+
tar.extractall(path=extract_path)
|
|
212
|
+
|
|
213
|
+
typer.secho("Successfully extracted.", fg=typer.colors.GREEN)
|
|
214
|
+
|
|
215
|
+
except tarfile.TarError as e:
|
|
216
|
+
typer.secho(f"Failed to extract archive: {e}", fg=typer.colors.RED)
|
|
217
|
+
|
|
218
|
+
except requests.exceptions.RequestException as e:
|
|
219
|
+
typer.secho(f"Error downloading file: {e}", fg=typer.colors.RED)
|
|
220
|
+
raise typer.Exit(1)
|
kleinkram/cli/_upload.py
CHANGED
|
@@ -11,8 +11,9 @@ import kleinkram.utils
|
|
|
11
11
|
from kleinkram.api.client import AuthenticatedClient
|
|
12
12
|
from kleinkram.api.query import MissionQuery
|
|
13
13
|
from kleinkram.api.query import ProjectQuery
|
|
14
|
+
from kleinkram.cli._file_validator import FileValidator
|
|
15
|
+
from kleinkram.cli._file_validator import _report_skipped_files
|
|
14
16
|
from kleinkram.config import get_shared_state
|
|
15
|
-
from kleinkram.errors import FileNameNotSupported
|
|
16
17
|
from kleinkram.errors import MissionNotFound
|
|
17
18
|
from kleinkram.utils import load_metadata
|
|
18
19
|
from kleinkram.utils import split_args
|
|
@@ -29,55 +30,86 @@ upload_typer = typer.Typer(
|
|
|
29
30
|
)
|
|
30
31
|
|
|
31
32
|
|
|
33
|
+
def _build_mission_query(mission: str, project: Optional[str]) -> MissionQuery:
|
|
34
|
+
"""Constructs the MissionQuery object from CLI args."""
|
|
35
|
+
mission_ids, mission_patterns = split_args([mission])
|
|
36
|
+
project_ids, project_patterns = split_args([project] if project else [])
|
|
37
|
+
|
|
38
|
+
project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
|
|
39
|
+
return MissionQuery(
|
|
40
|
+
ids=mission_ids,
|
|
41
|
+
patterns=mission_patterns,
|
|
42
|
+
project_query=project_query,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _handle_no_files_to_upload(original_count: int, uploaded_count: int) -> None:
|
|
47
|
+
"""Checks if any files are left to upload and exits if not."""
|
|
48
|
+
if uploaded_count > 0:
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
if original_count > 0:
|
|
52
|
+
typer.echo(
|
|
53
|
+
typer.style("All paths were skipped. No files to upload.", fg=typer.colors.RED),
|
|
54
|
+
err=True,
|
|
55
|
+
)
|
|
56
|
+
else:
|
|
57
|
+
typer.echo(typer.style("No files provided to upload.", fg=typer.colors.RED), err=True)
|
|
58
|
+
raise typer.Exit(code=1)
|
|
59
|
+
|
|
60
|
+
|
|
32
61
|
@upload_typer.callback()
|
|
33
62
|
def upload(
|
|
34
63
|
files: List[str] = typer.Argument(help="files to upload"),
|
|
35
|
-
project: Optional[str] = typer.Option(
|
|
36
|
-
None, "--project", "-p", help="project id or name"
|
|
37
|
-
),
|
|
64
|
+
project: Optional[str] = typer.Option(None, "--project", "-p", help="project id or name"),
|
|
38
65
|
mission: str = typer.Option(..., "--mission", "-m", help="mission id or name"),
|
|
39
66
|
create: bool = typer.Option(False, help="create mission if it does not exist"),
|
|
40
|
-
metadata: Optional[str] = typer.Option(
|
|
41
|
-
None, help="path to metadata file (json or yaml)"
|
|
42
|
-
),
|
|
67
|
+
metadata: Optional[str] = typer.Option(None, help="path to metadata file (json or yaml)"),
|
|
43
68
|
fix_filenames: bool = typer.Option(
|
|
44
69
|
False,
|
|
45
70
|
help="fix filenames before upload, this does not change the filenames locally",
|
|
46
71
|
),
|
|
72
|
+
skip: bool = typer.Option(
|
|
73
|
+
False,
|
|
74
|
+
"--skip",
|
|
75
|
+
"-s",
|
|
76
|
+
help="skip unsupported file types, badly named files, or directories instead of erroring",
|
|
77
|
+
),
|
|
78
|
+
experimental_datatypes: bool = typer.Option(False, help="allow experimental datatypes (yaml, svo2, db3, tum)"),
|
|
47
79
|
ignore_missing_tags: bool = typer.Option(False, help="ignore mission tags"),
|
|
48
80
|
) -> None:
|
|
49
|
-
|
|
50
|
-
|
|
81
|
+
original_file_paths = [Path(file) for file in files]
|
|
82
|
+
mission_query = _build_mission_query(mission, project)
|
|
51
83
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
project_query = ProjectQuery(ids=project_ids, patterns=project_patterns)
|
|
56
|
-
mission_query = MissionQuery(
|
|
57
|
-
ids=mission_ids,
|
|
58
|
-
patterns=mission_patterns,
|
|
59
|
-
project_query=project_query,
|
|
84
|
+
validator = FileValidator(
|
|
85
|
+
skip=skip,
|
|
86
|
+
experimental_datatypes=experimental_datatypes,
|
|
60
87
|
)
|
|
61
88
|
|
|
62
|
-
if
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
f"Consider using `--fix-filenames`"
|
|
69
|
-
)
|
|
89
|
+
# This function will raise an error if skip=False and a file is invalid
|
|
90
|
+
files_to_upload = validator.filter_files(original_file_paths)
|
|
91
|
+
|
|
92
|
+
_report_skipped_files(validator.skipped_files)
|
|
93
|
+
|
|
94
|
+
_handle_no_files_to_upload(original_count=len(original_file_paths), uploaded_count=len(files_to_upload))
|
|
70
95
|
|
|
71
96
|
try:
|
|
72
97
|
kleinkram.core.upload(
|
|
73
98
|
client=AuthenticatedClient(),
|
|
74
99
|
query=mission_query,
|
|
75
|
-
file_paths=
|
|
100
|
+
file_paths=files_to_upload,
|
|
76
101
|
create=create,
|
|
77
102
|
metadata=load_metadata(Path(metadata)) if metadata else None,
|
|
78
103
|
ignore_missing_metadata=ignore_missing_tags,
|
|
79
104
|
verbose=get_shared_state().verbose,
|
|
80
105
|
)
|
|
106
|
+
typer.echo(
|
|
107
|
+
typer.style(
|
|
108
|
+
f"\nSuccessfully uploaded {len(files_to_upload)} file(s).",
|
|
109
|
+
fg=typer.colors.GREEN,
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
81
113
|
except MissionNotFound:
|
|
82
114
|
if create:
|
|
83
115
|
raise # dont change the error message
|
kleinkram/cli/_verify.py
CHANGED
|
@@ -9,15 +9,14 @@ import typer
|
|
|
9
9
|
|
|
10
10
|
import kleinkram.core
|
|
11
11
|
from kleinkram.api.client import AuthenticatedClient
|
|
12
|
-
from kleinkram.
|
|
13
|
-
from kleinkram.
|
|
12
|
+
from kleinkram.cli._file_validator import FileValidator
|
|
13
|
+
from kleinkram.cli._file_validator import _report_skipped_files
|
|
14
|
+
from kleinkram.cli._upload import _build_mission_query
|
|
14
15
|
from kleinkram.config import get_shared_state
|
|
15
16
|
from kleinkram.printing import print_file_verification_status
|
|
16
|
-
from kleinkram.utils import split_args
|
|
17
17
|
|
|
18
18
|
logger = logging.getLogger(__name__)
|
|
19
19
|
|
|
20
|
-
|
|
21
20
|
HELP = """\
|
|
22
21
|
Verify if files were uploaded correctly.
|
|
23
22
|
"""
|
|
@@ -25,32 +24,76 @@ Verify if files were uploaded correctly.
|
|
|
25
24
|
verify_typer = typer.Typer(name="verify", invoke_without_command=True, help=HELP)
|
|
26
25
|
|
|
27
26
|
|
|
27
|
+
def _handle_no_files_to_process(original_count: int, processed_count: int, action: str = "verify") -> None:
|
|
28
|
+
"""Checks if any files are left and exits if not."""
|
|
29
|
+
if processed_count > 0:
|
|
30
|
+
return
|
|
31
|
+
|
|
32
|
+
if original_count > 0:
|
|
33
|
+
typer.echo(
|
|
34
|
+
typer.style(f"All paths were skipped. No files to {action}.", fg=typer.colors.RED),
|
|
35
|
+
err=True,
|
|
36
|
+
)
|
|
37
|
+
else:
|
|
38
|
+
typer.echo(
|
|
39
|
+
typer.style(f"No files provided to {action}.", fg=typer.colors.RED),
|
|
40
|
+
err=True,
|
|
41
|
+
)
|
|
42
|
+
raise typer.Exit(code=1)
|
|
43
|
+
|
|
44
|
+
|
|
28
45
|
@verify_typer.callback()
|
|
29
46
|
def verify(
|
|
30
|
-
files: List[str] = typer.Argument(help="files to
|
|
31
|
-
project: Optional[str] = typer.Option(
|
|
32
|
-
None, "--project", "-p", help="project id or name"
|
|
33
|
-
),
|
|
47
|
+
files: List[str] = typer.Argument(help="files to verify"),
|
|
48
|
+
project: Optional[str] = typer.Option(None, "--project", "-p", help="project id or name"),
|
|
34
49
|
mission: str = typer.Option(..., "--mission", "-m", help="mission id or name"),
|
|
35
|
-
|
|
50
|
+
skip: bool = typer.Option(
|
|
51
|
+
False,
|
|
52
|
+
"--skip",
|
|
53
|
+
"-s",
|
|
54
|
+
help="skip unsupported file types, badly named files, or directories instead of erroring",
|
|
55
|
+
),
|
|
56
|
+
experimental_datatypes: bool = typer.Option(False, help="allow experimental datatypes (yaml, svo2, db3, tum)"),
|
|
57
|
+
skip_hash: bool = typer.Option(None, help="skip hash check"),
|
|
58
|
+
check_file_hash: bool = typer.Option(
|
|
59
|
+
True,
|
|
60
|
+
help="check file hash. If True, file names and file hashes are checked.",
|
|
61
|
+
),
|
|
62
|
+
check_file_size: bool = typer.Option(
|
|
63
|
+
True,
|
|
64
|
+
help="check file size. If True, file names and file sizes are checked.",
|
|
65
|
+
),
|
|
36
66
|
) -> None:
|
|
37
67
|
# get all filepaths
|
|
38
|
-
|
|
68
|
+
original_file_paths = [Path(file) for file in files]
|
|
39
69
|
|
|
40
70
|
# get mission query
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
71
|
+
mission_query = _build_mission_query(mission, project)
|
|
72
|
+
|
|
73
|
+
validator = FileValidator(
|
|
74
|
+
skip=skip,
|
|
75
|
+
experimental_datatypes=experimental_datatypes,
|
|
76
|
+
)
|
|
77
|
+
files_to_verify = validator.filter_files(original_file_paths)
|
|
78
|
+
|
|
79
|
+
# Report skipped files (if any)
|
|
80
|
+
_report_skipped_files(validator.skipped_files)
|
|
81
|
+
|
|
82
|
+
# Check if we have anything left to do
|
|
83
|
+
_handle_no_files_to_process(
|
|
84
|
+
original_count=len(original_file_paths),
|
|
85
|
+
processed_count=len(files_to_verify),
|
|
86
|
+
action="verify",
|
|
46
87
|
)
|
|
47
88
|
|
|
48
89
|
verbose = get_shared_state().verbose
|
|
49
90
|
file_status = kleinkram.core.verify(
|
|
50
91
|
client=AuthenticatedClient(),
|
|
51
92
|
query=mission_query,
|
|
52
|
-
file_paths=
|
|
93
|
+
file_paths=files_to_verify,
|
|
53
94
|
skip_hash=skip_hash,
|
|
95
|
+
check_file_hash=check_file_hash,
|
|
96
|
+
check_file_size=check_file_size,
|
|
54
97
|
verbose=verbose,
|
|
55
98
|
)
|
|
56
99
|
print_file_verification_status(file_status, pprint=verbose)
|
kleinkram/cli/app.py
CHANGED
|
@@ -19,12 +19,14 @@ from kleinkram.api.client import AuthenticatedClient
|
|
|
19
19
|
from kleinkram.api.routes import _claim_admin
|
|
20
20
|
from kleinkram.api.routes import _get_api_version
|
|
21
21
|
from kleinkram.auth import login_flow
|
|
22
|
+
from kleinkram.cli._action import action_typer
|
|
22
23
|
from kleinkram.cli._download import download_typer
|
|
23
24
|
from kleinkram.cli._endpoint import endpoint_typer
|
|
24
25
|
from kleinkram.cli._file import file_typer
|
|
25
26
|
from kleinkram.cli._list import list_typer
|
|
26
27
|
from kleinkram.cli._mission import mission_typer
|
|
27
28
|
from kleinkram.cli._project import project_typer
|
|
29
|
+
from kleinkram.cli._run import run_typer
|
|
28
30
|
from kleinkram.cli._upload import upload_typer
|
|
29
31
|
from kleinkram.cli._verify import verify_typer
|
|
30
32
|
from kleinkram.cli.error_handling import ErrorHandledTyper
|
|
@@ -59,7 +61,7 @@ Kleinkram CLI
|
|
|
59
61
|
|
|
60
62
|
The Kleinkram CLI is a command line interface for Kleinkram.
|
|
61
63
|
For a list of available commands, run `klein --help` or visit \
|
|
62
|
-
https://docs.datasets.leggedrobotics.com/usage/python/
|
|
64
|
+
https://docs.datasets.leggedrobotics.com/usage/python/setup \
|
|
63
65
|
for more information.
|
|
64
66
|
"""
|
|
65
67
|
|
|
@@ -85,6 +87,7 @@ class CommandTypes(str, Enum):
|
|
|
85
87
|
AUTH = "Authentication Commands"
|
|
86
88
|
CORE = "Core Commands"
|
|
87
89
|
CRUD = "Create Update Delete Commands"
|
|
90
|
+
ACTION = "Kleinkram Action Commands"
|
|
88
91
|
|
|
89
92
|
|
|
90
93
|
class OrderCommands(TyperGroup):
|
|
@@ -110,6 +113,8 @@ app.add_typer(list_typer, name="list", rich_help_panel=CommandTypes.CORE)
|
|
|
110
113
|
app.add_typer(file_typer, name="file", rich_help_panel=CommandTypes.CRUD)
|
|
111
114
|
app.add_typer(mission_typer, name="mission", rich_help_panel=CommandTypes.CRUD)
|
|
112
115
|
app.add_typer(project_typer, name="project", rich_help_panel=CommandTypes.CRUD)
|
|
116
|
+
app.add_typer(action_typer, name="action", rich_help_panel=CommandTypes.ACTION)
|
|
117
|
+
app.add_typer(run_typer, name="run", rich_help_panel=CommandTypes.ACTION)
|
|
113
118
|
|
|
114
119
|
|
|
115
120
|
# attach error handler to app
|
|
@@ -127,10 +132,42 @@ def base_handler(exc: Exception) -> int:
|
|
|
127
132
|
|
|
128
133
|
@app.command(rich_help_panel=CommandTypes.AUTH)
|
|
129
134
|
def login(
|
|
135
|
+
oAuthProvider: str = typer.Option(
|
|
136
|
+
"auto",
|
|
137
|
+
"--oauth-provider",
|
|
138
|
+
"-p",
|
|
139
|
+
help="OAuth provider to use for login. Supported providers: google, github, fake-oauth.",
|
|
140
|
+
show_default=True,
|
|
141
|
+
),
|
|
130
142
|
key: Optional[str] = typer.Option(None, help="CLI key"),
|
|
131
143
|
headless: bool = typer.Option(False),
|
|
144
|
+
user: Optional[str] = typer.Option(
|
|
145
|
+
None,
|
|
146
|
+
"--user",
|
|
147
|
+
"-u",
|
|
148
|
+
help="Auto-select user ID for fake-oauth (e.g., 1, 2, 3). Only works with fake-oauth provider.",
|
|
149
|
+
),
|
|
132
150
|
) -> None:
|
|
133
|
-
|
|
151
|
+
|
|
152
|
+
# logic to resolve the "auto" default
|
|
153
|
+
if oAuthProvider == "auto":
|
|
154
|
+
config = get_config()
|
|
155
|
+
if config.selected_endpoint == "local":
|
|
156
|
+
oAuthProvider = "fake-oauth"
|
|
157
|
+
else:
|
|
158
|
+
oAuthProvider = "google"
|
|
159
|
+
|
|
160
|
+
# validate oAuthProvider
|
|
161
|
+
if oAuthProvider not in ["google", "github", "fake-oauth"]:
|
|
162
|
+
raise typer.BadParameter(
|
|
163
|
+
f"Unsupported OAuth provider '{oAuthProvider}'. Supported providers: google, github, fake-oauth."
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# validate that user parameter is only used with fake-oauth
|
|
167
|
+
if user is not None and oAuthProvider != "fake-oauth":
|
|
168
|
+
raise typer.BadParameter("--user parameter can only be used with fake-oauth provider")
|
|
169
|
+
|
|
170
|
+
login_flow(oAuthProvider=oAuthProvider, key=key, headless=headless, user=user)
|
|
134
171
|
|
|
135
172
|
|
|
136
173
|
@app.command(rich_help_panel=CommandTypes.AUTH)
|
|
@@ -156,29 +193,35 @@ def _version_callback(value: bool) -> None:
|
|
|
156
193
|
raise typer.Exit()
|
|
157
194
|
|
|
158
195
|
|
|
159
|
-
def
|
|
196
|
+
def check_version_compatibility() -> None:
|
|
160
197
|
cli_version = get_supported_api_version()
|
|
161
198
|
api_version = _get_api_version()
|
|
162
199
|
api_vers_str = ".".join(map(str, api_version))
|
|
163
200
|
|
|
164
201
|
if cli_version[0] != api_version[0]:
|
|
165
202
|
raise InvalidCLIVersion(
|
|
166
|
-
f"
|
|
203
|
+
f"You are using an unsupported CLI version ({__version__}). "
|
|
204
|
+
f"Please upgrade the CLI to version {api_vers_str} to continue using the CLI."
|
|
167
205
|
)
|
|
168
206
|
|
|
169
207
|
if cli_version[1] != api_version[1]:
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
208
|
+
if cli_version < api_version:
|
|
209
|
+
msg = f"You are using an outdated CLI version ({__version__}). "
|
|
210
|
+
msg += f"Please consider upgrading the CLI to version {api_vers_str}."
|
|
211
|
+
Console(file=sys.stderr).print(msg, style="red")
|
|
212
|
+
logger.warning(msg)
|
|
213
|
+
elif cli_version > api_version:
|
|
214
|
+
msg = f"You are using a CLI version ({__version__}) that is newer than the server version ({api_vers_str}). "
|
|
215
|
+
msg += "Please ask the admin to update the server."
|
|
216
|
+
Console(file=sys.stderr).print(msg, style="yellow")
|
|
217
|
+
logger.warning(msg)
|
|
173
218
|
|
|
174
219
|
|
|
175
220
|
@app.callback()
|
|
176
221
|
def cli(
|
|
177
222
|
verbose: bool = typer.Option(True, help="Enable verbose mode."),
|
|
178
223
|
debug: bool = typer.Option(False, help="Enable debug mode."),
|
|
179
|
-
version: Optional[bool] = typer.Option(
|
|
180
|
-
None, "--version", "-v", callback=_version_callback
|
|
181
|
-
),
|
|
224
|
+
version: Optional[bool] = typer.Option(None, "--version", "-v", callback=_version_callback),
|
|
182
225
|
log_level: Optional[LogLevel] = typer.Option(None, help="Set log level."),
|
|
183
226
|
max_lines: int = typer.Option(
|
|
184
227
|
MAX_TABLE_SIZE,
|
|
@@ -205,19 +248,15 @@ def cli(
|
|
|
205
248
|
log_level = LogLevel.WARNING
|
|
206
249
|
|
|
207
250
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
208
|
-
logging.basicConfig(
|
|
209
|
-
level=LOG_LEVEL_MAP[log_level], filename=LOG_FILE, format=LOG_FORMAT
|
|
210
|
-
)
|
|
251
|
+
logging.basicConfig(level=LOG_LEVEL_MAP[log_level], filename=LOG_FILE, format=LOG_FORMAT)
|
|
211
252
|
logger.info(f"CLI version: {__version__}")
|
|
212
253
|
|
|
213
254
|
try:
|
|
214
|
-
|
|
255
|
+
check_version_compatibility()
|
|
215
256
|
except InvalidCLIVersion as e:
|
|
216
257
|
logger.error(format_traceback(e))
|
|
217
258
|
raise
|
|
218
259
|
except Exception:
|
|
219
260
|
err = "failed to check version compatibility"
|
|
220
|
-
Console(file=sys.stderr).print(
|
|
221
|
-
err, style="yellow" if shared_state.verbose else None
|
|
222
|
-
)
|
|
261
|
+
Console(file=sys.stderr).print(err, style="yellow" if shared_state.verbose else None)
|
|
223
262
|
logger.error(err)
|
kleinkram/cli/error_handling.py
CHANGED
|
@@ -23,9 +23,7 @@ class ErrorHandledTyper(typer.Typer):
|
|
|
23
23
|
|
|
24
24
|
_error_handlers: OrderedDict[Type[Exception], ExceptionHandler]
|
|
25
25
|
|
|
26
|
-
def error_handler(
|
|
27
|
-
self, exc: Type[Exception]
|
|
28
|
-
) -> Callable[[ExceptionHandler], ExceptionHandler]:
|
|
26
|
+
def error_handler(self, exc: Type[Exception]) -> Callable[[ExceptionHandler], ExceptionHandler]:
|
|
29
27
|
def dec(func: ExceptionHandler) -> ExceptionHandler:
|
|
30
28
|
self._error_handlers[exc] = func
|
|
31
29
|
return func
|
kleinkram/config.py
CHANGED
|
@@ -120,13 +120,9 @@ def _get_default_credentials() -> Dict[str, Credentials]:
|
|
|
120
120
|
@dataclass
|
|
121
121
|
class Config:
|
|
122
122
|
version: str = __version__
|
|
123
|
-
selected_endpoint: str = field(
|
|
124
|
-
default_factory=lambda: _get_default_selected_endpoint().name
|
|
125
|
-
)
|
|
123
|
+
selected_endpoint: str = field(default_factory=lambda: _get_default_selected_endpoint().name)
|
|
126
124
|
endpoints: Dict[str, Endpoint] = field(default_factory=_get_default_endpoints)
|
|
127
|
-
endpoint_credentials: Dict[str, Credentials] = field(
|
|
128
|
-
default_factory=_get_default_credentials
|
|
129
|
-
)
|
|
125
|
+
endpoint_credentials: Dict[str, Credentials] = field(default_factory=_get_default_credentials)
|
|
130
126
|
|
|
131
127
|
@property
|
|
132
128
|
def endpoint(self) -> Endpoint:
|
|
@@ -149,9 +145,7 @@ def _config_to_dict(config: Config) -> Dict[str, Any]:
|
|
|
149
145
|
return {
|
|
150
146
|
"version": config.version,
|
|
151
147
|
"endpoints": {key: value._asdict() for key, value in config.endpoints.items()},
|
|
152
|
-
"endpoint_credentials": {
|
|
153
|
-
key: value._asdict() for key, value in config.endpoint_credentials.items()
|
|
154
|
-
},
|
|
148
|
+
"endpoint_credentials": {key: value._asdict() for key, value in config.endpoint_credentials.items()},
|
|
155
149
|
"selected_endpoint": config.endpoint.name,
|
|
156
150
|
}
|
|
157
151
|
|
|
@@ -161,16 +155,11 @@ def _config_from_dict(dct: Dict[str, Any]) -> Config:
|
|
|
161
155
|
dct["version"],
|
|
162
156
|
dct["selected_endpoint"],
|
|
163
157
|
{key: Endpoint(**value) for key, value in dct["endpoints"].items()},
|
|
164
|
-
{
|
|
165
|
-
key: Credentials(**value)
|
|
166
|
-
for key, value in dct["endpoint_credentials"].items()
|
|
167
|
-
},
|
|
158
|
+
{key: Credentials(**value) for key, value in dct["endpoint_credentials"].items()},
|
|
168
159
|
)
|
|
169
160
|
|
|
170
161
|
|
|
171
|
-
def _safe_config_write(
|
|
172
|
-
config: Config, path: Path, tmp_dir: Optional[Path] = None
|
|
173
|
-
) -> None:
|
|
162
|
+
def _safe_config_write(config: Config, path: Path, tmp_dir: Optional[Path] = None) -> None:
|
|
174
163
|
fd, temp_path = tempfile.mkstemp(dir=tmp_dir)
|
|
175
164
|
with os.fdopen(fd, "w") as f:
|
|
176
165
|
json.dump(_config_to_dict(config), f)
|
|
@@ -248,11 +237,7 @@ def endpoint_table(config: Config) -> Table:
|
|
|
248
237
|
table.add_column("S3", style="cyan")
|
|
249
238
|
|
|
250
239
|
for name, endpoint in config.endpoints.items():
|
|
251
|
-
display_name = (
|
|
252
|
-
Text(f"* {name}", style="bold yellow")
|
|
253
|
-
if name == config.selected_endpoint
|
|
254
|
-
else Text(f" {name}")
|
|
255
|
-
)
|
|
240
|
+
display_name = Text(f"* {name}", style="bold yellow") if name == config.selected_endpoint else Text(f" {name}")
|
|
256
241
|
table.add_row(display_name, endpoint.api, endpoint.s3)
|
|
257
242
|
return table
|
|
258
243
|
|