artefacts-cli 0.6.16__py3-none-any.whl → 0.6.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- artefacts/cli/__init__.py +40 -32
- artefacts/cli/app.py +69 -10
- artefacts/cli/app_containers.py +28 -7
- artefacts/cli/containers/__init__.py +11 -4
- artefacts/cli/containers/docker.py +20 -2
- artefacts/cli/containers/utils.py +11 -4
- artefacts/cli/errors.py +1 -0
- artefacts/cli/logger.py +10 -0
- artefacts/cli/parameters.py +1 -1
- artefacts/cli/ros1.py +7 -7
- artefacts/cli/ros2.py +2 -2
- artefacts/cli/utils.py +1 -1
- artefacts/cli/utils_ros.py +1 -1
- artefacts/cli/version.py +2 -2
- {artefacts_cli-0.6.16.dist-info → artefacts_cli-0.6.18.dist-info}/METADATA +10 -10
- artefacts_cli-0.6.18.dist-info/RECORD +23 -0
- {artefacts_cli-0.6.16.dist-info → artefacts_cli-0.6.18.dist-info}/WHEEL +1 -1
- artefacts_cli-0.6.16.dist-info/RECORD +0 -21
- {artefacts_cli-0.6.16.dist-info → artefacts_cli-0.6.18.dist-info}/entry_points.txt +0 -0
- {artefacts_cli-0.6.16.dist-info → artefacts_cli-0.6.18.dist-info}/top_level.txt +0 -0
artefacts/cli/__init__.py
CHANGED
@@ -2,7 +2,6 @@ from importlib.metadata import version, PackageNotFoundError
|
|
2
2
|
import json
|
3
3
|
import glob
|
4
4
|
from datetime import datetime, timezone
|
5
|
-
import logging
|
6
5
|
import os
|
7
6
|
import math
|
8
7
|
import requests
|
@@ -10,12 +9,11 @@ import copy
|
|
10
9
|
from typing import Optional
|
11
10
|
|
12
11
|
from .parameters import iter_grid
|
13
|
-
|
14
|
-
logging.basicConfig(level=logging.INFO)
|
12
|
+
from .logger import logger
|
15
13
|
|
16
14
|
|
17
15
|
try:
|
18
|
-
__version__ = version("
|
16
|
+
__version__ = version("artefacts-cli")
|
19
17
|
except PackageNotFoundError:
|
20
18
|
try:
|
21
19
|
# Package is not installed, most likely dev/test mode
|
@@ -23,7 +21,7 @@ except PackageNotFoundError:
|
|
23
21
|
|
24
22
|
__version__ = get_version()
|
25
23
|
except Exception as e:
|
26
|
-
|
24
|
+
logger.warning(f"Could not determine package version: {e}. Default to 0.0.0")
|
27
25
|
__version__ = "0.0.0"
|
28
26
|
|
29
27
|
|
@@ -42,6 +40,7 @@ class WarpJob:
|
|
42
40
|
jobconf,
|
43
41
|
dryrun=False,
|
44
42
|
nosim=False,
|
43
|
+
noupload=False,
|
45
44
|
noisolation=False,
|
46
45
|
context=None,
|
47
46
|
run_offset=0,
|
@@ -57,6 +56,7 @@ class WarpJob:
|
|
57
56
|
self.n_runs = run_offset
|
58
57
|
self.dryrun = dryrun
|
59
58
|
self.nosim = nosim
|
59
|
+
self.noupload = noupload
|
60
60
|
self.noisolation = noisolation
|
61
61
|
self.context = context
|
62
62
|
|
@@ -84,10 +84,10 @@ class WarpJob:
|
|
84
84
|
if response.status_code != 200:
|
85
85
|
if response.status_code == 403:
|
86
86
|
msg = response.json()["message"]
|
87
|
-
|
87
|
+
logger.warning(msg)
|
88
88
|
raise AuthenticationError(msg)
|
89
|
-
|
90
|
-
|
89
|
+
logger.warning(f"Error on job creation: {response.status_code}")
|
90
|
+
logger.warning(response.text)
|
91
91
|
raise AuthenticationError(str(response.status_code))
|
92
92
|
self.job_id = response.json()["job_id"]
|
93
93
|
self.output_path = self.params.get("output_path", f"/tmp/{self.job_id}")
|
@@ -108,7 +108,7 @@ class WarpJob:
|
|
108
108
|
"success": self.success, # need to be determined based on all runs, can be an AND in the API
|
109
109
|
"status": "finished", # need to be determined based on all runs
|
110
110
|
}
|
111
|
-
|
111
|
+
requests.put(
|
112
112
|
f"{self.api_conf.api_url}/{self.project_id}/job/{self.job_id}",
|
113
113
|
json=data,
|
114
114
|
headers=self.api_conf.headers,
|
@@ -155,10 +155,10 @@ class WarpRun:
|
|
155
155
|
if response.status_code != 200:
|
156
156
|
if response.status_code == 403:
|
157
157
|
msg = response.json()["message"]
|
158
|
-
|
158
|
+
logger.warning(msg)
|
159
159
|
raise AuthenticationError(msg)
|
160
|
-
|
161
|
-
|
160
|
+
logger.warning(f"Error on scenario creation: {response.status_code}")
|
161
|
+
logger.warning(response.text)
|
162
162
|
raise AuthenticationError(str(response.status_code))
|
163
163
|
return
|
164
164
|
|
@@ -170,7 +170,7 @@ class WarpRun:
|
|
170
170
|
|
171
171
|
def log_metrics(self):
|
172
172
|
metrics = self.params.get("metrics", None)
|
173
|
-
if type(metrics)
|
173
|
+
if type(metrics) is str:
|
174
174
|
with open(f"{self.output_path}/{metrics}") as f:
|
175
175
|
metric_values = json.load(f)
|
176
176
|
for k, v in metric_values.items():
|
@@ -182,7 +182,8 @@ class WarpRun:
|
|
182
182
|
metric_values = json.load(f)
|
183
183
|
for k, v in metric_values.items():
|
184
184
|
self.log_metric(k, v)
|
185
|
-
except:
|
185
|
+
except FileNotFoundError:
|
186
|
+
# if the metrics.json file does not exist, do nothing
|
186
187
|
pass
|
187
188
|
|
188
189
|
def log_tests_results(self, test_results, success):
|
@@ -225,7 +226,7 @@ class WarpRun:
|
|
225
226
|
"""log a single file filename"""
|
226
227
|
|
227
228
|
def _get_filename(path):
|
228
|
-
return path.split(
|
229
|
+
return path.split("/")[-1]
|
229
230
|
|
230
231
|
if prefix is not None:
|
231
232
|
self.uploads.update({f"{prefix}/{_get_filename(filename)}": filename})
|
@@ -255,23 +256,28 @@ class WarpRun:
|
|
255
256
|
headers=self.job.api_conf.headers,
|
256
257
|
)
|
257
258
|
# use s3 presigned urls to upload the artifacts
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
259
|
+
if self.job.noupload:
|
260
|
+
print(
|
261
|
+
"noupload: job artifacts are not uploaded to cloud, including the ones specified in output_dirs"
|
262
|
+
)
|
263
|
+
else:
|
264
|
+
upload_urls = response.json()["upload_urls"]
|
265
|
+
for key, file_name in self.uploads.items():
|
266
|
+
files = {"file": open(file_name, "rb")}
|
267
|
+
upload_info = upload_urls[key]
|
268
|
+
file_size_mb = os.path.getsize(file_name) / 1024 / 1024
|
269
|
+
try:
|
270
|
+
print(f"Uploading {file_name} ({file_size_mb:.2f} MB)")
|
271
|
+
# TODO: add a retry policy
|
272
|
+
requests.post(
|
273
|
+
upload_info["url"],
|
274
|
+
data=upload_info["fields"],
|
275
|
+
files=files,
|
276
|
+
)
|
277
|
+
except OverflowError:
|
278
|
+
logger.warning(f"File too large: {file_name} could not be uploaded")
|
279
|
+
except Exception as e:
|
280
|
+
logger.warning(f"Error uploading {file_name}: {e}, skipping")
|
275
281
|
|
276
282
|
|
277
283
|
def init_job(
|
@@ -281,6 +287,7 @@ def init_job(
|
|
281
287
|
jobconf: dict,
|
282
288
|
dryrun: bool = False,
|
283
289
|
nosim: bool = False,
|
290
|
+
noupload: bool = False,
|
284
291
|
noisolation: bool = False,
|
285
292
|
context: Optional[dict] = None,
|
286
293
|
run_offset=0,
|
@@ -292,6 +299,7 @@ def init_job(
|
|
292
299
|
jobconf,
|
293
300
|
dryrun,
|
294
301
|
nosim,
|
302
|
+
noupload,
|
295
303
|
noisolation,
|
296
304
|
context,
|
297
305
|
run_offset,
|
artefacts/cli/app.py
CHANGED
@@ -5,7 +5,6 @@ import os
|
|
5
5
|
import platform
|
6
6
|
import random
|
7
7
|
import subprocess
|
8
|
-
import sys
|
9
8
|
import tarfile
|
10
9
|
import tempfile
|
11
10
|
import time
|
@@ -22,7 +21,6 @@ from artefacts.cli import init_job, generate_scenarios, AuthenticationError, __v
|
|
22
21
|
from artefacts.cli import app_containers as containers
|
23
22
|
from artefacts.cli.constants import DEPRECATED_FRAMEWORKS, SUPPORTED_FRAMEWORKS
|
24
23
|
from artefacts.cli.utils import read_config, config_validation
|
25
|
-
import artefacts_copava as copava
|
26
24
|
|
27
25
|
HOME = os.path.expanduser("~")
|
28
26
|
CONFIG_DIR = f"{HOME}/.artefacts"
|
@@ -213,6 +211,12 @@ def hello(project_name):
|
|
213
211
|
default=False,
|
214
212
|
help="nosim: no simulator resource provided by Artefacts",
|
215
213
|
)
|
214
|
+
@click.option(
|
215
|
+
"--noupload",
|
216
|
+
is_flag=True,
|
217
|
+
default=False,
|
218
|
+
help="noupload: rosbags are not uploaded to cloud",
|
219
|
+
)
|
216
220
|
@click.option(
|
217
221
|
"--noisolation",
|
218
222
|
is_flag=True,
|
@@ -231,11 +235,65 @@ def hello(project_name):
|
|
231
235
|
is_eager=True, # Necessary for callbacks to see it.
|
232
236
|
help="Skip configuration validation, so that unsupported settings can be tried out, e.g. non-ROS settings or simulators like SAPIEN.",
|
233
237
|
)
|
238
|
+
@click.option(
|
239
|
+
"--in-container",
|
240
|
+
is_flag=True,
|
241
|
+
default=False,
|
242
|
+
help='[Experimental] Run the job inside a package container. The container image is build if it does not exist yet, with default name as "artefacts" (please use --with-image to override the image name). This option overrides (for now) --dryrun, --nosim, --noisolation and --description.',
|
243
|
+
)
|
244
|
+
@click.option(
|
245
|
+
"--with-image",
|
246
|
+
default="artefacts",
|
247
|
+
help="[Experimental] Run the job using the image name passed here. Only used when running with --in-container set.",
|
248
|
+
)
|
249
|
+
@click.option(
|
250
|
+
"--rebuild-container",
|
251
|
+
is_flag=True,
|
252
|
+
default=False,
|
253
|
+
help="[Experimental] Rebuild the container image before running. This flag guarantees that the run uses the latest code available when building the image, and usually takes more time before the run can start (time to compile and generate the image).",
|
254
|
+
)
|
255
|
+
@click.option(
|
256
|
+
"--with-gui",
|
257
|
+
is_flag=True,
|
258
|
+
default=False,
|
259
|
+
help="Show any GUI if any is created by the test runs. By default, UI elements are run but hidden---only test logs are returned. Please note GUI often assume X11 (e.g. ROS), typically with Qt, so this may not work without a appropriate environment.",
|
260
|
+
)
|
234
261
|
@click.argument("jobname")
|
262
|
+
@click.pass_context
|
235
263
|
def run(
|
236
|
-
|
264
|
+
ctx: click.Context,
|
265
|
+
config,
|
266
|
+
jobname,
|
267
|
+
dryrun,
|
268
|
+
nosim,
|
269
|
+
noupload,
|
270
|
+
noisolation,
|
271
|
+
description="",
|
272
|
+
skip_validation=False,
|
273
|
+
in_container: bool = False,
|
274
|
+
with_image: str = "artefacts",
|
275
|
+
rebuild_container: bool = False,
|
276
|
+
with_gui: bool = False,
|
237
277
|
):
|
238
|
-
"""
|
278
|
+
"""
|
279
|
+
Run JOBNAME locally
|
280
|
+
|
281
|
+
* Directly in the shell by default.
|
282
|
+
* Inside a packaged container when using the --in-container option.
|
283
|
+
"""
|
284
|
+
if in_container:
|
285
|
+
if rebuild_container or not ctx.invoke(containers.check, name=with_image):
|
286
|
+
ctx.invoke(
|
287
|
+
containers.build, path=".", dockerfile="Dockerfile", name=with_image
|
288
|
+
)
|
289
|
+
return ctx.invoke(
|
290
|
+
containers.run,
|
291
|
+
image=with_image,
|
292
|
+
jobname=jobname,
|
293
|
+
config=config,
|
294
|
+
with_gui=with_gui,
|
295
|
+
)
|
296
|
+
|
239
297
|
warpconfig = read_config(config)
|
240
298
|
|
241
299
|
project_id = warpconfig["project"]
|
@@ -284,6 +342,7 @@ def run(
|
|
284
342
|
jobconf,
|
285
343
|
dryrun,
|
286
344
|
nosim,
|
345
|
+
noupload,
|
287
346
|
noisolation,
|
288
347
|
context,
|
289
348
|
first,
|
@@ -296,7 +355,7 @@ def run(
|
|
296
355
|
job_success = True
|
297
356
|
for scenario_n, scenario in enumerate(scenarios):
|
298
357
|
click.echo(
|
299
|
-
f"Starting scenario {scenario_n+1}/{len(scenarios)}: {scenario['name']}"
|
358
|
+
f"Starting scenario {scenario_n + 1}/{len(scenarios)}: {scenario['name']}"
|
300
359
|
)
|
301
360
|
try:
|
302
361
|
run = warpjob.new_run(scenario)
|
@@ -357,7 +416,7 @@ def run(
|
|
357
416
|
results, success = run_other_tests(run)
|
358
417
|
if not success:
|
359
418
|
job_success = False
|
360
|
-
if type(run.params.get("metrics", []))
|
419
|
+
if type(run.params.get("metrics", [])) is str:
|
361
420
|
run.log_metrics()
|
362
421
|
|
363
422
|
run.stop()
|
@@ -427,7 +486,7 @@ def run_remote(config, description, jobname, skip_validation=False):
|
|
427
486
|
if "on" in run_config:
|
428
487
|
del run_config["on"]
|
429
488
|
|
430
|
-
click.echo(
|
489
|
+
click.echo("Packaging source...")
|
431
490
|
|
432
491
|
with tempfile.NamedTemporaryFile(
|
433
492
|
prefix=project_id.split("/")[-1], suffix=".tgz", delete=True
|
@@ -437,7 +496,7 @@ def run_remote(config, description, jobname, skip_validation=False):
|
|
437
496
|
try:
|
438
497
|
ignore_matches = parse_gitignore(ignore_file)
|
439
498
|
except FileNotFoundError:
|
440
|
-
ignore_matches = lambda x: False
|
499
|
+
ignore_matches = lambda x: False # noqa: E731
|
441
500
|
with tarfile.open(fileobj=temp_file, mode="w:gz") as tar_file:
|
442
501
|
for root, dirs, files in os.walk(project_folder):
|
443
502
|
for file in files:
|
@@ -551,7 +610,7 @@ def run_remote(config, description, jobname, skip_validation=False):
|
|
551
610
|
)
|
552
611
|
|
553
612
|
click.echo(
|
554
|
-
f"Uploading complete! The new job will show up shortly at {dashboard_url}
|
613
|
+
f"Uploading complete! The new job will show up shortly at {dashboard_url}"
|
555
614
|
)
|
556
615
|
|
557
616
|
|
@@ -573,7 +632,7 @@ artefacts.add_command(config)
|
|
573
632
|
artefacts.add_command(hello)
|
574
633
|
artefacts.add_command(run)
|
575
634
|
artefacts.add_command(run_remote)
|
576
|
-
artefacts.add_command(containers.
|
635
|
+
artefacts.add_command(containers.containers)
|
577
636
|
|
578
637
|
|
579
638
|
if __name__ == "__main__":
|
artefacts/cli/app_containers.py
CHANGED
@@ -10,12 +10,12 @@ from artefacts.cli.containers.utils import ContainerMgr
|
|
10
10
|
@click.group()
|
11
11
|
@click.option("--debug/--no-debug", default=False)
|
12
12
|
@click.pass_context
|
13
|
-
def
|
13
|
+
def containers(ctx: click.Context, debug: bool):
|
14
14
|
ctx.ensure_object(dict)
|
15
15
|
ctx.obj["debug"] = debug
|
16
16
|
|
17
17
|
|
18
|
-
@
|
18
|
+
@containers.command()
|
19
19
|
@click.option(
|
20
20
|
"--path",
|
21
21
|
default=".",
|
@@ -29,13 +29,13 @@ def package(ctx: click.Context, debug: bool):
|
|
29
29
|
@click.option(
|
30
30
|
"--name",
|
31
31
|
required=False,
|
32
|
-
help="Name for the generated
|
32
|
+
help="Name for the generated container",
|
33
33
|
)
|
34
34
|
@click.pass_context
|
35
35
|
def build(ctx: click.Context, path: str, dockerfile: str, name: str):
|
36
36
|
if not os.path.exists(os.path.join(path, dockerfile)):
|
37
37
|
raise click.ClickException(
|
38
|
-
f"No {dockerfile} found here. I cannot build the
|
38
|
+
f"No {dockerfile} found here. I cannot build the container."
|
39
39
|
)
|
40
40
|
if name is None:
|
41
41
|
name = "artefacts"
|
@@ -44,7 +44,21 @@ def build(ctx: click.Context, path: str, dockerfile: str, name: str):
|
|
44
44
|
print(f"Package complete in image: {image}")
|
45
45
|
|
46
46
|
|
47
|
-
@
|
47
|
+
@containers.command()
|
48
|
+
@click.argument("name")
|
49
|
+
@click.pass_context
|
50
|
+
def check(ctx: click.Context, name: str):
|
51
|
+
if name is None:
|
52
|
+
name = "artefacts"
|
53
|
+
handler = ContainerMgr()
|
54
|
+
result = handler.check(name)
|
55
|
+
if ctx.parent is None:
|
56
|
+
# Print only if the command is called directly.
|
57
|
+
print(f"Package {name} exists and ready to use.")
|
58
|
+
return result
|
59
|
+
|
60
|
+
|
61
|
+
@containers.command()
|
48
62
|
@click.argument("image")
|
49
63
|
@click.argument("jobname")
|
50
64
|
@click.option(
|
@@ -53,8 +67,14 @@ def build(ctx: click.Context, path: str, dockerfile: str, name: str):
|
|
53
67
|
default="artefacts.yaml",
|
54
68
|
help="Artefacts config file.",
|
55
69
|
)
|
70
|
+
@click.option(
|
71
|
+
"--with-gui",
|
72
|
+
"with_gui",
|
73
|
+
default=False,
|
74
|
+
help="Show any GUI if any is created by the test runs. By default, UI elements are run but hidden---only test logs are returned. Please note GUI often assume an X11 environment, typically with Qt, so this may not work without a appropriate environment.",
|
75
|
+
)
|
56
76
|
@click.pass_context
|
57
|
-
def run(ctx: click.Context, image: str, jobname: str, config: str):
|
77
|
+
def run(ctx: click.Context, image: str, jobname: str, config: str, with_gui: bool):
|
58
78
|
try:
|
59
79
|
artefacts_config = read_config(config)
|
60
80
|
except FileNotFoundError:
|
@@ -67,11 +87,12 @@ def run(ctx: click.Context, image: str, jobname: str, config: str):
|
|
67
87
|
jobname=jobname,
|
68
88
|
# Hidden setting primarily useful to Artefacts developers
|
69
89
|
api_url=os.environ.get("ARTEFACTS_API_URL", DEFAULT_API_URL),
|
90
|
+
with_gui=with_gui,
|
70
91
|
)
|
71
92
|
container, logs = handler.run(**params)
|
72
93
|
if container:
|
73
94
|
print(f"Package run complete: Container Id for inspection: {container['Id']}")
|
74
95
|
else:
|
75
|
-
print(
|
96
|
+
print("Package run failed:")
|
76
97
|
for entry in logs:
|
77
98
|
print("\t- " + entry)
|
@@ -13,7 +13,13 @@ class CMgr:
|
|
13
13
|
Returns the build image ID (e.g. sha256:abcdefghi)
|
14
14
|
and an iterator over the build log entries.
|
15
15
|
"""
|
16
|
-
raise
|
16
|
+
raise NotImplementedError()
|
17
|
+
|
18
|
+
def check(self, image: str) -> bool:
|
19
|
+
"""
|
20
|
+
Checks whether a target image exists locally.
|
21
|
+
"""
|
22
|
+
raise NotImplementedError()
|
17
23
|
|
18
24
|
def run(
|
19
25
|
self,
|
@@ -22,12 +28,13 @@ class CMgr:
|
|
22
28
|
jobname: str = None,
|
23
29
|
artefacts_dir: str = Path("~/.artefacts").expanduser(),
|
24
30
|
api_url: str = DEFAULT_API_URL,
|
31
|
+
with_gui: bool = False,
|
25
32
|
) -> Tuple[Any, Generator]:
|
26
33
|
"""
|
27
34
|
Returns a container (Any type as depends on the framework)
|
28
35
|
and an iterator over the container log entries.
|
29
36
|
"""
|
30
|
-
raise
|
37
|
+
raise NotImplementedError()
|
31
38
|
|
32
39
|
def _valid_artefacts_api_key(
|
33
40
|
self, project: str, path: Union[str, Path] = Path("~/.artefacts").expanduser()
|
@@ -45,7 +52,7 @@ class CMgr:
|
|
45
52
|
"""
|
46
53
|
if not path:
|
47
54
|
raise Exception(
|
48
|
-
"`path` must be a string, a Path object, or
|
55
|
+
"`path` must be a string, a Path object, or excluded from the kwargs"
|
49
56
|
)
|
50
57
|
if os.environ.get("ARTEFACTS_KEY", None):
|
51
58
|
return True
|
@@ -56,6 +63,6 @@ class CMgr:
|
|
56
63
|
else:
|
57
64
|
config.read(path)
|
58
65
|
try:
|
59
|
-
return config[project].get("apikey")
|
66
|
+
return config[project].get("apikey") is not None
|
60
67
|
except KeyError:
|
61
68
|
return False
|
@@ -2,6 +2,7 @@ from collections.abc import Generator
|
|
2
2
|
import json
|
3
3
|
import os
|
4
4
|
from pathlib import Path
|
5
|
+
import platform
|
5
6
|
from typing import Any, Tuple
|
6
7
|
from uuid import uuid4
|
7
8
|
|
@@ -11,8 +12,8 @@ from artefacts.cli.utils import ensure_available
|
|
11
12
|
|
12
13
|
ensure_available("docker")
|
13
14
|
|
14
|
-
import docker
|
15
|
-
from docker import APIClient
|
15
|
+
import docker # noqa: E402
|
16
|
+
from docker import APIClient # noqa: E402
|
16
17
|
|
17
18
|
|
18
19
|
class DockerManager(CMgr):
|
@@ -39,6 +40,12 @@ class DockerManager(CMgr):
|
|
39
40
|
img_id = self.client.inspect_image(kwargs["tag"])["Id"]
|
40
41
|
return img_id, iter(logs)
|
41
42
|
|
43
|
+
def check(
|
44
|
+
self,
|
45
|
+
image: str,
|
46
|
+
) -> bool:
|
47
|
+
return len(self.client.images(name=image)) > 0
|
48
|
+
|
42
49
|
def run(
|
43
50
|
self,
|
44
51
|
image: str,
|
@@ -46,6 +53,7 @@ class DockerManager(CMgr):
|
|
46
53
|
jobname: str = None,
|
47
54
|
artefacts_dir: str = Path("~/.artefacts").expanduser(),
|
48
55
|
api_url: str = DEFAULT_API_URL,
|
56
|
+
with_gui: bool = False,
|
49
57
|
) -> Tuple[Any, Generator]:
|
50
58
|
if not self._valid_artefacts_api_key(project, artefacts_dir):
|
51
59
|
return None, iter(
|
@@ -60,6 +68,15 @@ class DockerManager(CMgr):
|
|
60
68
|
"ARTEFACTS_API_URL": api_url,
|
61
69
|
}
|
62
70
|
|
71
|
+
if platform.system() in ["Darwin", "Windows"]:
|
72
|
+
# Assume we run in Docker Desktop
|
73
|
+
env["DISPLAY"] = "host.docker.internal:0"
|
74
|
+
else:
|
75
|
+
env["DISPLAY"] = os.environ.get("DISPLAY", ":0")
|
76
|
+
|
77
|
+
if not with_gui:
|
78
|
+
env["QT_QPA_PLATFORM"] = "offscreen"
|
79
|
+
|
63
80
|
container = self.client.create_container(
|
64
81
|
image,
|
65
82
|
environment=env,
|
@@ -72,6 +89,7 @@ class DockerManager(CMgr):
|
|
72
89
|
"mode": "ro",
|
73
90
|
},
|
74
91
|
},
|
92
|
+
network_mode="host",
|
75
93
|
),
|
76
94
|
)
|
77
95
|
self.client.start(container=container.get("Id"))
|
@@ -1,8 +1,11 @@
|
|
1
|
+
import sys
|
2
|
+
|
1
3
|
from collections.abc import Generator
|
2
|
-
import logging
|
3
4
|
from typing import Any, Tuple
|
4
5
|
|
6
|
+
from artefacts.cli import errors
|
5
7
|
from artefacts.cli.containers import CMgr
|
8
|
+
from artefacts.cli.logger import logger
|
6
9
|
|
7
10
|
|
8
11
|
class ContainerMgr:
|
@@ -12,12 +15,13 @@ class ContainerMgr:
|
|
12
15
|
}
|
13
16
|
|
14
17
|
def __init__(self):
|
15
|
-
self.logger =
|
18
|
+
self.logger = logger
|
16
19
|
self.mgr = self._configure()
|
17
20
|
if self.mgr is None:
|
18
|
-
|
19
|
-
f"Failed to find supported container stack. Please install and start one
|
21
|
+
self.logger.error(
|
22
|
+
f"Failed to find supported container stack. Please install and start one of {list(self.SUPPORTED_PRIORITISED_ENGINES.values())}, with default settings (custom sockets not supported at this stage)"
|
20
23
|
)
|
24
|
+
sys.exit(errors.CONTAINER_ENGINE_NOT_FOUND)
|
21
25
|
|
22
26
|
def _configure(self) -> CMgr:
|
23
27
|
manager = None
|
@@ -50,5 +54,8 @@ class ContainerMgr:
|
|
50
54
|
def build(self, **kwargs) -> Tuple[str, Generator]:
|
51
55
|
return self.mgr.build(**kwargs)
|
52
56
|
|
57
|
+
def check(self, image: str) -> bool:
|
58
|
+
return self.mgr.check(image)
|
59
|
+
|
53
60
|
def run(self, **kwargs) -> Tuple[Any, Generator]:
|
54
61
|
return self.mgr.run(**kwargs)
|
artefacts/cli/errors.py
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
CONTAINER_ENGINE_NOT_FOUND = 1000
|
artefacts/cli/logger.py
ADDED
artefacts/cli/parameters.py
CHANGED
@@ -17,7 +17,7 @@ def iter_grid(grid_spec: dict) -> Iterable[dict]:
|
|
17
17
|
items = sorted(grid_spec.items())
|
18
18
|
keys, values = zip(*items)
|
19
19
|
# Make sure single values are converted to lists
|
20
|
-
values = [x if type(x)
|
20
|
+
values = [x if type(x) is list else [x] for x in values]
|
21
21
|
for v in product(*values):
|
22
22
|
params = dict(zip(keys, v))
|
23
23
|
yield params
|
artefacts/cli/ros1.py
CHANGED
@@ -50,17 +50,17 @@ def generate_rosbag_args(scenario: dict) -> str:
|
|
50
50
|
return topics
|
51
51
|
else:
|
52
52
|
logging.warning(
|
53
|
-
|
53
|
+
"[warning in generate_rosbag_args] rosbag_record asks for 'subscriptions' but they are not specified. Falling back to default: no rosbag will be recorded"
|
54
54
|
)
|
55
55
|
return "none"
|
56
56
|
else:
|
57
|
-
assert (
|
58
|
-
|
59
|
-
)
|
57
|
+
assert type(rosbag_record) is list, (
|
58
|
+
"rosbag_record supports 'all', 'none', 'subscriptions' or a list of strings interpreted as a list of ROS topics, regex supported"
|
59
|
+
)
|
60
60
|
for e in rosbag_record:
|
61
|
-
assert (
|
62
|
-
|
63
|
-
)
|
61
|
+
assert type(e) is str, (
|
62
|
+
"Elements of the rosbag_record list must only be strings. They are interpreted as a list of ROS topics, regex supported"
|
63
|
+
)
|
64
64
|
return f"--regex {' '.join(rosbag_record)}"
|
65
65
|
|
66
66
|
|
artefacts/cli/ros2.py
CHANGED
@@ -114,9 +114,9 @@ def run_ros2_tests(run):
|
|
114
114
|
try:
|
115
115
|
last_value = bag.get_last_message(metric)[1].data
|
116
116
|
run.log_metric(metric, last_value)
|
117
|
-
except KeyError
|
117
|
+
except KeyError:
|
118
118
|
print(f"Metric {metric} not found in rosbag, skipping.")
|
119
|
-
except TypeError or IndexError
|
119
|
+
except TypeError or IndexError:
|
120
120
|
print(
|
121
121
|
f"Metric {metric} not found. Is it being published?. Skipping."
|
122
122
|
)
|
artefacts/cli/utils.py
CHANGED
artefacts/cli/utils_ros.py
CHANGED
@@ -49,7 +49,7 @@ def parse_tests_results(file):
|
|
49
49
|
|
50
50
|
except Exception as e:
|
51
51
|
print(f"[Exception in parse_tests_results] {e}")
|
52
|
-
print(
|
52
|
+
print("Test result xml could not be loaded, marking success as False")
|
53
53
|
results = [
|
54
54
|
{
|
55
55
|
"suite": "unittest.suite.TestSuite",
|
artefacts/cli/version.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.2
|
2
2
|
Name: artefacts_cli
|
3
|
-
Version: 0.6.
|
3
|
+
Version: 0.6.18
|
4
4
|
Author-email: FD <fabian@artefacts.com>, AGC <alejandro@artefacts.com>, TN <tomo@artefacts.com>, EP <eric@artefacts.com>
|
5
5
|
Project-URL: Homepage, https://github.com/art-e-fact/artefacts-client
|
6
6
|
Project-URL: Bug Tracker, https://github.com/art-e-fact/artefacts-client/issues
|
@@ -17,25 +17,25 @@ Requires-Dist: mcap
|
|
17
17
|
Requires-Dist: mcap-ros2-support
|
18
18
|
Requires-Dist: PyYAML>=6.0
|
19
19
|
Requires-Dist: requests>=2.27.1
|
20
|
-
Requires-Dist: setuptools-scm
|
21
20
|
Provides-Extra: dev
|
22
21
|
Requires-Dist: awscli; extra == "dev"
|
23
22
|
Requires-Dist: build; extra == "dev"
|
24
23
|
Requires-Dist: docker; extra == "dev"
|
25
24
|
Requires-Dist: lark; extra == "dev"
|
26
|
-
Requires-Dist: pyre-check; extra == "dev"
|
27
|
-
Requires-Dist: pytest; extra == "dev"
|
28
|
-
Requires-Dist: pytest-cov; extra == "dev"
|
29
|
-
Requires-Dist: pytest-env; extra == "dev"
|
30
|
-
Requires-Dist: pytest-mock; extra == "dev"
|
31
|
-
Requires-Dist: ruff; extra == "dev"
|
32
|
-
Requires-Dist: twine; extra == "dev"
|
33
25
|
Requires-Dist: mkdocs-click==0.8.0; extra == "dev"
|
34
26
|
Requires-Dist: mkdocs-material==8.5.6; extra == "dev"
|
35
27
|
Requires-Dist: mkdocs-mermaid2-plugin==0.6.0; extra == "dev"
|
36
28
|
Requires-Dist: mkdocs==1.4.2; extra == "dev"
|
29
|
+
Requires-Dist: numpy; extra == "dev"
|
37
30
|
Requires-Dist: pre-commit; extra == "dev"
|
31
|
+
Requires-Dist: pyre-check; extra == "dev"
|
32
|
+
Requires-Dist: pytest; extra == "dev"
|
33
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
34
|
+
Requires-Dist: pytest-env; extra == "dev"
|
35
|
+
Requires-Dist: pytest-mock; extra == "dev"
|
38
36
|
Requires-Dist: python-markdown-math; extra == "dev"
|
37
|
+
Requires-Dist: ruff>=0.9.2; extra == "dev"
|
38
|
+
Requires-Dist: twine; extra == "dev"
|
39
39
|
|
40
40
|
# Artefacts CLI
|
41
41
|
|
@@ -0,0 +1,23 @@
|
|
1
|
+
artefacts/cli/__init__.py,sha256=pt8OK66hMeQUxT9iLcvzYIIjFGrPS63ecWo8hS0T2qQ,11980
|
2
|
+
artefacts/cli/app.py,sha256=fHqq4N_JdSO8jgN4uc0puLSSJk_NU5srbztveipbqLw,22221
|
3
|
+
artefacts/cli/app_containers.py,sha256=dLQqcrRF-3NSpcmijGK6Q51CinumAXnJ1KLNSEKUIPQ,3043
|
4
|
+
artefacts/cli/bagparser.py,sha256=FE_QaztC9pg4hQzTjGSdyve6mzZbHJbyqa3wqvZSbxE,3702
|
5
|
+
artefacts/cli/constants.py,sha256=bvsVDwqkAc49IZN7j6k6IL6EG87bECHd_VINtKJqbv8,320
|
6
|
+
artefacts/cli/errors.py,sha256=BiCRo3IwVjtEotaFtmwsGTZiX-TRE69KqLrEQItLsag,34
|
7
|
+
artefacts/cli/logger.py,sha256=MP8WDImHA3BKVsn55BMWtGP5-aCmXl5ViVPtIo3jKk4,242
|
8
|
+
artefacts/cli/other.py,sha256=7NvzlspvG0zF7sryR-QznwdLupXLln1BKWxHB9VuEcc,1160
|
9
|
+
artefacts/cli/parameters.py,sha256=msf2aG-tmw0ahxwrPpB2W6KqdMj5A-nw9DPG9flkHTg,788
|
10
|
+
artefacts/cli/ros1.py,sha256=rKepZckAuy5O_qraF2CW5GiTmTZHar7LRD4pvESy6T0,9622
|
11
|
+
artefacts/cli/ros2.py,sha256=9Ax_WQIOV_cohKz3H1eo1LnWiahiaqxO8r99doMmhEc,4466
|
12
|
+
artefacts/cli/utils.py,sha256=bqADil7Aqvg-ci0244e-yf8G9KvIkYeWGNc_jMn6qv0,3151
|
13
|
+
artefacts/cli/utils_ros.py,sha256=3EFoMrzBdlhLc-wAL3mmS5sSw_pACkurYhssKHqYJsI,2089
|
14
|
+
artefacts/cli/version.py,sha256=hdhRgZhTA3pieFZY1w6wT4BsoNOPCUB21i3j8v6cYbA,413
|
15
|
+
artefacts/cli/containers/__init__.py,sha256=ga52A5dJUl3EzmrRAQ-ucsLVExPhL61cfcT4rnTgtzc,2258
|
16
|
+
artefacts/cli/containers/docker.py,sha256=ndidmuMUc8LvgGGF7MwUTCaSUzp9dSS56DE847krmJ8,3535
|
17
|
+
artefacts/cli/containers/utils.py,sha256=bILX0uvazUJq7hoqKk4ztRzI_ZerYs04XQdKdx1ltjk,2002
|
18
|
+
artefacts/wrappers/artefacts_ros1_meta.launch,sha256=9tN7_0xLH8jW27KYFerhF3NuWDx2dED3ks_qoGVZAPw,1412
|
19
|
+
artefacts_cli-0.6.18.dist-info/METADATA,sha256=r0BO8lNkG3yljvWpgU3R11x3WfpR3fGVdNIcK_Xu2-s,3035
|
20
|
+
artefacts_cli-0.6.18.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
21
|
+
artefacts_cli-0.6.18.dist-info/entry_points.txt,sha256=nlTXRzilNjccbi53FgaRWCQPkG-pv61HRkaCkrKjlec,58
|
22
|
+
artefacts_cli-0.6.18.dist-info/top_level.txt,sha256=FdaMV1C9m36MWa-2Stm5xVODv7hss_nRYNwR83j_7ow,10
|
23
|
+
artefacts_cli-0.6.18.dist-info/RECORD,,
|
@@ -1,21 +0,0 @@
|
|
1
|
-
artefacts/cli/__init__.py,sha256=9rTAfBoS9ALJxKSqaUo32ThbDshbvNN4-EmoyUsPhRI,11643
|
2
|
-
artefacts/cli/app.py,sha256=597uPtK4A_wujvqhk5h4KkrXV_XVkDLpymedd2lZB5k,20140
|
3
|
-
artefacts/cli/app_containers.py,sha256=7OAtc4n9YeK8ZYUKvteOMbZr7lYjx_YMa6tu8Pnu9Yc,2280
|
4
|
-
artefacts/cli/bagparser.py,sha256=FE_QaztC9pg4hQzTjGSdyve6mzZbHJbyqa3wqvZSbxE,3702
|
5
|
-
artefacts/cli/constants.py,sha256=bvsVDwqkAc49IZN7j6k6IL6EG87bECHd_VINtKJqbv8,320
|
6
|
-
artefacts/cli/other.py,sha256=7NvzlspvG0zF7sryR-QznwdLupXLln1BKWxHB9VuEcc,1160
|
7
|
-
artefacts/cli/parameters.py,sha256=MDhrM7ur95wKTLDteqz2f-sLdCPcepi5wk0XjeLo6TU,788
|
8
|
-
artefacts/cli/ros1.py,sha256=RbtirCGarD9a0ikfuGK-pdpWYSXfqJhEt4rpA0uFsyU,9625
|
9
|
-
artefacts/cli/ros2.py,sha256=YaCi3HRuCdvCTjM0Ftnm-SLgMOy1OneIr0aU7KVTiAM,4476
|
10
|
-
artefacts/cli/utils.py,sha256=6yQJRzv-xaLbtcA73Tty9C9QZeV06n5pjXMYl3xsLPE,3156
|
11
|
-
artefacts/cli/utils_ros.py,sha256=pYbhAU9fK2VbnWm3nSLBwUMVBzsZuCn10SFZmVW3-Zo,2090
|
12
|
-
artefacts/cli/version.py,sha256=zbqzBVUYJbK8o4vMJuNyIN5tBM9QXVZa7XiEM2PNdR8,413
|
13
|
-
artefacts/cli/containers/__init__.py,sha256=S3DXLsb0csUdfjOZIngTfhMHrmYAaWBeu5EctLEEXbM,2057
|
14
|
-
artefacts/cli/containers/docker.py,sha256=RtUOMbtWHQumTtDRa0ALjJ29tSaa7Pp497Ah1aJ7nZ4,2964
|
15
|
-
artefacts/cli/containers/utils.py,sha256=5yUoREHM9AxnTkDNCY-DFUPSR5OWtu1rNxIVs6n5zZo,1816
|
16
|
-
artefacts/wrappers/artefacts_ros1_meta.launch,sha256=9tN7_0xLH8jW27KYFerhF3NuWDx2dED3ks_qoGVZAPw,1412
|
17
|
-
artefacts_cli-0.6.16.dist-info/METADATA,sha256=OLy7Jmo_RMo9CiktBZSKiZRN1HVILJkuhz-W24YB0RA,3021
|
18
|
-
artefacts_cli-0.6.16.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
19
|
-
artefacts_cli-0.6.16.dist-info/entry_points.txt,sha256=nlTXRzilNjccbi53FgaRWCQPkG-pv61HRkaCkrKjlec,58
|
20
|
-
artefacts_cli-0.6.16.dist-info/top_level.txt,sha256=FdaMV1C9m36MWa-2Stm5xVODv7hss_nRYNwR83j_7ow,10
|
21
|
-
artefacts_cli-0.6.16.dist-info/RECORD,,
|
File without changes
|
File without changes
|