artefacts-cli 0.7.3__py3-none-any.whl → 0.9.1__py3-none-any.whl

artefacts/cli/app_containers.py

@@ -1,11 +1,13 @@
 import os
 from pathlib import Path
+from typing import Optional
 
 import click
 
 from c2d.core import Converter
 
 from artefacts.cli.constants import DEFAULT_API_URL
+from artefacts.cli.i18n import localise
 from artefacts.cli.utils import config_validation, read_config
 from artefacts.cli.containers.utils import ContainerMgr
 
@@ -22,28 +24,45 @@ def containers(ctx: click.Context, debug: bool):
 @click.option(
     "--path",
     default=".",
-    help="[Deprecated since 0.8.0; please see --root] Path to the root of the project.",
+    help=localise(
+        "[Deprecated since 0.8.0; please see --root] Path to the root of the project."
+    ),
 )
 @click.option(
     "--root",
     default=".",
-    help="Path to the root of the project.",
+    help=localise("Path to the root of the project."),
 )
 @click.option(
     "--dockerfile",
     default="Dockerfile",
-    help="Path to a custom Dockerfile. Defaults to Dockerfile under `path` (see option of the same name).",
+    help=localise(
+        "Path to a custom Dockerfile. Defaults to Dockerfile under `path` (see option of the same name)."
+    ),
 )
 @click.option(
     "--name",
     required=False,
-    help="[Deprecated since 0.8.0; not used and will disappear after 0.8.0] Name for the generated image",
+    help=localise(
+        "[Deprecated since 0.8.0; not used and will disappear after 0.8.0] Name for the generated image"
+    ),
 )
 @click.option(
     "--config",
     callback=config_validation,
     default="artefacts.yaml",
-    help="Path to the Artefacts configuration file. It defaults to `./artefacts.yaml`",
+    help=localise(
+        "Path to the Artefacts configuration file. It defaults to `./artefacts.yaml`"
+    ),
+)
+@click.option(
+    "--only",
+    required=False,
+    type=Optional[list],
+    default=None,
+    help=localise(
+        "Optional list of job names to process. The default is to process all jobs."
+    ),
 )
 @click.pass_context
 def build(
@@ -53,17 +72,26 @@ def build(
     dockerfile: str,
     name: str,
     config: str,
+    only: Optional[list] = None,
 ):
     try:
         artefacts_config = read_config(config)
     except FileNotFoundError:
         raise click.ClickException(
-            f"Project config file not found: {config}. Please provide an Artefacts configuration file to proceed (running `artefacts init` allows to generate one)."
+            localise(
+                "Project config file not found: {config}. Please provide an Artefacts configuration file to proceed (running `artefacts init` allows to generate one).".format(
+                    config=config
+                )
+            )
         )
     prefix = artefacts_config["project"].strip().lower()
     dockerfiles = []
     if os.path.exists(dockerfile):
-        for job_name in artefacts_config["jobs"]:
+        if only:
+            jobs = only
+        else:
+            jobs = artefacts_config["jobs"]
+        for job_name in jobs:
             dockerfiles.append(
                 dict(
                     path=root,
@@ -71,6 +99,15 @@ def build(
                     name=f"{prefix}/{job_name.strip().lower()}",
                 )
             )
+    elif dockerfile != "Dockerfile" and not os.path.exists(dockerfile):
+        # The user asks explicitly for using a specific Dockerfile, so fast fail if we cannot find it
+        raise click.ClickException(
+            localise(
+                "Dockerfile `{dockerfile}` not found. Please ensure the file exits. Automatic Dockerfile generation may also work by dropping the --dockerfile option.".format(
+                    dockerfile=dockerfile
+                )
+            )
+        )
     else:
         # The split on `prefix` is to ensure there is no slash (project names are org/project) confusing the path across supported OS.
         dest_root = (
@@ -82,7 +119,11 @@ def build(
         )
         if not dest_root.exists():
             click.echo(
-                f"No {dockerfile} found here. Let's generate one per scenario based on artefacts.yaml. They will be available under the `{dest_root}` folder and used from there."
+                localise(
+                    "No {dockerfile} found here. Let's generate one per scenario based on artefacts.yaml. They will be available under the `{dest_root}` folder and used from there.".format(
+                        dockerfile=dockerfile, dest_root=dest_root
+                    )
+                )
             )
         # No condition on generating the Dockerfiles as:
         #   - Fast
@@ -90,11 +131,20 @@ def build(
         scenarios = Converter().process(config, as_text=False)
         for idx, df in enumerate(scenarios.values()):
             job_name = df.job_name.strip().lower()
+            if only and job_name not in only:
+                continue
             dest = dest_root / Path(job_name)
             dest.mkdir(parents=True, exist_ok=True)
             _dockerfile = os.path.join(dest, "Dockerfile")
             df.dump(_dockerfile)
-            click.echo(f"[{job_name}] Using generated Dockerfile at: {_dockerfile}")
+            click.echo(
+                f"[{job_name}] "
+                + localise(
+                    "Using generated Dockerfile at: {dockerfile}".format(
+                        dockerfile=_dockerfile
+                    )
+                )
+            )
             dockerfiles.append(
                 dict(
                     path=root,
@@ -108,43 +158,78 @@ def build(
             # No condition on building the images, as relatively fast when already exists, and straightforward logic.
             image, _ = handler.build(**specs)
     else:
-        click.echo("No Dockerfile, nothing to do.")
+        click.echo(localise("No Dockerfile, nothing to do."))
 
 
 @containers.command()
-@click.argument("name")
+@click.argument("image_name", metavar=localise("IMAGE_NAME"))
 @click.pass_context
-def check(ctx: click.Context, name: str):
-    if name is None:
-        name = "artefacts"
+def check(ctx: click.Context, image_name: str):
+    if image_name is None:
+        image_name = "artefacts"
     handler = ContainerMgr()
-    result = handler.check(name)
+    result = handler.check(image_name)
     if ctx.parent is None:
         # Print only if the command is called directly.
-        print(f"Package {name} exists and ready to use.")
+        print(
+            localise("Container image {name} exists and ready to use.").format(
+                name=image_name
+            )
+        )
     return result
 
 
-@containers.command()
-@click.argument("jobname")
+@containers.command(
+    context_settings=dict(
+        ignore_unknown_options=True,
+    )
+)
+@click.argument("jobname", metavar=localise("JOBNAME"))
 @click.option(
     "--config",
     callback=config_validation,
     default="artefacts.yaml",
-    help="Path to the Artefacts configuration file. It defaults to `./artefacts.yaml`",
+    help=localise(
+        "Path to the Artefacts configuration file. It defaults to `./artefacts.yaml`"
+    ),
 )
 @click.option(
     "--with-gui",
     "with_gui",
     default=False,
-    help="Show any GUI if any is created by the test runs. By default, UI elements are run but hidden---only test logs are returned. Please note GUI often assume an X11 environment, typically with Qt, so this may not work without a appropriate environment.",
+    help=localise(
+        "Show any GUI if any is created by the test runs. By default, UI elements are run but hidden---only test logs are returned. Please note GUI often assume an X11 environment, typically with Qt, so this may not work without a appropriate environment."
+    ),
 )
+# Not ready, placeholder for the introduction of Podman
+# @click.option(
+#     "--backend",
+#     help="Container backend to use, mostly useful if you are using several at the same time. When this option is not specified, Artefacts will try to automatically detect available backends, with a preference for Docker if found. The `--engine-args` allows to pass engine arguments like `--gpu` for Docker, etc.",
+# )
+@click.argument("engine-args", nargs=-1, type=click.UNPROCESSED)
 @click.pass_context
-def run(ctx: click.Context, jobname: str, config: str, with_gui: bool):
+def run(
+    ctx: click.Context,
+    jobname: str,
+    config: str,
+    with_gui: bool,
+    #    backend: str,
+    engine_args: tuple,
+):
+    # Workaround for job names coming after engine arguments
+    #    Idea: Job names do not start with hyphens.
+    if jobname.startswith("-") and engine_args is not None:
+        _fix = list(engine_args)
+        _fix.insert(0, jobname)
+        jobname = _fix.pop()
+        engine_args = tuple(_fix)
+
     try:
         artefacts_config = read_config(config)
     except FileNotFoundError:
-        raise click.ClickException(f"Project config file not found: {config}")
+        raise click.ClickException(
+            localise("Project config file not found: {config}".format(config=config))
+        )
     project = artefacts_config["project"]
     handler = ContainerMgr()
     params = dict(
@@ -155,11 +240,18 @@ def run(ctx: click.Context, jobname: str, config: str, with_gui: bool):
         # Hidden settings primarily useful to Artefacts developers
         api_url=os.environ.get("ARTEFACTS_API_URL", DEFAULT_API_URL),
         api_key=os.environ.get("ARTEFACTS_KEY", None),
+        engine_args=list(engine_args),
     )
     container, logs = handler.run(**params)
     if container:
-        print(f"Package run complete: Container Id for inspection: {container['Id']}")
+        print(
+            localise(
+                "Container run complete: Container Id for inspection: {container_id}".format(
+                    container_id=container["Id"]
+                )
+            )
+        )
     else:
-        print("Package run failed:")
+        print(localise("Package run failed:"))
         for entry in logs:
             print("\t- " + entry)

artefacts/cli/app_containers.pyi

@@ -0,0 +1,3 @@
+from typing import Any
+
+def __getattr__(name: str) -> Any: ...

artefacts/cli/bagparser.py

@@ -1,5 +1,9 @@
 import sqlite3
+
+# pyre-fixme[21]
 from rosidl_runtime_py.utilities import get_message
+
+# pyre-fixme[21]
 from rclpy.serialization import deserialize_message
 from mcap.reader import make_reader
 from mcap_ros2.decoder import DecoderFactory

artefacts/cli/config.py

@@ -0,0 +1,62 @@
+import os
+import platform
+from typing import Optional
+
+import click
+
+from artefacts.cli.i18n import localise
+from artefacts.cli.helpers import (
+    get_conf_from_file,
+    get_artefacts_api_url,
+)
+
+
+class APIConf:
+    def __init__(
+        self, project_name: str, api_version: str, job_name: Optional[str] = None
+    ) -> None:
+        config = get_conf_from_file()
+        if project_name in config:
+            profile = config[project_name]
+        else:
+            profile = {}
+        self.api_url = get_artefacts_api_url(profile)
+        self.api_key = os.environ.get("ARTEFACTS_KEY", profile.get("ApiKey", None))
+        if self.api_key is None:
+            batch_id = os.environ.get("AWS_BATCH_JOB_ID", None)
+            job_id = os.environ.get("ARTEFACTS_JOB_ID", None)
+            if batch_id is None or job_id is None:
+                raise click.ClickException(
+                    localise(
+                        "No API KEY set. Please run `artefacts config add {project_name}`".format(
+                            project_name=project_name
+                        )
+                    )
+                )
+            auth_type = "Internal"
+            # Batch id for array jobs contains array index
+            batch_id = batch_id.split(":")[0]
+            self.headers = {"Authorization": f"{auth_type} {job_id}:{batch_id}"}
+        else:
+            auth_type = "ApiKey"
+            self.headers = {"Authorization": f"{auth_type} {self.api_key}"}
+        self.headers["User-Agent"] = (
+            f"ArtefactsClient/{api_version} ({platform.platform()}/{platform.python_version()})"
+        )
+        if job_name:
+            click.echo(
+                f"[{job_name}] "
+                + localise(
+                    "Connecting to {api_url} using {auth_type}".format(
+                        api_url=self.api_url, auth_type=auth_type
+                    )
+                )
+            )
+        else:
+            click.echo(
+                localise(
+                    "Connecting to {api_url} using {auth_type}".format(
+                        api_url=self.api_url, auth_type=auth_type
+                    )
+                )
+            )

artefacts/cli/constants.py

@@ -1,3 +1,6 @@
+import os
+
+
 DEFAULT_API_URL = "https://app.artefacts.com/api"
 
 SUPPORTED_FRAMEWORKS = [
@@ -16,3 +19,7 @@ DEPRECATED_FRAMEWORKS = {
     "ros2:0": "ros2:galactic",
     "ros1:0": "ros1:noetic",
 }
+
+HOME = os.path.expanduser("~")
+CONFIG_DIR = f"{HOME}/.artefacts"
+CONFIG_PATH = f"{CONFIG_DIR}/config"

artefacts/cli/containers/__init__.py

@@ -1,4 +1,4 @@
-from collections.abc import Generator
+from collections.abc import Iterator
 import configparser
 import os
 from pathlib import Path
@@ -8,7 +8,7 @@ from artefacts.cli.constants import DEFAULT_API_URL
 
 
 class CMgr:
-    def build(self, **kwargs) -> Tuple[str, Generator]:
+    def build(self, **kwargs) -> Tuple[str, Iterator]:
         """
         Returns the build image ID (e.g. sha256:abcdefghi)
         and an iterator over the build log entries.
@@ -25,11 +25,11 @@ class CMgr:
         self,
         image: str,
         project: str,
-        jobname: str = None,
-        artefacts_dir: str = Path("~/.artefacts").expanduser(),
+        jobname: Optional[str] = None,
+        artefacts_dir: Union[str, Path] = Path("~/.artefacts").expanduser(),
         api_url: str = DEFAULT_API_URL,
         with_gui: bool = False,
-    ) -> Tuple[Any, Generator]:
+    ) -> Tuple[Any, Iterator]:
         """
         Returns a container (Any type as depends on the framework)
         and an iterator over the container log entries.

artefacts/cli/containers/docker_cm.py

@@ -0,0 +1,175 @@
+from collections.abc import Iterator
+import json
+import os
+from pathlib import Path
+import platform
+from typing import Any, Optional, Tuple, Union
+from uuid import uuid4
+
+from artefacts.cli.constants import DEFAULT_API_URL
+from artefacts.cli.containers import CMgr
+from artefacts.cli.containers.docker_utils import cli2sdk
+from artefacts.cli.i18n import localise
+from artefacts.cli.utils import ensure_available
+
+ensure_available("docker")
+
+import docker  # noqa: E402
+from docker import APIClient  # noqa: E402
+
+
+class DockerManager(CMgr):
+    def __init__(self):
+        self.client = APIClient()
+
+    def build(self, **kwargs) -> Tuple[str, Iterator]:
+        kwargs["tag"] = kwargs.pop("name")
+        # Ensure `path` is a string, the Docker package does not support pathlib.
+        kwargs["path"] = str(kwargs.pop("path"))
+        # Remove intermediate containers
+        kwargs["rm"] = True
+        logs = []
+        img_id = None
+        for entry in self.client.build(**kwargs):
+            line_data = [
+                json.loads(v) for v in entry.decode("utf-8").split("\r\n") if len(v) > 0
+            ]
+            for data in line_data:
+                if "stream" in data:
+                    line = data["stream"].strip()
+                    if not line.startswith("---") and len(line) > 0:
+                        print(f"[{kwargs['tag'].split('/')[-1]}] {line}")
+                        logs.append(line)
+                elif "aux" in data and "ID" in data["aux"]:
+                    img_id = str(data["aux"]["ID"])
+        if img_id is None:
+            img_id = str(self.client.inspect_image(kwargs["tag"])["Id"])
+        return img_id, iter(logs)
+
+    def check(
+        self,
+        image: str,
+    ) -> bool:
+        return len(self.client.images(name=image)) > 0
+
+    def run(
+        self,
+        image: str,
+        project: str,
+        jobname: Optional[str] = None,
+        artefacts_dir: Union[str, Path] = Path("~/.artefacts").expanduser(),
+        api_url: str = DEFAULT_API_URL,
+        api_key: Optional[str] = None,
+        with_gui: bool = False,
+        engine_args: Optional[list] = None,
+    ) -> Tuple[Any, Iterator]:
+        """
+        Run an application as an Artefacts-enabled container in a Docker engine
+        """
+        env = {
+            "JOB_ID": str(uuid4()),
+            "ARTEFACTS_JOB_NAME": jobname,
+            "ARTEFACTS_API_URL": api_url,
+        }
+
+        env["ARTEFACTS_KEY"] = api_key or self._get_artefacts_api_key(
+            project, artefacts_dir
+        )
+        if env["ARTEFACTS_KEY"] is None:
+            return None, iter(
+                [
+                    localise(
+                        "Missing API key for the project. Does `{path}/config` exist and contain your key? Alternatively ARTEFACTS_KEY can be set with the key.".format(
+                            path=artefacts_dir
+                        )
+                    )
+                ]
+            )
+        try:
+            if platform.system() in ["Darwin", "Windows"]:
+                # Assume we run in Docker Desktop
+                env["DISPLAY"] = "host.docker.internal:0"
+            else:
+                env["DISPLAY"] = os.environ.get("DISPLAY", ":0")
+
+            if not with_gui:
+                env["QT_QPA_PLATFORM"] = "offscreen"
+
+            # Default configs
+            host_conf = dict(
+                network_mode="host",
+            )
+
+            container_conf = dict(
+                image=image,
+                environment=env,
+                detach=False,
+            )
+
+            # Apply user config and overrides, if any
+            if engine_args:
+                option = None
+                # Add a marker to detect end of args
+                engine_args.append("--end--")
+                for current in engine_args:
+                    is_option = current.startswith("-")
+                    if is_option and not option:
+                        option = current.lstrip("-")
+                        if "=" in option:
+                            option, value = option.split("=")
+                            cli2sdk(host_conf, container_conf, option, value)
+                            option = None
+                    elif not is_option and option:
+                        cli2sdk(host_conf, container_conf, option, current)
+                        option = None
+                    elif is_option and option:
+                        # Assuming detection of concatenated flags, all set.
+                        for flag in str(option):
+                            cli2sdk(host_conf, container_conf, flag, True)
+                        if current != "--end--":
+                            option = current
+
+            # Final container config
+            container_conf["host_config"] = self.client.create_host_config(**host_conf)
+
+            try:
+                container = self.client.create_container(**container_conf)
+            except Exception as e:
+                # SDK errors may have an explanation
+                # E.g. in 7.1.0 https://github.com/docker/docker-py/blob/7.1.0/docker/errors.py#L53
+                known = str(e)
+                if known.endswith(")"):
+                    # Error message: info ("explanation")
+                    # We return explanation
+                    detail = known[known.index("(") + 2 : -2]
+                else:
+                    # Error message: info
+                    # We return info
+                    detail = known[known.index(":") + 2 :]
+                raise Exception(f"Invalid container configuration: {detail}")
+            self.client.start(container=container.get("Id"))
+
+            for entry in self.client.logs(container=container.get("Id"), stream=True):
+                print(entry.decode("utf-8").strip())
+
+            return container, iter([])
+        except docker.errors.ImageNotFound:
+            return None, iter(
+                [
+                    localise(
+                        "Image {image} not found by Docker. Perhaps need to build first?".format(
+                            image=image
+                        )
+                    )
+                ]
+            )
+        except Exception as e:
+            return None, iter(
+                [
+                    localise(
+                        "Failed to run from {image}. All we know: {message}".format(
+                            image=image, message=e
+                        )
+                    )
+                ]
+            )

artefacts/cli/containers/docker_utils.py

@@ -0,0 +1,98 @@
+from typing import Any
+
+import docker  # noqa: E402
+
+from artefacts.cli.i18n import localise
+
+
+def _identity(v: Any) -> Any:
+    return v
+
+
+def _make_gpu_device_request(gpus: str) -> list:
+    """
+    Code based on Docker documented use, not on the source code.
+
+    https://docs.docker.com/reference/cli/docker/container/run/#gpus
+    """
+    if '"device=' in gpus:
+        try:
+            # Docs: `--gpus '"device=1,2"'`
+            _g = gpus.strip('"')
+            # There must be 2 double quotes.
+            assert len(_g) == len(gpus) - 2
+            ids = sorted(_g[_g.index("=") + 1 :].split(","))
+        except Exception as e:
+            raise Exception(
+                localise(
+                    'Invalid GPU device for Docker: {g} ({e}). Accepted device formats are all, device=1 and "device=1,2" (with a list of devices, quotes must be included'.format(
+                        g=gpus, e=e
+                    )
+                )
+            )
+    elif "device=" in gpus:
+        # Docs: `--gpus device=1` or `--gpus device=GPU-3faa8-219`
+        if "," in gpus:
+            raise Exception(
+                localise(
+                    'Invalid GPU device for Docker: {g}. List of devices must be double-quoted. Accepted device formats are all, device=1 and "device=1,2" (with a list of devices, quotes must be included'.format(
+                        g=gpus,
+                    )
+                )
+            )
+        ids = [gpus.split("=")[-1]]
+    elif "all" == gpus:
+        ids = [gpus]
+    else:
+        raise Exception(
+            localise(
+                'Invalid GPU device for Docker: {g}. Accepted device formats are all, device=1 and "device=1,2" (with a list of devices, quotes must be included'.format(
+                    g=gpus
+                )
+            )
+        )
+    return [docker.types.DeviceRequest(device_ids=ids, capabilities=[["gpu"]])]
+
+
+_cli_sdk_option_map = {
+    "net": {
+        "t": "host",
+        "o": "network_mode",
+    },
+    "gpus": {
+        "t": "host",
+        "o": "device_requests",
+        "f": _make_gpu_device_request,
+    },
+    "t": {
+        "t": "container",
+        "o": "tty",
+    },
+    "i": {
+        "t": "container",
+        "o": "stdin_open",
+    },
+    "tty": {
+        "t": "container",
+    },
+    "interactive": {
+        "t": "container",
+        "o": "stdin_open",
+    },
+}
+
+
+def cli2sdk(host: dict, container: dict, option: str, value: Any) -> None:
+    """
+    `host` and `container` are IO map arguments.
+    """
+    method = _cli_sdk_option_map.get(option)
+    if method:
+        if method["t"] == "host":
+            host[method.get("o") or option] = (method.get("f") or _identity)(value)
+        elif method["t"] == "container":
+            container[method.get("o") or option] = (method.get("f") or _identity)(value)
+    else:
+        # Based on current knowledge of the SDK, it seems
+        # we can opt for the host config as default.
+        host[option] = value