artefacts-cli 0.7.3__py3-none-any.whl → 0.8.0__py3-none-any.whl

artefacts/cli/__init__.py

@@ -133,6 +133,7 @@ class WarpRun:
         self.output_path = self.params.get(
             "output_path", f"{self.job.output_path}/{self.run_n}"
         )
+        self.logger = logger
         os.makedirs(self.output_path, exist_ok=True)
         data = {
             "job_id": job.job_id,
@@ -155,10 +156,10 @@ class WarpRun:
         if response.status_code != 200:
             if response.status_code == 403:
                 msg = response.json()["message"]
-                logger.warning(msg)
+                self.logger.warning(msg)
                 raise AuthenticationError(msg)
-            logger.warning(f"Error on scenario creation: {response.status_code}")
-            logger.warning(response.text)
+            self.logger.warning(f"Error on scenario creation: {response.status_code}")
+            self.logger.warning(response.text)
             raise AuthenticationError(str(response.status_code))
         return
 
@@ -275,9 +276,11 @@ class WarpRun:
                         files=files,
                     )
                 except OverflowError:
-                    logger.warning(f"File too large: {file_name} could not be uploaded")
+                    self.logger.warning(
+                        f"File too large: {file_name} could not be uploaded"
+                    )
                 except Exception as e:
-                    logger.warning(f"Error uploading {file_name}: {e}, skipping")
+                    self.logger.warning(f"Error uploading {file_name}: {e}, skipping")
 
 
 def init_job(

artefacts/cli/app.py

@@ -244,6 +244,11 @@ def hello(project_name):
     default=False,
     help='[Experimental] Run the job inside a package container. The container image is build if it does not exist yet, with default name as "artefacts" (please use --with-image to override the image name). This option overrides (for now) --dryrun, --nosim, --noisolation and --description.',
 )
+@click.option(
+    "--dockerfile",
+    default="Dockerfile",
+    help="[Experimental] Path to a custom Dockerfile. Defaults to Dockerfile in the run directory. This flag is only used together with `--in-container`",
+)
 @click.option(
     "--with-image",
     default=None,
@@ -274,6 +279,7 @@ def run(
     description="",
     skip_validation=False,
     in_container: bool = False,
+    dockerfile: str = "Dockerfile",
     with_image: str = "artefacts",
     no_rebuild: bool = False,
     with_gui: bool = False,
@@ -283,6 +289,12 @@ def run(
 
     * Directly in the shell by default.
     * Inside a packaged container when using the --in-container option.
+
+    In container mode:
+    * Images are built automatically if missing.
+    * Currently 1 image per job found in artefacts.yaml.
+    * Images are rebuilt at each run (relatively fast when no change).
+    * `dockerfile` allows to specify an alternative Dockerfile.
     """
     warpconfig = read_config(config)
     project_id = warpconfig["project"]
@@ -296,6 +308,8 @@ def run(
             ctx.invoke(
                 containers.build,
                 root=".",
+                dockerfile=dockerfile,
+                only=[jobname],
             )
             click.echo(f"[{jobname}] Container image ready")
         click.echo(f"[{jobname}] Run in container")

artefacts/cli/app_containers.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from typing import Optional
 
 import click
 
@@ -45,6 +46,13 @@ def containers(ctx: click.Context, debug: bool):
     default="artefacts.yaml",
     help="Path to the Artefacts configuration file. It defaults to `./artefacts.yaml`",
 )
+@click.option(
+    "--only",
+    required=False,
+    type=Optional[list],
+    default=None,
+    help="Optional list of job names to process. The default is to process all jobs.",
+)
 @click.pass_context
 def build(
     ctx: click.Context,
@@ -53,6 +61,7 @@ def build(
     dockerfile: str,
     name: str,
     config: str,
+    only: Optional[list] = None,
 ):
     try:
         artefacts_config = read_config(config)
@@ -63,7 +72,11 @@ def build(
     prefix = artefacts_config["project"].strip().lower()
     dockerfiles = []
     if os.path.exists(dockerfile):
-        for job_name in artefacts_config["jobs"]:
+        if only:
+            jobs = only
+        else:
+            jobs = artefacts_config["jobs"]
+        for job_name in jobs:
             dockerfiles.append(
                 dict(
                     path=root,
@@ -71,6 +84,11 @@ def build(
                     name=f"{prefix}/{job_name.strip().lower()}",
                 )
             )
+    elif dockerfile != "Dockerfile" and not os.path.exists(dockerfile):
+        # The user asks explicitly for using a specific Dockerfile, so fast fail if we cannot find it
+        raise click.ClickException(
+            f"Dockerfile `{dockerfile}` not found. Please ensure the file exits. Automatic Dockerfile generation may also work by dropping the --dockerfile option."
+        )
     else:
         # The split on `prefix` is to ensure there is no slash (project names are org/project) confusing the path across supported OS.
         dest_root = (
@@ -90,6 +108,8 @@ def build(
         scenarios = Converter().process(config, as_text=False)
         for idx, df in enumerate(scenarios.values()):
             job_name = df.job_name.strip().lower()
+            if only and job_name not in only:
+                continue
             dest = dest_root / Path(job_name)
             dest.mkdir(parents=True, exist_ok=True)
             _dockerfile = os.path.join(dest, "Dockerfile")

artefacts/cli/logger.py

@@ -1,10 +1,26 @@
 import logging
 import sys
 
-logger = logging.getLogger()
+logger = logging.getLogger("artefacts")
+logger.setLevel(logging.INFO)  # Allow INFO and above
 
-default_handler = logging.StreamHandler(stream=sys.stderr)
-default_handler.setLevel(logging.ERROR)
-default_handler.setFormatter(logging.Formatter())
 
-logger.addHandler(default_handler)
+# INFO and WARNING messages go to stdout
+class InfoWarningFilter(logging.Filter):
+    def filter(self, record):
+        return record.levelno in [logging.INFO, logging.WARNING]
+
+
+info_handler = logging.StreamHandler(stream=sys.stdout)
+info_handler.setLevel(logging.INFO)
+info_handler.addFilter(InfoWarningFilter())
+info_handler.setFormatter(logging.Formatter("%(message)s"))
+
+# ERROR and above go to stderr
+error_handler = logging.StreamHandler(stream=sys.stderr)
+error_handler.setLevel(logging.ERROR)
+error_handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+
+logger.addHandler(info_handler)
+logger.addHandler(error_handler)
+logger.propagate = False

artefacts/cli/other.py

@@ -16,6 +16,9 @@ def generate_parameter_output(params: dict):
 
 
 def run_other_tests(run):
+    """Note: parameter names will be set as environment variables
+    (must be letters, numbers and underscores), and saved into yaml and json files
+    """
     scenario = run.params
     if "params" in scenario:
         generate_parameter_output(scenario["params"])

artefacts/cli/ros2.py

@@ -2,7 +2,6 @@ import yaml
 from glob import glob
 import os
 import shutil
-
 from .utils import run_and_save_logs
 from .utils_ros import parse_tests_results, get_TestSuite_error_result
 from .parameters import TMP_SCENARIO_PARAMS_YAML
@@ -93,8 +92,11 @@ def run_ros2_tests(run):
         generate_scenario_parameter_output(
             run.params["params"], TMP_SCENARIO_PARAMS_YAML
         )
-
-    preexisting_rosbags = glob("rosbag2*")
+    # We look for the directory as drilling down will find both
+    # the directory as well as the rosbag itself.
+    preexisting_rosbags = [
+        path for path in glob("**/rosbag2*", recursive=True) if os.path.isdir(path)
+    ]
     test_result_file_path = f"{run.output_path}/tests_junit.xml"
     launch_arguments = [
         f"{k}:={v}" for k, v in run.params.get("launch_arguments", {}).items()
@@ -182,33 +184,40 @@ def run_ros2_tests(run):
     for output in scenario.get("output_dirs", []):
         run.log_artifacts(output)
 
-    # check if any rosbag was created
-    rosbags = glob("rosbag2*")
+    # Checks for a new rosbag after test completions. We look for the directory
+    # as drilling down will find both the directory as well as the rosbag itself.
+    rosbags = [
+        path for path in glob("**/rosbag2*", recursive=True) if os.path.isdir(path)
+    ]
     new_rosbags = set(rosbags).difference(set(preexisting_rosbags))
     from artefacts.cli.bagparser import BagFileParser
 
+    run.logger.info(f"Found new rosbags: {new_rosbags}")
     if len(new_rosbags) > 0:
-        rosbag_path = new_rosbags.pop()
-        run.log_artifacts(rosbag_path, "rosbag")
+        rosbag_dir = new_rosbags.pop()
+        run.log_artifacts(rosbag_dir, "rosbag")
         if "metrics" in run.params:
-            # TODO should go inside BagFileParser?
-            db_files = glob(f"{rosbag_path}/*.mcap")  # Ros2 Default
+            # Search for database files in the directory
+            # TODO: should go inside BagFileParser?
+            db_files = glob(f"{rosbag_dir}/*.mcap")  # Ros2 Default
             if not db_files:
-                db_files = glob(f"{rosbag_path}/*.db3")  # Legacy
+                db_files = glob(f"{rosbag_dir}/*.db3")  # Legacy
             if not db_files:
                 raise FileNotFoundError(
-                    "No .mcap or .db3 files found in the specified path."
+                    "No .mcap or .db3 files found in the specified path. Attempted to find in "
+                    f"{rosbag_dir}/*.mcap and {rosbag_dir}/*.db3"
                 )
             db_file = db_files[0]
+
             bag = BagFileParser(db_file)
             for metric in run.params["metrics"]:
                 try:
                     last_value = bag.get_last_message(metric)[1].data
                     run.log_metric(metric, last_value)
                 except KeyError:
-                    print(f"Metric {metric} not found in rosbag, skipping.")
+                    run.logger.error(f"Metric {metric} not found in rosbag, skipping.")
                 except TypeError or IndexError:
-                    print(
+                    run.logger.error(
                         f"Metric {metric} not found. Is it being published?. Skipping."
                     )
 

artefacts/cli/utils_ros.py

@@ -66,9 +66,9 @@ def parse_tests_results(file):
     except Exception as e:
         print(f"[Exception in parse_tests_results] {e}")
         print("Test result xml could not be loaded, marking success as False")
-        results = get_TestSuite_error_result(
+        result = get_TestSuite_error_result(
             "unittest.suite.TestSuite",
             "Error parsing XML test results",
             f"The test may have timed out. Exception: {e}",
         )
-        return results, None
+        return [result], None

artefacts/cli/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.7.3'
-__version_tuple__ = version_tuple = (0, 7, 3)
+__version__ = version = '0.8.0'
+__version_tuple__ = version_tuple = (0, 8, 0)

{artefacts_cli-0.7.3.dist-info → artefacts_cli-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: artefacts_cli
-Version: 0.7.3
+Version: 0.8.0
 Author-email: FD <fabian@artefacts.com>, AGC <alejandro@artefacts.com>, TN <tomo@artefacts.com>, EP <eric@artefacts.com>
 Project-URL: Homepage, https://github.com/art-e-fact/artefacts-client
 Project-URL: Bug Tracker, https://github.com/art-e-fact/artefacts-client/issues

{artefacts_cli-0.7.3.dist-info → artefacts_cli-0.8.0.dist-info}/RECORD

@@ -1,24 +1,24 @@
 artefacts/__init__.py,sha256=VLmogtpRQeJjQjAORV8ClSJ5qF-57Hxx3apvgy9H1zk,76
-artefacts/cli/__init__.py,sha256=pt8OK66hMeQUxT9iLcvzYIIjFGrPS63ecWo8hS0T2qQ,11980
-artefacts/cli/app.py,sha256=lTdfJsNCdUh8A0V-MvvYUNoB1rgHAMpXxInqPgPeC4g,24241
-artefacts/cli/app_containers.py,sha256=AOTiXjyJzYZn6JqfW26N3ehGQWcEVQ5mEmKDdp9a6tQ,5636
+artefacts/cli/__init__.py,sha256=qIHUUf51UoviMtChkO5Qn-9ZzC5nH7vPE-QlqC30cRE,12080
+artefacts/cli/app.py,sha256=2ogpU_zZuynCu3lz5Ai2PqFiVFiGVdBZojeD7qNZqyk,24831
+artefacts/cli/app_containers.py,sha256=sRjm-z1Bbp81UE-GFrXv-0XiB6bCDsiMcsEOuT24dww,6413
 artefacts/cli/bagparser.py,sha256=FE_QaztC9pg4hQzTjGSdyve6mzZbHJbyqa3wqvZSbxE,3702
 artefacts/cli/constants.py,sha256=bvsVDwqkAc49IZN7j6k6IL6EG87bECHd_VINtKJqbv8,320
 artefacts/cli/errors.py,sha256=BiCRo3IwVjtEotaFtmwsGTZiX-TRE69KqLrEQItLsag,34
-artefacts/cli/logger.py,sha256=MP8WDImHA3BKVsn55BMWtGP5-aCmXl5ViVPtIo3jKk4,242
-artefacts/cli/other.py,sha256=7NvzlspvG0zF7sryR-QznwdLupXLln1BKWxHB9VuEcc,1160
+artefacts/cli/logger.py,sha256=PklhdEs-T8p9eogX7O_i6Gmw3v707y1Z0vgVwxBu96Q,792
+artefacts/cli/other.py,sha256=Jp15jLjttMk0JW15EAU5hgQNAkbX7vMGyjdWectehlw,1317
 artefacts/cli/parameters.py,sha256=msf2aG-tmw0ahxwrPpB2W6KqdMj5A-nw9DPG9flkHTg,788
 artefacts/cli/ros1.py,sha256=rKepZckAuy5O_qraF2CW5GiTmTZHar7LRD4pvESy6T0,9622
-artefacts/cli/ros2.py,sha256=mXBURNHosG7WtqxjmydfQiPoGCNXOHe9reM-FRLwaHU,7887
+artefacts/cli/ros2.py,sha256=UMC1-6h0LhUhvXZist134WG18Tgm0y1DZeHqFNDOEKk,8495
 artefacts/cli/utils.py,sha256=oy56o3N361srwhIvbMxwSPg8I_-tC7xcWtTSINTF2rE,4125
-artefacts/cli/utils_ros.py,sha256=ucJrIMLcTh26ioduj3xiozgxqXZghkyTMHWI9BsHNjI,2156
-artefacts/cli/version.py,sha256=HuvheJ09dqzSRYh_uHb_Af7PEi7XiVW5Tk0cjtnAa6I,511
+artefacts/cli/utils_ros.py,sha256=D6NSsF-5EfkQjCa6KAL7pFNRpu3JyH3nb0UEEz2SEqQ,2156
+artefacts/cli/version.py,sha256=fSm5pLlwHxfTD7vBTVEqChJUua9ilUsdQYNN_V3u3iE,511
 artefacts/cli/containers/__init__.py,sha256=K0efkJXNCqXH-qYBqhCE_8zVUCHbVmeuKH-y_fE8s4M,2254
 artefacts/cli/containers/docker.py,sha256=R0yA-aIZCyYWN7gzim_Dhn1owpKI9ekMu6qbz5URYbQ,4311
 artefacts/cli/containers/utils.py,sha256=bILX0uvazUJq7hoqKk4ztRzI_ZerYs04XQdKdx1ltjk,2002
 artefacts/wrappers/artefacts_ros1_meta.launch,sha256=9tN7_0xLH8jW27KYFerhF3NuWDx2dED3ks_qoGVZAPw,1412
-artefacts_cli-0.7.3.dist-info/METADATA,sha256=ByqmiKWu6jggqXcAsF_7wa1kl2iVJl5Dq_x9sD-I4Xw,3183
-artefacts_cli-0.7.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-artefacts_cli-0.7.3.dist-info/entry_points.txt,sha256=nlTXRzilNjccbi53FgaRWCQPkG-pv61HRkaCkrKjlec,58
-artefacts_cli-0.7.3.dist-info/top_level.txt,sha256=FdaMV1C9m36MWa-2Stm5xVODv7hss_nRYNwR83j_7ow,10
-artefacts_cli-0.7.3.dist-info/RECORD,,
+artefacts_cli-0.8.0.dist-info/METADATA,sha256=D2FpCO_zKGpEeeoRiUnaMLV5mcZS14Wu-rc4MpVq7zs,3183
+artefacts_cli-0.8.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+artefacts_cli-0.8.0.dist-info/entry_points.txt,sha256=nlTXRzilNjccbi53FgaRWCQPkG-pv61HRkaCkrKjlec,58
+artefacts_cli-0.8.0.dist-info/top_level.txt,sha256=FdaMV1C9m36MWa-2Stm5xVODv7hss_nRYNwR83j_7ow,10
+artefacts_cli-0.8.0.dist-info/RECORD,,