artefacts-cli 0.6.8__py3-none-any.whl

artefacts/cli/bagparser.py

@@ -0,0 +1,98 @@
+import sqlite3
+from rosidl_runtime_py.utilities import get_message
+from rclpy.serialization import deserialize_message
+from mcap.reader import make_reader
+from mcap_ros2.decoder import DecoderFactory
+
+
+class BagFileParser:
+    def __init__(self, bag_file):
+        self.bag_file = bag_file
+        if bag_file.endswith(".db3"):
+            self.conn = sqlite3.connect(bag_file)
+            self.cursor = self.conn.cursor()
+
+            # create a message type map
+            topics_data = self.cursor.execute(
+                "SELECT id, name, type FROM topics"
+            ).fetchall()
+            self.topic_type = {
+                name_of: type_of for id_of, name_of, type_of in topics_data
+            }
+            self.topic_id = {name_of: id_of for id_of, name_of, type_of in topics_data}
+            self.topic_msg_message = {
+                name_of: get_message(type_of) for id_of, name_of, type_of in topics_data
+            }
+        elif bag_file.endswith(".mcap"):
+            self.mcap_file = open(bag_file, "rb")
+            self.mcap_reader = make_reader(
+                self.mcap_file, decoder_factories=[DecoderFactory()]
+            )
+            self.topic_type = {}
+            self.topic_msg_message = {}
+            for schema, channel, _, _ in self.mcap_reader.iter_decoded_messages():
+                if channel.topic not in self.topic_type:
+                    self.topic_type[channel.topic] = schema.name
+                    self.topic_msg_message[channel.topic] = get_message(schema.name)
+
+    def __del__(self):
+        if hasattr(self, "conn"):
+            self.conn.close()
+        elif hasattr(self, "mcap_file"):
+            self.mcap_file.close()
+
+    def get_messages(self, topic_name):
+        """
+        Return [(timestamp0, message0), (timestamp1, message1), ...]
+        """
+        if hasattr(self, "conn"):
+            topic_id = self.topic_id[topic_name]
+            rows = self.cursor.execute(
+                "SELECT timestamp, data FROM messages WHERE topic_id = {}".format(
+                    topic_id
+                )
+            ).fetchall()
+            # Deserialize all and timestamp them
+            return [
+                (
+                    timestamp,
+                    deserialize_message(data, self.topic_msg_message[topic_name]),
+                )
+                for timestamp, data in rows
+            ]
+        elif hasattr(self, "mcap_reader"):
+            messages = []
+            for (
+                schema,
+                channel,
+                message,
+                ros_msg,
+            ) in self.mcap_reader.iter_decoded_messages():
+                if channel.topic == topic_name:
+                    timestamp = message.log_time
+                    messages.append((timestamp, ros_msg))
+            return messages
+
+    def get_last_message(self, topic_name):
+        if hasattr(self, "conn"):
+            topic_id = self.topic_id[topic_name]
+            timestamp, msg = self.cursor.execute(
+                "SELECT timestamp, data FROM messages WHERE topic_id = {} ORDER BY timestamp DESC LIMIT 1".format(
+                    topic_id
+                )
+            ).fetchone()
+            return (
+                timestamp,
+                deserialize_message(msg, self.topic_msg_message[topic_name]),
+            )
+        elif hasattr(self, "mcap_reader"):
+            last_message = None
+            for (
+                schema,
+                channel,
+                message,
+                ros_msg,
+            ) in self.mcap_reader.iter_decoded_messages():
+                if channel.topic == topic_name:
+                    last_message = (message.log_time, ros_msg)
+            return last_message

artefacts/cli/constants.py

@@ -0,0 +1,16 @@
+SUPPORTED_FRAMEWORKS = [
+    "ros2:iron",
+    "ros2:humble",
+    "ros2:galactic",
+    "ros1:noetic",
+    "maniskill:challenge2022",
+    "None",
+    "none",
+    "null",
+    None,
+]
+
+DEPRECATED_FRAMEWORKS = {
+    "ros2:0": "ros2:galactic",
+    "ros1:0": "ros1:noetic",
+}

artefacts/cli/other.py

@@ -0,0 +1,40 @@
+import json
+import yaml
+import os
+from .utils import run_and_save_logs
+from .parameters import TMP_SCENARIO_PARAMS_YAML, TMP_SCENARIO_PARAMS_JSON
+
+
+def generate_parameter_output(params: dict):
+    """Store `params` in both json and yaml temporary files
+    Note: fixed filenames will lead concurent executions to overwrite each other
+    """
+    with open(TMP_SCENARIO_PARAMS_JSON, "w") as f:
+        json.dump(params, f)
+    with open(TMP_SCENARIO_PARAMS_YAML, "w") as f:
+        yaml.dump(params, f)
+
+
+def run_other_tests(run):
+    scenario = run.params
+    if "params" in scenario:
+        generate_parameter_output(scenario["params"])
+    full_env = {**os.environ, **scenario.get("params", {})}
+
+    command = scenario["run"]
+    run_and_save_logs(
+        command,
+        shell=True,
+        env={k: str(v) for k, v in full_env.items()},
+        output_path=os.path.join(run.output_path, "test_process_log.txt"),
+    )
+
+    results = []
+    success = True
+    run.log_artifacts(run.output_path)
+
+    for output in scenario.get("output_dirs", []):
+        run.log_artifacts(output)
+
+    run.log_tests_results(results, success)
+    return results, success

artefacts/cli/parameters.py

@@ -0,0 +1,23 @@
+from itertools import product
+from typing import Iterable
+
+# constants for internal temporary files
+
+TMP_RUNTIME_PARAMS_YAML = "/tmp/runtime_params.yaml"
+TMP_SCENARIO_PARAMS_YAML = "/tmp/scenario_params.yaml"
+TMP_SCENARIO_PARAMS_JSON = "/tmp/scenario_params.json"
+
+
+def iter_grid(grid_spec: dict) -> Iterable[dict]:
+    """Iterate over the points defined by the `grid_spec`"""
+    if not grid_spec:
+        yield {}
+    else:
+        # Sort the keys of the dictionary, for reproducibility
+        items = sorted(grid_spec.items())
+        keys, values = zip(*items)
+        # Make sure single values are converted to lists
+        values = [x if type(x) == list else [x] for x in values]
+        for v in product(*values):
+            params = dict(zip(keys, v))
+            yield params

artefacts/cli/ros1.py

@@ -0,0 +1,240 @@
+import yaml
+import subprocess
+from glob import glob
+import os
+import logging
+
+from .utils import run_and_save_logs
+from .utils_ros import parse_tests_results
+from .parameters import TMP_RUNTIME_PARAMS_YAML, TMP_SCENARIO_PARAMS_YAML
+
+logging.basicConfig(level=logging.INFO)
+
+
+def generate_scenario_parameter_output(params: dict, param_file: str) -> None:
+    """
+    Write `params` in `param_file` as YAML format
+    (supports namespace nesting via forward slashes)
+    roslaunch will then load them
+    """
+    with open(param_file, "w") as f:
+        yaml.dump(params, f)
+
+
+def generate_runtime_parameter_output(params: dict, param_file: str) -> None:
+    """
+    Write `params` in `param_file` as YAML format
+    A strict dump is performed to avoid ambiguity:
+    whatever the user specifies in artefacts.yaml/runtime/params will be made available in param_file
+    """
+    with open(param_file, "w") as f:
+        yaml.dump(params, f)
+
+
+def generate_rosbag_args(scenario: dict) -> str:
+    if "rosbag_record" in scenario.keys():
+        rosbag_record = scenario["rosbag_record"]
+    else:
+        # default behavior: do not record any rosbag
+        rosbag_record = "none"
+
+    # return the rosbag args as a string with the proper format
+    if rosbag_record == "none":
+        return "none"
+    elif rosbag_record == "all":
+        return "--all"
+    elif rosbag_record == "subscriptions":
+        if "subscriptions" in scenario.keys():
+            sub = scenario["subscriptions"]
+            topics = " ".join(list(sub.values()))
+            return topics
+        else:
+            logging.warning(
+                f"[warning in generate_rosbag_args] rosbag_record asks for 'subscriptions' but they are not specified. Falling back to default: no rosbag will be recorded"
+            )
+            return "none"
+    else:
+        assert (
+            type(rosbag_record) == list
+        ), f"rosbag_record supports 'all', 'none', 'subscriptions' or a list of strings interpreted as a list of ROS topics, regex supported"
+        for e in rosbag_record:
+            assert (
+                type(e) == str
+            ), f"Elements of the rosbag_record list must only be strings. They are interpreted as a list of ROS topics, regex supported"
+        return f"--regex {' '.join(rosbag_record)}"
+
+
+def get_result_path(scenario, PKGDIR, PACKAGE):
+    """Can't choose the unittest output .xml filename with rostest.
+    instead re-create here the hardcoded naming logic of rostest:
+    need to find out the name of the test method
+    """
+    try:
+        import rospkg
+        import ast
+        import xml.etree.ElementTree as ET
+
+        # find the path of the user's launch file
+        launch_file = (
+            rospkg.RosPack().get_path(scenario["ros_testpackage"])
+            + "/launch/"
+            + scenario["ros_testfile"]
+        )
+        # find the user's test file and test package
+        xml_root = ET.parse(launch_file).getroot()
+        test_package = xml_root.find("test").get("pkg")
+        test_file = xml_root.find("test").get("type")
+        # find the path of the user's test file
+        full_path = rospkg.RosPack().get_path(test_package) + "/src/" + test_file
+        # parse the python file
+        with open(full_path) as file:
+            node = ast.parse(file.read())
+        # find the class and method that match unittest convention naming (start with 'Test')
+        # note: returns the first match
+        test_class = [
+            n for n in node.body if isinstance(n, ast.ClassDef) and "Test" in n.name
+        ][0]
+        test_class_suffix = test_class.name.split("Test")[-1].lower()
+        # Finally, build the unittest result file path
+        test_result_file_path = os.path.expanduser(
+            f"{PKGDIR}/{PACKAGE}/rosunit-{test_class_suffix}.xml"
+        )
+        return test_result_file_path
+    except Exception as e:
+        logging.error(f"[Exception in get_result_path()] {e}")
+        logging.error(
+            f"Unable to parse the ros1 .launch specified ({scenario['ros_testfile']}) and the <test> tag within to find the unittest test method's name.",
+            "Please ensure all ROS and unittest naming conventions are respected. Exiting..",
+        )
+        return None
+
+
+def get_wrapper_path():
+    """Get the absolute path of artefacts_ros1_meta.launch wrapper inside our warp-client repo"""
+    current_dir = os.path.abspath(os.path.dirname(__file__))
+    wrapper_path = current_dir + "/wrappers/artefacts_ros1_meta.launch"
+    return wrapper_path
+
+
+def generate_sim_args(simulator_value, nosim_flag):
+    if nosim_flag:
+        return "none"
+    else:
+        return simulator_value
+
+
+def run_ros1_tests(run):
+    scenario = run.params
+    job = run.job.params
+
+    # ROS1 specific naming conventions for outputting results (can be chosen arbitrarily)
+    PKGDIR = f"{os.path.expanduser(os.getenv('ROS_HOME', '~/.ros'))}/test_results"
+    PACKAGE = "artefacts"
+
+    # dump params specified by the user in the scenario section of the artefacts.yaml
+    # to load them onto the rosparam server
+    if "params" in scenario:
+        param_file = TMP_SCENARIO_PARAMS_YAML  # note: fixed filename will lead concurent executions to overwrite each other
+        generate_scenario_parameter_output(scenario["params"], param_file)
+    else:
+        param_file = "none"
+
+    # dump params specified by the user in the runtime section of the artefacts.yaml
+    # to make them available for the prelaunch script
+    if "params" in job["runtime"]:
+        # note: fixed filename will lead concurent executions to overwrite each other
+        generate_runtime_parameter_output(
+            job["runtime"]["params"], TMP_RUNTIME_PARAMS_YAML
+        )
+
+    # take note of previous rosbags
+    rosbag_path = os.path.expanduser("~/.ros/*.bag")
+    preexisting_rosbags = glob(rosbag_path)
+
+    # get the unittest result path
+    test_result_file_path = get_result_path(scenario, PKGDIR, PACKAGE)
+    if test_result_file_path is None:
+        return {}, False
+
+    ## Main launch of the test sequence:
+    # command line arguments control user specified settings for resource provisions
+    # get_wrapper_path() is the absolute path of artefacts_ros1_meta.launch inside the warp-client repo
+    command = [
+        "rostest",
+        get_wrapper_path(),
+        "--package",
+        PACKAGE,
+        "--pkgdir",
+        PKGDIR,
+        f"rosbag_args:='{generate_rosbag_args(scenario)}'",
+        f"simulator:={generate_sim_args(job['runtime']['simulator'], run.job.nosim)}",
+        f"param_file:={param_file}",
+        f"ros_testfile:={scenario['ros_testfile']}",
+        f"ros_testpackage:={scenario['ros_testpackage']}",
+    ]
+    # for debugging: break rostest isolation from the host's ROSmaster
+    if run.job.noisolation:
+        command.insert(1, "--reuse-master")
+    # last step to prepare the command for execution with subprocess.run():
+    if "pre_launch" in job["runtime"].keys():
+        # join the user's specified pre_launch command with the regular rostest command
+        # note that a single subprocess.run() call is required for any environment variables sourced in the pre_launch command to be available for the rostest command
+        command = f"{job['runtime']['pre_launch']} && {' '.join(command)}"
+    else:
+        command = " ".join(command)
+
+    # Main: test execution
+    # shell=True required to execute two commands in the same shell environment
+    run_and_save_logs(
+        command,
+        shell=True,
+        executable="/bin/bash",
+        output_path=os.path.join(run.output_path, "test_process_log.txt"),
+    )
+
+    # parse xml generated by rostest
+    results, success = parse_tests_results(test_result_file_path)
+
+    # upload artefacts generated by rostest
+    run.log_artifacts(os.path.expanduser(f"{PKGDIR}/{PACKAGE}"))
+
+    # upload files from the general output folder
+    run.log_artifacts(run.output_path)
+
+    # upload any additional files in the folders specified by the user in artefacts.yaml
+    for output in scenario.get("output_dirs", []):
+        run.log_artifacts(output)
+
+    # check if any rosbag was created
+    rosbags = glob(rosbag_path)
+    new_rosbags = set(rosbags).difference(set(preexisting_rosbags))
+    if len(new_rosbags) > 0:
+        new_rosbag = new_rosbags.pop()
+        logging.info(f"new rosbag found: {new_rosbag}")
+        # upload rosbag to dashboard
+        run.log_single_artifact(new_rosbag, "rosbag")
+        # perform any post processing, using the rosbag
+        if "rosbag_postprocess" in scenario.keys():
+            logging.info("starting rosbag postprocess")
+            post_process_folder = os.path.expanduser(f"{PKGDIR}/{PACKAGE}_postprocess")
+            os.makedirs(post_process_folder, exist_ok=True)
+            existing_files = glob(f"{post_process_folder}/*")
+            for f in existing_files:
+                os.remove(f)
+            command = [
+                "rosrun",
+                scenario["ros_testpackage"],
+                scenario["rosbag_postprocess"],
+                "--bag_path",
+                f"{new_rosbag}",
+                "--out_folder",
+                f"{post_process_folder}",
+            ]
+            # shell=True required to support command list items that are strings with spaces
+            # (allows passing additional arguments from the convenience of artefacts.yaml)
+            subprocess.run(" ".join(command), shell=True, executable="/bin/bash")
+            run.log_artifacts(post_process_folder)
+            run.log_post_process_metrics(post_process_folder)
+
+    run.log_tests_results(results, success)
+    return results, success

artefacts/cli/ros2.py

@@ -0,0 +1,125 @@
+import yaml
+from glob import glob
+import os
+import shutil
+
+from .utils import run_and_save_logs
+from .utils_ros import parse_tests_results
+from .parameters import TMP_SCENARIO_PARAMS_YAML
+
+
+def generate_scenario_parameter_output(params: dict, param_file: str):
+    """
+    Store `params` in `param_file` and convert to ros2 param file nested format,
+    to be used by the launch file
+    """
+    content = {}
+    for k, v in params.items():
+        try:
+            node, pname = k.split("/")
+        except Exception:
+            print("ensure params are in the format `{node_name}/{param_name}`")
+            return
+        if node not in content:
+            content[node] = {"ros__parameters": {}}
+        content[node]["ros__parameters"][pname] = v
+    with open(param_file, "w") as f:
+        yaml.dump(content, f)
+
+
+def run_ros2_tests(run):
+    scenario = run.params
+    # TODO: HOW TO ADD  NODE to launch
+    # TODO: set params from conf
+    # TODO: get params to log
+    # TODO: where is the rosbag
+    if "params" in run.params:
+        # note: fixed filename will lead concurent executions to overwrite each other
+        generate_scenario_parameter_output(
+            run.params["params"], TMP_SCENARIO_PARAMS_YAML
+        )
+
+    preexisting_rosbags = glob("rosbag2*")
+    test_result_file_path = f"{run.output_path}/tests_junit.xml"
+    launch_arguments = [
+        f"{k}:={v}" for k, v in run.params.get("launch_arguments", {}).items()
+    ]
+    command = [
+        "launch_test",
+        "--junit-xml",
+        test_result_file_path,
+        scenario["ros_testfile"],
+    ] + launch_arguments
+
+    # save ROS logs in the output dir
+    ros_log_dir = os.path.join(run.output_path, "ros_logs")
+
+    # Main: test execution
+    # shell=True required to support command list items that are strings with spaces
+    # (this way, scenario["ros_testfile"] can be either a path to the launch file or '<package_name> <launch_name>')
+    return_code = run_and_save_logs(
+        " ".join(command),
+        shell=True,
+        executable="/bin/bash",
+        env={
+            **os.environ,
+            **{
+                "ROS_LOG_DIR": ros_log_dir,
+                "ARTEFACTS_SCENARIO_PARAMS_FILE": TMP_SCENARIO_PARAMS_YAML,
+            },
+        },
+        output_path=os.path.join(run.output_path, "test_process_log.txt"),
+    )
+    if return_code == 2:
+        raise Exception(
+            f"Running {scenario['ros_testfile']} failed. Please check that the launch file exists."
+        )
+
+    # zip ROS logs and delete the original folder
+    if os.path.exists(ros_log_dir):
+        shutil.make_archive(ros_log_dir, "zip", ros_log_dir)
+        shutil.rmtree(ros_log_dir)
+
+    # parse xml generated by launch_test
+    results, success = parse_tests_results(test_result_file_path)
+    if success is None:
+        run.log_tests_results(results, False)
+        return results, success
+    run.log_artifacts(run.output_path)
+
+    # upload any additional files in the folders specified by the user in artefacts.yaml
+    for output in scenario.get("output_dirs", []):
+        run.log_artifacts(output)
+
+    # check if any rosbag was created
+    rosbags = glob("rosbag2*")
+    new_rosbags = set(rosbags).difference(set(preexisting_rosbags))
+    from artefacts.bagparser import BagFileParser
+
+    if len(new_rosbags) > 0:
+        rosbag_path = new_rosbags.pop()
+        run.log_artifacts(rosbag_path, "rosbag")
+        if "metrics" in run.params:
+            # TODO should go inside BagFileParser?
+            db_files = glob(f"{rosbag_path}/*.mcap")  # Ros2 Default
+            if not db_files:
+                db_files = glob(f"{rosbag_path}/*.db3")  # Legacy
+            if not db_files:
+                raise FileNotFoundError(
+                    "No .mcap or .db3 files found in the specified path."
+                )
+            db_file = db_files[0]
+            bag = BagFileParser(db_file)
+            for metric in run.params["metrics"]:
+                try:
+                    last_value = bag.get_last_message(metric)[1].data
+                    run.log_metric(metric, last_value)
+                except KeyError as e:
+                    print(f"Metric {metric} not found in rosbag, skipping.")
+                except TypeError or IndexError as e:
+                    print(
+                        f"Metric {metric} not found. Is it being published?. Skipping."
+                    )
+
+    run.log_tests_results(results, success)
+    return results, success

artefacts/cli/utils.py

@@ -0,0 +1,35 @@
+import subprocess
+import sys
+
+
+def run_and_save_logs(
+    args, output_path, shell=False, executable=None, env=None, cwd=None
+):
+    """
+    Run a command and save stdout and stderr to a file in output_path
+
+    Note: explicitly list used named params instead of using **kwargs to avoid typing issue: https://github.com/microsoft/pyright/issues/455#issuecomment-780076232
+    """
+    output_file = open(output_path, "wb")
+    proc = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,  # Capture stdout
+        stderr=subprocess.PIPE,  # Capture stderr
+        shell=shell,
+        executable=executable,
+        env=env,
+        cwd=cwd,
+    )
+    # write test-process stdout and stderr into file and stdout
+    if proc.stdout:
+        for line in proc.stdout:
+            decoded_line = line.decode()
+            sys.stdout.write(decoded_line)
+            output_file.write(line)
+    if proc.stderr:
+        for line in proc.stderr:
+            decoded_line = line.decode()
+            sys.stderr.write(decoded_line)
+            output_file.write(line)
+    proc.wait()
+    return proc.returncode

artefacts/cli/utils_ros.py

@@ -0,0 +1,68 @@
+from junitparser import JUnitXml, Attr, Element
+
+
+class FailureElement(Element):
+    _tag = "failure"
+    message = Attr()
+
+
+def parse_tests_results(file):
+    def parse_suite(suite):
+        nonlocal success, results
+        suite_results = {
+            "suite": suite.name,
+            "errors": suite.errors,
+            "failures": suite.failures,
+            "tests": suite.tests,
+        }
+        details = []
+        for case in suite:
+            case_details = {
+                "name": case.name,
+            }
+            try:
+                case_details["failure_message"] = case.child(FailureElement).message
+                case_details["result"] = "failure"
+                success = False
+            except AttributeError:
+                case_details["result"] = "success"
+            details.append(case_details)
+        suite_results["details"] = details
+        results.append(suite_results)
+
+    try:
+        xml = JUnitXml.fromfile(file)
+
+        results = []
+        success = True
+        # some xml files do not have the <testsuites> tag, just a single <tessuite>
+        if xml._tag == "testsuite":
+            # handle single suite
+            suite = xml
+            parse_suite(suite)
+        elif xml._tag == "testsuites":
+            # handle suites
+            for suite in xml:
+                parse_suite(suite)
+        # else: TODO
+        return results, success
+
+    except Exception as e:
+        print(f"[Exception in parse_tests_results] {e}")
+        print(f"Test result xml could not be loaded, marking success as False")
+        results = [
+            {
+                "suite": "unittest.suite.TestSuite",
+                "errors": 1,
+                "failures": 0,
+                "tests": 1,
+                "details": [
+                    {
+                        "name": "Error parsing XML test results",
+                        "failure_message": f"The test may have timed out. Exception: {e}",
+                        "result": "failure",
+                    }
+                ],
+            }
+        ]
+        return results, None

artefacts/cli/version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.6.8'
+__version_tuple__ = version_tuple = (0, 6, 8)