sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/execenv/prefectserver/flow.py

@@ -0,0 +1,66 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Glue for bridging SIERRA <-> prefect for executing local builds."""
+
+
+# Core packages
+import pathlib
+import subprocess
+
+
+# 3rd party packages
+import prefect
+
+# Project packages
+
+
+@prefect.task(tags=["sierra-exec-jobs-per-node"])
+def exec_exp_run(
+    cmd: list[str],
+    scratch_path: pathlib.Path,
+) -> None:
+    """Execute the given command.
+
+    Arguments:
+        cmd: Command to execute a single :term:`Experimental Run`.
+
+        scratch_path: Where to write stdout/stderr to.
+    """
+    flow_run_id = prefect.runtime.task_run.id
+
+    with (
+        open(str(scratch_path) + f"_{flow_run_id}_stdout", "w") as stdout,
+        open(str(scratch_path) + f"_{flow_run_id}_stderr", "w") as stderr,
+    ):
+        subprocess.run(
+            cmd,
+            stdout=stdout,
+            stderr=stderr,
+            shell=True,
+            check=True,
+        )
+
+
+@prefect.flow
+def sierra(
+    input_root: pathlib.Path,
+    scratch_root: pathlib.Path,
+) -> None:
+    """Generate commands, execute the simulation, and package the data.
+
+    Arguments:
+        input_root: Path to the input directory for the :term:`Experiment`.
+
+        scratch_root: Path to the scratch directory for the :term:`Experiment`.
+    """
+    commands = []
+    with open(input_root / "commands.txt") as f:
+        commands = [line.strip() for line in f.readlines()]
+
+    scratch_stdouts = [scratch_root / f"_run{i}" for i in range(0, len(commands))]
+
+    sim_results = exec_exp_run.map(commands, scratch_stdouts)
+    sim_results.wait()

sierra/plugins/execenv/prefectserver/local/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for the local execution environment.
+
+See :ref:`plugins/execenv/prefectserver/local`.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/execenv/prefectserver/local/cmdline.py

@@ -0,0 +1,29 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import typing as tp
+import argparse
+
+# 3rd party packages
+
+# Project packages
+from sierra.plugins.execenv import prefectserver
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+def build(
+    parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+) -> PluginCmdline:
+    """
+    Get a cmdline parser supporting the ``prefectserver.local`` execution environment.
+    """
+    return prefectserver.cmdline.PrefectCmdline(parents, stages)
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    return prefectserver.cmdline.to_cmdopts(args)

sierra/plugins/execenv/prefectserver/local/plugin.py

@@ -0,0 +1,133 @@
+# Copyright 2025 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""Prefect plugin for running SIERRA locally."""
+
+# Core packages
+import typing as tp
+import pathlib
+
+# 3rd party packages
+import implements
+
+# Project packages
+from sierra.core import types
+from sierra.core.experiment import bindings
+
+
+@implements.implements(bindings.IBatchShellCmdsGenerator)
+class BatchShellCmdsGenerator:
+    """
+    Generate commands to invoke :term:`Prefect` for local computing.
+    """
+
+    def __init__(self, cmdopts: types.Cmdopts) -> None:
+        self.cmdopts = cmdopts
+        self.api_url = "http://127.0.0.1:4200/api"
+
+    def pre_batch_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        flow_path = pathlib.Path(__file__).parent / "../flow.py"
+        no_prompt = "PREFECT_CLI_PROMPT=false"
+
+        return [
+            # We obviously have to start a local server. We don't have to wait
+            # for the command, because it is backgrounded.
+            types.ShellCmdSpec(
+                cmd=f"env {no_prompt} prefect server start &",
+                shell=True,
+                wait=False,
+            ),
+            types.ShellCmdSpec(
+                cmd="sleep 5",
+                shell=True,
+                wait=True,
+            ),
+            types.ShellCmdSpec(
+                cmd="prefect config set PREFECT_API_URL={0}".format(self.api_url),
+                shell=True,
+                wait=True,
+            ),
+            # 2025-07-29 [JRH]: This doesn't work (workers ignore it and start #
+            # simultaneous sims corresponding to # cores available on the
+            # machine), despite all the docs saying it should. I THINK it's
+            # because global concurrency limits/task concurrency limits only
+            # work with prefect cloud.
+            #
+            # Concurrency limit has to be set BEFORE starting workers.
+            types.ShellCmdSpec(
+                cmd="echo 'y' | prefect gcl delete sierra-exec-jobs-per-node || true",
+                shell=True,
+                wait=True,
+            ),
+            types.ShellCmdSpec(
+                cmd="prefect gcl create sierra-exec-jobs-per-node --limit {0}".format(
+                    self.cmdopts["exec_jobs_per_node"]
+                ),
+                shell=True,
+                wait=True,
+            ),
+            types.ShellCmdSpec(
+                cmd="prefect work-pool create --type process {0}".format(
+                    self.cmdopts["work_pool"],
+                ),
+                shell=True,
+                wait=True,
+            ),
+            # We only need to start 1 worker, since each worker can
+            # handle arbitrary concurrency limits. We use a unique name for the pool
+            # and queue names to avoid conflicts if we are on a shared server and
+            # trying to run prefect locally.
+            types.ShellCmdSpec(
+                cmd="{0} prefect worker start "
+                "--pool={1} "
+                "--work-queue={2} &".format(
+                    no_prompt,
+                    self.cmdopts["work_pool"],
+                    self.cmdopts["work_queue"],
+                ),
+                shell=True,
+                wait=False,
+            ),
+            types.ShellCmdSpec(
+                cmd="until prefect work-pool ls | grep -q 'sierra'; do sleep 1; done",
+                shell=True,
+                wait=True,
+            ),
+            # Deploy SIERRA's local experiment execution flow. This flow
+            # executes all runs in all experiments in the batch in parallel. We
+            # use a unique name for the pool name to avoid conflicts if we are
+            # on a shared server and trying to run prefect locally.
+            types.ShellCmdSpec(
+                cmd="{0} prefect deploy {1}:sierra --name sierra/local --pool {2} --work-queue {3}".format(
+                    no_prompt,
+                    flow_path,
+                    self.cmdopts["work_pool"],
+                    self.cmdopts["work_queue"],
+                ),
+                shell=True,
+                wait=True,
+            ),
+        ]
+
+    def exec_batch_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
+        # The needed flow is already present on the server as a deployment, so
+        # we can just run it directly as many times as we want. You don't need
+        # to set the queue/pool--it is inferred from the flow you are running.
+        flow_cmd = (
+            "prefect deployment run sierra/local "
+            '--params=\'{{"input_root": "{0}","scratch_root": "{1}"}}\' --watch'
+        )
+        # Since this execenv is batch level, so are the outputs and scratch
+        # dirs.
+        flow = flow_cmd.format(exec_opts["batch_root"], exec_opts["batch_scratch_root"])
+        spec = types.ShellCmdSpec(cmd=flow, shell=True, wait=True)
+
+        return [spec]
+
+    def post_batch_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        return [
+            types.ShellCmdSpec(cmd="killall prefect || true", shell=True, wait=True),
+        ]
+
+
+__all__ = ["BatchShellCmdsGenerator"]

sierra/plugins/{hpc/adhoc → execenv/robot}/__init__.py

@@ -1,6 +1,7 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
+"""Container module for plugins related to real robot execution environments."""
 
 # Core packages
 

sierra/plugins/execenv/robot/turtlebot3/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for the Turtlebot3 robot execution environment plugin.
+
+See :ref:`plugins/execenv/realrobot/turtlebot3`.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/execenv/robot/turtlebot3/plugin.py

@@ -0,0 +1,204 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Robot plugin for running SIERRA with a set of Turtlebot3 robots.
+
+"""
+
+# Core packages
+import os
+import logging
+import typing as tp
+import argparse
+import shutil
+import pathlib
+
+# 3rd party packages
+import implements
+
+# Project packages
+from sierra.core import types, execenv, utils
+from sierra.core.experiment import bindings
+
+_logger = logging.getLogger(__name__)
+
+
+def cmdline_postparse_configure(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    Configure SIERRA for the turtlebot3 execution environment.
+
+    May use the following environment variables:
+
+    - :envvar:`SIERRA_NODEFILE` - If this is not defined ``--nodefile`` must be
+      passed.
+    """
+    if args.nodefile is None:
+        assert "SIERRA_NODEFILE" in os.environ, (
+            "Non-robot.turtlebot3 environment detected: --nodefile not "
+            "passed and 'SIERRA_NODEFILE' not found"
+        )
+        args.nodefile = os.environ["SIERRA_NODEFILE"]
+
+    assert utils.path_exists(
+        args.nodefile
+    ), f"SIERRA_NODEFILE '{args.nodefile}' does not exist"
+    _logger.info("Using '%s' as robot hostnames file", args.nodefile)
+
+    assert not args.engine_vc, "Engine visual capture not supported on robot.turtlebot3"
+
+    return args
+
+
+@implements.implements(bindings.IExpShellCmdsGenerator)
+class ExpShellCmdsGenerator:
+    """Generate the cmds to invoke GNU Parallel to launch ROS on the turtlebots."""
+
+    def __init__(self, cmdopts: types.Cmdopts, exp_num: int) -> None:
+        self.cmdopts = cmdopts
+        self.exp_num = exp_num
+
+    def pre_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        return []
+
+    def post_exp_cmds(self) -> tp.List[types.ShellCmdSpec]:
+        return []
+
+    def exec_exp_cmds(self, exec_opts: types.StrDict) -> tp.List[types.ShellCmdSpec]:
+        jobid = os.getpid()
+
+        # Even if we are passed --nodelist, we still make our own copy of it, so
+        # that the user can safely modify it (if they want to) after running
+        # stage 1.
+        nodelist = pathlib.Path(
+            exec_opts["exp_input_root"], "{0}-nodelist.txt".format(jobid)
+        )
+
+        resume = ""
+        # This can't be --resume, because then GNU parallel looks at the results
+        # directory, and if there is stuff in it, (apparently) assumes that the
+        # job finished...
+        if exec_opts["exec_resume"]:
+            resume = "--resume-failed"
+
+        # Make sure there are no duplicate nodes
+        unique_nodes = types.ShellCmdSpec(
+            cmd="sort -u {0} > {1}".format(exec_opts["nodefile"], nodelist),
+            shell=True,
+            wait=True,
+        )
+
+        # Make sure GNU parallel uses the right shell, because it seems to
+        # defaults to /bin/sh since all cmds are run in a python shell which
+        # does not have $SHELL set.
+        use_bash = types.ShellCmdSpec(
+            cmd="export PARALLEL_SHELL={0}".format(shutil.which("bash")),
+            shell=True,
+            env=True,
+            wait=True,
+        )
+
+        ret = [use_bash, unique_nodes]
+
+        # 1 GNU parallel command to launch each experimental run, because each
+        # run might use all available nodes/robots.
+        for i in range(self.cmdopts["n_runs"]):
+            ret.extend(self._generate_for_run(i, exec_opts, resume, nodelist))
+
+        return ret
+
+    def _generate_for_run(
+        self, i: int, exec_opts: dict, resume: str, nodelist: pathlib.Path
+    ) -> tp.List[types.ShellCmdSpec]:
+        ret = []
+        # GNU parallel cmd for robots (slaves)
+        robots = (
+            "parallel {2} "
+            "--jobs {1} "
+            "--results {4} "
+            "--joblog {3} "
+            "--sshloginfile {0} "
+            '--workdir {4} < "{5}"'
+        )
+
+        robots_ipath = (
+            exec_opts["cmdfile_stem_path"] + f"_run{i}_slave" + exec_opts["cmdfile_ext"]
+        )
+
+        robot_log = pathlib.Path(
+            exec_opts["exp_scratch_root"], f"parallel-slaves-run{i}.log"
+        )
+
+        robots = robots.format(
+            nodelist,
+            exec_opts["n_jobs"],
+            resume,
+            robot_log,
+            exec_opts["exp_scratch_root"],
+            robots_ipath,
+        )
+
+        # If no master is spawned, then we need to wait for this GNU
+        # parallel cmd. If the master is spawned, then we wait for THAT
+        # command; waiting for both results in the master never starting
+        # because that cmd is never run.
+        robots_spec = types.ShellCmdSpec(
+            cmd=robots, shell=True, wait=self.cmdopts["no_master_node"]
+        )
+        ret.append(robots_spec)
+
+        if not self.cmdopts["no_master_node"]:
+            ros_master = "parallel {3} --results {1} --joblog {0} '--workdir {1} < {2}'"
+
+            ros_master_ipath = (
+                exec_opts["cmdfile_stem_path"]
+                + f"_run{i}_master"
+                + exec_opts["cmdfile_ext"]
+            )
+
+            master_log = pathlib.Path(
+                exec_opts["exp_scratch_root"], f"parallel-master-run{i}.log"
+            )
+            ros_master = ros_master.format(
+                master_log, exec_opts["exp_scratch_root"], ros_master_ipath, resume
+            )
+
+            master_spec = types.ShellCmdSpec(
+                cmd=ros_master, shell=True, wait=not self.cmdopts["no_master_node"]
+            )
+            ret.append(master_spec)
+
+        wait = 'echo "{0} seconds until launching next run!"; ' "sleep {0}s ;".format(
+            self.cmdopts["exec_inter_run_pause"]
+        )
+        wait_spec = types.ShellCmdSpec(cmd=wait, shell=True, wait=True)
+        ret.append(wait_spec)
+
+        return ret
+
+
+def execenv_check(cmdopts: types.Cmdopts) -> None:
+    """
+    Verify execution environment in stage 2 for the :term:`ROS1+Robot` engine.
+
+    Checks that a valid list of IPs for robots is set/passed, and checks that
+    they are reachable.
+    """
+    nodes = execenv.parse_nodefile(cmdopts["nodefile"])
+    for node in nodes:
+        if int(node.n_cores) != 1:
+            _logger.warning(
+                (
+                    "Nodefile %s, host %s has multiple "
+                    "cores; turtlebots are single core"
+                ),
+                cmdopts["nodefile"],
+                node.hostname,
+            )
+        if not cmdopts["skip_online_check"]:
+            execenv.check_connectivity(
+                cmdopts, node.login, node.hostname, node.port, "turtlebot3"
+            )
+
+
+__all__ = ["cmdline_postparse_configure", "execenv_check", "ExpShellCmdsGenerator"]

sierra/plugins/expdef/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for plugins related to :term:`Experiment` definitions.
+
+Driven by ``--expdef``.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages

sierra/plugins/expdef/json/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""Container module for the JSON expdef plugin."""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"