sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/main.py

@@ -11,139 +11,307 @@ from collections.abc import Iterable
 import os
 import multiprocessing as mp
 import pathlib
+import argparse
+import typing as tp
 
 # 3rd party packages
 
 # Project packages
-import sierra.core.cmdline as cmd
-from sierra.core import platform, plugin
+import sierra.core.cmdline as corecmd
+from sierra import version
+from sierra.core import engine, startup, batchroot, execenv, utils
 from sierra.core.pipeline.pipeline import Pipeline
-from sierra.core.generators.controller_generator_parser import ControllerGeneratorParser
-import sierra.core.root_dirpath_generator as rdg
-import sierra.core.plugin_manager as pm
+import sierra.core.plugin as pm
 import sierra.core.logging  # type: ignore
-import sierra.core.startup
-import sierra.version
+from sierra.core import expdef, prod, proc, storage, compare
 
 kIssuesURL = "https://github.com/jharwell/sierra/issues"
 
 
-class SIERRA():
+class SIERRA:
     """Initialize SIERRA and then launch the pipeline."""
 
-    def __init__(self, bootstrap: cmd.BootstrapCmdline) -> None:
+    def __init__(self, bootstrap: corecmd.BootstrapCmdline) -> None:
+        bootstrap_args, other_args = self._bootstrap(bootstrap)
+        manager = self._load_plugins(bootstrap_args)
+        self._verify_plugins(manager, bootstrap_args)
+        self.args = self._load_cmdline(bootstrap_args, other_args)
+
+        # Configure cmdopts for engine + execution environment by modifying
+        # arguments/adding new arguments as needed, and perform additional
+        # validation.
+        self.args = execenv.cmdline_postparse_configure(
+            bootstrap_args.execenv, self.args
+        )
+        self.args = engine.cmdline_postparse_configure(
+            bootstrap_args.engine, bootstrap_args.execenv, self.args
+        )
+
+        # Inject bootstrap arguments into the namespace for non-bootstrap
+        # arguments to make setting up the pipeline downstream much more
+        # uniform.
+        self.args.__dict__["project"] = bootstrap_args.project
+        self.args.__dict__["engine"] = bootstrap_args.engine
+        self.args.__dict__["execenv"] = bootstrap_args.execenv
+        self.args.__dict__["expdef"] = bootstrap_args.expdef
+        self.args.__dict__["storage"] = bootstrap_args.storage
+        self.args.__dict__["proc"] = bootstrap_args.proc
+        self.args.__dict__["prod"] = bootstrap_args.prod
+        self.args.__dict__["compare"] = bootstrap_args.compare
+
+    def __call__(self) -> None:
+        # If only 1 pipeline stage is passed, then the list of stages to run is
+        # parsed as a non-iterable integer, which can cause the generator to
+        # fail to be created. So make it iterable in that case as well.
+        if not isinstance(self.args.pipeline, Iterable):
+            self.args.pipeline = [self.args.pipeline]
+
+        if 5 not in self.args.pipeline:
+            self.logger.info(
+                "Controller=%s, Scenario=%s", self.args.controller, self.args.scenario
+            )
+            pathset = batchroot.from_cmdline(self.args)
+
+            pipeline = Pipeline(self.args, self.args.controller, pathset)
+        else:
+            pipeline = Pipeline(self.args, None)
+
+        try:
+            pipeline.run()
+        except KeyboardInterrupt:
+            self.logger.info("Exiting on user cancel")
+            sys.exit()
+
+    def _bootstrap(
+        self, bootstrap: corecmd.BootstrapCmdline
+    ) -> tp.Tuple[argparse.Namespace, tp.List[str]]:
         # Bootstrap the cmdline
         bootstrap_args, other_args = bootstrap.parser.parse_known_args()
+        if bootstrap_args.rcfile:
+            bootstrap_args.rcfile = os.path.expanduser(bootstrap_args.rcfile)
 
         # Setup logging customizations
         sierra.core.logging.initialize(bootstrap_args.log_level)
         self.logger = logging.getLogger(__name__)
-        self.logger.info("This is SIERRA %s.", sierra.version.__version__)
+        self.logger.info("This is SIERRA %s.", version.__version__)
+
+        bootstrap_args = self._handle_rc(bootstrap_args.rcfile, bootstrap_args)
 
         # Check SIERRA runtime environment
-        sierra.core.startup.startup_checks(not bootstrap_args.skip_pkg_checks)
-        self.logger.info("Using python=%s.", sys.version.replace('\n', ''))
+        startup.startup_checks(not bootstrap_args.skip_pkg_checks)
+        self.logger.info("Using python=%s.", sys.version.replace("\n", ""))
+
+        return bootstrap_args, other_args
+
+    def _load_cmdline(
+        self, bootstrap_args: argparse.Namespace, other_args: tp.List[str]
+    ) -> argparse.Namespace:
+        """Build the cmdline from the selected plugins and parse args.
+
+        This is one of the places where the SIERRA magic happens. This function
+        dynamically combines declared cmdlines from each active plugin, in the
+        following partial dependency order::
+
+               --project -> {--expdef, --proc, --prod, --compare, --storage} -> --execenv --> --engine
+
+        Plugins in the {} could *probably* be reordered without breaking things,
+        but the other plugins (--project, --execenv, --engine) need to be where
+        they are in the chain. Change the order at your own risk!
+        """
+
+        self.logger.info("Dynamically building cmdline from selected plugins")
+        parents = [corecmd.CoreCmdline([], [-1, 1, 2, 3, 4, 5]).parser]
+        stages = [-1, 1, 2, 3, 4, 5]
+
+        # For plugin options which don't take lists, we can just iterate over
+        # them with some simple conf and build their portions of the cmdline.
+        simple = {
+            "engine": {"arg": "--engine", "module": engine},
+            "execenv": {"arg": "--execenv", "module": execenv},
+            "expdef": {"arg": "--expdef", "module": expdef},
+            "storage": {"arg": "--storage", "module": storage},
+        }
+        for name, conf in simple.items():
+            if parser := conf["module"].cmdline_parser(
+                bootstrap_args.__dict__[name], parents, stages
+            ):
+                self.logger.debug(
+                    "Loaded %s=%s cmdline", conf["arg"], bootstrap_args.__dict__[name]
+                )
+                parents = [parser]
+
+        # These plugin options take lists of plugins to use, and so take
+        # slightly more complicated processing.
+        lists = {
+            "proc": {"arg": "--proc", "module": proc},
+            "prod": {"arg": "--prod", "module": prod},
+            "compare": {"arg": "--compare", "module": compare},
+        }
+        for name, conf in lists.items():
+            for to_load in bootstrap_args.__dict__[name]:
+                if parser := conf["module"].cmdline_parser(to_load, parents, stages):
+                    self.logger.debug("Loaded %s=%s cmdline", conf["arg"], to_load)
+                    parents = [parser]
+
+        path = f"{bootstrap_args.project}.cmdline"
+        module = pm.module_load(path)
 
+        nonbootstrap_cmdline = module.build(parents, stages)
+        args = nonbootstrap_cmdline.parser.parse_args(other_args)
+        args.sierra_root = os.path.expanduser(args.sierra_root)
+        args = self._handle_rc(bootstrap_args.rcfile, args)
+        return args
+
+    def _load_plugins(self, bootstrap_args: argparse.Namespace):
         this_file = pathlib.Path(__file__)
         install_root = pathlib.Path(this_file.parent)
 
         # Load plugins
         self.logger.info("Loading plugins")
-        project = bootstrap_args.project
-        plugin_core_path = [install_root / 'plugins' / 'hpc',
-                            install_root / 'plugins' / 'storage',
-                            install_root / 'plugins' / 'robot',
-                            install_root / 'plugins' / 'platform']
-        plugin_search_path = plugin_core_path
-        env = os.environ.get('SIERRA_PLUGIN_PATH')
-        if env is not None:
+        plugin_search_path = [install_root / "plugins"]
+        if env := os.environ.get("SIERRA_PLUGIN_PATH"):
             for p in env.split(os.pathsep):
                 plugin_search_path.append(pathlib.Path(p))
 
         manager = pm.pipeline
-        manager.initialize(project, plugin_search_path)
+        manager.initialize(bootstrap_args.project, plugin_search_path)
 
+        # 2025-06-14 [JRH]: All found plugins are loaded/executed as python
+        # modules, even if they are not currently selected. I don't know if this
+        # is a good idea or not.
         for p in manager.available_plugins():
             manager.load_plugin(p)
 
-        # Verify platform plugin
-        module = manager.get_plugin_module(bootstrap_args.platform)
-        plugin.platform_sanity_checks(module)
+        return manager
+
+    def _verify_plugins(
+        self,
+        manager,
+        bootstrap_args: argparse.Namespace,
+    ) -> None:
+        # Verify engine plugin
+        module = manager.get_plugin_module(bootstrap_args.engine)
+        pm.engine_sanity_checks(bootstrap_args.engine, module)
 
         # Verify execution environment plugin
-        module = manager.get_plugin_module(bootstrap_args.exec_env)
-        plugin.exec_env_sanity_checks(module)
+        module = manager.get_plugin_module(bootstrap_args.execenv)
+        pm.execenv_sanity_checks(bootstrap_args.execenv, module)
 
-        # Load platform cmdline extensions
-        platform_parser = platform.CmdlineParserGenerator(
-            bootstrap_args.platform)()
+        # Verify processing plugins
+        for p in bootstrap_args.proc:
+            module = manager.get_plugin_module(p)
+            pm.proc_sanity_checks(p, module)
 
-        # Load project cmdline extensions
-        self.logger.info("Loading cmdline extensions from project '%s'",
-                         project)
-        path = f"{project}.cmdline"
-        module = pm.module_load(path)
+        # Verify product plugins
+        for p in bootstrap_args.prod:
+            module = manager.get_plugin_module(p)
+            pm.prod_sanity_checks(p, module)
 
-        # Validate cmdline args
-        self.args = module.Cmdline([bootstrap.parser, platform_parser],
-                                   [-1, 1, 2, 3, 4, 5]).parser.parse_args(other_args)
-        module.CmdlineValidator()(self.args)
+        # Verify comparison plugins
+        for p in bootstrap_args.compare:
+            module = manager.get_plugin_module(p)
+            pm.compare_sanity_checks(p, module)
 
-        # Verify storage plugin (declared as part of core cmdline arguments
-        # rather than bootstrap, so we have to wait until after all arguments
-        # are parsed to verify it)
-        module = manager.get_plugin_module(self.args.storage_medium)
-        plugin.storage_sanity_checks(module)
+        # Verify expdef plugin
+        module = manager.get_plugin_module(bootstrap_args.expdef)
+        pm.expdef_sanity_checks(bootstrap_args.expdef, module)
 
-        # Configure cmdopts for platform + execution environment by modifying
-        # arguments/adding new arguments as needed.
-        configurer = platform.ParsedCmdlineConfigurer(bootstrap_args.platform,
-                                                      bootstrap_args.exec_env)
-        self.args = configurer(self.args)
-        self.args.__dict__['project'] = project
+        # Verify storage plugin
+        module = manager.get_plugin_module(bootstrap_args.storage)
+        pm.storage_sanity_checks(bootstrap_args.storage, module)
 
-    def __call__(self) -> None:
-        # If only 1 pipeline stage is passed, then the list of stages to run is
-        # parsed as a non-iterable integer, which can cause the generator to
-        # fail to be created. So make it iterable in that case as well.
-        if not isinstance(self.args.pipeline, Iterable):
-            self.args.pipeline = [self.args.pipeline]
+    def _handle_rc(
+        self, rcfile_path: tp.Optional[str], args: argparse.Namespace
+    ) -> argparse.Namespace:
+        """
+        Populate cmdline arguments from a .sierrarc file.
 
-        if 5 not in self.args.pipeline:
-            controller = ControllerGeneratorParser()(self.args)
-            sgp = pm.module_load_tiered(project=self.args.project,
-                                        path='generators.scenario_generator_parser')
-            scenario = sgp.ScenarioGeneratorParser().to_scenario_name(self.args)
-
-            self.logger.info("Controller=%s, Scenario=%s", controller, scenario)
-            cmdopts = rdg.from_cmdline(self.args)
-            pipeline = Pipeline(self.args, controller, cmdopts)
-        else:
-            pipeline = Pipeline(self.args, None, {})
+        In order of priority:
 
-        try:
-            pipeline.run()
-        except KeyboardInterrupt:
-            self.logger.info("Exiting on user cancel")
-            sys.exit()
+        #. ``--rcfile``
+
+        #. ``SIERRA_RCFILE``
+
+        #. ``~/.sierrarc``
+
+
+        Anything passed on the cmdline overrides, if both are present.
+
+        """
+        # Check rcfile envvar first, so that you can override it on cmdline if
+        # desired.
+        realpath = os.getenv("SIERRA_RCFILE", None)
+
+        if realpath:
+            self.logger.debug("Reading rcfile from envvar")
+
+        if rcfile_path:
+            self.logger.debug("Reading rcfile from cmdline")
+            realpath = rcfile_path
+
+        if not realpath and os.path.exists(os.path.expanduser("~/.sierrarc")):
+            self.logger.debug("Reading rcfile from ~/.sierrarc")
+            realpath = "~/.sierrarc"
+
+        if not realpath:
+            return args
+
+        path = pathlib.Path(realpath).expanduser()
+
+        with utils.utf8open(path, "r") as rcfile:
+            for line in rcfile.readlines():
+                # There are 3 ways to pass arguments in the rcfile:
+                #
+                # 1. --arg
+                # 2. --arg=foo
+                # 3. --arg foo
+                #
+                # If you encounter a ~, we assume its a path, so we expand it to
+                # match cmdline behavior.
+                line = line.strip("\n")
+                components = line.split()
+
+                if len(components) == 1 and "=" not in components[0]:  # boolean
+                    key = line[2:].replace("-", "_")
+                    args.__dict__[key] = True
+
+                elif len(components) == 1 and "=" in components[0]:
+                    key = line.split("=")[0][2:].replace("-", "_")
+                    value = os.path.expanduser(line.split("=")[1])
+
+                    args.__dict__[key] = value
+                else:
+                    key = line.split()[0][2:].replace("-", "_")
+                    value = os.path.expanduser(line.split()[1])
+                    args.__dict__[key] = value
+
+                self.logger.trace(
+                    "Applied cmdline arg from rcfile='%s': %s", path, line
+                )
+
+        return args
 
 
 def excepthook(exc_type, exc_value, exc_traceback):
-    logging.fatal(("SIERRA has encountered an unexpected error and will now "
-                   "terminate.\n\n"
-                   "If you think this is a bug, please report it at:\n\n%s\n\n"
-                   "When reporting, please include as much information as you "
-                   "can. Ideally:\n\n"
-                   "1. What you were trying to do in SIERRA.\n"
-                   "2. The terminal output of sierra-cli, including the "
-                   "below traceback.\n"
-                   "3. The exact command you used to run SIERRA.\n"
-                   "\n"
-                   "In some cases, creating a Minimum Working Example (MWE) "
-                   "reproducing the error with specific input files and/or "
-                   "data is also helpful for quick triage and fix.\n"),
-                  kIssuesURL,
-                  exc_info=(exc_type, exc_value, exc_traceback))
+    logging.fatal(
+        (
+            "SIERRA has encountered an unexpected error and will now "
+            "terminate.\n\n"
+            "If you think this is a bug, please report it at:\n\n%s\n\n"
+            "When reporting, please include as much information as you "
+            "can. Ideally:\n\n"
+            "1. What you were trying to do in SIERRA.\n"
+            "2. The terminal output of sierra-cli, including the "
+            "below traceback.\n"
+            "3. The exact command you used to run SIERRA.\n"
+            "\n"
+            "In some cases, creating a Minimum Working Example (MWE) "
+            "reproducing the error with specific input files and/or "
+            "data is also helpful for quick triage and fix.\n"
+        ),
+        kIssuesURL,
+        exc_info=(exc_type, exc_value, exc_traceback),
+    )
 
 
 def main():
@@ -156,11 +324,11 @@ def main():
     sys.excepthook = excepthook
 
     # Bootstrap the cmdline to print version if needed
-    bootstrap = cmd.BootstrapCmdline()
+    bootstrap = corecmd.BootstrapCmdline()
     bootstrap_args, _ = bootstrap.parser.parse_known_args()
 
     if bootstrap_args.version:
-        sys.stdout.write(cmd.kVersionMsg)
+        sys.stdout.write(corecmd.kVersionMsg)
     else:
         app = SIERRA(bootstrap)
         app()

sierra/plugins/__init__.py

@@ -1,9 +1,128 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
+"""Container module for all plugins in SIERRA."""
 
 # Core packages
+import typing as tp
+import argparse
 
 # 3rd party packages
 
 # Project packages
+from sierra.core import cmdline
+
+
+class PluginCmdline(cmdline.BaseCmdline):
+    """Base class for plugin cmdlines using :class:`argparse`.
+
+    Note that this can't be in a cmdline.py in this folder because SIERRA will
+    erroneouly pick it up as a plugin cmdline module.
+    """
+
+    def __init__(
+        self,
+        parents: tp.List[argparse.ArgumentParser],
+        stages: tp.List[int],
+    ) -> None:
+        super().__init__()
+        self._scaffold_cli(parents)
+        self._init_cli(stages)
+
+    def _scaffold_cli(self, parents: tp.List[argparse.ArgumentParser]) -> None:
+        """
+        Scaffold CLI by defining the parser and common argument groups.
+        """
+        if parents:
+            self.parser = argparse.ArgumentParser(
+                prog="sierra-cli", parents=parents, add_help=False, allow_abbrev=False
+            )
+        else:
+            self.parser = argparse.ArgumentParser(
+                prog="sierra-cli", add_help=False, allow_abbrev=False
+            )
+
+        self.multistage = self.parser.add_argument_group(
+            self.multistage_desc[0], self.multistage_desc[1]
+        )
+        self.stage1 = self.parser.add_argument_group(
+            self.stage1_desc[0], self.stage1_desc[1]
+        )
+        self.stage2 = self.parser.add_argument_group(
+            self.stage2_desc[0], self.stage2_desc[1]
+        )
+        self.stage3 = self.parser.add_argument_group(
+            self.stage3_desc[0], self.stage3_desc[1]
+        )
+        self.stage4 = self.parser.add_argument_group(
+            self.stage4_desc[0], self.stage4_desc[1]
+        )
+        self.stage5 = self.parser.add_argument_group(
+            self.stage5_desc[0], self.stage5_desc[1]
+        )
+        self.shortforms = self.parser.add_argument_group(
+            title="Shortform aliases",
+            description="""
+                        Most cmdline options to SIERRA are longform (i.e.,
+                        ``--option``), but some families of options have
+                        shortforms (i.e., ``-o`` for ``--option``) as
+                        well. Shortform arguments behave the same as their
+                        longform counterparts.
+                        """,
+        )
+
+    def _init_cli(self, stages: tp.List[int]) -> None:
+        """Define cmdline arguments for stages 1-5."""
+        if -1 in stages:
+            self.init_shortforms()
+            self.init_multistage()
+
+        if 1 in stages:
+            self.init_stage1()
+
+        if 2 in stages:
+            self.init_stage2()
+
+        if 3 in stages:
+            self.init_stage3()
+
+        if 4 in stages:
+            self.init_stage4()
+
+        if 5 in stages:
+            self.init_stage5()
+
+    def init_shortforms(self) -> None:
+        """
+        Define cmdline shortform arguments for all pipeline stages.
+        """
+
+    def init_multistage(self) -> None:
+        """
+        Define cmdline arguments for all pipeline stages.
+        """
+
+    def init_stage1(self) -> None:
+        """
+        Define cmdline arguments for stage 1.
+        """
+
+    def init_stage2(self) -> None:
+        """
+        Define cmdline arguments for stage 2.
+        """
+
+    def init_stage3(self) -> None:
+        """
+        Define cmdline arguments for stage 3.
+        """
+
+    def init_stage4(self) -> None:
+        """
+        Define cmdline arguments for stage 4.
+        """
+
+    def init_stage5(self) -> None:
+        """
+        Define cmdline arguments for stage 5.
+        """

sierra/plugins/compare/__init__.py

@@ -0,0 +1,14 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Container module for plugins related to product comparison.
+
+Driven by ``--compare``.
+"""
+# Core packages
+
+# 3rd party packages
+
+# Project packages

sierra/plugins/compare/graphs/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Container module for the graph product comparison plugin.
+
+See :ref:`plugins/compare/graphs`.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"