sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/core/plugin.py

@@ -1,112 +1,579 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
-"""Sanity checks for verifying selected plugins.
-
-Checks that selected plugins implement the necessary classes and
- functions. Currently checkes: ``--storage-medium``, ``--exec-env``, and
- ``--platform``.
+"""SIERRA plugin management to make SIERRA OPEN/CLOSED.
 
+Also contains checks that selected plugins implement the necessary classes and
+functions.  Currently checkes: ``--storage``, ``--execenv``, and ``--engine``.
 """
 
 # Core packages
-import inspect
+# Core packages
+import importlib.util
+import importlib
+import typing as tp
+import sys
 import logging
+import pathlib
+import inspect
 
 # 3rd party packages
 
+# 3rd party packages
+import json
+
 # Project packages
+from sierra.core import types
+
+
+class BasePluginManager:
+    """Base class for common functionality."""
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.loaded = {}  # type: tp.Dict[str, tp.Dict]
+
+    def available_plugins(self):
+        raise NotImplementedError
+
+    def loaded_plugins(self):
+        return self.loaded
+
+    def load_plugin(self, name: str) -> None:
+        """Load a plugin module."""
+        plugins = self.available_plugins()
+        if name not in plugins:
+            self.logger.fatal("Cannot locate plugin %s", name)
+            self.logger.fatal(
+                "Loaded plugins: %s\n",
+                json.dumps(self.loaded, default=lambda x: "<ModuleSpec>", indent=4),
+            )
+            raise Exception(f"Cannot locate plugin '{name}'")
+
+        init = importlib.util.module_from_spec(plugins[name]["init_spec"])
+        plugins[name]["init_spec"].loader.exec_module(init)
+
+        if not hasattr(init, "sierra_plugin_type"):
+            self.logger.warning(
+                "Cannot load plugin %s: __init__.py does not define sierra_plugin_type()",
+                name,
+            )
+            return
+
+        plugin_type = init.sierra_plugin_type()
+
+        # The name of the module is only needed for pipeline plugins, not
+        # project plugins.
+        if plugins[name]["module_spec"] is None and plugin_type == "pipeline":
+            if not hasattr(init, "sierra_plugin_module"):
+                self.logger.warning(
+                    "Cannot load plugin %s: __init__.py does not define sierra_plugin_module()",
+                    name,
+                )
+                return
+
+            modname = init.sierra_plugin_module()
+            fpath = (
+                plugins[name]["parent_dir"] / name.replace(".", "/") / f"{modname}.py"
+            )
+            plugins[name]["module_spec"] = importlib.util.spec_from_file_location(
+                modname, fpath
+            )
+
+        if plugin_type == "pipeline":
+            self._load_pipeline_plugin(name)
+
+        elif plugin_type == "project":
+            self._load_project_plugin(name)
+        elif plugin_type == "model":
+            if not hasattr(init, "sierra_models"):
+                self.logger.warning(
+                    "Cannot load plugin %s: __init__.py does not define sierra_models()",
+                    name,
+                )
+                return
+
+            self._load_model_plugin(name)
+        else:
+            self.logger.warning(
+                "Unknown plugin type '%s' for %s: cannot load", plugin_type, name
+            )
+
+    def get_plugin(self, name: str) -> dict:
+        try:
+            return self.loaded[name]
+        except KeyError:
+            self.logger.fatal("No such plugin %s", name)
+            self.logger.fatal(
+                "Loaded plugins: %s\n",
+                json.dumps(self.loaded, default=lambda x: "<ModuleSpec>", indent=4),
+            )
+            raise
+
+    def get_plugin_module(self, name: str) -> types.ModuleType:
+        try:
+            return self.loaded[name]["module"]
+        except KeyError:
+            self.logger.fatal("No such plugin %s", name)
+            self.logger.fatal(
+                "Loaded plugins: %s\n",
+                json.dumps(self.loaded, default=lambda x: "<ModuleSpec>", indent=4),
+            )
+            raise
+
+    def has_plugin(self, name: str) -> bool:
+        return name in self.loaded
+
+    def _load_pipeline_plugin(self, name: str) -> None:
+        if name in self.loaded:
+            self.logger.warning("Pipeline plugin %s already loaded", name)
+            return
+
+        plugins = self.available_plugins()
+
+        # The parent directory of the plugin must be on sys.path so it can be
+        # imported, so we put in on there if it isn't.
+        new = str(plugins[name]["parent_dir"])
+        if new not in sys.path:
+            sys.path = [new] + sys.path[0:]
+            self.logger.debug("Updated sys.path with %s", [new])
+
+        module = importlib.util.module_from_spec(plugins[name]["module_spec"])
+        plugins[name]["module_spec"].loader.exec_module(module)
+
+        # When importing with importlib, the module is not automatically added
+        # to sys.modules. This means that trying to pickle anything in it will
+        # fail with a rather cryptic 'AttributeError', so we explicitly add the
+        # last path component of the plugin name--which is the actual name of
+        # the module the plugin lives in--to sys.modules so that pickling will
+        # work.
+        sys_modname = name.split(".")[1]
+        if sys_modname not in sys.modules:
+            sys.modules[sys_modname] = module
+
+        self.loaded[name] = {
+            "spec": plugins[name]["module_spec"],
+            "parent_dir": plugins[name]["parent_dir"],
+            "module": module,
+            "type": "pipeline",
+        }
+        self.logger.debug(
+            "Loaded pipeline plugin %s from %s -> %s",
+            name,
+            plugins[name]["parent_dir"],
+            name,
+        )
+
+    def _load_project_plugin(self, name: str) -> None:
+        if name in self.loaded:
+            self.logger.warning("Project plugin %s already loaded", name)
+            return
 
+        plugins = self.available_plugins()
 
-def storage_sanity_checks(module) -> None:
+        # The parent directory of the plugin must be on sys.path so it can be
+        # imported, so we put in on there if it isn't.
+        new = str(plugins[name]["parent_dir"])
+        if new not in sys.path:
+            sys.path = [new] + sys.path[0:]
+            self.logger.debug("Updated sys.path with %s", [new])
+
+        self.loaded[name] = {
+            "spec": plugins[name]["module_spec"],
+            "parent_dir": plugins[name]["parent_dir"],
+            "type": "project",
+        }
+
+        self.logger.debug(
+            ("Loaded project plugin %s from %s -> %s"),
+            name,
+            plugins[name]["parent_dir"],
+            name,
+        )
+
+    def _load_model_plugin(self, name: str) -> None:
+        if name in self.loaded:
+            self.logger.warning("Model plugin %s already loaded", name)
+            return
+
+        plugins = self.available_plugins()
+
+        # The parent directory of the plugin must be on sys.path so it can be
+        # imported, so we put in on there if it isn't.
+        new = str(plugins[name]["parent_dir"])
+        if new not in sys.path:
+            sys.path = [new] + sys.path[0:]
+            self.logger.debug("Updated sys.path with %s", [new])
+
+        self.loaded[name] = {
+            "spec": plugins[name]["module_spec"],
+            "parent_dir": plugins[name]["parent_dir"],
+            "type": "model",
+        }
+
+        self.logger.debug(
+            ("Loaded model plugin %s from %s -> %s"),
+            name,
+            plugins[name]["parent_dir"],
+            name,
+        )
+
+
+class DirectoryPluginManager(BasePluginManager):
+    """Container for managing directory-based plugins."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.plugins = {}  # type: tp.Dict[str, tp.Dict]
+
+    def initialize(self, project: str, search_path: tp.List[pathlib.Path]) -> None:
+        self.logger.debug(
+            "Initializing with plugin search path %s", [str(p) for p in search_path]
+        )
+
+        for path in search_path:
+            if not path.exists():
+                self.logger.warning(
+                    "Non-existent path '%s' on SIERRA_PLUGIN_PATH", path
+                )
+                continue
+
+            self.logger.debug("Searching for plugins in '%s'", path)
+
+            def recursive_search(root: pathlib.Path) -> None:
+                try:
+                    for f in root.iterdir():
+                        if not f.is_dir():
+                            continue
+                        recursive_search(f)
+
+                        plugin = f / "plugin.py"
+                        init = f / "__init__.py"
+                        cookie = f / ".sierraplugin"
+
+                        if not (init.exists() and (plugin.exists() or cookie.exists())):
+                            continue
+
+                        name = f"{f.parent.name}.{f.name}"
+
+                        if plugin.exists():
+                            module_spec = importlib.util.spec_from_file_location(
+                                f.name, plugin
+                            )
+                        else:
+                            module_spec = None
+
+                        init_spec = importlib.util.spec_from_file_location(
+                            "__init__", init
+                        )
+
+                        self.logger.debug("Found plugin in '%s' -> %s", f, name)
+
+                        self.plugins[name] = {
+                            "parent_dir": root.parent,
+                            "module_spec": module_spec,
+                            "init_spec": init_spec,
+                        }
+                except FileNotFoundError:
+                    self.logger.warning(
+                        "Malformed plugin in %s: not loading", f.relative_to(path)
+                    )
+
+            recursive_search(path)
+
+    def available_plugins(self):
+        return self.plugins
+
+
+def module_exists(name: str) -> bool:
     """
-    Check the selected ``--storage-medium`` plugin.
+    Check if a module exists before trying to import it.
     """
-    logging.trace("Verifying --storage-medium plugin interface")  # type: ignore
+    try:
+        _ = __import__(name)
+    except ImportError:
+        return False
+    else:
+        return True
+
+
+def module_load(name: str) -> types.ModuleType:
+    """
+    Import the specified module.
+    """
+    return __import__(name, fromlist=["*"])
+
 
-    functions = ['df_read',
-                 'df_write']
+def bc_load(cmdopts: types.Cmdopts, category: str):
+    """
+    Load the specified :term:`Batch Criteria`.
+    """
+    path = f"variables.{category}"
+    return module_load_tiered(
+        project=cmdopts["project"], engine=cmdopts["engine"], path=path
+    )
+
+
+def module_load_tiered(
+    path: str, project: tp.Optional[str] = None, engine: tp.Optional[str] = None
+) -> types.ModuleType:
+    """Attempt to load the specified python module with tiered precedence.
+
+    Generally, the precedence is project -> project submodule -> engine module
+    -> SIERRA core module, to allow users to override SIERRA core functionality
+    with ease.  Specifically:
+
+        #. Check if the requested module directly exists.  If it does, return
+           it.
+
+        #. Check if the requested module is a part of a project (i.e.,
+           ``<project>.<path>`` exists).  If it does, return it.  This requires
+           that :envvar:`SIERRA_PLUGIN_PATH` to be set properly.
+
+        #. Check if the requested module is provided by the engine plugin (i.e.,
+           ``sierra.engine.<engine>.<path>`` exists).  If it does, return it.
+
+        #. Check if the requested module is part of the SIERRA core (i.e.,
+           ``sierra.core.<path>`` exists).  If it does, return it.
+
+    If no match was found using any of these, throw an error.
+    """
+    # First, see if the requested module is a project/directly exists as
+    # specified.
+    if module_exists(path):
+        logging.trace("Using direct path %s", path)  # type: ignore
+        return module_load(path)
+
+    # Next, check if the requested module is part of the project plugin
+    if project is not None:
+        component_path = f"{project}.{path}"
+        if module_exists(component_path):
+            logging.trace(
+                "Using project component path %s", component_path  # type: ignore
+            )
+            return module_load(component_path)
+        else:
+            logging.trace(
+                "Project component path %s does not exist",  # type: ignore
+                component_path,
+            )
+
+    # If that didn't work, check the engine plugin
+    if engine is not None:
+        engine_path = f"{engine}.{path}"
+        if module_exists(engine_path):
+            logging.trace("Using engine component path %s", engine_path)  # type: ignore
+            return module_load(engine_path)
+
+    # If that didn't work, then check the SIERRA core
+    core_path = f"sierra.core.{path}"
+    if module_exists(core_path):
+        logging.trace("Using SIERRA core path %s", core_path)  # type: ignore
+        return module_load(core_path)
+    else:
+        logging.trace("SIERRA core path %s does not exist", core_path)  # type: ignore
+
+    # Module does not exist
+    error = (
+        f"project: '{project}' "
+        f"engine: '{engine}' "
+        f"path: '{path}' "
+        f"sys.path: {sys.path}"
+    )
+    raise ImportError(error)
+
+
+def storage_sanity_checks(medium: str, module) -> None:
+    """
+    Check the selected ``--storage`` plugin.
+    """
+    logging.trace("Verifying --storage=%s plugin interface", medium)  # type: ignore
+
+    functions = ["df_read", "df_write", "suffixes"]
     in_module = inspect.getmembers(module, inspect.isfunction)
 
     for f in functions:
-        assert (any(f in name for (name, _) in in_module)),\
-            f"Storage medium plugin does not define {f}"
+        assert any(
+            f == name for (name, _) in in_module
+        ), f"Storage medium {medium} does not define {f}()"
 
 
-def exec_env_sanity_checks(module) -> None:
+def expdef_sanity_checks(expdef: str, module) -> None:
     """
-    Check the selected ``--exec-env`` plugin.
+    Check the selected ``--expdef`` plugin.
     """
-    logging.trace("Verifying --exec-env plugin interface")  # type: ignore
+    logging.trace("Verifying --expdef=%s plugin interface", expdef)  # type: ignore
+
+    functions = ["root_querypath", "unpickle"]
+    module_funcs = inspect.getmembers(module, inspect.isfunction)
+    module_classes = inspect.getmembers(module, inspect.isclass)
+    classes = ["ExpDef", "Writer"]
+
+    for c in classes:
+        assert any(
+            c == name for (name, _) in module_classes
+        ), f"Expdef plugin {expdef} does not define {c}"
+
+    for f in functions:
+        assert any(
+            f == name for (name, _) in module_funcs
+        ), f"Expdef  {expdef} does not define {f}()"
+
+
+def proc_sanity_checks(proc: str, module) -> None:
+    """
+    Check the selected ``--proc`` plugins.
+    """
+    logging.trace("Verifying --proc=%s plugin interface", proc)  # type: ignore
+
+    functions = ["proc_batch_exp"]
+    in_module = inspect.getmembers(module, inspect.isfunction)
+
+    for f in functions:
+        assert any(
+            f == name for (name, _) in in_module
+        ), f"Processing plugin  {proc} does not define {f}()"
+
+
+def prod_sanity_checks(prod: str, module) -> None:
+    """
+    Check the selected ``--prod`` plugins.
+    """
+    logging.trace("Verifying --prod=%s plugin interface", prod)  # type: ignore
+
+    functions = ["proc_batch_exp"]
+    in_module = inspect.getmembers(module, inspect.isfunction)
+
+    for f in functions:
+        assert any(
+            f == name for (name, _) in in_module
+        ), f"Product plugin  {prod} does not define {f}()"
+
+
+def compare_sanity_checks(compare: str, module) -> None:
+    """
+    Check the selected ``--compare`` plugins.
+    """
+    logging.trace("Verifying --compare=%s plugin interface", compare)  # type: ignore
+
+    functions = ["proc_exps"]
+    in_module = inspect.getmembers(module, inspect.isfunction)
+
+    for f in functions:
+        assert any(
+            f == name for (name, _) in in_module
+        ), f"Comparison plugin  {compare} does not define {f}()"
+
+
+def execenv_sanity_checks(execenv: str, module) -> None:
+    """
+    Check the selected ``--execenv`` plugin.
+    """
+    logging.trace("Verifying --execenv=%s plugin interface", execenv)  # type: ignore
 
     in_module = inspect.getmembers(module, inspect.isclass)
 
-    opt_classes = ['ParsedCmdlineConfigurer',
-                   'ExpRunShellCmdsGenerator',
-                   'ExpShellCmdsGenerator',
-                   'ExecEnvChecker']
+    opt_functions = ["cmdline_postparse_configure", "execenv_check"]
+    opt_classes = ["ExpRunShellCmdsGenerator", "ExpShellCmdsGenerator"]
 
     for c in opt_classes:
-        if not any(c in name for (name, _) in in_module):
-            logging.debug(("Execution environment plugin '%s' does not define "
-                           "'%s'"),
-                          module.__name__,
-                          c)
+        if not any(c == name for (name, _) in in_module):
+            logging.debug(
+                (
+                    "Execution environment plugin %s does not define "
+                    "%s--some SIERRA functionality may not be "
+                    "available. See docs for details."
+                ),
+                execenv,
+                c,
+            )
 
+    for f in opt_functions:
+        if not any(f in name for (name, _) in in_module):
+            logging.debug(
+                ("Execution environment plugin %s does not define %s()."),
+                execenv,
+                f,
+            )
 
-def platform_sanity_checks(module) -> None:
+
+def engine_sanity_checks(engine: str, module) -> None:
     """
-    Check the selected ``--platform`` plugin.
+    Check the selected ``--engine`` plugin.
     """
-    logging.trace("Verifying --platform plugin interface")  # type: ignore
+    logging.trace("Verifying --engine=%s plugin interface", engine)  # type: ignore
 
-    req_classes = ['ExpConfigurer',
-                   'CmdlineParserGenerator'
-                   ]
+    req_classes = [
+        "ExpConfigurer",
+    ]
 
-    req_functions = ['population_size_from_def',
-                     'population_size_from_pickle',
-                     ]
+    req_functions = [
+        "population_size_from_def",
+        "population_size_from_pickle",
+    ]
 
-    opt_classes = ['ParsedCmdlineConfigurer',
-                   'ExpRunShellCmdsGenerator',
-                   'ExpShellCmdsGenerator',
-                   'ExecEnvChecker']
+    opt_classes = ["ExpRunShellCmdsGenerator", "ExpShellCmdsGenerator"]
 
-    opt_functions = ['robot_prefix_extract',
-                     'arena_dims_from_criteria']
+    opt_functions = [
+        "cmdline_postparse_configure",
+        "execenv_check",
+        "agent_prefix_extract",
+        "arena_dims_from_criteria",
+    ]
 
     in_module = inspect.getmembers(module, inspect.isclass)
 
     for c in req_classes:
-        assert (any(c in name for (name, _) in in_module)),\
-            f"Platform plugin '{module.__name__}' does not define '{c}'"
+        assert any(
+            c == name for (name, _) in in_module
+        ), f"Engine plugin {engine} does not define {c}"
 
     for f in opt_classes:
         if not any(f in name for (name, _) in in_module):
-            logging.debug(("Platform plugin '%s' not define define '%s'"
-                           "--some SIERRA functionality may not be available. "
-                           "See docs for details."),
-                          module.__name__,
-                          f)
+            logging.debug(
+                (
+                    "Engine plugin %s does not define %s"
+                    "--some SIERRA functionality may not be available. "
+                    "See docs for details."
+                ),
+                engine,
+                f,
+            )
 
     in_module = inspect.getmembers(module, inspect.isfunction)
 
     for f in req_functions:
-        assert (any(f in name for (name, _) in in_module)),\
-            f"Platform plugin '{module.__name__}' does not define '{f}()'"
+        assert any(
+            f == name for (name, _) in in_module
+        ), f"Engine plugin {engine} does not define {f}()"
 
     for f in opt_functions:
-        if not any(f in name for (name, _) in in_module):
-            logging.debug(("Platform plugin '%s' not define define '%s()'"
-                           "--some SIERRA functionality may not be available. "
-                           "See docs for details."),
-                          module.__name__,
-                          f)
-
-
-__api__ = {
-    'storage_sanity_checks',
-    'exec_env_sanity_checks',
-    'platform_sanity_checks'
-}
+        if not any(f == name for (name, _) in in_module):
+            logging.debug(
+                (
+                    "Engine plugin %s does not define %s()"
+                    "--some SIERRA functionality may not be available. "
+                    "See docs for details."
+                ),
+                engine,
+                f,
+            )
+
+
+# Singletons
+pipeline = DirectoryPluginManager()
+models = DirectoryPluginManager()
+
+__all__ = [
+    "DirectoryPluginManager",
+    "module_exists",
+    "module_load",
+    "bc_load",
+    "module_load_tiered",
+    "storage_sanity_checks",
+    "execenv_sanity_checks",
+    "engine_sanity_checks",
+    "proc_sanity_checks",
+    "prod_sanity_checks",
+    "compare_sanity_checks",
+]

sierra/core/proc.py

@@ -0,0 +1,11 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""Trampoline bindings for the various ``--proc`` plugins that come with SIERRA."""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+from sierra.core.trampoline import cmdline_parser  # noqa: F401

sierra/core/prod.py

@@ -0,0 +1,11 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""Trampoline bindings for the various ``--prod`` plugins that come with SIERRA."""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+from sierra.core.trampoline import cmdline_parser  # noqa: F401

sierra/core/ros1/__init__.py

@@ -1,10 +1,14 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
+"""Container module for things related to :term:`ROS1`."""
 
 # Core packages
 
 # 3rd party packages
 
 # Project packages
-from . import cmdline, callbacks, variables, generators
+from . import cmdline as cmdline
+from . import callbacks as callbacks
+from . import variables as variables
+from . import generators as generators