sierra-research 1.3.11__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/core/plugin_manager.py

@@ -1,369 +0,0 @@
-# Copyright 2020 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-"""Simple plugin managers to make SIERRA OPEN/CLOSED.
-
-"""
-# Core packages
-import importlib.util
-import importlib
-import os
-import typing as tp
-import sys
-import logging
-import pathlib
-
-# 3rd party packages
-import json
-
-# Project packages
-from sierra.core import types, utils
-
-
-class BasePluginManager():
-    """
-    Base class for common functionality.
-    """
-
-    def __init__(self) -> None:
-        self.logger = logging.getLogger(__name__)
-        self.loaded = {}  # type: tp.Dict[str, tp.Dict]
-
-    def available_plugins(self):
-        raise NotImplementedError
-
-    def load_plugin(self, name: str) -> None:
-        """Load a plugin module.
-
-        """
-        plugins = self.available_plugins()
-        if name not in plugins:
-            self.logger.fatal("Cannot locate plugin '%s'", name)
-            self.logger.fatal('Loaded plugins: %s\n',
-                              json.dumps(self.loaded,
-                                         default=lambda x: '<ModuleSpec>',
-                                         indent=4))
-            raise Exception(f"Cannot locate plugin '{name}'")
-
-        if plugins[name]['type'] == 'pipeline':
-            parent_scope = pathlib.Path(plugins[name]['parent_dir']).name
-            scoped_name = f'{parent_scope}.{name}'
-
-            if name not in self.loaded:
-                module = importlib.util.module_from_spec(plugins[name]['spec'])
-
-                plugins[name]['spec'].loader.exec_module(module)
-                self.loaded[scoped_name] = {
-                    'spec': plugins[name]['spec'],
-                    'parent_dir': plugins[name]['parent_dir'],
-                    'module': module
-                }
-                self.logger.debug("Loaded pipeline plugin '%s' from '%s'",
-                                  scoped_name,
-                                  plugins[name]['parent_dir'])
-            else:
-                self.logger.warning("Pipeline plugin '%s' already loaded", name)
-        elif plugins[name]['type'] == 'project':
-            # Projects are addressed directly without scoping. Only one project
-            # is loaded at a time, so this should be fine.
-            scoped_name = name
-            if name not in self.loaded:
-                self.loaded[scoped_name] = {
-                    'spec': plugins[name]['spec'],
-                    'parent_dir': plugins[name]['parent_dir'],
-                }
-                self.logger.debug("Loaded project plugin '%s' from '%s'",
-                                  scoped_name,
-                                  plugins[name]['parent_dir'])
-            else:
-                self.logger.warning("Project plugin '%s' already loaded", name)
-
-    def get_plugin(self, name: str) -> dict:
-        try:
-            return self.loaded[name]
-        except KeyError:
-            self.logger.fatal("No such plugin '%s'", name)
-            self.logger.fatal('Loaded plugins: %s\n',
-                              json.dumps(self.loaded,
-                                         default=lambda x: '<ModuleSpec>',
-                                         indent=4))
-            raise
-
-    def get_plugin_module(self, name: str) -> types.ModuleType:
-        try:
-            return self.loaded[name]['module']
-        except KeyError:
-            self.logger.fatal("No such plugin '%s'", name)
-            self.logger.fatal('Loaded plugins: %s\n',
-                              json.dumps(self.loaded,
-                                         default=lambda x: '<ModuleSpec>',
-                                         indent=4))
-            raise
-
-    def has_plugin(self, name: str) -> bool:
-        return name in self.loaded
-
-
-class FilePluginManager(BasePluginManager):
-    """Plugins are ``.py`` files within a root plugin directory.
-
-    Intended for use with :term:`models <Model>`.
-
-    """
-
-    def __init__(self) -> None:
-        super().__init__()
-        self.search_root = None  # type: tp.Optional[pathlib.Path]
-
-    def initialize(self, project: str, search_root: pathlib.Path) -> None:
-        self.search_root = search_root
-
-    def available_plugins(self) -> tp.Dict[str, tp.Dict]:
-        """Get the available plugins in the configured plugin root.
-
-        """
-        plugins = {}
-        assert self.search_root is not None, \
-            "FilePluginManager not initialized!"
-
-        for candidate in self.search_root.iterdir():
-            if candidate.is_file() and '.py' in candidate.name:
-                name = candidate.stem
-                spec = importlib.util.spec_from_file_location(name, candidate)
-                plugins[name] = {
-                    'spec': spec,
-                    'parent_dir': self.search_root,
-                    'type': 'pipeline'
-                }
-        return plugins
-
-
-class DirectoryPluginManager(BasePluginManager):
-    """Plugins are `directories` found in a root plugin directory.
-
-    Intended for use with :term:`Pipeline plugins <plugin>`.
-
-    """
-
-    def __init__(self, search_root: pathlib.Path) -> None:
-        super().__init__()
-        self.search_root = search_root
-        self.main_module = 'plugin'
-
-    def initialize(self, project: str) -> None:
-        pass
-
-    def available_plugins(self):
-        """
-        Find all pipeline plugins in all directories within the search root.
-        """
-        plugins = {}
-        try:
-            for location in self.search_root.iterdir():
-                plugin = location / (self.main_module + '.py')
-
-                if location.is_dir() and plugin in location.iterdir():
-                    spec = importlib.util.spec_from_file_location(location.name,
-                                                                  plugin)
-                    plugins[location.name] = {
-                        'parent_dir': self.search_root,
-                        'spec': spec,
-                        'type': 'pipeline'
-                    }
-        except FileNotFoundError:
-            pass
-
-        return plugins
-
-
-class ProjectPluginManager(BasePluginManager):
-    """Plugins are `directories` found in a root plugin directory.
-
-    Intended for use with :term:`Project plugins <plugin>`.
-
-    """
-
-    def __init__(self, search_root: pathlib.Path, project: str) -> None:
-        super().__init__()
-
-        self.search_root = search_root
-        self.project = project
-
-    def initialize(self, project: str) -> None:
-        # Update PYTHONPATH with the directory containing the project so imports
-        # of the form 'import project.module' work.
-        #
-        # 2021/07/19: If you put the entries at the end of sys.path it
-        # doesn't work for some reason...
-        sys.path = [str(self.search_root)] + sys.path[0:]
-
-    def available_plugins(self):
-        """
-        Find all pipeline plugins in all directories within the search root.
-        """
-        plugins = {}
-        try:
-            for location in self.search_root.iterdir():
-                if self.project in location.name:
-                    plugins[location.name] = {
-                        'parent_dir': self.search_root,
-                        'spec': None,
-                        'type': 'project'
-                    }
-
-        except FileNotFoundError:
-            pass
-
-        return plugins
-
-
-class CompositePluginManager(BasePluginManager):
-    def __init__(self) -> None:
-        super().__init__()
-        self.components = []  # type: tp.List[tp.Union[DirectoryPluginManager,ProjectPluginManager]]
-
-    def initialize(self,
-                   project: str,
-                   search_path: tp.List[pathlib.Path]) -> None:
-        self.logger.debug("Initializing with plugin search path %s",
-                          [str(p) for p in search_path])
-        for d in search_path:
-            project_path = d / project
-
-            if utils.path_exists(project_path):
-                project_plugin = ProjectPluginManager(d, project)
-                self.components.append(project_plugin)
-            else:
-                pipeline_plugin = DirectoryPluginManager(d)
-                self.components.append(pipeline_plugin)
-
-        for c in self.components:
-            c.initialize(project)
-
-    def available_plugins(self):
-        plugins = {}
-
-        for c in self.components:
-            plugins.update(c.available_plugins())
-
-        return plugins
-
-
-# Singletons
-pipeline = CompositePluginManager()
-models = FilePluginManager()
-
-
-def module_exists(name: str) -> bool:
-    """
-    Check if a module exists before trying to import it.
-    """
-    try:
-        _ = __import__(name)
-    except ImportError:
-        return False
-    else:
-        return True
-
-
-def module_load(name: str) -> types.ModuleType:
-    """
-    Import the specified module.
-    """
-    return __import__(name, fromlist=["*"])
-
-
-def bc_load(cmdopts: types.Cmdopts, category: str):
-    """
-    Load the specified :term:`Batch Criteria`.
-    """
-    path = f'variables.{category}'
-    return module_load_tiered(project=cmdopts['project'],
-                              platform=cmdopts['platform'],
-                              path=path)
-
-
-def module_load_tiered(path: str,
-                       project: tp.Optional[str] = None,
-                       platform: tp.Optional[str] = None) -> types.ModuleType:
-    """Attempt to load the specified python module with tiered precedence.
-
-    Generally, the precedence is project -> project submodule -> platform module
-    -> SIERRA core module, to allow users to override SIERRA core functionality
-    with ease. Specifically:
-
-    #. Check if the requested module is a project. If it is, return it.
-
-    #. Check if the requested module is a part of a project (i.e.,
-       ``<project>.<path>`` exists). If it does, return it. This requires that
-       :envvar:`SIERRA_PLUGIN_PATH` to be set properly.
-
-    #. Check if the requested module is provided by the platform plugin (i.e.,
-       ``sierra.platform.<platform>.<path>`` exists). If it does, return it.
-
-    #. Check if the requested module is part of the SIERRA core (i.e.,
-       ``sierra.core.<path>`` exists). If it does, return it.
-
-    If no match was found using any of these, throw an error.
-
-    """
-    # First, see if the requested module is a project
-    if module_exists(path):
-        logging.trace("Using project path '%s'", path)  # type: ignore
-        return module_load(path)
-
-    # First, see if the requested module is part of the project plugin
-    if project is not None:
-        component_path = f'{project}.{path}'
-        if module_exists(component_path):
-            logging.trace("Using project component path '%s'",  # type: ignore
-                          component_path)
-            return module_load(component_path)
-        else:
-            logging.trace("Project component path '%s' does not exist",  # type: ignore
-                          component_path)
-
-    # If that didn't work, check the platform plugin
-    if platform is not None:
-        # We manually add 'sierra.plugins' here, rather than adding the
-        # necessary directory to PYTHONPATH so that we don't accidentally get
-        # the files from other non-platform plugins with the same name as the
-        # platform plugin file we are interested in getting picked.
-        platform_path = f'sierra.plugins.{platform}.{path}'
-        if module_exists(platform_path):
-            logging.trace("Using platform component path '%s'",  # type: ignore
-                          platform_path)
-            return module_load(platform_path)
-        else:
-            logging.trace("Platform component path '%s' does not exist",  # type: ignore
-                          platform_path)
-
-    # If that didn't work, then check the SIERRA core
-    core_path = f'sierra.core.{path}'
-    if module_exists(core_path):
-        logging.trace("Using SIERRA core path '%s'",  # type: ignore
-                      core_path)
-        return module_load(core_path)
-    else:
-        logging.trace("SIERRA core path '%s' does not exist",  # type: ignore
-                      core_path)
-
-    # Module does not exist
-    error = (f"project: '{project}' "
-             f"platform: '{platform}' "
-             f"path: '{path}' "
-             f"sys.path: {sys.path}")
-    raise ImportError(error)
-
-
-__api__ = [
-    'BasePluginManager',
-    'FilePluginManager',
-    'DirectoryPluginManager',
-    'ProjectPluginManager',
-    'CompositePluginManager',
-    'module_exists',
-    'module_load',
-    'bc_load',
-    'module_load_tiered'
-]

sierra/core/root_dirpath_generator.py

@@ -1,241 +0,0 @@
-# Copyright 2018 John Harwell, All rights reserved.
-#
-#  SPDX-License-Identifier: MIT
-
-"""Functions for generating root directory paths for a batch experiment.
-
-- The batch experiment root. ALL files (inputs and outputs) are written to this
-  directory, which will be under ``--sierra-root``. Named using a combination of
-  ``--scenario`` (block distribution + arena dimensions) and
-  ``--batch-criteria`` in order to guarantee uniqueness among batch roots
-  anytime the batch criteria or scenario change.
-
-- The batch input root. All input files will be generated under this root
-  directory. Named ``<batch experiment root>/exp-inputs``.
-
-- The batch output root. All output files will accrue under this root
-  directory. Each experiment will get their own directory in this root for its
-  outputs to accrue into. Named ``<batch experiment root>/exp-outputs``.
-
-- The batch graph root. All generated graphs will acrrue under this root
-  directory. Each experiment will get their own directory in this root for their
-  graphs to accrue into. Named ``<batch experiment root>/graphs``.
-
-- The batch model root. All model outputs will accrue under this root
-  directory. Each experiment will get their own directory in this root for their
-  model outputs to accrue into. Named ``<batch experiment root>/models``.
-
-- The batch statistics root. All statistics generated during stage 3 will accrue
-  under this root directory. Each experiment will get their own directory in
-  this root for their statistics. Named
-  ``<batch experiment root>/statistics``.
-
-- The batch imagizing root. All images generated during stage 3 will accrue
-  under this root directory. Each experiment will get their own directory in
-  this root for their images. Named
-  ``<batch experiment root>/images``.
-
-- The batch video root. All videos rendered during stage 4 will accrue
-  under this root directory. Each experiment will get their own directory in
-  this root for their videos. Named
-  ``<batch experiment root>/videos``.
-
-- The batch scratch root. All GNU parallel outputs, ``--exec-env`` artifacts
-  will appear under here. Each experiment will get their own directory in this
-  root for their own scratch. This root is separate from experiment inputs to
-  make checking for segfaults, tar-ing experiments, etc. easier. Named ``<batch
-  experiment root>/scratch``.
-
-"""
-# Core packages
-import os
-import typing as tp
-import logging
-import argparse
-import pathlib
-
-# 3rd party packages
-
-# Project packages
-
-
-def from_cmdline(args: argparse.Namespace) -> tp.Dict[str, pathlib.Path]:
-    """Generate directory paths directly from cmdline arguments.
-
-    """
-    template = pathlib.Path(args.template_input_file)
-
-    # Remove all '-' from the template input file stem so we know the only '-'
-    # that are in it are ones that we put there.
-    template_stem = template.stem.replace('-', '')
-
-    batch_leaf = gen_batch_leaf(args.batch_criteria,
-                                str(template_stem),
-                                args.scenario)
-
-    return regen_from_exp(args.sierra_root,
-                          args.project,
-                          batch_leaf,
-                          args.controller)
-
-
-def parse_batch_leaf(root: str) -> tp.Tuple[str, str, tp.List[str]]:
-    """Parse a batch root (dirpath leaf).
-
-    Parsed into (template input file basename, scenario, batch criteria list)
-    string components as they would have been specified on the cmdline.
-
-    """
-    template_stem = ''.join(root.split('-')[0])
-    scenario_and_bc = root.split('-')[1].split('+')
-    scenario = scenario_and_bc[0]
-    bc = scenario_and_bc[1:]  # type: tp.Union[tp.List[str],str]
-
-    if not isinstance(bc, list):  # Univariate batch criteria
-        bc = [bc]
-
-    return (template_stem, scenario, bc)
-
-
-def gen_batch_leaf(criteria: tp.List[str],
-                   template_stem: str,
-                   scenario: str) -> str:
-    leaf = template_stem + '-' + scenario + '+' + '+'.join(criteria)
-    logging.debug("Generated batch leaf %s", leaf)
-
-    return leaf
-
-
-def regen_from_exp(sierra_rpath: str,
-                   project: str,
-                   batch_leaf: str,
-                   controller: str) -> tp.Dict[str, pathlib.Path]:
-    """Regenerate directory paths from a previously created batch experiment.
-
-    Arguments:
-
-        sierra_rpath: The path to the root directory where SIERRA should store
-                      everything.
-
-        project: The name of the project plugin used.
-
-        criteria: List of strings from the cmdline specification of the batch
-                  criteria.
-
-        batch_root: The name of the directory that will be the root of the batch
-                    experiment (not including its parent).
-
-        controller: The name of the controller used.
-
-    """
-    template_stem, scenario, bc = parse_batch_leaf(batch_leaf)
-
-    root = gen_batch_root(sierra_rpath,
-                          project,
-                          bc,
-                          scenario,
-                          controller,
-                          template_stem)
-    logging.info('Generated batch root %s', root)
-    return {
-        'batch_root': root,
-        'batch_input_root': _gen_input_root(root),
-        'batch_output_root': _gen_output_root(root),
-        'batch_graph_root': _gen_graph_root(root),
-        'batch_model_root': _gen_model_root(root),
-        'batch_stat_root': _gen_statistics_root(root),
-        'batch_imagize_root': _gen_imagize_root(root),
-        'batch_video_root': _gen_video_root(root),
-        'batch_stat_collate_root': _gen_stat_collate_root(root),
-        'batch_graph_collate_root': _gen_graph_collate_root(root),
-        'batch_scratch_root': _gen_scratch_root(root)
-    }
-
-
-def gen_batch_root(root: str,
-                   project: str,
-                   criteria: tp.List[str],
-                   scenario: str,
-                   controller: str,
-                   template_stem: str) -> pathlib.Path:
-    """Generate the directory path for the batch root directory.
-
-    The directory path depends on all of the input arguments to this function,
-    and if ANY of the arguments change, so will the generated path.
-
-    Batch root is:
-    <sierra_root>/<project>/<template_basename>-<scenario>+<criteria0>+<criteria1>
-
-    Arguments:
-
-        root: The path to the root directory where SIERRA should store
-              everything.
-
-        project: The name of the project plugin used.
-
-        criteria: List of strings from the cmdline specification of the batch
-                  criteria.
-
-        scenario: The cmdline specification of ``--scenario``
-
-        batch_root: The name of the directory that will be the root of the batch
-                    experiment (not including its parent).
-
-        controller: The name of the controller used.
-
-    """
-    batch_leaf = gen_batch_leaf(criteria, template_stem, scenario)
-
-    # Don't reslove() the path--that makes symlinked dirs under $HOME through
-    # errors which are fatal from pathlib's POV, but actually harmless.
-    sierra_root = pathlib.Path(root)
-
-    return sierra_root / project / controller / batch_leaf
-
-
-def _gen_output_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "exp-outputs"
-
-
-def _gen_input_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "exp-inputs"
-
-
-def _gen_graph_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "graphs"
-
-
-def _gen_model_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "models"
-
-
-def _gen_statistics_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "statistics"
-
-
-def _gen_scratch_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "scratch"
-
-
-def _gen_imagize_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "imagize"
-
-
-def _gen_video_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "videos"
-
-
-def _gen_stat_collate_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "statistics" / "collated"
-
-
-def _gen_graph_collate_root(root: pathlib.Path) -> pathlib.Path:
-    return root / "graphs" / "collated"
-
-
-__api__ = [
-    'from_cmdline',
-    'regen_from_exp',
-    'parse_batch_leaf',
-    'gen_batch_root'
-]