sierra-research 1.3.11__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/prod/graphs/intra/generate.py

@@ -0,0 +1,202 @@
+# Copyright 2018 London Lowmanstone, John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+#
+"""
+Generate graphs within a single :term:`Experiment`.
+"""
+
+# Core packages
+import os
+import copy
+import typing as tp
+import logging
+import pathlib
+
+# 3rd party packages
+import yaml
+
+# Project packages
+
+import sierra.core.plugin as pm
+from sierra.core import types, utils, batchroot, exproot, config
+from sierra.plugins.prod.graphs import intra
+from sierra.core.variables import batch_criteria as bc
+
+
+_logger = logging.getLogger(__name__)
+
+
+def proc_batch_exp(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """
+    Generate intra-experiment graphs for a :term:`Batch Experiment`.
+
+    Arguments:
+        main_config: Parsed dictionary of main YAML configuration
+
+
+        criteria:  The :term:`Batch Criteria` used for the batch
+                   experiment.
+    """
+    info = criteria.graph_info(cmdopts, batch_output_root=pathset.output_root)
+    exp_to_gen = utils.exp_range_calc(
+        cmdopts["exp_range"], pathset.output_root, info.exp_names
+    )
+
+    if not exp_to_gen:
+        return
+
+    loader = pm.module_load_tiered(project=cmdopts["project"], path="pipeline.yaml")
+
+    graphs_config = loader.load_config(cmdopts, config.kYAML.graphs)
+
+    if "intra-exp" not in graphs_config:
+        _logger.warning(
+            "Cannot generate graphs: 'intra-exp' key not found in YAML config"
+        )
+        return
+
+    project_config_root = pathlib.Path(cmdopts["project_config_root"])
+    controllers_yaml = project_config_root / config.kYAML.controllers
+
+    if controllers_yaml.exists():
+        with utils.utf8open(controllers_yaml) as f:
+            controller_config = yaml.load(f, yaml.FullLoader)
+    else:
+        controller_config = None
+
+    generator = IntraExpGraphGenerator(
+        main_config, controller_config, graphs_config["intra-exp"], cmdopts
+    )
+    for exp in exp_to_gen:
+        exproots = exproot.PathSet(pathset, exp.name)
+
+        if os.path.isdir(exproots.stat_root):
+            generator(exproots)
+        else:
+            _logger.warning(
+                "Skipping experiment '%s': %s does not exist, or isn't a directory",
+                exp,
+                exproots.stat_root,
+            )
+
+
+class IntraExpGraphGenerator:
+    """Generates graphs from :term:`Processed Output Data` files.
+
+    Which graphs are generated is controlled by YAML configuration files parsed
+    in stage 4.
+
+    Attributes:
+        cmdopts: Dictionary of parsed cmdline attributes.
+
+        main_config: Parsed dictionary of main YAML configuration
+
+        controller_config: Parsed dictionary of controller YAML
+                           configuration.
+
+        graphs_config: Parsed dictionary of intra-experiment graph
+                       configuration.
+
+        logger: The handle to the logger for this class. If you extend this
+               class, you should save/restore this variable in tandem with
+               overriding it in order to get logging messages have unique logger
+               names between this class and your derived class, in order to
+               reduce confusion.
+
+    """
+
+    def __init__(
+        self,
+        main_config: types.YAMLDict,
+        controller_config: tp.Optional[types.YAMLDict],
+        graphs_config: types.YAMLDict,
+        cmdopts: types.Cmdopts,
+    ) -> None:
+        # Copy because we are modifying it and don't want to mess up the
+        # arguments for graphs that are generated after us
+        self.cmdopts = copy.deepcopy(cmdopts)
+        self.main_config = main_config
+        self.graphs_config = graphs_config
+        self.controller_config = controller_config
+        self.logger = logging.getLogger(__name__)
+
+    def __call__(self, pathset: exproot.PathSet) -> None:
+        """
+        Generate graphs.
+
+        Performs the following steps:
+
+        #. :func:`~sierra.plugins.prod.graphs.intra.line.generate()`
+            to generate linegraphs for each experiment in the batch.
+
+        #. :func:`~sierra.plugins.prod.graphs.intra.heatmap.generate()`
+            to generate heatmaps for each experiment in the batch.
+        """
+        utils.dir_create_checked(pathset.graph_root, exist_ok=True)
+
+        LN_targets, HM_targets = self._calc_targets()
+
+        if not self.cmdopts["project_no_LN"]:
+            intra.line.generate(self.cmdopts, pathset, LN_targets)
+
+        if not self.cmdopts["project_no_HM"]:
+            intra.heatmap.generate(self.cmdopts, pathset, HM_targets)
+
+    def _calc_targets(
+        self,
+    ) -> tp.Tuple[tp.List[types.YAMLDict], tp.List[types.YAMLDict]]:
+        """Calculate what intra-experiment graphs should be generated.
+
+        Uses YAML configuration for controller and intra-experiment graphs.
+        Returns a tuple of dictionaries: (intra-experiment linegraphs,
+        intra-experiment heatmaps) defined what graphs to generate.  The enabled
+        graphs exist in their respective YAML configuration *and* are enabled by
+        the YAML configuration for the selected controller.
+        """
+        keys = []
+        if self.controller_config:
+            for category in list(self.controller_config.keys()):
+                if category not in self.cmdopts["controller"]:
+                    continue
+                for controller in self.controller_config[category]["controllers"]:
+                    if controller["name"] not in self.cmdopts["controller"]:
+                        continue
+
+                    # valid to specify no graphs, and only to inherit graphs
+                    keys = controller.get("graphs", [])
+                    if "graphs_inherit" in controller:
+                        for inherit in controller["graphs_inherit"]:
+                            keys.extend(inherit)  # optional
+
+        else:
+            keys = list(k for k in self.graphs_config)
+            self.logger.warning(
+                "Missing controller graph config--generating all enabled "
+                "intra-experiment graphs for all controllers: %s",
+                keys,
+            )
+
+        # Get keys for enabled graphs
+        LN_keys = [k for k in self.graphs_config if k in keys]
+        self.logger.debug("Enabled linegraph categories: %s", LN_keys)
+
+        HM_keys = [k for k in self.graphs_config if k in keys]
+        self.logger.debug("Enabled heatmap categories: %s", HM_keys)
+
+        # Strip out all configured graphs which are not enabled
+        LN_targets = [self.graphs_config[k] for k in LN_keys]
+        HM_targets = [self.graphs_config[k] for k in HM_keys]
+
+        return LN_targets, HM_targets
+
+
+__all__ = [
+    "proc_batch_exp",
+    "IntraExpGraphGenerator",
+]

sierra/plugins/prod/graphs/intra/heatmap.py

@@ -0,0 +1,74 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Generate heatmaps within a single :term:`Experiment`."""
+
+# Core packages
+import typing as tp
+import logging
+
+# 3rd party packages
+import json
+
+# Project packages
+from sierra.core import types, exproot, graphs
+
+_logger = logging.getLogger(__name__)
+
+
+def generate(
+    cmdopts: types.Cmdopts,
+    pathset: exproot.PathSet,
+    targets: tp.List[types.YAMLDict],
+) -> None:
+    """
+    Generate heatmaps from: term:`Processed Output Data` files.
+    """
+    large_text = cmdopts["plot_large_text"]
+
+    _logger.info(
+        "Heatmaps from <batch_root>/%s", pathset.stat_root.relative_to(pathset.parent)
+    )
+
+    # For each category of heatmaps we are generating
+    for category in targets:
+
+        # For each graph in each category
+        for graph in category:
+            # Only try to create heatmaps (duh)
+            if graph["type"] != "heatmap":
+                continue
+
+            _logger.trace("\n" + json.dumps(graph, indent=4))  # type: ignore
+
+            graph_pathset = graphs.PathSet(
+                input_root=pathset.stat_root,
+                output_root=pathset.graph_root,
+                batchroot=pathset.parent.parent,
+                model_root=None,
+            )
+            # 2025-06-05 [JRH]: We always write stage {3,4} output data files as
+            # .csv because that is currently SIERRA's 'native' format; this may
+            # change in the future.
+            graphs.heatmap(
+                pathset=graph_pathset,
+                input_stem=graph["src_stem"],
+                output_stem=graph["dest_stem"],
+                medium="storage.csv",
+                title=graph.get("title", None),
+                xlabel=graph.get("xlabel", None),
+                ylabel=graph.get("ylabel", None),
+                zlabel=graph.get("zlabel", None),
+                backend=graph.get("backend", cmdopts["graphs_backend"]),
+                colnames=(
+                    graph.get("x", "x"),
+                    graph.get("y", "y"),
+                    graph.get("z", "z"),
+                ),
+                large_text=large_text,
+            )
+
+
+__all__ = ["generate"]

sierra/plugins/prod/graphs/intra/line.py

@@ -0,0 +1,114 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Generate linegraphs within a single :term:`Experiment`."""
+
+# Core packages
+import typing as tp
+import logging
+
+# 3rd party packages
+import json
+import numpy as np
+
+# Project packages
+from sierra.core import types, exproot, graphs, config
+from sierra.core import plugin as pm
+
+_logger = logging.getLogger(__name__)
+
+
+def generate(
+    cmdopts: types.Cmdopts, pathset: exproot.PathSet, targets: tp.List[types.YAMLDict]
+) -> None:
+    """
+    Generate linegraphs from: term:`Processed Output Data` files.
+    """
+
+    _logger.info(
+        "Linegraphs from <batch_root>/%s", pathset.stat_root.relative_to(pathset.parent)
+    )
+
+    # For each category of linegraphs we are generating
+    for category in targets:
+        # For each graph in each category
+        for graph in category:
+
+            # Only try to create linegraphs (duh)
+            if graph["type"] != "stacked_line":
+                continue
+
+            _logger.trace("\n" + json.dumps(graph, indent=4))  # type: ignore
+
+            paths = graphs.PathSet(
+                input_root=pathset.stat_root,
+                output_root=pathset.graph_root,
+                batchroot=pathset.parent.parent,
+                model_root=pathset.model_root,
+            )
+
+            try:
+                # 2025-06-05 [JRH]: We always write stage {3,4} output data
+                # files as .csv because that is currently SIERRA's 'native'
+                # format; this may change in the future.
+                module = pm.pipeline.get_plugin_module(cmdopts["engine"])
+                if hasattr(module, "expsetup_from_def"):
+                    module2 = pm.pipeline.get_plugin_module(cmdopts["expdef"])
+                    pkl_def = module2.unpickle(pathset.input_root / config.kPickleLeaf)
+
+                    info = module.expsetup_from_def(pkl_def)
+                    xticks = np.linspace(
+                        0,
+                        info["duration"],
+                        int(
+                            info["duration"]
+                            * info["n_ticks_per_sec"]
+                            * cmdopts["exp_n_datapoints_factor"]
+                        ),
+                    )
+                else:
+                    xticks = None
+
+                graphs.stacked_line(
+                    paths=paths,
+                    input_stem=graph["src_stem"],
+                    output_stem=graph["dest_stem"],
+                    medium="storage.csv",
+                    backend=graph.get("backend", cmdopts["graphs_backend"]),
+                    xticks=xticks,
+                    stats=cmdopts["dist_stats"],
+                    cols=graph.get("cols", None),
+                    title=graph.get("title", ""),
+                    legend=graph.get("legend", graph.get("cols", None)),
+                    xlabel=graph.get("xlabel", ""),
+                    ylabel=graph.get("ylabel", ""),
+                    points=graph.get("points", False),
+                    logyscale=graph.get("logy", cmdopts["plot_log_yscale"]),
+                    large_text=cmdopts["plot_large_text"],
+                )
+            except KeyError:
+                _logger.fatal(
+                    ("Could not generate linegraph. Possible reasons include: ")
+                )
+
+                _logger.fatal(
+                    ("1. The YAML configuration entry is missing required fields")
+                )
+                missing_cols = graph.get("cols", "MISSING_KEY")
+                missing_stem = graph.get("src_stem", "MISSING_KEY")
+                _logger.fatal(
+                    (
+                        "2. 'cols' is present in YAML "
+                        "configuration but some of %s are "
+                        "missing from %s"
+                    ),
+                    missing_cols,
+                    missing_stem,
+                )
+
+                raise
+
+
+__all__ = ["generate"]

sierra/plugins/prod/graphs/plugin.py

@@ -0,0 +1,103 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import logging
+import pathlib
+import time
+import datetime
+
+# 3rd party packages
+import yaml
+
+# Project packages
+import sierra.core.variables.batch_criteria as bc
+from sierra.core import types, config, utils, batchroot
+
+from sierra.plugins.prod.graphs import inter, intra, collate
+
+_logger = logging.getLogger(__name__)
+
+
+def proc_batch_exp(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """
+    Generate graphs from the :term:`Batch Experiment`.
+
+    Intra-experiment graph generation: if intra-experiment graphs should be
+    generated, according to configuration, the following is run:
+
+        #. :py:func:`~sierra.plugins.prod.graphs.intra.generate` to
+           generate graphs for each experiment in the batch, or a subset.
+
+    Inter-experiment graph generation: if inter-experiment graphs should be
+    generated according to cmdline configuration, the following is run:
+
+        #. :class:`~sierra.plugins.prod.graphs.collate.GraphCollator`.
+
+        #. :py:func:`~sierra.plugins.prod.graphs.inter.generate` to perform
+           graph generation from collated CSV files.
+    """
+    graphs_path = pathlib.Path(cmdopts["project_config_root"]) / pathlib.Path(
+        config.kYAML.graphs
+    )
+    if utils.path_exists(graphs_path):
+        _logger.info("Loading graphs config for project=%s", cmdopts["project"])
+        graphs_config = yaml.load(utils.utf8open(graphs_path), yaml.FullLoader)
+    else:
+        _logger.warning("%s does not exist--cannot generate graphs", graphs_path)
+        return
+
+    if (
+        (cmdopts["exp_graphs"] == "all" or cmdopts["exp_graphs"] == "intra")
+        and graphs_config is not None
+        and "intra-exp" in graphs_config
+    ):
+        _logger.info("Generating intra-experiment graphs...")
+        start = time.time()
+        intra.generate.proc_batch_exp(
+            main_config,
+            cmdopts,
+            pathset,
+            criteria,
+        )
+        elapsed = int(time.time() - start)
+        sec = datetime.timedelta(seconds=elapsed)
+        _logger.info("Intra-experiment graph generation complete: %s", str(sec))
+
+    # Collation must be after intra-experiment graph generation, so that all
+    # .csv files to be collated have been generated/modified according to
+    # parameters.
+    if (
+        (cmdopts["exp_graphs"] == "all" or cmdopts["exp_graphs"] == "inter")
+        and graphs_config is not None
+        and "inter-exp" in graphs_config
+    ):
+        if not cmdopts["skip_collate"]:
+            _logger.info("Collating inter-experiment files...")
+            start = time.time()
+            collate.proc_batch_exp(main_config, cmdopts, pathset, criteria)
+            elapsed = int(time.time() - start)
+            sec = datetime.timedelta(seconds=elapsed)
+            _logger.info("Collating inter-experiment files complete: %s", str(sec))
+
+        _logger.info("Generating inter-experiment graphs...")
+        start = time.time()
+
+        inter.generate.proc_batch_exp(
+            main_config,
+            cmdopts,
+            pathset,
+            criteria,
+        )
+        elapsed = int(time.time() - start)
+        sec = datetime.timedelta(seconds=elapsed)
+
+        _logger.info("Inter-experiment graph generation complete: %s", str(sec))

sierra/plugins/prod/graphs/targets.py

@@ -0,0 +1,63 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import typing as tp
+import logging
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import types
+
+_logger = logging.getLogger(__name__)
+
+
+def inter_exp_calc(
+    loaded_graphs: types.YAMLDict,
+    controller_config: tp.Optional[types.YAMLDict],
+    cmdopts: types.Cmdopts,
+) -> tp.List[types.YAMLDict]:
+    """Calculate what inter-experiment graphs to generate.
+
+    This also defines what CSV files need to be collated, as one graph is
+    always generated from one CSV file. Uses YAML configuration for
+    controllers and inter-experiment graphs.
+
+    """
+    keys = []
+
+    if controller_config:
+        for category in list(controller_config.keys()):
+            if category not in cmdopts["controller"]:
+                continue
+            for controller in controller_config[category]["controllers"]:
+                if dict(controller)["name"] not in cmdopts["controller"]:
+                    continue
+
+                # valid to specify no graphs, and only to inherit graphs
+                keys = controller.get("graphs", [])
+                if "graphs_inherit" in controller:
+                    for inherit in dict(controller)["graphs_inherit"]:
+                        keys.extend(inherit)  # optional
+        _logger.debug("Loaded %s inter-experiment categories: %s", len(keys), keys)
+    else:
+        keys = list(k for k in loaded_graphs)
+        _logger.debug(
+            "Missing controller graph config--generating all "
+            "inter-experiment graphs for all controllers: %s",
+            keys,
+        )
+
+    filtered_keys = [k for k in loaded_graphs if k in keys]
+    targets = [loaded_graphs[k] for k in filtered_keys]
+
+    _logger.debug(
+        "Enabled %s inter-experiment categories: %s",
+        len(filtered_keys),
+        filtered_keys,
+    )
+    return targets

sierra/plugins/prod/render/__init__.py

@@ -0,0 +1,18 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""
+Container module for rendering in stage 4.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/prod/render/cmdline.py

@@ -0,0 +1,72 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import typing as tp
+import argparse
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+def build(
+    parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+) -> PluginCmdline:
+    """
+    Get a cmdline parser supporting the ``prod.render`` product plugin.
+    """
+    cmdline = PluginCmdline(parents, stages)
+    cmdline.stage4.add_argument(
+        "--render-cmd-opts",
+        help="""
+                 Specify the :program:`ffmpeg` options to appear between the
+                 specification of the input image files and the specification of
+                 the output file.  The default is suitable for use with ARGoS
+                 frame grabbing set to a frames size of 1600x1200 to output a
+                 reasonable quality video.
+                 """
+        + cmdline.stage_usage_doc([4]),
+        default="-r 10 -s:v 800x600 -c:v libx264 -crf 25 -filter:v scale=-2:956 -pix_fmt yuv420p",
+    )
+
+    cmdline.stage4.add_argument(
+        "--project-rendering",
+        help="""
+                 Enable generation of videos from imagized CSV files created as
+                 a result of running the ``proc.imagize`` plugin.  See
+                 :ref:`plugins/prod/render` for details.
+                 """
+        + cmdline.stage_usage_doc([4]),
+        action="store_true",
+    )
+
+    cmdline.stage4.add_argument(
+        "--bc-rendering",
+        help="""
+                 Enable generation of videos from generated graphs, such as
+                 heatmaps.  Bivariate batch criteria only.
+
+                 .. versionadded:: 1.2.20
+                 """
+        + cmdline.stage_usage_doc([4]),
+        action="store_true",
+    )
+    return cmdline
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    return {
+        "render_cmd_opts": args.render_cmd_opts,
+        "project_rendering": args.project_rendering,
+        "bc_rendering": args.bc_rendering,
+    }
+
+
+def sphinx_cmdline_stage4():
+    return build([], [4]).parser