sierra-research 1.3.11__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/compare/graphs/inter_scenario.py

@@ -0,0 +1,297 @@
+# Copyright 2019 John Harwell, All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+
+"""Classes for comparing deliverables across a set of scenarios.
+
+Univariate batch criteria only. The same controller must be used for all
+scenarios.
+
+"""
+
+# Core packages
+import typing as tp
+import argparse
+import logging
+import pathlib
+
+# 3rd party packages
+
+# Project packages
+from sierra.core.variables import batch_criteria as bc
+import sierra.core.plugin as pm
+from sierra.core import types, utils, config, storage, batchroot, graphs
+from sierra.plugins.compare.graphs import outputroot, preprocess, namecalc, comparator
+
+
+class UnivarInterScenarioComparator(comparator.BaseComparator):
+    """Compares a single controller across a set of scenarios.
+
+    Graph generation is controlled via a config file parsed in
+    :class:`~sierra.core.pipeline.stage5.pipeline_stage5.PipelineStage5`.
+
+    Univariate batch criteria only.
+
+    Attributes:
+        controller: Controller to use.
+
+        scenarios: List of scenario names to compare ``controller`` across.
+
+        sc_csv_root: Absolute directory path to the location scenario CSV
+                     files should be output to.
+
+        sc_graph_root: Absolute directory path to the location the generated
+                       graphs should be output to.
+
+        cmdopts: Dictionary of parsed cmdline parameters.
+
+        cli_args: :class:`argparse` object containing the cmdline
+                  parameters. Needed for
+                  :class:`~sierra.core.variables.batch_criteria.XVarBatchCriteria`
+                  generation for each scenario controllers are compared within,
+                  as batch criteria is dependent on controller+scenario
+                  definition, and needs to be re-generated for each scenario in
+                  order to get graph labels/axis ticks to come out right in all
+                  cases.
+
+    """
+
+    def __init__(
+        self,
+        controller: str,
+        scenarios: tp.List[str],
+        stage5_roots: outputroot.PathSet,
+        cmdopts: types.Cmdopts,
+        cli_args: argparse.Namespace,
+        main_config: types.YAMLDict,
+    ) -> None:
+        super().__init__(scenarios, stage5_roots, cmdopts, cli_args, main_config)
+        self.logger = logging.getLogger(__name__)
+        self.controller = controller
+
+    def exp_select(self) -> tp.List[batchroot.ExpRoot]:
+        """
+        Determine if a scenario can be include in the comparison for a controller.
+
+        """
+        selected = []
+        for scenario in self.things:
+            for candidate in (self.project_root / self.controller / scenario).iterdir():
+                root = batchroot.ExpRoot(
+                    sierra_root=self.cmdopts["sierra_root"],
+                    project=self.cmdopts["project"],
+                    controller=self.controller,
+                    leaf=batchroot.ExpRootLeaf.from_name(candidate.name),
+                    scenario=scenario,
+                )
+                if root.to_path().exists():
+                    selected.append(root)
+        return selected
+
+    def compare(
+        self,
+        cmdopts: types.Cmdopts,
+        graph: types.YAMLDict,
+        roots: tp.List[batchroot.ExpRoot],
+        legend: tp.List[str],
+    ) -> None:
+        for scenario in self.things:
+            valid_configurations = sum(r.scenario == scenario for r in roots)
+            if valid_configurations > 1:
+                self.logger.warning(
+                    "Skipping ambiguous comparison for scenario %s: was run on multiple selected batch roots %s",
+                    scenario,
+                    [r.to_str() for r in roots],
+                )
+                continue
+
+            if valid_configurations == 0:
+                self.logger.warning(
+                    "Skipping comparison for scenario %s: not run on any selected batch roots %s",
+                    scenario,
+                    [r.to_str() for r in roots],
+                )
+                continue
+
+        # Each controller should have been run on exactly ONE batch experiment
+        # that we selected for controller comparison, by definition.
+        root = next(r for r in roots if r.scenario == scenario)
+
+        # We need to generate the root directory paths for each batch experiment
+        # (which lives inside of the scenario dir), because they are all
+        # different. We need generate these paths for EACH controller, because
+        # the controller is part of the batch root path.
+        pathset = batchroot.from_exp(
+            sierra_root=self.cli_args.sierra_root,
+            project=self.cli_args.project,
+            batch_leaf=root.leaf,
+            controller=self.controller,
+            scenario=root.scenario,
+        )
+
+        # For each scenario, we have to create the batch criteria for it,
+        # because they are all different.
+        criteria = bc.factory(
+            self.main_config,
+            cmdopts,
+            pathset.input_root,
+            self.cli_args,
+            root.scenario,
+        )
+
+        self._gen_csvs(
+            criteria=criteria,
+            pathset=pathset,
+            project=self.cli_args.project,
+            root=root,
+            src_stem=graph["src_stem"],
+            dest_stem=graph["dest_stem"],
+            index=graph.get("index", -1),
+        )
+
+        self._gen_graph(
+            batch_leaf=root.leaf,
+            criteria=criteria,
+            cmdopts=cmdopts,
+            batch_output_root=pathset.output_root,
+            dest_stem=graph["dest_stem"],
+            title=graph.get("title", None),
+            label=graph["label"],
+            legend=legend,
+            backend=graph.get("backend", cmdopts["graphs_backend"]),
+        )
+
+    def _gen_graph(
+        self,
+        batch_leaf: batchroot.ExpRootLeaf,
+        criteria: bc.XVarBatchCriteria,
+        cmdopts: types.Cmdopts,
+        batch_output_root: pathlib.Path,
+        dest_stem: str,
+        title: str,
+        label: str,
+        legend: tp.List[str],
+        backend: str,
+        inc_exps: tp.Optional[str] = None,
+    ) -> None:
+        """Generate graph comparing the specified controller across scenarios."""
+        opath_leaf = namecalc.for_sc(batch_leaf, self.things, dest_stem, None)
+        info = criteria.graph_info(cmdopts, batch_output_root=batch_output_root)
+
+        xticklabels = info.xticklabels
+        xticks = info.xticks
+        if inc_exps is not None:
+            xticklabels = utils.exp_include_filter(
+                inc_exps, info.xticklabels, criteria.n_exp()
+            )
+            xticks = utils.exp_include_filter(inc_exps, info.xticks, criteria.n_exp())
+
+        paths = graphs.PathSet(
+            input_root=self.stage5_roots.csv_root,
+            output_root=self.stage5_roots.graph_root,
+            batchroot=pathlib.Path(
+                self.cmdopts["sierra_root"], self.cmdopts["project"]
+            ),
+            model_root=None,
+        )
+
+        graphs.summary_line(
+            paths=paths,
+            input_stem=opath_leaf,
+            stats=cmdopts["dist_stats"],
+            medium=cmdopts["storage"],
+            output_stem=opath_leaf,
+            title=title,
+            xlabel=info.xlabel,
+            ylabel=label,
+            backend=backend,
+            xticks=xticks,
+            xticklabels=xticklabels,
+            logyscale=cmdopts["plot_log_yscale"],
+            large_text=cmdopts["plot_large_text"],
+            legend=legend,
+        )
+
+    def _gen_csvs(
+        self,
+        criteria: bc.XVarBatchCriteria,
+        pathset: batchroot.PathSet,
+        project: str,
+        root: batchroot.ExpRoot,
+        index: int,
+        src_stem: str,
+        dest_stem: str,
+    ) -> None:
+        """Generate a set of CSV files for use in inter-scenario graph generation.
+
+        Generates:
+
+        - ``.mean`` CSV file containing results for each scenario the controller
+           is being compared across, 1 per line.
+
+        - Stastics CSV files containing various statistics for the ``.mean`` CSV
+          file, 1 per line.
+
+        - ``.model`` file containing model predictions for controller behavior
+          during each scenario, 1 per line (not generated if models were not run
+          the performance measures we are generating graphs for).
+
+        - ``.legend`` file containing legend values for models to plot (not
+          generated if models were not run for the performance measures we are
+          generating graphs for).
+
+        """
+
+        csv_ipath_stem = pathset.stat_interexp_root / src_stem
+
+        # Some experiments might not generate the necessary performance measure
+        # CSVs for graph generation, which is OK.
+        csv_ipath_mean = csv_ipath_stem.with_suffix(config.kStats["mean"].exts["mean"])
+        if not utils.path_exists(csv_ipath_mean):
+            self.logger.warning(
+                "%s missing for controller %s", csv_ipath_mean, self.controller
+            )
+            return
+
+        preparer = preprocess.IntraExpPreparer(
+            ipath_stem=pathset.stat_interexp_root,
+            ipath_leaf=src_stem,
+            opath_stem=self.stage5_roots.csv_root,
+            criteria=criteria,
+        )
+        opath_leaf = namecalc.for_sc(root.leaf, self.things, dest_stem, None)
+
+        preparer.for_sc(
+            scenario=root.scenario,
+            opath_leaf=opath_leaf,
+            index=index,
+            inc_exps=None,
+        )
+
+        # Collect performance results models and legends. Append to existing
+        # dataframes if they exist, otherwise start new ones.
+        # Can't use with_suffix() for opath, because that path contains the
+        # controller, which already has a '.' in it.
+        # model_istem = pathlib.Path(pathset.model_root, src_stem)
+        model_ostem = self.stage5_roots.model_root / (dest_stem + "-" + self.controller)
+
+        # model_ipath = model_istem.with_suffix(config.kModelsExt["model"])
+        model_opath = model_ostem.with_name(
+            model_ostem.name + config.kModelsExt["model"]
+        )
+        # model_df = self._accum_df(criteria, model_ipath, model_opath, src_stem)
+        model_df = None
+        legend_opath = model_ostem.with_name(
+            model_ostem.name + config.kModelsExt["legend"]
+        )
+
+        if model_df is not None:
+            storage.df_write(model_df, model_opath, "storage.csv", index=False)
+
+            with utils.utf8open(legend_opath, "a") as f:
+                sgp = pm.module_load_tiered(project=project, path="generators.scenario")
+                kw = sgp.to_dict(root.scenario)
+                f.write("{0} Model Prediction\n".format(kw["scenario_tag"]))
+
+
+__all__ = ["UnivarInterScenarioComparator"]

sierra/plugins/compare/graphs/namecalc.py

@@ -0,0 +1,53 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""Utility module for calculating path names."""
+
+# Core packages
+import typing as tp
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import batchroot
+
+
+def for_cc(
+    leaf: batchroot.ExpRootLeaf, new_stem: str, indices: tp.Union[tp.List[int], None]
+) -> str:
+    """
+    Calculate a name suitable for CSVs/graphs in stage 5 to ensure uniqueness.
+
+    "Name" here is in pathlib terminology. Targets controller comparisons. Since
+    controller name is part of the default batchroot path, AND each batchroot
+    path is unique, this case is easy.
+    """
+    name = "{0}-{1}".format(new_stem, leaf.to_path().name)
+
+    if indices is not None:
+        name += "_" + "".join([str(i) for i in indices])
+
+    return name
+
+
+def for_sc(
+    leaf: batchroot.ExpRootLeaf,
+    scenarios: tp.List[str],
+    new_stem: str,
+    indices: tp.Union[tp.List[int], None],
+) -> str:
+    """
+    Calculate a name suitable for CSVs/graphs in stage 5 to ensure uniqueness.
+
+    "Name" here is in pathlib terminology. Targets scenario comparisons, so we
+    need a list of all scenarios in the path to eliminate path collisions in all
+    cases.
+    """
+    name = "{0}-{1}+{2}".format(new_stem, "+".join(scenarios), leaf.to_path())
+
+    if indices is not None:
+        name += "_" + "".join([str(i) for i in indices])
+
+    return name

sierra/plugins/compare/graphs/outputroot.py

@@ -0,0 +1,73 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""
+Utility module for functionality for managing all the paths used in stage5.
+"""
+# Core packages
+import pathlib
+import typing as tp
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import types
+
+
+class PathSet:
+    """
+    The set of filesystem paths used during stage 5.
+
+    Collected here in the interest of DRY.
+
+    Attributes:
+        graph_root: The path where all graphs will be created.
+
+        csv_root: The path where all collated CSVs will be stored.
+
+        model_root: The path where all CSVs resulting from model execution will
+                    be stored.
+    """
+
+    def __init__(self,
+                 cmdopts: types.Cmdopts,
+                 controllers: tp.List[str],
+                 scenarios: tp.List[str]) -> None:
+        assert not (controllers and scenarios)
+
+        # We add the controller list to the directory path for the .csv
+        # and graph directories so that multiple runs of stage5 with
+        # different controller sets do not overwrite each other
+        # (i.e. make stage5 more idempotent).
+        if controllers:
+            self.graph_root = pathlib.Path(cmdopts['sierra_root'],
+                                           cmdopts['project'],
+                                           '+'.join(controllers) + "-cc-graphs")
+
+            self.csv_root = pathlib.Path(cmdopts['sierra_root'],
+                                         cmdopts['project'],
+                                         '+'.join(controllers) + "-cc-csvs")
+
+            self.model_root = None
+
+        if scenarios:
+            # We add the scenario list to the directory path for the .csv
+            # and graph directories so that multiple runs of stage5 with
+            # different scenario sets do not overwrite each other (i.e. make
+            # stage5 idempotent).
+            self.graph_root = pathlib.Path(cmdopts['sierra_root'],
+                                           cmdopts['project'],
+                                           '+'.join(scenarios) + "-sc-graphs")
+            self.csv_root = pathlib.Path(cmdopts['sierra_root'],
+                                         cmdopts['project'],
+                                         '+'.join(scenarios) + "-sc-csvs")
+            self.model_root = pathlib.Path(cmdopts['sierra_root'],
+                                           cmdopts['project'],
+                                           '+'.join(scenarios) + "-sc-models")
+
+
+__all__ = [
+    "PathSet"
+]

sierra/plugins/compare/graphs/plugin.py

@@ -0,0 +1,147 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""
+Stage5 plugin to compare graphs across {controllers, scenarios, criterias}.
+"""
+
+# Core packages
+import logging
+import argparse
+import pathlib
+
+# 3rd party packages
+import yaml
+
+# Project packages
+from sierra.core import types, utils, config
+from sierra.plugins.compare.graphs import outputroot
+from sierra.plugins.compare.graphs import inter_controller as intercc
+from sierra.plugins.compare.graphs import inter_scenario as intersc
+
+_logger = logging.getLogger(__name__)
+
+
+def proc_exps(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    cli_args: argparse.Namespace,
+) -> None:
+    stage5_roots = outputroot.PathSet(
+        cmdopts,
+        cmdopts["things"].split(",") if cmdopts["across"] == "controllers" else None,
+        cmdopts["things"].split(",") if cmdopts["across"] == "scenarios" else None,
+    )
+
+    if stage5_roots.model_root is not None:
+        utils.dir_create_checked(stage5_roots.model_root, True)
+
+    # Create directories for .csv files and graphs
+    utils.dir_create_checked(stage5_roots.graph_root, True)
+    utils.dir_create_checked(stage5_roots.csv_root, True)
+
+    assert (
+        cmdopts["bc_cardinality"] <= 2
+    ), "This plugin only supports batch criteria with cardinality <=2"
+
+    path = pathlib.Path(cmdopts["project_config_root"], config.kYAML.graphs)
+    with utils.utf8open(path) as f:
+        graphs_config = yaml.load(f, yaml.FullLoader)
+
+    if cmdopts["across"] == "controllers":
+        _run_cc(main_config, cmdopts, cli_args, stage5_roots, graphs_config)
+    elif cmdopts["across"] == "scenarios":
+        _run_sc(main_config, cmdopts, cli_args, stage5_roots, graphs_config)
+    elif cmdopts["across"] == "criterias":
+        raise RuntimeError("Inter-criteria comparison not implemented yet!")
+
+
+def _run_cc(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    cli_args: argparse.Namespace,
+    stage5_roots: outputroot.PathSet,
+    graphs_config: types.YAMLDict,
+) -> None:
+    controllers = cmdopts["things"].split(",")
+
+    # Use nice controller names on graph legends if configured
+    if cmdopts["things_legend"] is not None:
+        legend = cmdopts["things_legend"].split(",")
+    else:
+        legend = controllers
+
+    _logger.info("Inter-batch controller comparison of %s...", controllers)
+
+    if cmdopts["bc_cardinality"] == 1:
+        univar = intercc.UnivarInterControllerComparator(
+            controllers,
+            stage5_roots,
+            cmdopts,
+            cli_args,
+            main_config,
+        )
+        univar(
+            target_graphs=list(graphs_config["inter-controller"]),
+            legend=list(legend),
+        )
+    elif cmdopts["bc_cardinality"] == 2:
+        bivar = intercc.BivarInterControllerComparator(
+            controllers,
+            stage5_roots,
+            cmdopts,
+            cli_args,
+            main_config,
+        )
+        bivar(
+            target_graphs=list(graphs_config["inter-controller"]),
+            legend=list(legend),
+        )
+
+    _logger.info("Inter-batch controller comparison complete")
+
+
+def _run_sc(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    cli_args: argparse.Namespace,
+    stage5_roots: outputroot.PathSet,
+    graphs_config: types.YAMLDict,
+) -> None:
+    scenarios = cmdopts["things"].split(",")
+
+    # Use nice scenario names on graph legends if configured
+    if cmdopts["things_legend"] is not None:
+        legend = cmdopts["things_legend"].split(",")
+    else:
+        legend = scenarios
+
+    controller = cmdopts["controller"]
+
+    _logger.info("Inter-batch comparison of %s across %s...", controller, scenarios)
+
+    assert (
+        cmdopts["bc_cardinality"] == 1
+    ), "inter-scenario controller comparison only valid for univariate batch criteria"
+
+    comparator = intersc.UnivarInterScenarioComparator(
+        controller,
+        scenarios,
+        stage5_roots,
+        cmdopts,
+        cli_args,
+        main_config,
+    )
+
+    comparator(
+        target_graphs=list(graphs_config["inter-scenario"]),
+        legend=list(legend),
+    )
+
+    _logger.info(
+        "Inter-batch  comparison of %s across %s complete",
+        controller,
+        scenarios,
+    )