sierra-research 1.3.6__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/proc/statistics/plugin.py

@@ -0,0 +1,390 @@
+# Copyright 2019 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+
+"""
+Classes for generating statistics within and across experiments in a batch.
+"""
+
+# Core packages
+import multiprocessing as mp
+import typing as tp
+import queue
+import logging
+import pathlib
+import os
+
+# 3rd party packages
+import pandas as pd
+import yaml
+
+# Project packages
+import sierra.core.variables.batch_criteria as bc
+from sierra.core import types, utils, stat_kernels, storage, batchroot, config
+from sierra.core.pipeline.stage3 import gather
+import sierra.core.plugin as pm
+
+_logger = logging.getLogger(__name__)
+
+
+class DataGatherer(gather.BaseGatherer):
+    """Gather :term:`Raw Output Data` files from all runs.
+
+    The configured output directory for each run is searched recursively for
+    files to gather.  To be eligible for gathering and later processing, files
+    must:
+
+        - Be non-empty
+
+        - Have a suffix which supported by the selected ``--storage`` plugin.
+
+        - Match an intra/inter experiment graph in ``graphs.yaml``.
+    """
+
+    def __init__(
+        self,
+        main_config: types.YAMLDict,
+        gather_opts: types.SimpleDict,
+        processq: mp.Queue,
+    ) -> None:
+        super().__init__(main_config, gather_opts, processq)
+        self.logger = logging.getLogger(__name__)
+        config_path = pathlib.Path(gather_opts["project_config_root"]) / pathlib.Path(
+            config.kYAML.graphs
+        )
+        if utils.path_exists(config_path):
+            _logger.debug("Filtering gathered data by graph generation targets")
+            self.config = yaml.load(utils.utf8open(config_path), yaml.FullLoader)
+        else:
+            _logger.debug(
+                "%s does not exist for project: not filtering gathered data",
+                config.kYAML.graphs,
+            )
+
+    def calc_gather_items(
+        self, run_output_root: pathlib.Path, exp_name: str
+    ) -> tp.List[gather.GatherSpec]:
+        to_gather = []
+        proj_output_root = run_output_root / str(self.run_metrics_leaf)
+        plugin = pm.pipeline.get_plugin_module(self.gather_opts["storage"])
+
+        for item in proj_output_root.rglob("*"):
+            if (
+                item.is_dir()
+                or not any(s in plugin.suffixes() for s in item.suffixes)
+                or item.stat().st_size == 0
+            ):
+                continue
+
+            filter_by_intra = "intra-exp" in self.config
+            filter_by_inter = "inter-exp" in self.config
+
+            filtered_intra = any(
+                g["src_stem"] in str(item.relative_to(proj_output_root))
+                for category in self.config["intra-exp"]
+                for g in self.config["intra-exp"][category]
+            )
+
+            filtered_inter = any(
+                g["src_stem"] in str(item.relative_to(proj_output_root))
+                for category in self.config["inter-exp"]
+                for g in self.config["inter-exp"][category]
+            )
+
+            # If both are present, we gather from it if there is a positive
+            # match in either graph type category.
+            if (
+                filter_by_intra
+                and filter_by_inter
+                and (filtered_intra or filtered_inter)
+            ):
+                self.logger.trace(
+                    "Gathering %s: match in %s [intra/inter]",
+                    item.relative_to(proj_output_root),
+                    config.kYAML.graphs,
+                )
+                to_gather.append(
+                    gather.GatherSpec(
+                        exp_name=exp_name,
+                        item_stem_path=item.relative_to(proj_output_root),
+                        collate_col=None,
+                    )
+                )
+                continue
+
+            # If only intra-exp graphs are present, we gather from it if
+            # there is a positive match in that category.
+            if filter_by_intra and filtered_intra:
+                self.logger.trace(
+                    "Gathering %s: match in %s [intra]",
+                    item.relative_to(proj_output_root),
+                    config.kYAML.graphs,
+                )
+                to_gather.append(
+                    gather.GatherSpec(
+                        exp_name=exp_name,
+                        item_stem_path=item.relative_to(proj_output_root),
+                        collate_col=None,
+                    )
+                )
+                continue
+
+            # If only inter-exp graphs are are present, we gather from it if
+            # there is a positive match in that category.
+            if filter_by_inter and filtered_inter:
+                self.logger.trace(
+                    "Gathering %s: match in %s [inter]",
+                    item.relative_to(proj_output_root),
+                    config.kYAML.graphs,
+                )
+                to_gather.append(
+                    gather.GatherSpec(
+                        exp_name=exp_name,
+                        item_stem_path=item.relative_to(proj_output_root),
+                        collate_col=None,
+                    )
+                )
+                continue
+
+        return to_gather
+
+
+def proc_batch_exp(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+    gatherer_type=DataGatherer,
+) -> None:
+    """Process :term:`Raw Output Data` files for each :term:`Experiment`.
+
+    Ideally this is done in parallel across experiments, but this can be changed
+    to serial if memory on the SIERRA host machine is limited via
+    ``--processing-parallelism``.
+
+    It *IS* faster to do all the gathering at once and THEN do all the
+    processing, but that doesn't work for extremely large amounts of data
+    generated per :term:`Experimental Run`.
+    """
+    exp_to_proc = utils.exp_range_calc(
+        cmdopts["exp_range"], pathset.output_root, criteria.gen_exp_names()
+    )
+
+    template_input_leaf = pathlib.Path(cmdopts["expdef_template"]).stem
+
+    stat_opts = {
+        "template_input_leaf": template_input_leaf,
+        "df_verify": cmdopts["df_verify"],
+        "dist_stats": cmdopts["dist_stats"],
+        "processing_mem_limit": cmdopts["processing_mem_limit"],
+        "storage": cmdopts["storage"],
+        "project_config_root": cmdopts["project_config_root"],
+        "df_homogenize": cmdopts["df_homogenize"],
+    }
+
+    pool_opts = {}
+    parallelism = cmdopts["processing_parallelism"]
+
+    # Aways need to have at least one of each! If SIERRA is invoked on a machine
+    # with 2 or less logical cores, the calculation with psutil.cpu_count() will
+    # return 0 for # gatherers.
+    pool_opts["n_gatherers"] = max(1, int(parallelism * 0.25))
+    pool_opts["n_processors"] = max(1, int(parallelism * 0.75))
+
+    with mp.Pool(
+        processes=pool_opts["n_gatherers"] + pool_opts["n_processors"]
+    ) as pool:
+        _execute_for_batch(
+            main_config, pathset, exp_to_proc, stat_opts, pool_opts, gatherer_type, pool
+        )
+
+        pool.close()
+        pool.join()
+
+
+def _execute_for_batch(
+    main_config: types.YAMLDict,
+    pathset: batchroot.PathSet,
+    exp_to_proc: tp.List[pathlib.Path],
+    stat_opts: types.SimpleDict,
+    pool_opts: types.SimpleDict,
+    gatherer_type,
+    pool,
+) -> None:
+    """
+    Perform statistics generation on the :term:`Batch Experiment`.
+
+    Gathers all :term:`Raw Output Data` files FIRST, and *then* does
+    processing. This is almost 50% faster than doing a true producer-consumer
+    queue, probably because there is much less traffic across processes and/or
+    better disk I/O performance.
+    """
+    m = mp.Manager()
+    gatherq = m.Queue()
+    processq = m.Queue()
+
+    for exp in exp_to_proc:
+        gatherq.put(exp)
+
+    _logger.debug(
+        "Starting %d gatherers, method=%s",
+        pool_opts["n_gatherers"],
+        mp.get_start_method(),
+    )
+
+    gathered = [
+        pool.apply_async(
+            _gather_worker,
+            (gatherer_type, gatherq, processq, main_config, stat_opts),
+        )
+        for i in range(0, pool_opts["n_gatherers"])
+    ]
+
+    _logger.debug(
+        "Starting %d processors, method=%s",
+        pool_opts["n_processors"],
+        mp.get_start_method(),
+    )
+
+    processed = [
+        pool.apply_async(_process_worker, (processq, main_config, pathset, stat_opts))
+        for i in range(0, pool_opts["n_processors"])
+    ]
+
+    _logger.debug("Waiting for workers to finish")
+
+    # To capture the otherwise silent crashes when something goes wrong in
+    # worker threads. Any assertions will show and any exceptions will be
+    # re-raised.
+    for g in gathered:
+        g.get()
+
+    for p in processed:
+        p.get()
+
+    _logger.debug("All workers finished")
+
+    assert (
+        gatherq.empty()
+    ), f"Finished processing but gather queue has {gatherq.qsize()} items?"
+
+    assert (
+        processq.empty()
+    ), f"Finished processing but process queue has {processq.qsize()} items?"
+
+
+def _gather_worker(
+    gatherer_type,
+    gatherq: mp.Queue,
+    processq: mp.Queue,
+    main_config: types.YAMLDict,
+    stat_opts: tp.Dict[str, str],
+) -> None:
+    gatherer = gatherer_type(main_config, stat_opts, processq)
+
+    # Wait for 2 seconds after the queue is empty before bailing, at the
+    # start. If that is not long enough then exponentially increase from
+    # there until you find how long it takes to get the first item in the
+    # queue, and use that as the appropriate timeout (plus a little
+    # margin).
+    timeout = 3
+    got_item = False
+    n_tries = 0
+    while n_tries < config.kGatherWorkerRetries:
+        try:
+            exp_output_root = gatherq.get(True, timeout)
+            gatherer(exp_output_root)
+            gatherq.task_done()
+            got_item = True
+
+        except queue.Empty:
+            if got_item:
+                break
+
+            timeout *= 2
+            n_tries += 1
+
+    _logger.trace(f"Gather worker {os.getpid()} exit")
+
+
+def _process_worker(
+    processq: mp.Queue,
+    main_config: types.YAMLDict,
+    pathset: batchroot.PathSet,
+    stat_opts: tp.Dict[str, str],
+) -> None:
+    # Wait for 2 seconds after the queue is empty before bailing, at the
+    # start. If that is not long enough then exponentially increase from
+    # there until you find how long it takes to get the first item in the
+    # queue, and use that as the appropriate timeout (plus a little
+    # margin).
+    timeout = 3
+    got_item = False
+    n_tries = 0
+    while n_tries < config.kProcessWorkerRetries:
+        try:
+            spec = processq.get(True, timeout)
+
+            _proc_single_exp(main_config, stat_opts, pathset, spec)
+            processq.task_done()
+            got_item = True
+
+        except queue.Empty:
+            if got_item:
+                break
+
+            timeout *= 2
+            n_tries += 1
+    _logger.trace(f"Process worker {os.getpid()} exit")
+
+
+def _proc_single_exp(
+    main_config: types.YAMLDict,
+    stat_opts: types.StrDict,
+    pathset: batchroot.PathSet,
+    spec: gather.ProcessSpec,
+) -> None:
+    """Generate statistics from output files for all runs within an experiment.
+
+    .. IMPORTANT:: You *CANNOT* use logging ANYWHERE during processing .csv
+                   files. Why ? I *think* because of a bug in the logging module
+                   it If you get unlucky enough to spawn the process which
+                   enters the __call__() method in this class while another
+                   logging statement is in progress (and is therefore holding an
+                   internal logging module lock), then the underlying fork()
+                   call will copy the lock in the acquired state. Then, when
+                   this class goes to try to log something, it deadlocks with
+                   it.
+
+                   You also can't just create loggers with unique names, as this
+                   seems to be something like the GIL, but for the logging
+                   module. Sometimes python sucks.
+    """
+    csv_concat = pd.concat(spec.dfs)
+    exp_stat_root = pathset.stat_root / spec.gather.exp_name
+
+    utils.dir_create_checked(exp_stat_root, exist_ok=True)
+
+    by_row_index = csv_concat.groupby(csv_concat.index)
+
+    dfs = {}
+
+    if stat_opts["dist_stats"] in ["none", "all"]:
+        dfs.update(stat_kernels.mean.from_groupby(by_row_index))
+
+    if stat_opts["dist_stats"] in ["conf95", "all"]:
+        dfs.update(stat_kernels.conf95.from_groupby(by_row_index))
+
+    if stat_opts["dist_stats"] in ["bw", "all"]:
+        dfs.update(stat_kernels.bw.from_groupby(by_row_index))
+
+    for ext, df in dfs.items():
+        opath = exp_stat_root / spec.gather.item_stem_path
+        utils.dir_create_checked(opath.parent, exist_ok=True)
+        opath = opath.with_suffix(ext)
+
+        df = utils.df_fill(df, stat_opts["df_homogenize"])
+        storage.df_write(df, opath, "storage.csv", index=False)
+
+
+__all__ = ["proc_batch_exp"]

sierra/plugins/{hpc → prod}/__init__.py

@@ -1,6 +1,7 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
+"""Common functionality ``--prod`` plugins."""
 
 # Core packages
 

sierra/plugins/prod/graphs/__init__.py

@@ -0,0 +1,18 @@
+#
+# Copyright 2024 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+"""
+Container module for graph generation in stage 4.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/prod/graphs/cmdline.py

@@ -0,0 +1,269 @@
+#
+# Copyright 2025 John Harwell, All rights reserved.
+#
+# SPDX-License Identifier: MIT
+#
+
+# Core packages
+import typing as tp
+import argparse
+
+# 3rd party packages
+
+# Project packages
+from sierra.core import types
+from sierra.plugins import PluginCmdline
+
+
+def build(
+    parents: tp.List[argparse.ArgumentParser], stages: tp.List[int]
+) -> PluginCmdline:
+    """
+    Get a cmdline parser supporting the ``prod.graphs`` product plugin.
+    """
+    cmdline = PluginCmdline(parents, stages)
+    _build_multistage(cmdline)
+    _build_stage4(cmdline)
+    return cmdline
+
+
+def _build_multistage(cmdline: PluginCmdline) -> PluginCmdline:
+    # Plotting options
+    cmdline.multistage.add_argument(
+        "--plot-log-xscale",
+        help="""
+             Place the set of X values used to generate intra- and
+             inter-experiment graphs into the logarithmic space.  Mainly useful
+             when the batch criteria involves large system sizes, so that the
+             plots are more readable.
+             """
+        + cmdline.graphs_applicable_doc(
+            [":py:func:`Summary Line <sierra.core.graphs.summary_line.generate>`"]
+        )
+        + cmdline.stage_usage_doc([4, 5]),
+        action="store_true",
+    )
+
+    cmdline.multistage.add_argument(
+        "--plot-enumerated-xscale",
+        help="""
+             Instead of using the values generated by a given batch criteria for
+             the X values, use an enumerated list[0, ..., len(X value) - 1].
+             Mainly useful when the batch criteria involves large system sizes,
+             so that the plots are more readable.
+             """
+        + cmdline.graphs_applicable_doc(
+            [":py:func:`Summary Line <sierra.core.graphs.summary_line.generate>`"]
+        )
+        + cmdline.stage_usage_doc([4, 5]),
+        action="store_true",
+    )
+
+    cmdline.multistage.add_argument(
+        "--plot-log-yscale",
+        help="""
+             Place the set of Y values used to generate intra - and
+             inter-experiment graphs into the logarithmic space.  Mainly useful
+             when the batch criteria involves large system sizes, so that the
+             plots are more readable.
+             """
+        + cmdline.graphs_applicable_doc(
+            [
+                ":py:func:`Summary Line <sierra.core.graphs.summary_line.generate>`",
+                ":py:func:`Stacked Line <sierra.core.graphs.stacked_line.generate>`",
+            ]
+        )
+        + cmdline.stage_usage_doc([4, 5]),
+        action="store_true",
+    )
+
+    cmdline.multistage.add_argument(
+        "--plot-primary-axis",
+        type=int,
+        help="""
+             This option allows you to override the primary axis, which is
+             normally is computed based on the batch criteria.
+
+             For example, in a bivariate batch criteria composed of
+
+                 - :ref:`plugins/engine/argos/bc/population-size` on the X axis
+                   (rows)
+
+                 - Another batch criteria which does not affect system size
+                   (columns)
+
+             Metrics will be calculated by `computing` across .csv rows and
+             `projecting` down the columns by default, since system size will
+             only vary within a row.  Passing a value of 1 to this option will
+             override this calculation, which can be useful in bivariate batch
+             criteria in which you are interested in the effect of the OTHER
+             non-size criteria on various performance measures.
+
+             0=criteria of interest varies across `rows`.
+
+             1=criteria of interest varies across `columns`.
+
+             This option only affects generating graphs from bivariate batch
+             criteria.
+             """
+        + cmdline.graphs_applicable_doc(
+            [
+                ":py:func:`Heatmap <sierra.core.graphs.heatmap.generate>`",
+                ":py:func:`Stacked Line <sierra.core.graphs.stacked_line.generate>`",
+            ]
+        )
+        + cmdline.stage_usage_doc([4, 5]),
+        default=None,
+    )
+
+    cmdline.multistage.add_argument(
+        "--plot-large-text",
+        help="""
+             This option specifies that the title, X/Y axis labels/tick labels
+             should be larger than the SIERRA default.  This is useful when
+        generating graphs suitable for two column paper format where the
+             default text size for rendered graphs will be too small to see
+             easily.  The SIERRA defaults are generally fine for the one
+             column/journal paper format.
+             """
+        + cmdline.stage_usage_doc([4, 5]),
+        action="store_true",
+    )
+
+    cmdline.multistage.add_argument(
+        "--plot-transpose-graphs",
+        help="""
+             Transpose the X, Y axes in generated graphs.  Useful as a general
+             way to tweak graphs for best use of space within a paper.
+
+             .. versionchanged:: 1.2.20
+
+             Renamed from ``--transpose-graphs`` to make its relation to other
+             plotting options clearer.
+             """
+        + cmdline.graphs_applicable_doc(
+            [":py:func:`Heatmap <sierra.core.graphs.heatmap.generate>`"]
+        )
+        + cmdline.stage_usage_doc([4, 5]),
+        action="store_true",
+    )
+    return cmdline
+
+
+def _build_stage4(cmdline: PluginCmdline) -> PluginCmdline:
+    cmdline.stage4.add_argument(
+        "--graphs-backend",
+        choices=["matplotlib", "bokeh"],
+        help="""
+             Specify the default backend to be used when generating plots.  Can
+             be overriden on a per-graph basis.
+
+                 - ``matplotlib`` - Use matplotlib to generate static PNG
+                   images.
+
+                 - ``bokeh`` - Use bokeh to generate stand-alone HTML files
+                   containing interactive bokeh visualizations.  Files are
+                   suitable for inclusion in static webpages, viewing in a
+                   browser, etc.
+
+             See :ref:`plugins/prod/graphs` for more information.
+             """,
+        default="matplotlib",
+    )
+    cmdline.stage4.add_argument(
+        "--exp-n-datapoints-factor",
+        type=float,
+        help="""
+             Specify an additional multiplicative factor for computing the # of
+             datapoints captured duration an :term:`Experiment` to modify the
+             duration * ticks_per_sec default.
+             """,
+        default=1.0,
+    )
+    cmdline.stage4.add_argument(
+        "--exp-graphs",
+        choices=["intra", "inter", "all", "none"],
+        help="""
+             Specify which types of graphs should be generated from experimental
+             results:
+
+                 - ``intra`` - Generate intra-experiment graphs from the results
+                   of a single experiment within a batch, for each experiment in
+                   the batch(this can take a long time with large batch
+                   experiments).  If any intra-experiment models are defined and
+                   enabled, those are run and the results placed on appropriate
+                   graphs.
+
+                 - ``inter`` - Generate inter-experiment graphs _across_ the
+                   results of all experiments in a batch.  These are very fast
+                   to generate, regardless of batch experiment size.  If any
+                   inter-experiment models are defined and enabled, those are
+                   run and the results placed on appropriate graphs.
+
+                 - ``all`` - Generate all types of graphs.
+
+                 - ``none`` - Skip graph generation.
+             """
+        + cmdline.stage_usage_doc([4]),
+        default="all",
+    )
+
+    cmdline.stage4.add_argument(
+        "--project-no-LN",
+        help="""
+             Specify that the intra-experiment and inter-experiment linegraphs
+             defined in project YAML configuration should not be generated.
+             Useful if you are working on something which results in the
+             generation of other types of graphs, and the generation of those
+             linegraphs is not currently needed only slows down your development
+             cycle.
+
+             Model linegraphs are still generated, if applicable.
+             """,
+        action="store_true",
+    )
+
+    cmdline.stage4.add_argument(
+        "--project-no-HM",
+        help="""
+             Specify that the intra-experiment heatmaps defined in project YAML
+             configuration should not be generated.  Useful if:
+
+                 - You are working on something which results in the generation
+                   of other types of graphs, and the generation of heatmaps only
+                   slows down your development cycle.
+
+                 - You are working on stage5 comparison graphs for bivariate
+                   batch criteria, and re-generating many heatmaps during stage4
+                   is taking too long.
+
+             Model heatmaps are still generated, if applicable.
+
+             .. versionadded:: 1.2.20
+             """,
+        action="store_true",
+    )
+
+    return cmdline
+
+
+def to_cmdopts(args: argparse.Namespace) -> types.Cmdopts:
+    return {
+        # multistage
+        "plot_log_xscale": args.plot_log_xscale,
+        "plot_enumerated_xscale": args.plot_enumerated_xscale,
+        "plot_log_yscale": args.plot_log_yscale,
+        "plot_primary_axis": args.plot_primary_axis,
+        "plot_large_text": args.plot_large_text,
+        "plot_transpose_graphs": args.plot_transpose_graphs,
+        # stage 4
+        "graphs_backend": args.graphs_backend,
+        "exp_n_datapoints_factor": args.exp_n_datapoints_factor,
+        "exp_graphs": args.exp_graphs,
+        "project_no_LN": args.project_no_LN,
+        "project_no_HM": args.project_no_HM,
+    }
+
+
+def sphinx_cmdline_multistage():
+    return build([], [3, 4, 5]).parser