sierra-research 1.3.11__py3-none-any.whl → 1.5.0__py3-none-any.whl

sierra/plugins/prod/render/plugin.py

@@ -0,0 +1,282 @@
+# Copyright 2019 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+
+"""Classes for rendering frames (images) into videos.
+
+Frames can be:
+
+- Captured by by the ``--engine`` during stage 2.
+
+- Generated during stage 3 of SIERRA via imagizing.
+
+- Generated inter-experiment heatmaps from bivariate experiments.
+
+"""
+
+# Core packages
+import subprocess
+import typing as tp
+import multiprocessing as mp
+import queue
+import copy
+import shutil
+import logging
+import pathlib
+import time
+import datetime
+
+# 3rd party packages
+
+# Project packages
+import sierra.core.variables.batch_criteria as bc
+from sierra.core import types, config, utils, batchroot
+from sierra.core import plugin as pm
+
+_logger = logging.getLogger(__name__)
+
+
+def proc_batch_exp(
+    main_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """
+    Render videos.
+
+        #. From :term:`Engine` if ``--engine-vc`` was passed.
+
+        #. From imagized images if ``proc.imagize`` was run previously to
+           generate frames, and ``--project-rendering`` is passed.
+    """
+    if (not cmdopts["engine_vc"]) and (not cmdopts["project_rendering"]):
+        return
+
+    _logger.info("Rendering videos...")
+    start = time.time()
+
+    graphs_path = pathlib.Path(cmdopts["project_config_root"]) / pathlib.Path(
+        config.kYAML.graphs
+    )
+    if utils.path_exists(graphs_path):
+        _logger.info("Loading render config for project=%s", cmdopts["project"])
+        loader = pm.module_load_tiered(project=cmdopts["project"], path="pipeline.yaml")
+        render_config = loader.load_config(cmdopts, config.kYAML.graphs)
+
+    else:
+        _logger.warning("%s does not exist--cannot generate render", graphs_path)
+        return
+
+    if cmdopts["engine_vc"]:
+        _from_engine(render_config["intra-exp"], cmdopts, pathset, criteria)
+    else:
+        _logger.debug(
+            (
+                "--engine-vc not passed--(possibly) skipping "
+                "rendering frames captured by the engine"
+            )
+        )
+
+    if cmdopts["project_rendering"] and "imagize" in render_config:
+        _from_project_imagized(render_config["imagize"], cmdopts, pathset, criteria)
+    else:
+        _logger.debug(
+            (
+                "--project-rendering not passed--(possibly) "
+                "skipping rendering frames captured by the "
+                "project"
+            )
+        )
+
+    elapsed = int(time.time() - start)
+    sec = datetime.timedelta(seconds=elapsed)
+    _logger.info("Rendering complete in %s", str(sec))
+
+
+def _from_engine(
+    render_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """Render frames (images) captured in by a engine into videos.
+
+    Frames are stitched together to make a video using :program:`ffmpeg`.  Output
+    format controlled via configuration.
+
+    Targets to render are found in::
+
+      <batch_root>/<exp_name>/<run_name>/<frames_leaf>
+
+    Videos are output in::
+
+      <batch_root>/videos/<exp_name>
+
+    ``<frames_leaf>`` is controlled via configuration.  For more
+    details, see :ref:`plugins/prod/render`.
+
+    .. note:: This currently only works with PNG images.
+    """
+    exp_to_render = utils.exp_range_calc(
+        cmdopts["exp_range"], pathset.output_root, criteria.gen_exp_names()
+    )
+
+    inputs = []
+    for exp in exp_to_render:
+        output_dir = pathset.video_root / exp.name
+
+        for run in exp.iterdir():
+            engine = cmdopts["engine"].split(".")[1]
+            frames_leaf = config.kRendering[engine]["frames_leaf"]
+            output_path = output_dir / (run.name + config.kRenderFormat)
+            opts = {
+                "exp_root": pathset.imagize_root / exp.name,
+                "output_path": output_path,
+                "input_dir": str(exp / run / frames_leaf),
+                "ffmpeg_opts": cmdopts["render_cmd_opts"],
+            }
+            inputs.append(copy.deepcopy(opts))
+
+    _parallel(render_config, cmdopts, inputs)
+
+
+def _from_project_imagized(
+    render_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    pathset: batchroot.PathSet,
+    criteria: bc.XVarBatchCriteria,
+) -> None:
+    """Render THINGS previously imagized in a project in stage 3 into videos.
+
+    Frames (images) in the imagize root (see :ref:`usage/run-time-tree`) are
+    stitched together to make a video using :program:`ffmpeg`.  Output format
+    controlled via configuration.
+
+    Targets to render are found in::
+
+        <batch_root>/imagize/<subdir_path>
+
+    Videos are output in::
+
+        <batch_root>/videos/<exp>/<subdir_path>
+
+    For more details, see :ref:`plugins/prod/render`.
+
+    .. note:: This currently only works with PNG images.
+    """
+    exp_to_render = utils.exp_range_calc(
+        cmdopts["exp_range"], pathset.output_root, criteria.gen_exp_names()
+    )
+
+    inputs = []
+    # For each category of imagized/rendered graphs
+    # For each graph in each category
+    for graph in render_config:
+        # Across all experiments
+        for exp in exp_to_render:
+            exp_imagize_root = pathset.imagize_root / exp.name
+            if not exp_imagize_root.exists():
+                continue
+
+            # Check all directories recursively
+            for candidate in exp_imagize_root.rglob("*"):
+                path = pathlib.Path(dict(graph)["src_stem"])
+                fragment = path.parent / path.name
+                if candidate.is_file():
+                    continue
+
+                if str(fragment) not in str(candidate):
+                    continue
+
+                output_path = (
+                    pathset.video_root
+                    / exp.name
+                    / candidate.relative_to(exp_imagize_root)
+                ) / (candidate.name + config.kRenderFormat)
+                inputs.append(
+                    {
+                        "input_dir": candidate,
+                        "exp_root": pathset.imagize_root / exp.name,
+                        "output_path": output_path,
+                        "ffmpeg_opts": cmdopts["render_cmd_opts"],
+                    }
+                )
+
+    _parallel(render_config, cmdopts, inputs)
+
+
+def _parallel(
+    render_config: types.YAMLDict,
+    cmdopts: types.Cmdopts,
+    inputs: tp.List[types.SimpleDict],
+) -> None:
+    """Perform the requested rendering in parallel.
+
+    Unless disabled with ``--proccessing-serial``, then it is done serially.
+    """
+    q = mp.JoinableQueue()  # type: mp.JoinableQueue
+
+    for spec in inputs:
+        q.put(spec)
+
+    # Render videos in parallel--waaayyyy faster
+    parallelism = cmdopts["processing_parallelism"]
+
+    for _ in range(0, parallelism):
+        p = mp.Process(target=_worker, args=(q, render_config))
+        p.start()
+
+    q.join()
+
+
+def _worker(q: mp.Queue, render_config: types.YAMLDict) -> None:
+    assert shutil.which("ffmpeg") is not None, "ffmpeg not found"
+
+    while True:
+        # Wait for 3 seconds after the queue is empty before bailing
+        try:
+            render_opts = q.get(True, 3)
+
+            _logger.info("Rendering images in %s...", render_opts["exp_root"].name)
+
+            opts = render_opts["ffmpeg_opts"].split(" ")
+
+            ipaths = "'{0}/*.{1}'".format(
+                render_opts["input_dir"], config.kStaticImageType
+            )
+            cmd = ["ffmpeg", "-y", "-pattern_type", "glob", "-i", ipaths]
+            cmd.extend(opts)
+            cmd.extend([str(render_opts["output_path"])])
+
+            to_run = " ".join(cmd)
+            _logger.trace("Run cmd: %s", to_run)  # type: ignore
+
+            utils.dir_create_checked(render_opts["output_path"].parent, exist_ok=True)
+
+            with subprocess.Popen(
+                to_run, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE
+            ) as proc:
+                proc.wait()
+
+            # We use communicate(), not wait() to avoid issues with IO buffers
+            # becoming full (e.g., you get deadlocks with wait() regularly).
+            stdout_raw, stderr_raw = proc.communicate()
+
+            # Only show output if the process failed (i.e., did not return 0)
+            if proc.returncode != 0:
+                _logger.error("Cmd '%s' failed!", to_run)
+                stdout_str = stdout_raw.decode("ascii")
+                stderr_str = stderr_raw.decode("ascii")
+
+                _logger.error("Return code=%d", proc.returncode)
+                _logger.error("stdout: %s", stdout_str)
+                _logger.error("stderr: %s", stderr_str)
+
+            q.task_done()
+
+        except queue.Empty:
+            break
+
+
+__all__ = ["proc_batch_exp"]

sierra/plugins/storage/__init__.py

@@ -1,6 +1,11 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
+"""
+Container module for plugins related to storage media.
+
+Driven by ``--storage``.
+"""
 
 # Core packages
 

sierra/plugins/storage/arrow/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2021 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Container module for the arrow storage plugin.
+
+See :ref:`plugins/storage/arrow`.
+"""
+
+# Core packages
+
+# 3rd party packages
+
+# Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/storage/arrow/plugin.py

@@ -0,0 +1,38 @@
+# Copyright 2025 John Harwell, All rights reserved.
+#
+#  SPDX-License-Identifier: MIT
+"""
+Plugin for reading/writing apache .arrow files.
+"""
+
+# Core packages
+import pathlib
+import typing as tp
+
+# 3rd party packages
+from retry import retry
+import pandas as pd
+
+# Project packages
+
+
+def suffixes() -> tp.Set[str]:
+    return {".arrow"}
+
+
+@retry(pd.errors.ParserError, tries=10, delay=0.100, backoff=1.1)  # type:ignore
+def df_read(
+    path: pathlib.Path, run_output_root: tp.Optional[pathlib.Path] = None, **kwargs
+) -> pd.DataFrame:
+    """
+    Read a pandas dataframe from an apache .arrow file.
+    """
+    return pd.read_feather(path)
+
+
+@retry(pd.errors.ParserError, tries=10, delay=0.100, backoff=1.1)  # type:ignore
+def df_write(df: pd.DataFrame, path: pathlib.Path, **kwargs) -> None:
+    """
+    Write a pandas dataframe to a apache .arrow file.
+    """
+    df.to_feather(path, **kwargs)

sierra/plugins/storage/csv/__init__.py

@@ -1,9 +1,18 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
+"""
+Container module for the CSV storage plugin.
+
+See :ref:`plugins/storage/csv`.
+"""
 
 # Core packages
 
 # 3rd party packages
 
 # Project packages
+
+
+def sierra_plugin_type() -> str:
+    return "pipeline"

sierra/plugins/storage/csv/plugin.py

@@ -1,12 +1,13 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
-"""Plugin for reading from CSV files when processing experimental run results.
-
+"""
+Plugin for reading/writing CSV files.
 """
 
 # Core packages
 import pathlib
+import typing as tp
 
 # 3rd party packages
 from retry import retry
@@ -15,14 +16,20 @@ import pandas as pd
 # Project packages
 
 
+def suffixes() -> tp.Set[str]:
+    return {".csv"}
+
+
 @retry(pd.errors.ParserError, tries=10, delay=0.100, backoff=1.1)  # type:ignore
-def df_read(path: pathlib.Path, **kwargs) -> pd.DataFrame:
+def df_read(
+    path: pathlib.Path, run_output_root: tp.Optional[pathlib.Path] = None, **kwargs
+) -> pd.DataFrame:
     """
     Read a dataframe from a CSV file using pandas.
     """
     # Always specify the datatype so pandas does not have to infer it--much
     # faster.
-    return pd.read_csv(path, sep=';', float_precision='high', **kwargs)
+    return pd.read_csv(path, sep=",", **kwargs)
 
 
 @retry(pd.errors.ParserError, tries=10, delay=0.100, backoff=1.1)  # type:ignore
@@ -30,4 +37,4 @@ def df_write(df: pd.DataFrame, path: pathlib.Path, **kwargs) -> None:
     """
     Write a dataframe to a CSV file using pandas.
     """
-    df.to_csv(path, sep=';', float_format='%.8f', **kwargs)
+    df.to_csv(path, sep=",", float_format="%.8f", **kwargs)

sierra/version.py

@@ -1,12 +1,13 @@
 # Copyright 2021 John Harwell, All rights reserved.
 #
 #  SPDX-License-Identifier: MIT
-"""The one and only place where the current SIERRA version is recorded."""
+"""Provide access to the SIERRA version from the pyproject.toml in source code."""
 
 # Core packages
+import importlib.metadata
 
 # 3rd party packages
 
 # Project packages
 
-__version__ = "1.3.11"
+__version__ = importlib.metadata.version("sierra_research")

sierra_research-1.5.0.dist-info/METADATA

@@ -0,0 +1,238 @@
+Metadata-Version: 2.4
+Name: sierra-research
+Version: 1.5.0
+Summary: Automation framework for the scientific method in R&D
+Project-URL: Homepage, https://github.com/jharwell/sierra
+Author-email: John Harwell <john.r.harwell@gmail.com>
+License: Copyright 2022 John Harwell
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of
+        this software and associated documentation files (the "Software"), to deal in
+        the Software without restriction, including without limitation the rights to
+        use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+        the Software, and to permit persons to whom the Software is furnished to do so,
+        subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+        FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+        COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+        IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+        CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+License-File: LICENSE
+Keywords: agent-based modeling,automation,parameter sweeps,reproducibility,research,reusability,robotics
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Python: >=3.9
+Requires-Dist: coloredlogs
+Requires-Dist: distro
+Requires-Dist: haggis
+Requires-Dist: hatchling
+Requires-Dist: holoviews
+Requires-Dist: hvplot
+Requires-Dist: implements
+Requires-Dist: jsonpath-ng
+Requires-Dist: matplotlib
+Requires-Dist: netifaces
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: prefect
+Requires-Dist: prefect[docker]
+Requires-Dist: psutil
+Requires-Dist: pyarrow
+Requires-Dist: pyyaml
+Requires-Dist: retry
+Requires-Dist: strictyaml
+Requires-Dist: sympy
+Description-Content-Type: text/x-rst
+
+===========================================================================
+SIERRA (reSearch pIpEline for Reproducibility, Reusability, and Automation)
+===========================================================================
+
+.. |pepy-downloads| image:: https://pepy.tech/badge/sierra-research
+                    :target: https://pepy.tech/project/sierra-research
+
+.. |pypi-version| image:: https://img.shields.io/pypi/v/sierra-research.svg
+                  :target: https://pypi.python.org/pypi/sierra-research/
+
+.. |supported-pythons| image:: https://img.shields.io/pypi/pyversions/sierra-research.svg
+
+.. |os-supported| image:: https://img.shields.io/badge/os-linux%20%7C%20macOS-blue
+
+.. |ci-analysis-master| image:: https://github.com/jharwell/sierra/actions/workflows/analysis-top.yml/badge.svg?branch=master
+.. |ci-coverage-master| image:: https://coveralls.io/repos/github/jharwell/sierra/badge.svg?branch=master
+
+.. |ci-analysis-devel| image:: https://github.com/jharwell/sierra/actions/workflows/analysis-top.yml/badge.svg?branch=devel
+.. |ci-coverage-devel| image:: https://coveralls.io/repos/github/jharwell/sierra/badge.svg?branch=devel
+
+.. |license| image:: https://img.shields.io/badge/License-MIT-blue.svg
+
+.. |doi| image:: https://zenodo.org/badge/125774567.svg
+         :target: https://zenodo.org/badge/latestdoi/125774567
+
+.. |docs| image:: https://readthedocs.org/projects/sierra/badge/?version=master
+          :target: https://sierra.readthedocs.io/en/master/
+
+.. |maintenance| image:: https://img.shields.io/badge/Maintained%3F-yes-green.svg
+
+
++---------------+--------------------------------------------------------------------+
+| Usage         | |pepy-downloads| |pypi-version| |supported-pythons| |os-supported| |
++---------------+--------------------------------------------------------------------+
+| Release       | |ci-analysis-master| |ci-coverage-master|                          |
++---------------+--------------------------------------------------------------------+
+| Development   | |ci-analysis-devel| |ci-coverage-devel|                            |
++---------------+--------------------------------------------------------------------+
+| Miscellaneous |    |license| |doi| |docs| |maintenance|                            |
++---------------+--------------------------------------------------------------------+
+
+
+TL;DR
+=====
+
+What is SIERRA? See `What is SIERRA?`_
+
+Why should you use SIERRA? See `Why SIERRA?`_
+
+To install SIERRA (requires python 3.9+):
+
+::
+
+   pip3 install sierra-research
+
+
+SIERRA requires a recent OSX (tested with 13+) or Linux (tested with ubuntu
+20.04+) and python >= 3.9. For more details, including the requirements for
+project code, see the `SIERRA requirements
+<https://sierra.readthedocs.io/en/master/src/requirements.html>`_.
+
+To get started using SIERRA, see `getting started
+<https://sierra.readthedocs.io/en/master/src/getting_started.html>`_.
+
+Want to cite SIERRA? See `Citing`_.
+
+Have an issue using SIERRA? See `Troubleshooting`_.
+
+What is SIERRA?
+===============
+
+.. figure:: https://raw.githubusercontent.com/jharwell/sierra/master/docs/figures/architecture.png
+
+   SIERRA architecture, organized by pipeline stage, left to right. High-level
+   inputs/outputs and active plugins and shown for each stage. “...”  indicates
+   areas of further extensibility and customization via new plugins. “Host
+   machine” indicates the machine SIERRA was invoked on. The active plugins in
+   each stage and what they cumulatively enable are highlighted in red.
+
+SIERRA is a command line tool and plugin framework for:
+
+- Automating R&D, providing faculties for seamless experiment
+  generation, execution, and results processing.
+
+- Accelerating R&D cycles by allowing researchers/developers to focus on the
+  “science” aspects: developing new things and designing experiments to test
+  them, rather than the engineering aspects (writing scripts, configuring
+  environments, etc.).
+
+- Improving the reproducibility of scientific research, particularly in AI.
+
+Why SIERRA?
+===========
+
+- It changes the paradigm of the engineering tasks researchers must perform
+  from manual and procedural to declarative and automated. That is, from::
+
+    "I need to perform these steps to run the experiment, process the data and
+    generate the graphs I want."
+
+  to::
+
+    "Here is the environment and simulator/platforms(s) I want to use, the
+    deliverables I want to generate, and the data I want to appear on them for
+    my research query--GO!"
+
+  Essentially, SIERRA handles the “engineering” parts of research on the
+  backend, acting as a compiler of sorts, turning research queries into
+  executable objects, running the "compiled" experiments, and processing results
+  into visualizations or other deliverables.
+
+- It has deep support for arbitrary parameter sweeps: numeric, categorical, or
+  any combination thereof.
+
+- It supports a wide range of execution engines/environments, and experiment
+  input/output formats via plugins. SIERRA supports mix-and-match between all
+  plugin types, subject to restrictions within the plugins themselves. This is
+  and makes it very easy to run experiments on different hardware, targeting
+  different simulators, generating different outputs, etc., all with little to
+  no configuration changes by the user.
+
+- SIERRA maximizes reusability of code and configuration; it is designed so that
+  *no* copy-pasting is ever needed, improving code quality with no additional
+  effort from users.
+
+- SIERRA has a rich model framework allowing you to run arbitrary models,
+  generate data, and plot it on the same figure as empirical
+  results--automatically.
+
+- Why use SIERRA over something like `prefect <https://www.prefect.io>`_,
+  `dagster <https://www.dagster.io>`_, or `airflow
+  <https://airflow.apache.org>`_ ? Briefly, because SIERRA provides a common
+  pipeline which is tested and can accommodate most use cases; SIERRA is not as
+  feature complete as these other frameworks, though. For most use cases (but
+  not all), that delta doesn't matter. In addition, with the other frameworks,
+  you have to create your own pipelines from scratch.
+
+Not sure if SIERRA makes sense for you? Check out some of the `use cases
+<https://sierra.readthedocs.io/en/master/src/use-cases.html>`_ for which SIERRA
+was designed.  If aspects of any sound familiar, then there is a strong chance
+SIERRA could help you! See the `SIERRA docs
+<https://sierra.readthedocs.io/en/master/>`_ to get started.
+
+Citing
+======
+
+If you use SIERRA and have found it helpful, please cite the following paper::
+
+  @inproceedings{Harwell2022a-SIERRA,
+  author = {Harwell, John and Lowmanstone, London and Gini, Maria},
+  title = {SIERRA: A Modular Framework for Research Automation},
+  year = {2022},
+  isbn = {9781450392136},
+  publisher = {International Foundation for Autonomous Agents and Multiagent Systems},
+  booktitle = {Proceedings of the 21st International Conference on Autonomous Agents and Multiagent Systems},
+  pages = {1905–1907}
+  }
+
+You can also cite the specific version of SIERRA used with the DOI at the top of
+this page, to help facilitate reproducibility.
+
+Troubleshooting
+===============
+
+If you have problems using SIERRA, please open an issue or post in the Github
+forum and I'll be happy to help you work through it.
+
+Contributing
+============
+
+I welcome all types of contributions, no matter how large or how small, and if
+you have an idea, I'm happy to talk about it at any point :-). See the
+`contributing guide
+<https://sierra.readthedocs.io/en/master/src/contributing.html>`_ for the
+general procedure.