matensemble 0.3.0__tar.gz → 0.3.4__tar.gz

{matensemble-0.3.0 → matensemble-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matensemble
-Version: 0.3.0
+Version: 0.3.4
 Summary: An adaptive and highly asynchronous ensemble simulation workflow manager MatEnsemble (https://github.com/Q-CAD/MatEnsemble) built jointly on top of the hierarchical graph based scheduler FLUX and concurrent-futures infrastructure of python
 Author: Soumendu Bagchi, Kaleb Duchesneau
 Author-email: Soumendu Bagchi <soumendubagchi@gmail.com>, Kaleb Duchesneau <kalebduchesneau@gmail.com>
@@ -53,17 +53,23 @@ An optional in-tree **dynopro** stack supports streaming dynamics and on-the-fly
 </p>
 
 
+## Installation
+
 OCI images are published to GitHub Container Registry
 
 `ghcr.io/freddude2004/matensemble:baseline-vX.Y.Z`
 
 See the [container packages](https://github.com/FredDude2004/MatEnsemble/pkgs/container/matensemble) and the [Quick start](https://matensemble.readthedocs.io/en/latest/quickstart.html) in the docs for Apptainer/Singularity and site-specific notes.
 
-### Development install
+### Anaconda
+
+You can build a Conda environment with MatEnsemble and dependencies installed using the environment.yaml file.
 
-<!-- TODO: Add instructions for how to install and run here -->
+```bash
+conda env create -f environment.yaml
+```
 
-## Quick example
+## Example
 
 ```python
 from matensemble.pipeline import Pipeline

{matensemble-0.3.0 → matensemble-0.3.4}/README.md

@@ -30,17 +30,23 @@ An optional in-tree **dynopro** stack supports streaming dynamics and on-the-fly
 </p>
 
 
+## Installation
+
 OCI images are published to GitHub Container Registry
 
 `ghcr.io/freddude2004/matensemble:baseline-vX.Y.Z`
 
 See the [container packages](https://github.com/FredDude2004/MatEnsemble/pkgs/container/matensemble) and the [Quick start](https://matensemble.readthedocs.io/en/latest/quickstart.html) in the docs for Apptainer/Singularity and site-specific notes.
 
-### Development install
+### Anaconda
+
+You can build a Conda environment with MatEnsemble and dependencies installed using the environment.yaml file.
 
-<!-- TODO: Add instructions for how to install and run here -->
+```bash
+conda env create -f environment.yaml
+```
 
-## Quick example
+## Example
 
 ```python
 from matensemble.pipeline import Pipeline

{matensemble-0.3.0 → matensemble-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "matensemble"
-version = "0.3.0"
+version = "0.3.4"
 description = "An adaptive and highly asynchronous ensemble simulation workflow manager MatEnsemble (https://github.com/Q-CAD/MatEnsemble) built jointly on top of the hierarchical graph based scheduler FLUX and concurrent-futures infrastructure of python"
 readme = "README.md"
 license-files = ["LICENSE"]

{matensemble-0.3.0 → matensemble-0.3.4}/src/matensemble/TODO.md

@@ -171,6 +171,17 @@
 - [x] Convert Scaffold into PowerPoint Presentation
 - [x] Email coordinator about length of presentation and audience
 
+# === May 29, 2026 ===
+- [x] Fix the strategy error
+- [x] Make sure conda env is working
+- [ ] Make sure all containers are working and stable
+    * Frontier
+    * Perlmutter
+    * Pathfinder
+    * Linux
+- [ ] Update tutorials for all systems
+- [ ] Make all figures for JOSS paper and explain all of them
+
 ## === AFTER EVERYTHING ABOVE IS DONE AND STABLE ===
 
 ## --- Model Context Protocol ---
@@ -197,12 +208,12 @@
 
 ## --- Create first draft for JOSS ---
 - [ ] Read some example papers
-- [ ] Create draft and show Dr. Bagchi
-- [ ] Polish the repository to be ready for review
+- [x] Create draft and show Dr. Bagchi
+- [x] Polish the repository to be ready for review
 - [ ] Make sure that the tests work
 - [ ] Make sure that the example workflows work correctly
-- [ ] Make sure that they can easily test the code and
-- [ ] Create a conda package that they can easily test the code without having
+- [ ] Make sure that they can easily test the code
+- [x] Create a conda package that they can easily test the code without having
       to compile flux and flux-sched themselves
 
 ## --- Reading List ---

{matensemble-0.3.0 → matensemble-0.3.4}/src/matensemble/pipeline.py

@@ -133,7 +133,7 @@ class Pipeline:
         the module will use to find the *pickled* python object containing all
         of the data on the chore, and it will use it to import the function and
         call it with its respective arguments. The result will then be stored
-        in the flux KVS
+        in the chores respective directory.
 
         Parameters
         ----------
@@ -228,7 +228,79 @@ class Pipeline:
 
         return decorator
 
-    # def strategy_callback(self, ) -> Chore | None:
+    # WARNING: This is deprecated and only here for compatibility
+    def add_user_strat(self, chore_name: str, bolo_list: list[str]):
+        self._strategy_spec = {"name": chore_name, "bolo_list": bolo_list}
+
+    def strategy(
+        self,
+        bolo_list: list[str],
+        name: str | None = None,
+        num_tasks: int = 1,
+        cores_per_task: int = 1,
+        gpus_per_task: int = 0,
+        mpi: bool = False,
+        env: dict[str, str] | None = None,
+        inherit_env: bool = True,
+    ):
+        """
+        The strategy function creates a strategy, which is essentially a
+        callback function to another chore. But the callback function is itself
+        a chore. This function is expected to return an :obj:`ChoreSpec` which
+        will then dynamically spawn a new chore into the queue based on the
+        specification that is returned.
+
+        Parameters
+        ----------
+        bolo_list : list[str]
+            The names of the chores that you want this to callback on
+        name : str, optional
+            The name  that will be assigned to the chore_id, defaults to the
+            name of the function.
+        num_tasks : int, optional
+            The number of tasks that will be launched with flux, defaults to 1
+        cores_per_task : int, optional
+            The number of CPU cores that are required to submit the chore,
+            defaults to 1
+        gpus_per_task : int, optional
+            The number of GPUs that are required to submit the chore, defaults
+            to 0
+        mpi : bool, optional
+            When True, sets Flux shell option ``mpi=pmi2`` on the chorespec
+            (default False).
+        env : dict[str, str], optional
+            Extra environment variables for the task. For Python chores,
+            ``PYTHONPATH`` is merged to include the workflow parent directory.
+        inherit_env : bool
+            If True (default), the Flux jobspec starts from the submitting
+            process environment and applies ``env`` overrides.
+
+        Returns
+        -------
+        Callable
+            A dummy function that just prints a warning to the stdout. The
+            actual function is stored in the registry
+        """
+
+        def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+            for chore in bolo_list:
+                if chore not in self._registry:
+                    raise Exception(
+                        f"Error: The chore '{chore}' was not found in the registry"
+                    )
+
+            registry_key = name or str(func.__qualname__)
+            self._registry[registry_key] = func
+
+            def disabled_wrapper(*args: Any, **kwargs: Any) -> None:
+                raise RuntimeError(
+                    f"Do not call '{registry_key}' directly. "
+                    "This strategy is managed internally by the workflow engine."
+                )
+
+            return disabled_wrapper
+
+        return decorator
 
     def exec(
         self,
@@ -532,9 +604,6 @@ class Pipeline:
                 self._finished = True
             return self._collect_results()
 
-    def add_user_strat(self, chore_name: str, bolo_list: list[str]):
-        self._strategy_spec = {"name": chore_name, "bolo_list": bolo_list}
-
     def graph(self) -> nx.DiGraph:
         return self._create_graph()
 

{matensemble-0.3.0 → matensemble-0.3.4}/src/matensemble/strategy.py

@@ -224,9 +224,6 @@ class NonAdaptiveStrategy(FutureProcessingStrategy):
             return
 
 
-# TODO: Make the strategy look through the bolo list and spawn a new chore with the output of another
-#       and then make sure that the chore that is acting as a processing strat is a on the bolo list
-#       and if that is done then you need to get the results and spawn a new chore from the ChoreSpec
 class UserStrategy(FutureProcessingStrategy):
     def __init__(self, manager, pipeline, processing_chore, bolo_list) -> None:
         super().__init__(manager)
@@ -314,12 +311,13 @@ class UserStrategy(FutureProcessingStrategy):
                     # untrusted paths or third-party producers.
                     with (chore.workdir / "result.pickle").open("rb") as f:
                         chore_spec = pickle.load(f)
-                    new_chore, new_out = self.pipeline._spawn_chore_from_spec(
-                        chore_spec
-                    )
-                    self.pipeline._admit_spawned_chore(
-                        new_chore, new_out, self.manager
-                    )
+                    if chore_spec:
+                        new_chore, new_out = self.pipeline._spawn_chore_from_spec(
+                            chore_spec
+                        )
+                        self.pipeline._admit_spawned_chore(
+                            new_chore, new_out, self.manager
+                        )
                 except Exception as e:
                     self.manager._logger.exception(
                         f"FAILED TO SPAWN CHORE: chore={self.proc_chore} | due the following Exception ->\n{e}"