hpcflow-new2 0.2.0a176__py3-none-any.whl → 0.2.0a178__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a176"
+__version__ = "0.2.0a178"

hpcflow/sdk/cli.py

@@ -31,8 +31,13 @@ from hpcflow.sdk.cli_common import (
     zip_path_opt,
     zip_overwrite_opt,
     zip_log_opt,
+    zip_include_execute_opt,
+    zip_include_rechunk_backups_opt,
     unzip_path_opt,
     unzip_log_opt,
+    rechunk_backup_opt,
+    rechunk_chunk_size_opt,
+    rechunk_status_opt,
 )
 from hpcflow.sdk.helper.cli import get_helper_CLI
 from hpcflow.sdk.log import TimeIt
@@ -425,11 +430,28 @@ def _make_workflow_CLI(app):
     @zip_path_opt
     @zip_overwrite_opt
     @zip_log_opt
+    @zip_include_execute_opt
+    @zip_include_rechunk_backups_opt
     @click.pass_context
-    def zip_workflow(ctx, path, overwrite, log):
+    def zip_workflow(
+        ctx,
+        path,
+        overwrite,
+        log,
+        include_execute,
+        include_rechunk_backups,
+    ):
         """Generate a copy of the workflow in the zip file format in the current working
         directory."""
-        click.echo(ctx.obj["workflow"].zip(path=path, overwrite=overwrite, log=log))
+        click.echo(
+            ctx.obj["workflow"].zip(
+                path=path,
+                overwrite=overwrite,
+                log=log,
+                include_execute=include_execute,
+                include_rechunk_backups=include_rechunk_backups,
+            )
+        )
 
     @workflow.command(name="unzip")
     @unzip_path_opt
@@ -440,6 +462,37 @@ def _make_workflow_CLI(app):
         current working directory."""
         click.echo(ctx.obj["workflow"].unzip(path=path, log=log))
 
+    @workflow.command(name="rechunk")
+    @rechunk_backup_opt
+    @rechunk_chunk_size_opt
+    @rechunk_status_opt
+    @click.pass_context
+    def rechunk(ctx, backup, chunk_size, status):
+        """Rechunk metadata/runs and parameters/base arrays."""
+        ctx.obj["workflow"].rechunk(backup=backup, chunk_size=chunk_size, status=status)
+
+    @workflow.command(name="rechunk-runs")
+    @rechunk_backup_opt
+    @rechunk_chunk_size_opt
+    @rechunk_status_opt
+    @click.pass_context
+    def rechunk_runs(ctx, backup, chunk_size, status):
+        """Rechunk the metadata/runs array."""
+        ctx.obj["workflow"].rechunk_runs(
+            backup=backup, chunk_size=chunk_size, status=status
+        )
+
+    @workflow.command(name="rechunk-parameter-base")
+    @rechunk_backup_opt
+    @rechunk_chunk_size_opt
+    @rechunk_status_opt
+    @click.pass_context
+    def rechunk_parameter_base(ctx, backup, chunk_size, status):
+        """Rechunk the parameters/base array."""
+        ctx.obj["workflow"].rechunk_parameter_base(
+            backup=backup, chunk_size=chunk_size, status=status
+        )
+
     workflow.help = workflow.help.format(app_name=app.name)
 
     workflow.add_command(_make_workflow_submission_CLI(app))
@@ -711,8 +764,18 @@ def _make_zip_CLI(app):
     @zip_path_opt
     @zip_overwrite_opt
     @zip_log_opt
+    @zip_include_execute_opt
+    @zip_include_rechunk_backups_opt
     @workflow_ref_type_opt
-    def zip_workflow(workflow_ref, path, overwrite, log, ref_type):
+    def zip_workflow(
+        workflow_ref,
+        path,
+        overwrite,
+        log,
+        include_execute,
+        include_rechunk_backups,
+        ref_type,
+    ):
         """Generate a copy of the specified workflow in the zip file format in the
         current working directory.
 
@@ -721,7 +784,15 @@ def _make_zip_CLI(app):
         """
         workflow_path = app._resolve_workflow_reference(workflow_ref, ref_type)
         wk = app.Workflow(workflow_path)
-        click.echo(wk.zip(path=path, overwrite=overwrite, log=log))
+        click.echo(
+            wk.zip(
+                path=path,
+                overwrite=overwrite,
+                log=log,
+                include_execute=include_execute,
+                include_rechunk_backups=include_rechunk_backups,
+            )
+        )
 
     return zip_workflow
 
@@ -760,6 +831,27 @@ def _make_cancel_CLI(app):
     return cancel
 
 
+def _make_rechunk_CLI(app):
+    @click.command(name="rechunk")
+    @click.argument("workflow_ref")
+    @workflow_ref_type_opt
+    @rechunk_backup_opt
+    @rechunk_chunk_size_opt
+    @rechunk_status_opt
+    def rechunk(workflow_ref, ref_type, backup, chunk_size, status):
+        """Rechunk metadata/runs and parameters/base arrays.
+
+        WORKFLOW_REF is the local ID (that provided by the `show` command}) or the
+        workflow path.
+
+        """
+        workflow_path = app._resolve_workflow_reference(workflow_ref, ref_type)
+        wk = app.Workflow(workflow_path)
+        wk.rechunk(backup=backup, chunk_size=chunk_size, status=status)
+
+    return rechunk
+
+
 def _make_open_CLI(app):
     @click.group(name="open")
     def open_file():
@@ -1129,6 +1221,7 @@ def make_cli(app):
     new_CLI.add_command(_make_cancel_CLI(app))
     new_CLI.add_command(_make_zip_CLI(app))
     new_CLI.add_command(_make_unzip_CLI(app))
+    new_CLI.add_command(_make_rechunk_CLI(app))
     for cli_cmd in _make_API_CLI(app):
         new_CLI.add_command(cli_cmd)
 

hpcflow/sdk/cli_common.py

@@ -141,6 +141,8 @@ zip_overwrite_opt = click.option(
     help="If set, any existing file will be overwritten.",
 )
 zip_log_opt = click.option("--log", help="Path to a log file to use during zipping.")
+zip_include_execute_opt = click.option("--include-execute", is_flag=True)
+zip_include_rechunk_backups_opt = click.option("--include-rechunk-backups", is_flag=True)
 unzip_path_opt = click.option(
     "--path",
     default=".",
@@ -151,3 +153,23 @@ unzip_path_opt = click.option(
     ),
 )
 unzip_log_opt = click.option("--log", help="Path to a log file to use during unzipping.")
+rechunk_backup_opt = click.option(
+    "--backup/--no-backup",
+    default=True,
+    help=("First copy a backup of the array to a directory ending in `.bak`."),
+)
+rechunk_chunk_size_opt = click.option(
+    "--chunk-size",
+    type=click.INT,
+    default=None,
+    help=(
+        "New chunk size (array items per chunk). If unset (as by default), the array "
+        "will be rechunked to a single chunk array (i.e with a chunk size equal to the "
+        "array's shape)."
+    ),
+)
+rechunk_status_opt = click.option(
+    "--status/--no-status",
+    default=True,
+    help="If True, display a live status to track rechunking progress.",
+)

hpcflow/sdk/core/cache.py

@@ -0,0 +1,142 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Set, Dict
+
+from hpcflow.sdk.log import TimeIt
+
+
+@dataclass
+class DependencyCache:
+    """Class to bulk-retrieve dependencies between elements, iterations, and runs."""
+
+    run_dependencies: Dict[int, Set]
+    run_dependents: Dict[int, Set]
+    iter_run_dependencies: Dict[int, Set]
+    iter_iter_dependencies: Dict[int, Set]
+    elem_iter_dependencies: Dict[int, Set]
+    elem_elem_dependencies: Dict[int, Set]
+    elem_elem_dependents: Dict[int, Set]
+    elem_elem_dependents_rec: Dict[int, Set]
+
+    elements: Dict
+    iterations: Dict
+
+    @classmethod
+    @TimeIt.decorator
+    def build(cls, workflow):
+        num_iters = workflow.num_element_iterations
+        num_elems = workflow.num_elements
+        num_runs = workflow.num_EARs
+
+        all_store_runs = workflow._store.get_EARs(list(range(num_runs)))
+        all_store_iters = workflow._store.get_element_iterations(list(range(num_iters)))
+        all_store_elements = workflow._store.get_elements(list(range(num_elems)))
+        all_param_sources = workflow.get_all_parameter_sources()
+        all_data_idx = [
+            {
+                k: v if isinstance(v, list) else [v]
+                for k, v in i.data_idx.items()
+                if k not in ("repeats.",)
+            }
+            for i in all_store_runs
+        ]
+
+        # run dependencies and dependents
+        run_dependencies = {}
+        run_dependents = defaultdict(set)
+        for idx, i in enumerate(all_data_idx):
+            run_i_sources = set()
+            for j in i.values():
+                for k in j:
+                    run_k = all_param_sources[k].get("EAR_ID")
+                    if run_k is not None and run_k != idx:
+                        run_i_sources.add(run_k)
+            run_dependencies[idx] = run_i_sources
+            for m in run_i_sources:
+                run_dependents[m].add(idx)
+
+        # add missing:
+        for k in range(num_runs):
+            run_dependents[k]
+
+        run_dependents = dict(run_dependents)
+
+        # iteration dependencies
+        all_iter_run_IDs = {
+            i.id_: [k for j in i.EAR_IDs.values() for k in j] for i in all_store_iters
+        }
+        # for each iteration, which runs does it depend on?
+        iter_run_dependencies = {
+            k: set(j for i in v for j in run_dependencies[i])
+            for k, v in all_iter_run_IDs.items()
+        }
+
+        # for each run, which iteration does it belong to?
+        all_run_iter_IDs = {}
+        for iter_ID, run_IDs in all_iter_run_IDs.items():
+            for run_ID in run_IDs:
+                all_run_iter_IDs[run_ID] = iter_ID
+
+        # for each iteration, which iterations does it depend on?
+        iter_iter_dependencies = {
+            k: set(all_run_iter_IDs[i] for i in v)
+            for k, v in iter_run_dependencies.items()
+        }
+
+        all_elem_iter_IDs = {i.id_: i.iteration_IDs for i in all_store_elements}
+
+        elem_iter_dependencies = {
+            k: set(j for i in v for j in iter_iter_dependencies[i])
+            for k, v in all_elem_iter_IDs.items()
+        }
+
+        # for each iteration, which element does it belong to?
+        all_iter_elem_IDs = {}
+        for elem_ID, iter_IDs in all_elem_iter_IDs.items():
+            for iter_ID in iter_IDs:
+                all_iter_elem_IDs[iter_ID] = elem_ID
+
+        # element dependencies
+        elem_elem_dependencies = {
+            k: set(all_iter_elem_IDs[i] for i in v)
+            for k, v in elem_iter_dependencies.items()
+        }
+
+        # for each element, which elements depend on it (directly)?
+        elem_elem_dependents = defaultdict(set)
+        for k, v in elem_elem_dependencies.items():
+            for i in v:
+                elem_elem_dependents[i].add(k)
+
+        # for each element, which elements depend on it (recursively)?
+        elem_elem_dependents_rec = defaultdict(set)
+        for k in list(elem_elem_dependents):
+            for i in elem_elem_dependents[k]:
+                elem_elem_dependents_rec[k].add(i)
+                elem_elem_dependents_rec[k].update(
+                    {m for m in elem_elem_dependents[i] if m != k}
+                )
+
+        # add missing keys:
+        for k in range(num_elems):
+            elem_elem_dependents[k]
+            elem_elem_dependents_rec[k]
+
+        elem_elem_dependents = dict(elem_elem_dependents)
+        elem_elem_dependents_rec = dict(elem_elem_dependents_rec)
+
+        elements = workflow.get_all_elements()
+        iterations = workflow.get_all_element_iterations()
+
+        return cls(
+            run_dependencies=run_dependencies,
+            run_dependents=run_dependents,
+            iter_run_dependencies=iter_run_dependencies,
+            iter_iter_dependencies=iter_iter_dependencies,
+            elem_iter_dependencies=elem_iter_dependencies,
+            elem_elem_dependencies=elem_elem_dependencies,
+            elem_elem_dependents=elem_elem_dependents,
+            elem_elem_dependents_rec=elem_elem_dependents_rec,
+            elements=elements,
+            iterations=iterations,
+        )

hpcflow/sdk/core/element.py

@@ -675,6 +675,7 @@ class ElementIteration:
             default=default,
         )
 
+    @TimeIt.decorator
     def get_EAR_dependencies(
         self,
         as_objects: Optional[bool] = False,
@@ -708,6 +709,7 @@ class ElementIteration:
             out = self.workflow.get_EARs_from_IDs(out)
         return out
 
+    @TimeIt.decorator
     def get_element_iteration_dependencies(
         self, as_objects: bool = False
     ) -> List[Union[int, app.ElementIteration]]:
@@ -719,6 +721,7 @@ class ElementIteration:
             out = self.workflow.get_element_iterations_from_IDs(out)
         return out
 
+    @TimeIt.decorator
     def get_element_dependencies(
         self,
         as_objects: Optional[bool] = False,
@@ -769,6 +772,7 @@ class ElementIteration:
 
         return out
 
+    @TimeIt.decorator
     def get_dependent_EARs(
         self, as_objects: bool = False
     ) -> List[Union[int, app.ElementActionRun]]:
@@ -793,6 +797,7 @@ class ElementIteration:
 
         return deps
 
+    @TimeIt.decorator
     def get_dependent_element_iterations(
         self, as_objects: bool = False
     ) -> List[Union[int, app.ElementIteration]]:
@@ -816,6 +821,7 @@ class ElementIteration:
 
         return deps
 
+    @TimeIt.decorator
     def get_dependent_elements(
         self,
         as_objects: bool = False,
@@ -1246,6 +1252,7 @@ class Element:
         """Get tasks that depend on the most recent iteration of this element."""
         return self.latest_iteration.get_dependent_tasks(as_objects=as_objects)
 
+    @TimeIt.decorator
     def get_dependent_elements_recursively(self, task_insert_ID=None):
         """Get downstream elements that depend on this element, including recursive
         dependencies.