executorlib 1.9.4__py3-none-any.whl → 1.10.0__py3-none-any.whl

executorlib/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.9.4'
-__version_tuple__ = version_tuple = (1, 9, 4)
+__version__ = version = '1.10.0'
+__version_tuple__ = version_tuple = (1, 10, 0)
 
 __commit_id__ = commit_id = None

executorlib/standalone/hdf.py

@@ -38,7 +38,10 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
                     with contextlib.suppress(ValueError):
                         fname.create_dataset(
                             name="/" + group_dict[data_key],
-                            data=np.void(cloudpickle.dumps(data_value)),
+                            data=np.frombuffer(
+                                cloudpickle.dumps(data_value), dtype=np.uint8
+                            ),
+                            compression="gzip",
                         )
 
 
@@ -55,26 +58,26 @@ def load(file_name: str) -> dict:
     with h5py.File(file_name, "r") as hdf:
         data_dict = {}
         if "function" in hdf:
-            data_dict["fn"] = cloudpickle.loads(np.void(hdf["/function"]))
+            data_dict["fn"] = cloudpickle.loads(hdf["/function"][()].tobytes())
         else:
             raise TypeError("Function not found in HDF5 file.")
         if "input_args" in hdf:
-            data_dict["args"] = cloudpickle.loads(np.void(hdf["/input_args"]))
+            data_dict["args"] = cloudpickle.loads(hdf["/input_args"][()].tobytes())
         else:
             data_dict["args"] = ()
         if "input_kwargs" in hdf:
-            data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"]))
+            data_dict["kwargs"] = cloudpickle.loads(hdf["/input_kwargs"][()].tobytes())
         else:
             data_dict["kwargs"] = {}
         if "resource_dict" in hdf:
             data_dict["resource_dict"] = cloudpickle.loads(
-                np.void(hdf["/resource_dict"])
+                hdf["/resource_dict"][()].tobytes()
             )
         else:
             data_dict["resource_dict"] = {}
         if "error_log_file" in hdf:
             data_dict["error_log_file"] = cloudpickle.loads(
-                np.void(hdf["/error_log_file"])
+                hdf["/error_log_file"][()].tobytes()
             )
         return data_dict
 
@@ -93,9 +96,9 @@ def get_output(file_name: str) -> tuple[bool, bool, Any]:
     def get_output_helper(file_name: str) -> tuple[bool, bool, Any]:
         with h5py.File(file_name, "r") as hdf:
             if "output" in hdf:
-                return True, True, cloudpickle.loads(np.void(hdf["/output"]))
+                return True, True, cloudpickle.loads(hdf["/output"][()].tobytes())
             elif "error" in hdf:
-                return True, False, cloudpickle.loads(np.void(hdf["/error"]))
+                return True, False, cloudpickle.loads(hdf["/error"][()].tobytes())
             else:
                 return False, False, None
 
@@ -122,7 +125,7 @@ def get_runtime(file_name: str) -> float:
     """
     with h5py.File(file_name, "r") as hdf:
         if "runtime" in hdf:
-            return cloudpickle.loads(np.void(hdf["/runtime"]))
+            return cloudpickle.loads(hdf["/runtime"][()].tobytes())
         else:
             return 0.0
 
@@ -140,7 +143,7 @@ def get_queue_id(file_name: Optional[str]) -> Optional[int]:
     if file_name is not None and os.path.exists(file_name):
         with h5py.File(file_name, "r") as hdf:
             if "queue_id" in hdf:
-                return cloudpickle.loads(np.void(hdf["/queue_id"]))
+                return cloudpickle.loads(hdf["/queue_id"][()].tobytes())
     return None
 
 
@@ -230,7 +233,7 @@ def _get_content_of_file(file_name: str) -> dict:
     """
     with h5py.File(file_name, "r") as hdf:
         return {
-            key: cloudpickle.loads(np.void(hdf["/" + key]))
+            key: cloudpickle.loads(hdf["/" + key][()].tobytes())
             for key in group_dict.values()
             if key in hdf
         }

{executorlib-1.9.4.dist-info → executorlib-1.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.9.4
+Version: 1.10.0
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -218,6 +218,25 @@ given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primar
 recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux) 
 as hierarchical job scheduler within the allocations. 
 
+## Which Executor should I use?
+executorlib provides five `Executor` classes. They all share the same `submit()` / `map()` interface and only differ in
+*where* the Python functions are executed and *how* the resources are requested. A common workflow is to develop and 
+test with the `SingleNodeExecutor` on a laptop and then switch to one of the HPC executors by changing only the class 
+name:
+
+| Executor | Where it runs | Scheduler command | Best for |
+|---|---|---|---|
+| [`SingleNodeExecutor`](https://executorlib.readthedocs.io/en/latest/1-single-node.html) | laptop, workstation or single compute node | `subprocess` | developing and testing a workflow |
+| [`SlurmClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm) | HPC login node | `sbatch` (one job per function) | long-running functions that should outlive the Python session |
+| [`SlurmJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm) | inside a SLURM allocation | `srun` (job steps) | many functions within one existing allocation |
+| [`FluxClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux) | HPC login node or Flux instance | `flux submit` | long-running functions; disconnecting and reconnecting |
+| [`FluxJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux) | inside a Flux allocation | `flux run` | high-throughput execution of many short functions |
+
+The **Cluster** executors submit each Python function as an individual job and communicate via the file system, so the
+Python process which created the executor can be closed and the results reloaded later. The **Job** executors run inside
+an existing allocation and communicate via sockets, which has lower overhead and is the better choice for many short
+function calls.
+
 ## Documentation
 * [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
   * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
@@ -243,6 +262,9 @@ as hierarchical job scheduler within the allocations.
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
   * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)
+  * [Disconnecting and Reconnecting](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#disconnecting-and-reconnecting)
+  * [Combine both](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#combine-both)
+  * [Cleaning Cache](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#cleaning-cache)
 * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm)
   * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
@@ -250,6 +272,12 @@ as hierarchical job scheduler within the allocations.
 * [Application](https://executorlib.readthedocs.io/en/latest/application.html)
   * [GPAW](https://executorlib.readthedocs.io/en/latest/4-1-gpaw.html)
   * [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/4-2-quantum-espresso.html)
+* [Integration](https://executorlib.readthedocs.io/en/latest/coupling.html)
+  * [emcee](https://executorlib.readthedocs.io/en/latest/coupling.html#emcee)
+  * [pipefunc](https://executorlib.readthedocs.io/en/latest/coupling.html#pipefunc)
+  * [omp4py](https://executorlib.readthedocs.io/en/latest/coupling.html#omp4py)
+  * [pylammpsmpi](https://executorlib.readthedocs.io/en/latest/coupling.html#pylammpsmpi)
+  * [General Pattern](https://executorlib.readthedocs.io/en/latest/coupling.html#general-pattern)
 * [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
   * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
   * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)

{executorlib-1.9.4.dist-info → executorlib-1.10.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 executorlib/__init__.py,sha256=IY_DkT706LPIv2_iIe3whrQNGoDzaDFYBvXyUepoZpw,4780
-executorlib/_version.py,sha256=VdLE9NNjOWcm4_Kh_W9w_lFN3bh0oesrwyTL3nQnUKg,520
+executorlib/_version.py,sha256=w2O97JYq0sLxpegu_TQkAiqEbJF97ZGrOe1EIpfRGZA,522
 executorlib/api.py,sha256=F6ibDJnVcPGxxPe-UScFJHprWHiVsETGhgAvS1T-9ZI,1365
 executorlib/backend/__init__.py,sha256=2b_iwds4VUrJe92nKtQXkoGxkJiXeEDSlbvCRN_-de8,456
 executorlib/backend/cache_parallel.py,sha256=ldb2iYzrhcXJZ0LcH75FpTLO_6F51mleU9JGAilF00k,2020
@@ -15,7 +15,7 @@ executorlib/standalone/__init__.py,sha256=n_Q4NUFu35Fn0qlWIhmoC_FyUkmFGlWn1cNc_7
 executorlib/standalone/batched.py,sha256=o3TtsFJir8nP1O_NZFfW0Q78CoZAFgHMyroWlg520nE,1446
 executorlib/standalone/command.py,sha256=4Xndog84zJdku1ke1DwdUamkWvzRrMz-VDCi5cRXi0M,6548
 executorlib/standalone/error.py,sha256=aaMQoRsoLzLCbYT-E4LZxGM9pm_AhjMesRjVt0d_LDk,714
-executorlib/standalone/hdf.py,sha256=nE2mTH1U8O7sm2TPJkfRU0uEgWJOhLnYxI6yZord8Eo,7623
+executorlib/standalone/hdf.py,sha256=dRePOCYXbPG_i__ASUcD4qs5AAU2fPbP5o_5ytwaOjw,7805
 executorlib/standalone/inputcheck.py,sha256=kSWUzqdMOhILYXW4Isumn1EGH3SegxPUMWRkUWfEleU,10041
 executorlib/standalone/queue.py,sha256=KFEBFQXGvcaf5qcuCjT5kaZg7ucWOTGOtrD96tIBlq0,589
 executorlib/standalone/scheduler.py,sha256=LhCuIdaw9BjrP3GerbHUIqwsvxERdtH5JQoeyAqASIU,2325
@@ -44,7 +44,7 @@ executorlib/task_scheduler/interactive/shared.py,sha256=-krntmZFOOE5U0HdFwuU8JZP
 executorlib/task_scheduler/interactive/spawner_flux.py,sha256=M7CPWvzY7MDkplJ4awymh7WbMYwKsDPTKxA2Op9wfNo,7075
 executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=96NqUfoIBUFd7KjY02dpIL9ZoS_u5beRuNw_1FUKH6Q,10393
 executorlib/task_scheduler/interactive/spawner_slurm.py,sha256=K5mj63PE8FYbkw9WWm4uEjoA5q0HM42z1d2wjAmO5YA,3713
-executorlib-1.9.4.dist-info/METADATA,sha256=4augHmAbGQ-7H0r4Zh6vaFvTh-yTwZMSyuLVYsp7-Qo,18249
-executorlib-1.9.4.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-executorlib-1.9.4.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
-executorlib-1.9.4.dist-info/RECORD,,
+executorlib-1.10.0.dist-info/METADATA,sha256=F0WJv16IxqNE7831mIuOePPRPd4CM4y9O9QLZj2tawg,20915
+executorlib-1.10.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+executorlib-1.10.0.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
+executorlib-1.10.0.dist-info/RECORD,,