earthkit-workflows 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

cascade/benchmarks/__main__.py

@@ -23,10 +23,10 @@ Make sure you correctly configure:
 
 import logging
 import logging.config
+import multiprocessing
 import os
 import subprocess
 from concurrent.futures import ThreadPoolExecutor
-from multiprocessing import Process
 from time import perf_counter_ns
 
 import fire
@@ -139,7 +139,8 @@ def run_locally(
             gpu_count = get_gpu_count()
         else:
             gpu_count = 0
-        p = Process(
+        # NOTE forkserver/spawn seem to forget venv, we need fork
+        p = multiprocessing.get_context("fork").Process(
             target=launch_executor,
             args=(job, c, workers, portBase + 1 + i * 10, i, None, gpu_count),
         )

cascade/gateway/client.py

@@ -56,6 +56,8 @@ def request_response(
             raise TimeoutError  # NOTE consider setting `err` on the response instead
         else:
             rr = s.recv()
+    except TimeoutError:
+        raise
     except Exception as e:
         logger.exception(f"failed to communicate on {url=}")
         raise ValueError(f"failed to communicate on {url=} => {repr(e)[:32]}")

cascade/gateway/router.py

@@ -120,7 +120,11 @@ class JobRouter:
 
     def spawn_job(self, job_spec: JobSpec) -> JobId:
         job_id = next_uuid(self.jobs.keys(), lambda: str(uuid.uuid4()))
-        base_addr = f"tcp://{getfqdn()}"
+        if job_spec.use_slurm:
+            base_addr = f"tcp://{getfqdn()}"
+        else:
+            # NOTE on macos, it seems getfqdn does not give zmq-bindable addr
+            base_addr = "tcp://localhost"
         socket = get_context().socket(zmq.PULL)
         port = socket.bind_to_random_port(base_addr)
         full_addr = f"{base_addr}:{port}"

earthkit/workflows/_version.py

@@ -1,2 +1,2 @@
 # Do not change! Do not track in version control!
-__version__ = "0.3.3"
+__version__ = "0.3.5"

earthkit/workflows/fluent.py

@@ -424,12 +424,8 @@ class Action:
             )
 
         # Applies operation to every node, keeping node array structure
-        new_nodes = xr.DataArray(
-            np.empty(self.nodes.shape, dtype=object),
-            coords=self.nodes.coords,
-            dims=self.nodes.dims,
-            attrs=self.nodes.attrs,
-        )
+        new_nodes = np.empty(self.nodes.shape, dtype=object)
+
         it = np.nditer(self.nodes, flags=["multi_index", "refs_ok"])
         node_payload = payload
         for node in it:
@@ -441,7 +437,14 @@ class Action:
                 num_outputs=len(yields[1]) if yields else 1,
             )
 
-        return type(self)(new_nodes, yields)
+        new_nodes_xr = xr.DataArray(
+            new_nodes,
+            coords=self.nodes.coords,
+            dims=self.nodes.dims,
+            attrs=self.nodes.attrs,
+        )
+
+        return type(self)(new_nodes_xr, yields)
 
     def reduce(
         self,

{earthkit_workflows-0.3.3.dist-info → earthkit_workflows-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: earthkit-workflows
-Version: 0.3.3
+Version: 0.3.5
 Summary: Earthkit Workflows is a Python library for declaring earthkit task DAGs, as well as scheduling and executing them on heterogeneous computing systems.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License-Expression: Apache-2.0

{earthkit_workflows-0.3.3.dist-info → earthkit_workflows-0.3.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 cascade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cascade/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cascade/benchmarks/__init__.py,sha256=Gu8kEApmJ2zsIhT2zpm1-6n84-OwWnz-0vO8UHYtBzo,528
-cascade/benchmarks/__main__.py,sha256=M9MSOYdsOQZdXjB_0sfl5ltv0cuAatip_-e-Wi78FFQ,6897
+cascade/benchmarks/__main__.py,sha256=euamIHoDdqo2VC6gBUoOWuzlK1DttYCprDBWAVKGjnA,6986
 cascade/benchmarks/anemoi.py,sha256=qtAI03HdtAmcksCgjIEZyNyUNzMp370KF4lAh5g4cOk,1077
 cascade/benchmarks/generators.py,sha256=NK4fFisWsZdMkA2Auzrn-P7G5D9AKpo2JVnqXE44YT8,2169
 cascade/benchmarks/job1.py,sha256=NY1k9PvkUZODCIDO_zSNwC9sFiMYpBwOaWB7FMSkt2o,4594
@@ -28,8 +28,8 @@ cascade/executor/runner/runner.py,sha256=zqpkvxdWLbwyUFaUbZmSj0KQEBNRpmF8gwVotia
 cascade/gateway/__init__.py,sha256=1EzMKdLFXEucj0YWOlyVqLx4suOntitwM03T_rRubIk,829
 cascade/gateway/__main__.py,sha256=OcT5Amo0tE1-3StHuTOeQCaABmMBO3XLDK4t4b8oeeQ,647
 cascade/gateway/api.py,sha256=-7HTUhK9idszVCwiVwyHMcNx7n6qRcyPWsLx2e19n3A,2511
-cascade/gateway/client.py,sha256=tc-BXDXb0h7gjLMveATR8yOFqFT9fd7fXdx3PjIcTHs,4022
-cascade/gateway/router.py,sha256=p1QT7cMAzLgmfvNYqEtENsnHPPzDbIXcnNuWoI5aejM,7154
+cascade/gateway/client.py,sha256=1p4Tvrf-BH0LQHOES5rY1z3JNIfmXcqWG2kYl4rpcE0,4061
+cascade/gateway/router.py,sha256=KBlw5U-QOcZ8QN6Ls3WLS7gO0X2apupAWX6pS7A7mrs,7323
 cascade/gateway/server.py,sha256=srqmtOuzzon5GV5jR1wUVh744Ct336BQs5Gzd3mhsPA,3733
 cascade/low/__init__.py,sha256=5cw2taOGITK_gFbICftzK2YLdEAnLUY5OzblFzdHss4,769
 cascade/low/builders.py,sha256=_u5X8G_EF00hFt8Anv9AXo6yPf1O8MHDmqs2kKmREl0,7073
@@ -53,9 +53,9 @@ cascade/shm/disk.py,sha256=Fdl_pKOseaXroRp01OwqWVsdI-sSmiFizIFCdxBuMWM,2653
 cascade/shm/func.py,sha256=ZWikgnSLCmbSoW2LDRJwtjxdwTxkR00OUHAsIRQ-ChE,638
 cascade/shm/server.py,sha256=5Ub9bnBmDto9BwfjX3h3sJeiLzZN4lawgtLfvK-vcMU,5036
 earthkit/workflows/__init__.py,sha256=f17AdiV9g4eRN8m4dUnSU58RoLRqk1e6iMRrQiBUSKk,1880
-earthkit/workflows/_version.py,sha256=JTDTUt1BYSv9yiR48Riwpm5YvK2xpFLe9Yl8KX1W8i0,72
+earthkit/workflows/_version.py,sha256=y917q-_1kG-gTBVkeELzqyHU_FN3NUWwP8pqRDzG4Yw,72
 earthkit/workflows/decorators.py,sha256=DM4QAtQ2glUUcDecwPkXcdlu4dio7MvgpcdmU5LYvD8,937
-earthkit/workflows/fluent.py,sha256=OpN2ncU0UeIyvDcdMe4woH7ZGtUhxFMB3ZQTU0nCshU,30141
+earthkit/workflows/fluent.py,sha256=IN_sqwr7W8wbwP7wTOklgnjVe34IUCmv1ku-DWVTCJc,30179
 earthkit/workflows/mark.py,sha256=PdsXmRfhw1SyyJ74mzFPsLRqMCdlYv556fFX4bqlh9Y,1319
 earthkit/workflows/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 earthkit/workflows/taskgraph.py,sha256=RsT1Qlng1uPZSaSBNqE8vFsoI5J8DDcQl468YPX-kCY,4460
@@ -83,8 +83,8 @@ earthkit/workflows/graph/split.py,sha256=t-Sji5eZb01QO1szqmDNTodDDALqdo-0R0x1ESs
 earthkit/workflows/graph/transform.py,sha256=BZ8n7ePUnuGgoHkMqZC3SLzifu4oq6q6t6vka0khFtg,3842
 earthkit/workflows/graph/visit.py,sha256=MP-aFSqOl7aqJY2i7QTgY4epqb6yM7_lK3ofvOqfahw,1755
 earthkit/workflows/plugins/__init__.py,sha256=WcX4qbEhgTXabIbogydtzNmZ2tB_SuW6NzNkOYQfS-Y,61
-earthkit_workflows-0.3.3.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
-earthkit_workflows-0.3.3.dist-info/METADATA,sha256=DVMvtcARGr5s402T0-LXONNHc-OZyZgT-8fkT6s-sVg,1571
-earthkit_workflows-0.3.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-earthkit_workflows-0.3.3.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
-earthkit_workflows-0.3.3.dist-info/RECORD,,
+earthkit_workflows-0.3.5.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
+earthkit_workflows-0.3.5.dist-info/METADATA,sha256=o-BNwkFAKcA9NyyryCTovDEeMs1uNNde9IyChoUq_Lc,1571
+earthkit_workflows-0.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+earthkit_workflows-0.3.5.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
+earthkit_workflows-0.3.5.dist-info/RECORD,,