earthkit-workflows 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

cascade/benchmarks/__main__.py

@@ -28,12 +28,12 @@ import os
 import subprocess
 import sys
 from concurrent.futures import ThreadPoolExecutor
-from socket import getfqdn
 from time import perf_counter_ns
 
 import fire
 import orjson
 
+import cascade.executor.platform as platform
 import cascade.low.into
 from cascade.controller.impl import run
 from cascade.executor.bridge import Bridge
@@ -140,20 +140,25 @@ def launch_executor(
         logging.config.dictConfig(logging_config_filehandler(log_path))
     else:
         logging.config.dictConfig(logging_config)
-    logger.info(f"will set {gpu_count} gpus on host {i}")
-    os.environ["CASCADE_GPU_COUNT"] = str(gpu_count)
-    executor = Executor(
-        job_instance,
-        controller_address,
-        workers_per_host,
-        f"h{i}",
-        portBase,
-        shm_vol_gb,
-        log_base,
-        url_base,
-    )
-    executor.register()
-    executor.recv_loop()
+    try:
+        logger.info(f"will set {gpu_count} gpus on host {i}")
+        os.environ["CASCADE_GPU_COUNT"] = str(gpu_count)
+        executor = Executor(
+            job_instance,
+            controller_address,
+            workers_per_host,
+            f"h{i}",
+            portBase,
+            shm_vol_gb,
+            log_base,
+            url_base,
+        )
+        executor.register()
+        executor.recv_loop()
+    except Exception:
+        # NOTE we log this to get the stacktrace into the logfile
+        logger.exception("executor failure")
+        raise
 
 
 def run_locally(
@@ -169,31 +174,47 @@ def run_locally(
         logging.config.dictConfig(logging_config_filehandler(log_path))
     else:
         logging.config.dictConfig(logging_config)
+    logger.debug(f"local run starting with {hosts=} and {workers=} on {portBase=}")
     launch = perf_counter_ns()
-    preschedule = precompute(job)
     c = f"tcp://localhost:{portBase}"
     m = f"tcp://localhost:{portBase+1}"
     ps = []
-    for i, executor in enumerate(range(hosts)):
-        gpu_count = get_gpu_count(i, workers)
-        # NOTE forkserver/spawn seem to forget venv, we need fork
-        p = multiprocessing.get_context("fork").Process(
-            target=launch_executor,
-            args=(
-                job,
-                c,
-                workers,
-                portBase + 1 + i * 10,
-                i,
-                None,
-                gpu_count,
-                log_base,
-                "tcp://localhost",
-            ),
-        )
-        p.start()
-        ps.append(p)
     try:
+        # executors forking
+        for i, executor in enumerate(range(hosts)):
+            gpu_count = get_gpu_count(i, workers)
+            # NOTE forkserver/spawn seem to forget venv, we need fork
+            logger.debug(f"forking into executor on host {i}")
+            p = multiprocessing.get_context("fork").Process(
+                target=launch_executor,
+                args=(
+                    job,
+                    c,
+                    workers,
+                    portBase + 1 + i * 10,
+                    i,
+                    None,
+                    gpu_count,
+                    log_base,
+                    "tcp://localhost",
+                ),
+            )
+            p.start()
+            ps.append(p)
+
+        # compute preschedule
+        preschedule = precompute(job)
+
+        # check processes started healthy
+        for i, p in enumerate(ps):
+            if not p.is_alive():
+                # TODO ideally we would somehow connect this with the Register message
+                # consumption in the Controller -- but there we don't assume that
+                # executors are on the same physical host
+                raise ValueError(f"executor {i} failed to live due to {p.exitcode}")
+
+        # start bridge itself
+        logger.debug("starting bridge")
         b = Bridge(c, hosts)
         start = perf_counter_ns()
         run(job, b, preschedule, report_address=report_address)
@@ -201,7 +222,9 @@ def run_locally(
         print(
             f"compute took {(end-start)/1e9:.3f}s, including startup {(end-launch)/1e9:.3f}s"
         )
-    except:
+    except Exception:
+        # NOTE we log this to get the stacktrace into the logfile
+        logger.exception("controller failure, proceed with executor shutdown")
         for p in ps:
             if p.is_alive():
                 callback(m, ExecutorShutdown())
@@ -272,7 +295,7 @@ def main_dist(
             idx,
             shm_vol_gb,
             gpu_count,
-            f"tcp://{getfqdn()}",
+            f"tcp://{platform.get_bindabble_self()}",
         )
 
 

cascade/executor/config.py

@@ -21,6 +21,7 @@ logging_config = {
         "forecastbox.worker": {"level": "DEBUG"},
         "forecastbox.executor": {"level": "DEBUG"},
         "cascade": {"level": "INFO"},
+        "cascade.benchmarks": {"level": "DEBUG"},
         "cascade.low": {"level": "DEBUG"},
         "cascade.shm": {"level": "DEBUG"},
         "cascade.controller": {"level": "DEBUG"},

cascade/executor/executor.py

@@ -18,11 +18,11 @@ the tasks themselves.
 import atexit
 import logging
 import os
-import socket
 from multiprocessing import get_context
 from multiprocessing.process import BaseProcess
 from typing import Iterable
 
+import cascade.executor.platform as platform
 import cascade.shm.api as shm_api
 import cascade.shm.client as shm_client
 from cascade.executor.comms import GraceWatcher, Listener, ReliableSender, callback
@@ -58,7 +58,7 @@ heartbeat_grace_ms = 2 * comms_default_timeout_ms
 
 
 def address_of(port: int) -> BackboneAddress:
-    return f"tcp://{socket.gethostname()}:{port}"
+    return f"tcp://{platform.get_bindabble_self()}:{port}"
 
 
 class Executor:
@@ -86,6 +86,7 @@ class Executor:
         self.heartbeat_watcher = GraceWatcher(grace_ms=heartbeat_grace_ms)
 
         self.terminating = False
+        logger.debug("register terminate function")
         atexit.register(self.terminate)
         # NOTE following inits are with potential side effects
         self.mlistener = Listener(address_of(portBase))
@@ -99,6 +100,7 @@ class Executor:
             shm_logging = logging_config_filehandler(f"{log_base}.shm.txt")
         else:
             shm_logging = logging_config
+        logger.debug("about to fork into shm process")
         self.shm_process = ctx.Process(
             target=shm_server,
             args=(
@@ -114,6 +116,7 @@ class Executor:
             dsr_logging = logging_config_filehandler(f"{log_base}.dsr.txt")
         else:
             dsr_logging = logging_config
+        logger.debug("about to fork into data server")
         self.data_server = ctx.Process(
             target=start_data_server,
             args=(

cascade/executor/platform.py

@@ -0,0 +1,24 @@
+# (C) Copyright 2025- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+"""Macos-vs-Linux specific code"""
+
+import socket
+import sys
+
+
+def get_bindabble_self():
+    """Returns a hostname such that zmq can bind to it"""
+
+    if sys.platform == "darwin":
+        # NOTE on macos, getfqdn usually returns like '66246.local', which can't then be bound to
+        # This is a stopper for running a cluster of macos devices -- but we don't plan that yet
+        return "localhost"
+    else:
+        # NOTE not sure if fqdn or hostname is better -- all we need is for it to be resolvable within cluster
+        return socket.gethostname()  # socket.getfqdn()

cascade/gateway/router.py

@@ -14,12 +14,12 @@ import os
 import subprocess
 import uuid
 from dataclasses import dataclass
-from socket import getfqdn
 from typing import Iterable
 
 import orjson
 import zmq
 
+import cascade.executor.platform as platform
 from cascade.controller.report import JobId, JobProgress, JobProgressStarted
 from cascade.executor.comms import get_context
 from cascade.gateway.api import JobSpec
@@ -131,11 +131,7 @@ class JobRouter:
 
     def spawn_job(self, job_spec: JobSpec) -> JobId:
         job_id = next_uuid(self.jobs.keys(), lambda: str(uuid.uuid4()))
-        if job_spec.use_slurm:
-            base_addr = f"tcp://{getfqdn()}"
-        else:
-            # NOTE on macos, it seems getfqdn does not give zmq-bindable addr
-            base_addr = "tcp://localhost"
+        base_addr = f"tcp://{platform.get_bindabble_self()}"
         socket = get_context().socket(zmq.PULL)
         port = socket.bind_to_random_port(base_addr)
         full_addr = f"{base_addr}:{port}"

cascade/shm/server.py

@@ -115,5 +115,5 @@ def entrypoint(
         server.start()
     except Exception as e:
         # we always get a Bad file descriptor due to sigterm handler calling sock close mid-read
-        logger.warning(f"shutdown issue: {e}")
+        logger.warning(f"shutdown issue: {repr(e)}")
     server.atexit(0, None)

earthkit/workflows/_version.py

@@ -1,2 +1,2 @@
 # Do not change! Do not track in version control!
-__version__ = "0.4.1"
+__version__ = "0.4.3"

{earthkit_workflows-0.4.1.dist-info → earthkit_workflows-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: earthkit-workflows
-Version: 0.4.1
+Version: 0.4.3
 Summary: Earthkit Workflows is a Python library for declaring earthkit task DAGs, as well as scheduling and executing them on heterogeneous computing systems.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License-Expression: Apache-2.0

{earthkit_workflows-0.4.1.dist-info → earthkit_workflows-0.4.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 cascade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cascade/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cascade/benchmarks/__init__.py,sha256=Gu8kEApmJ2zsIhT2zpm1-6n84-OwWnz-0vO8UHYtBzo,528
-cascade/benchmarks/__main__.py,sha256=9g2F-EH-M2DOgSEm_K3teFcJMzRV7cB5Kp3DWviyIkY,8729
+cascade/benchmarks/__main__.py,sha256=UnIGFykZNw_WsuBbgWIxSEUZB_U4_Yra0VrmyMTEFQ8,9914
 cascade/benchmarks/anemoi.py,sha256=qtAI03HdtAmcksCgjIEZyNyUNzMp370KF4lAh5g4cOk,1077
 cascade/benchmarks/dist.py,sha256=ngXJJzegnMUVwDFPvGMG6997lamB-aSEHi74oBbayrE,4116
 cascade/benchmarks/generators.py,sha256=NK4fFisWsZdMkA2Auzrn-P7G5D9AKpo2JVnqXE44YT8,2169
@@ -17,10 +17,11 @@ cascade/controller/notify.py,sha256=5eSPKcxqrv9kHy7St-iIm1NttsyzcvwLhZI5dvr4cEY,
 cascade/controller/report.py,sha256=FD-MAWZq6pwSw2CP2m4OUBw4hzrX46vKE_FZO5NpjDU,3670
 cascade/executor/bridge.py,sha256=WDE-GM2Bv7nUk1-nV-otMGuaRYw1-Vmd7PWploXBp6Y,8267
 cascade/executor/comms.py,sha256=-9qrKwva6WXkHRQtzSnLFy5gB3bOWuxYJP5fL6Uavw8,8736
-cascade/executor/config.py,sha256=IXBheZCp8DYKMd42e8whR6Gar1jnJyT9BCl6_X2c0Wk,1521
+cascade/executor/config.py,sha256=8azy_sXdvDGO0zTNqA0pdtkXsyihM4FQ4U1W_3Dhua0,1571
 cascade/executor/data_server.py,sha256=xLIbLkWn8PnJl4lMP8ADHa2S0EgPwr0-bH7_Sib_Y70,13701
-cascade/executor/executor.py,sha256=MHG_b4GCFjx-E9nLYvIqQicoOfNcP6JMNtn1Q0l4VIY,13322
+cascade/executor/executor.py,sha256=ufl20jq1lObNONHiYkXDFFCNfv4xKVvFsC_gzhTLdKQ,13524
 cascade/executor/msg.py,sha256=7HI0rKeCRaV1ONR4HWEa64nHbu-p6-QdBwJNitmst48,4340
+cascade/executor/platform.py,sha256=Ym-J6v0Wx56r2z_1ZfK5cX1Zs8eMuu7l1SsNtVvT1Go,959
 cascade/executor/serde.py,sha256=z6klTOZqW_BVGrbIRNz4FN0_XTfRiKBRQuvgsQIuyAo,2827
 cascade/executor/runner/__init__.py,sha256=30BM80ZyA7w3IrGiKKLSFuhRehbR2Mm99OJ8q5PJ63c,1547
 cascade/executor/runner/entrypoint.py,sha256=32i2U4fmEvQnsV1MTl0Xf8mK_1nbk1BEVJqIidd6MRM,8042
@@ -31,7 +32,7 @@ cascade/gateway/__init__.py,sha256=1EzMKdLFXEucj0YWOlyVqLx4suOntitwM03T_rRubIk,8
 cascade/gateway/__main__.py,sha256=x6-DQin6ICvalHT9YcghGyVMoykEATOdN5ON9IeHPYA,862
 cascade/gateway/api.py,sha256=-7HTUhK9idszVCwiVwyHMcNx7n6qRcyPWsLx2e19n3A,2511
 cascade/gateway/client.py,sha256=1p4Tvrf-BH0LQHOES5rY1z3JNIfmXcqWG2kYl4rpcE0,4061
-cascade/gateway/router.py,sha256=iN-dc3L46aEy0EV57NNKYwaqIu0Au9kImu1pg-UbxwE,7680
+cascade/gateway/router.py,sha256=81kS_ZtWaDV-WLcxaYgsYv-1WHy8aQxYGLmVL5CzOto,7549
 cascade/gateway/server.py,sha256=tsOyKtVFs5EZmWrjKdi9JwWxK0DG207oSa9OQ-4zN3M,3772
 cascade/low/__init__.py,sha256=5cw2taOGITK_gFbICftzK2YLdEAnLUY5OzblFzdHss4,769
 cascade/low/builders.py,sha256=_u5X8G_EF00hFt8Anv9AXo6yPf1O8MHDmqs2kKmREl0,7073
@@ -53,9 +54,9 @@ cascade/shm/client.py,sha256=pnod_dmUROJZRtipCpoeCuAEuynW0IgSfgjrp21CH2M,5893
 cascade/shm/dataset.py,sha256=Z2ewpnW7mVDJB9GylIVoOWV0DYOF7FWLIXkIvV-Y7sI,12347
 cascade/shm/disk.py,sha256=Fdl_pKOseaXroRp01OwqWVsdI-sSmiFizIFCdxBuMWM,2653
 cascade/shm/func.py,sha256=ZWikgnSLCmbSoW2LDRJwtjxdwTxkR00OUHAsIRQ-ChE,638
-cascade/shm/server.py,sha256=5Ub9bnBmDto9BwfjX3h3sJeiLzZN4lawgtLfvK-vcMU,5036
+cascade/shm/server.py,sha256=LnnNX0F6QJt5V_JLfmC3ZMHGNL5WpLY44wpB_pYDr7Y,5042
 earthkit/workflows/__init__.py,sha256=-p4anEn0YQbYWM2tbXb0Vc3wq4-m6kFhcNEgAVu5Jis,1948
-earthkit/workflows/_version.py,sha256=y47bDxdCo8QZJg-t8qedLIvdjhzndkJxexb9k-2JS1g,72
+earthkit/workflows/_version.py,sha256=42UO3Zhb1-CO_KjQGniDtgDR8xL_a28l7aAWQcA9o9Y,72
 earthkit/workflows/decorators.py,sha256=DM4QAtQ2glUUcDecwPkXcdlu4dio7MvgpcdmU5LYvD8,937
 earthkit/workflows/fluent.py,sha256=IN_sqwr7W8wbwP7wTOklgnjVe34IUCmv1ku-DWVTCJc,30179
 earthkit/workflows/mark.py,sha256=PdsXmRfhw1SyyJ74mzFPsLRqMCdlYv556fFX4bqlh9Y,1319
@@ -85,8 +86,8 @@ earthkit/workflows/graph/split.py,sha256=t-Sji5eZb01QO1szqmDNTodDDALqdo-0R0x1ESs
 earthkit/workflows/graph/transform.py,sha256=BZ8n7ePUnuGgoHkMqZC3SLzifu4oq6q6t6vka0khFtg,3842
 earthkit/workflows/graph/visit.py,sha256=MP-aFSqOl7aqJY2i7QTgY4epqb6yM7_lK3ofvOqfahw,1755
 earthkit/workflows/plugins/__init__.py,sha256=nhMAC0eMLxoJamjqB5Ns0OWy0OuxEJ_YvaDFGEQITls,129
-earthkit_workflows-0.4.1.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
-earthkit_workflows-0.4.1.dist-info/METADATA,sha256=wntnQCnb78Ashzd9p3OHSzDlB4yH6Np9yd4jKO00EOI,1571
-earthkit_workflows-0.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-earthkit_workflows-0.4.1.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
-earthkit_workflows-0.4.1.dist-info/RECORD,,
+earthkit_workflows-0.4.3.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
+earthkit_workflows-0.4.3.dist-info/METADATA,sha256=whowFjotkctfsy1v7mH-CsoCIucWMZpFcf8bqajhmQk,1571
+earthkit_workflows-0.4.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+earthkit_workflows-0.4.3.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
+earthkit_workflows-0.4.3.dist-info/RECORD,,