earthkit-workflows 0.4.1__py3-none-any.whl → 0.4.2__py3-none-any.whl

cascade/benchmarks/__main__.py

@@ -140,20 +140,25 @@ def launch_executor(
         logging.config.dictConfig(logging_config_filehandler(log_path))
     else:
         logging.config.dictConfig(logging_config)
-    logger.info(f"will set {gpu_count} gpus on host {i}")
-    os.environ["CASCADE_GPU_COUNT"] = str(gpu_count)
-    executor = Executor(
-        job_instance,
-        controller_address,
-        workers_per_host,
-        f"h{i}",
-        portBase,
-        shm_vol_gb,
-        log_base,
-        url_base,
-    )
-    executor.register()
-    executor.recv_loop()
+    try:
+        logger.info(f"will set {gpu_count} gpus on host {i}")
+        os.environ["CASCADE_GPU_COUNT"] = str(gpu_count)
+        executor = Executor(
+            job_instance,
+            controller_address,
+            workers_per_host,
+            f"h{i}",
+            portBase,
+            shm_vol_gb,
+            log_base,
+            url_base,
+        )
+        executor.register()
+        executor.recv_loop()
+    except Exception:
+        # NOTE we log this to get the stacktrace into the logfile
+        logger.exception("executor failure")
+        raise
 
 
 def run_locally(
@@ -169,31 +174,47 @@ def run_locally(
         logging.config.dictConfig(logging_config_filehandler(log_path))
     else:
         logging.config.dictConfig(logging_config)
+    logger.debug(f"local run starting with {hosts=} and {workers=} on {portBase=}")
     launch = perf_counter_ns()
-    preschedule = precompute(job)
     c = f"tcp://localhost:{portBase}"
     m = f"tcp://localhost:{portBase+1}"
     ps = []
-    for i, executor in enumerate(range(hosts)):
-        gpu_count = get_gpu_count(i, workers)
-        # NOTE forkserver/spawn seem to forget venv, we need fork
-        p = multiprocessing.get_context("fork").Process(
-            target=launch_executor,
-            args=(
-                job,
-                c,
-                workers,
-                portBase + 1 + i * 10,
-                i,
-                None,
-                gpu_count,
-                log_base,
-                "tcp://localhost",
-            ),
-        )
-        p.start()
-        ps.append(p)
     try:
+        # executors forking
+        for i, executor in enumerate(range(hosts)):
+            gpu_count = get_gpu_count(i, workers)
+            # NOTE forkserver/spawn seem to forget venv, we need fork
+            logger.debug(f"forking into executor on host {i}")
+            p = multiprocessing.get_context("fork").Process(
+                target=launch_executor,
+                args=(
+                    job,
+                    c,
+                    workers,
+                    portBase + 1 + i * 10,
+                    i,
+                    None,
+                    gpu_count,
+                    log_base,
+                    "tcp://localhost",
+                ),
+            )
+            p.start()
+            ps.append(p)
+
+        # compute preschedule
+        preschedule = precompute(job)
+
+        # check processes started healthy
+        for i, p in enumerate(ps):
+            if not p.is_alive():
+                # TODO ideally we would somehow connect this with the Register message
+                # consumption in the Controller -- but there we don't assume that
+                # executors are on the same physical host
+                raise ValueError(f"executor {i} failed to live due to {p.exitcode}")
+
+        # start bridge itself
+        logger.debug("starting bridge")
         b = Bridge(c, hosts)
         start = perf_counter_ns()
         run(job, b, preschedule, report_address=report_address)
@@ -201,7 +222,9 @@ def run_locally(
         print(
             f"compute took {(end-start)/1e9:.3f}s, including startup {(end-launch)/1e9:.3f}s"
         )
-    except:
+    except Exception:
+        # NOTE we log this to get the stacktrace into the logfile
+        logger.exception("controller failure, proceed with executor shutdown")
         for p in ps:
             if p.is_alive():
                 callback(m, ExecutorShutdown())

cascade/executor/config.py

@@ -21,6 +21,7 @@ logging_config = {
         "forecastbox.worker": {"level": "DEBUG"},
         "forecastbox.executor": {"level": "DEBUG"},
         "cascade": {"level": "INFO"},
+        "cascade.benchmarks": {"level": "DEBUG"},
         "cascade.low": {"level": "DEBUG"},
         "cascade.shm": {"level": "DEBUG"},
         "cascade.controller": {"level": "DEBUG"},

cascade/executor/executor.py

@@ -86,6 +86,7 @@ class Executor:
         self.heartbeat_watcher = GraceWatcher(grace_ms=heartbeat_grace_ms)
 
         self.terminating = False
+        logger.debug("register terminate function")
         atexit.register(self.terminate)
         # NOTE following inits are with potential side effects
         self.mlistener = Listener(address_of(portBase))
@@ -99,6 +100,7 @@ class Executor:
             shm_logging = logging_config_filehandler(f"{log_base}.shm.txt")
         else:
             shm_logging = logging_config
+        logger.debug("about to fork into shm process")
         self.shm_process = ctx.Process(
             target=shm_server,
             args=(
@@ -114,6 +116,7 @@ class Executor:
             dsr_logging = logging_config_filehandler(f"{log_base}.dsr.txt")
         else:
             dsr_logging = logging_config
+        logger.debug("about to fork into data server")
         self.data_server = ctx.Process(
             target=start_data_server,
             args=(

cascade/shm/server.py

@@ -115,5 +115,5 @@ def entrypoint(
         server.start()
     except Exception as e:
         # we always get a Bad file descriptor due to sigterm handler calling sock close mid-read
-        logger.warning(f"shutdown issue: {e}")
+        logger.warning(f"shutdown issue: {repr(e)}")
     server.atexit(0, None)

earthkit/workflows/_version.py

@@ -1,2 +1,2 @@
 # Do not change! Do not track in version control!
-__version__ = "0.4.1"
+__version__ = "0.4.2"

{earthkit_workflows-0.4.1.dist-info → earthkit_workflows-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: earthkit-workflows
-Version: 0.4.1
+Version: 0.4.2
 Summary: Earthkit Workflows is a Python library for declaring earthkit task DAGs, as well as scheduling and executing them on heterogeneous computing systems.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License-Expression: Apache-2.0

{earthkit_workflows-0.4.1.dist-info → earthkit_workflows-0.4.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 cascade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cascade/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cascade/benchmarks/__init__.py,sha256=Gu8kEApmJ2zsIhT2zpm1-6n84-OwWnz-0vO8UHYtBzo,528
-cascade/benchmarks/__main__.py,sha256=9g2F-EH-M2DOgSEm_K3teFcJMzRV7cB5Kp3DWviyIkY,8729
+cascade/benchmarks/__main__.py,sha256=g03xRzp58dXLHDj8kTPyPnbBOS5sRIAMTthFtFjDRbs,9876
 cascade/benchmarks/anemoi.py,sha256=qtAI03HdtAmcksCgjIEZyNyUNzMp370KF4lAh5g4cOk,1077
 cascade/benchmarks/dist.py,sha256=ngXJJzegnMUVwDFPvGMG6997lamB-aSEHi74oBbayrE,4116
 cascade/benchmarks/generators.py,sha256=NK4fFisWsZdMkA2Auzrn-P7G5D9AKpo2JVnqXE44YT8,2169
@@ -17,9 +17,9 @@ cascade/controller/notify.py,sha256=5eSPKcxqrv9kHy7St-iIm1NttsyzcvwLhZI5dvr4cEY,
 cascade/controller/report.py,sha256=FD-MAWZq6pwSw2CP2m4OUBw4hzrX46vKE_FZO5NpjDU,3670
 cascade/executor/bridge.py,sha256=WDE-GM2Bv7nUk1-nV-otMGuaRYw1-Vmd7PWploXBp6Y,8267
 cascade/executor/comms.py,sha256=-9qrKwva6WXkHRQtzSnLFy5gB3bOWuxYJP5fL6Uavw8,8736
-cascade/executor/config.py,sha256=IXBheZCp8DYKMd42e8whR6Gar1jnJyT9BCl6_X2c0Wk,1521
+cascade/executor/config.py,sha256=8azy_sXdvDGO0zTNqA0pdtkXsyihM4FQ4U1W_3Dhua0,1571
 cascade/executor/data_server.py,sha256=xLIbLkWn8PnJl4lMP8ADHa2S0EgPwr0-bH7_Sib_Y70,13701
-cascade/executor/executor.py,sha256=MHG_b4GCFjx-E9nLYvIqQicoOfNcP6JMNtn1Q0l4VIY,13322
+cascade/executor/executor.py,sha256=egPhfqhzYfeM77Hu10-mGHNVsQAdqmZOA7hmjFP1Q8M,13484
 cascade/executor/msg.py,sha256=7HI0rKeCRaV1ONR4HWEa64nHbu-p6-QdBwJNitmst48,4340
 cascade/executor/serde.py,sha256=z6klTOZqW_BVGrbIRNz4FN0_XTfRiKBRQuvgsQIuyAo,2827
 cascade/executor/runner/__init__.py,sha256=30BM80ZyA7w3IrGiKKLSFuhRehbR2Mm99OJ8q5PJ63c,1547
@@ -53,9 +53,9 @@ cascade/shm/client.py,sha256=pnod_dmUROJZRtipCpoeCuAEuynW0IgSfgjrp21CH2M,5893
 cascade/shm/dataset.py,sha256=Z2ewpnW7mVDJB9GylIVoOWV0DYOF7FWLIXkIvV-Y7sI,12347
 cascade/shm/disk.py,sha256=Fdl_pKOseaXroRp01OwqWVsdI-sSmiFizIFCdxBuMWM,2653
 cascade/shm/func.py,sha256=ZWikgnSLCmbSoW2LDRJwtjxdwTxkR00OUHAsIRQ-ChE,638
-cascade/shm/server.py,sha256=5Ub9bnBmDto9BwfjX3h3sJeiLzZN4lawgtLfvK-vcMU,5036
+cascade/shm/server.py,sha256=LnnNX0F6QJt5V_JLfmC3ZMHGNL5WpLY44wpB_pYDr7Y,5042
 earthkit/workflows/__init__.py,sha256=-p4anEn0YQbYWM2tbXb0Vc3wq4-m6kFhcNEgAVu5Jis,1948
-earthkit/workflows/_version.py,sha256=y47bDxdCo8QZJg-t8qedLIvdjhzndkJxexb9k-2JS1g,72
+earthkit/workflows/_version.py,sha256=nkd71CReR3pz5TZ9pcVgB2cP1MDj4YK6VH9UGJYzXDM,72
 earthkit/workflows/decorators.py,sha256=DM4QAtQ2glUUcDecwPkXcdlu4dio7MvgpcdmU5LYvD8,937
 earthkit/workflows/fluent.py,sha256=IN_sqwr7W8wbwP7wTOklgnjVe34IUCmv1ku-DWVTCJc,30179
 earthkit/workflows/mark.py,sha256=PdsXmRfhw1SyyJ74mzFPsLRqMCdlYv556fFX4bqlh9Y,1319
@@ -85,8 +85,8 @@ earthkit/workflows/graph/split.py,sha256=t-Sji5eZb01QO1szqmDNTodDDALqdo-0R0x1ESs
 earthkit/workflows/graph/transform.py,sha256=BZ8n7ePUnuGgoHkMqZC3SLzifu4oq6q6t6vka0khFtg,3842
 earthkit/workflows/graph/visit.py,sha256=MP-aFSqOl7aqJY2i7QTgY4epqb6yM7_lK3ofvOqfahw,1755
 earthkit/workflows/plugins/__init__.py,sha256=nhMAC0eMLxoJamjqB5Ns0OWy0OuxEJ_YvaDFGEQITls,129
-earthkit_workflows-0.4.1.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
-earthkit_workflows-0.4.1.dist-info/METADATA,sha256=wntnQCnb78Ashzd9p3OHSzDlB4yH6Np9yd4jKO00EOI,1571
-earthkit_workflows-0.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-earthkit_workflows-0.4.1.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
-earthkit_workflows-0.4.1.dist-info/RECORD,,
+earthkit_workflows-0.4.2.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
+earthkit_workflows-0.4.2.dist-info/METADATA,sha256=LWW-xDc0sq8cOdu6IpY335_MSFfe7Lmg1SHYT9cXjWA,1571
+earthkit_workflows-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+earthkit_workflows-0.4.2.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
+earthkit_workflows-0.4.2.dist-info/RECORD,,