earthkit-workflows 0.4.7__py3-none-any.whl → 0.5.1__py3-none-any.whl

cascade/benchmarks/tests.py

@@ -0,0 +1,173 @@
+# (C) Copyright 2025- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+"""Module for simplifying writing tests
+
+Similar to util, but not enough to unify
+
+It is capable, for a single given task, to spin an shm server, put all task's inputs into it, execute the task, store outputs in memory, and retrieve the result.
+See the `demo()` function at the very end
+"""
+
+import logging
+from contextlib import contextmanager
+from dataclasses import dataclass
+from time import perf_counter_ns
+from typing import Any, Callable
+
+import cloudpickle
+
+import cascade.executor.platform as platform
+import cascade.shm.api as shm_api
+import cascade.shm.client as shm_client
+from cascade.executor.comms import Listener as ZmqListener
+from cascade.executor.config import logging_config
+from cascade.executor.msg import BackboneAddress, DatasetPublished
+from cascade.executor.runner.memory import Memory, ds2shmid
+from cascade.executor.runner.packages import PackagesEnv
+from cascade.executor.runner.runner import ExecutionContext, run
+from cascade.low.builders import TaskBuilder
+from cascade.low.core import DatasetId
+from cascade.shm.server import entrypoint as shm_server
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def setup_shm(testId: str):
+    mp_ctx = platform.get_mp_ctx("executor-aux")
+    shm_socket = f"/tmp/tcShm-{testId}"
+    shm_api.publish_socket_addr(shm_socket)
+    shm_process = mp_ctx.Process(
+        target=shm_server,
+        kwargs={
+            "logging_config": logging_config,
+            "shm_pref": f"tc{testId}",
+        },
+    )
+    shm_process.start()
+    shm_client.ensure()
+    try:
+        yield
+    except Exception as e:
+        # NOTE we log like this in case shm shutdown freezes
+        logger.exception(f"gotten {repr(e)}, proceed with shm shutdown")
+        raise
+    finally:
+        shm_client.shutdown(timeout_sec=1.0)
+        shm_process.join(1)
+        if shm_process.is_alive():
+            shm_process.terminate()
+            shm_process.join(1)
+        if shm_process.is_alive():
+            shm_process.kill()
+            shm_process.join(1)
+
+
+def simple_runner(callback: BackboneAddress, executionContext: ExecutionContext):
+    tasks = list(executionContext.tasks.keys())
+    if len(tasks) != 1:
+        raise ValueError(f"expected 1 task, gotten {len(tasks)}")
+    taskId = tasks[0]
+    taskInstance = executionContext.tasks[taskId]
+    with Memory(callback, "testWorker") as memory, PackagesEnv() as pckg:
+        # for key, value in taskSequence.extra_env.items():
+        #    os.environ[key] = value
+
+        pckg.extend(taskInstance.definition.environment)
+        run(taskId, executionContext, memory)
+        memory.flush()
+
+
+@dataclass
+class CallableInstance:
+    func: Callable
+    kwargs: dict[str, Any]
+    args: list[tuple[int, Any]]
+    env: list[str]
+    exp_output: Any
+
+
+def callable2ctx(
+    callableInstance: CallableInstance, callback: BackboneAddress
+) -> ExecutionContext:
+    taskInstance = TaskBuilder.from_callable(
+        callableInstance.func, callableInstance.env
+    )
+    param_source = {}
+    params = [
+        (key, DatasetId("taskId", f"kwarg.{key}"), value)
+        for key, value in callableInstance.kwargs.items()
+    ] + [
+        (key, DatasetId("taskId", f"pos.{key}"), value)
+        for key, value in callableInstance.args
+    ]
+    for key, ds_key, value in params:
+        raw = cloudpickle.dumps(value)
+        L = len(raw)
+        buf = shm_client.allocate(ds2shmid(ds_key), L, "cloudpickle.loads")
+        buf.view()[:L] = raw
+        buf.close()
+        param_source[key] = (ds_key, "Any")
+
+    return ExecutionContext(
+        tasks={"taskId": taskInstance},
+        param_source={"taskId": param_source},
+        callback=callback,
+        publish={
+            DatasetId("taskId", output)
+            for output, _ in taskInstance.definition.output_schema
+        },
+    )
+
+
+def run_test(
+    callableInstance: CallableInstance, testId: str, max_runtime_sec: int
+) -> Any:
+    with setup_shm(testId):
+        addr = f"ipc:///tmp/tc{testId}"
+        listener = ZmqListener(addr)
+        ec_ctx = callable2ctx(callableInstance, addr)
+        mp_ctx = platform.get_mp_ctx("executor-aux")
+        runner = mp_ctx.Process(target=simple_runner, args=(addr, ec_ctx))
+        runner.start()
+        output = DatasetId("taskId", "0")
+
+        end = perf_counter_ns() + max_runtime_sec * int(1e9)
+        while perf_counter_ns() < end:
+            mess = listener.recv_messages()
+            if mess == [
+                DatasetPublished(origin="testWorker", ds=output, transmit_idx=None)
+            ]:
+                break
+            elif not mess:
+                continue
+            else:
+                raise ValueError(mess)
+
+        runner.join()
+        output_buf = shm_client.get(ds2shmid(output))
+        output_des = cloudpickle.loads(output_buf.view())
+        output_buf.close()
+    assert output_des == callableInstance.exp_output
+
+
+def demo():
+    def myfunc(l: int) -> float:
+        import numpy as np
+
+        return np.arange(l).sum()
+
+    ci = CallableInstance(
+        func=myfunc, kwargs={"l": 4}, args=[], env=["numpy"], exp_output=6
+    )
+    run_test(ci, "numpyTest1", 2)
+
+
+if __name__ == "__main__":
+    demo()

cascade/benchmarks/util.py

@@ -17,6 +17,7 @@ import subprocess
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from time import perf_counter_ns
+from typing import Any
 
 import orjson
 
@@ -28,7 +29,7 @@ from cascade.executor.comms import callback
 from cascade.executor.config import logging_config, logging_config_filehandler
 from cascade.executor.executor import Executor
 from cascade.executor.msg import BackboneAddress, ExecutorShutdown
-from cascade.low.core import JobInstance
+from cascade.low.core import DatasetId, JobInstance
 from cascade.low.func import msum
 from cascade.scheduler.precompute import precompute
 from earthkit.workflows.graph import Graph, deduplicate_nodes
@@ -159,7 +160,7 @@ def run_locally(
     portBase: int = 12345,
     log_base: str | None = None,
     report_address: str | None = None,
-) -> None:
+) -> dict[DatasetId, Any]:
     if log_base is not None:
         log_path = f"{log_base}.controller.txt"
         logging.config.dictConfig(logging_config_filehandler(log_path))
@@ -216,6 +217,7 @@ def run_locally(
         if os.environ.get("CASCADE_DEBUG_PRINT"):
             for key, value in result.outputs.items():
                 print(f"{key} => {value}")
+        return result.outputs
     except Exception:
         # NOTE we log this to get the stacktrace into the logfile
         logger.exception("controller failure, proceed with executor shutdown")

cascade/executor/data_server.py

@@ -20,7 +20,6 @@ from concurrent.futures import Executor as PythonExecutor
 from concurrent.futures import Future, ThreadPoolExecutor, wait
 from time import time_ns
 
-import cascade.shm.api as shm_api
 import cascade.shm.client as shm_client
 from cascade.executor.comms import Listener, callback, send_data
 from cascade.executor.msg import (
@@ -48,7 +47,6 @@ class DataServer:
         maddress: BackboneAddress,
         daddress: BackboneAddress,
         host: str,
-        shm_port: int,
         logging_config: dict,
     ):
         logging.config.dictConfig(logging_config)
@@ -58,7 +56,6 @@ class DataServer:
         self.daddress = daddress
         self.dlistener = Listener(daddress)
         self.terminating = False
-        shm_api.publish_client_port(shm_port)
         self.cap = 2
         self.ds_proc_tp: PythonExecutor = ThreadPoolExecutor(max_workers=self.cap)
         self.futs_in_progress: dict[
@@ -305,8 +302,7 @@ def start_data_server(
     maddress: BackboneAddress,
     daddress: BackboneAddress,
     host: str,
-    shm_port: int,
     logging_config: dict,
 ):
-    server = DataServer(maddress, daddress, host, shm_port, logging_config)
+    server = DataServer(maddress, daddress, host, logging_config)
     server.recv_loop()

cascade/executor/executor.py

@@ -18,6 +18,7 @@ the tasks themselves.
 import atexit
 import logging
 import os
+import uuid
 from multiprocessing.process import BaseProcess
 from typing import Iterable
 
@@ -94,8 +95,8 @@ class Executor:
         self.sender = ReliableSender(self.mlistener.address, resend_grace_ms)
         self.sender.add_host("controller", controller_address)
         # TODO make the shm server params configurable
-        shm_port = portBase + 2
-        shm_api.publish_client_port(shm_port)
+        shm_port = f"/tmp/cascShmSock-{uuid.uuid4()}"  # portBase + 2
+        shm_api.publish_socket_addr(shm_port)
         ctx = platform.get_mp_ctx("executor-aux")
         if log_base:
             shm_logging = logging_config_filehandler(f"{log_base}.shm.txt")
@@ -104,12 +105,11 @@ class Executor:
         logger.debug("about to start an shm process")
         self.shm_process = ctx.Process(
             target=shm_server,
-            args=(
-                shm_port,
-                shm_vol_gb * (1024**3) if shm_vol_gb else None,
-                shm_logging,
-                f"sCasc{host}",
-            ),
+            kwargs={
+                "capacity": shm_vol_gb * (1024**3) if shm_vol_gb else None,
+                "logging_config": shm_logging,
+                "shm_pref": f"sCasc{host}",
+            },
         )
         self.shm_process.start()
         self.daddress = address_of(portBase + 1)
@@ -124,7 +124,6 @@ class Executor:
                 self.mlistener.address,
                 self.daddress,
                 self.host,
-                shm_port,
                 dsr_logging,
             ),
         )

cascade/executor/runner/memory.py

@@ -12,6 +12,7 @@ Interaction with shm
 
 import hashlib
 import logging
+import sys
 from contextlib import AbstractContextManager
 from typing import Any, Literal
 
@@ -119,27 +120,29 @@ class Memory(AbstractContextManager):
             self.local.pop(inputId)
 
         # NOTE poor man's gpu mem management -- currently torch only. Given the task sequence limitation,
-        # this may not be the best place to invoke. Additionally, we may want to check first whether
-        # the worker is gpu aware, etc
-        try:
-            import torch
-
-            if torch.cuda.is_available():
-                free, total = torch.cuda.mem_get_info()
-                logger.debug(f"cuda mem avail: {free/total:.2%}")
-                if free / total < 0.8:
-                    torch.cuda.empty_cache()
+        # this may not be the best place to invoke.
+        if (
+            "torch" in sys.modules
+        ):  # if no task on this worker imported torch, no need to flush
+            try:
+                import torch
+
+                if torch.cuda.is_available():
                     free, total = torch.cuda.mem_get_info()
-                    logger.debug(f"cuda mem avail post cache empty: {free/total:.2%}")
+                    logger.debug(f"cuda mem avail: {free/total:.2%}")
                     if free / total < 0.8:
-                        # NOTE this ofc makes low sense if there is any other application (like browser or ollama)
-                        # that the user may be running
-                        logger.warning("cuda mem avail low despite cache empty!")
-                        logger.debug(torch.cuda.memory_summary())
-        except ImportError:
-            return
-        except Exception:
-            logger.exception("failed to free cuda cache")
+                        torch.cuda.empty_cache()
+                        free, total = torch.cuda.mem_get_info()
+                        logger.debug(
+                            f"cuda mem avail post cache empty: {free/total:.2%}"
+                        )
+                        if free / total < 0.8:
+                            # NOTE this ofc makes low sense if there is any other application (like browser or ollama)
+                            # that the user may be running
+                            logger.warning("cuda mem avail low despite cache empty!")
+                            logger.debug(torch.cuda.memory_summary())
+            except Exception:
+                logger.exception("failed to free cuda cache")
 
     def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
         # this is required so that the Shm can be properly freed, otherwise you get 'pointers cannot be closed'

cascade/executor/runner/packages.py

@@ -12,6 +12,7 @@ Note that venv itself is left untouched after the run finishes -- we extend sys
 with a temporary directory and install in there
 """
 
+import importlib
 import logging
 import os
 import site
@@ -24,6 +25,51 @@ from typing import Literal
 logger = logging.getLogger(__name__)
 
 
+class Commands:
+    venv_command = lambda name: ["uv", "venv", name]
+    install_command = lambda name: [
+        "uv",
+        "pip",
+        "install",
+        "--prefix",
+        name,
+        "--prerelease",
+        "explicit",
+    ]
+
+
+def run_command(command: list[str]) -> None:
+    try:
+        result = subprocess.run(command, check=False, capture_output=True)
+    except FileNotFoundError as ex:
+        raise ValueError(f"command failure: {ex}")
+    if result.returncode != 0:
+        msg = f"command failed with {result.returncode}. Stderr: {result.stderr}, Stdout: {result.stdout}, Args: {result.args}"
+        logger.error(msg)
+        raise ValueError(msg)
+
+
+def new_venv() -> tempfile.TemporaryDirectory:
+    """1. Creates a new temporary directory with a venv inside.
+    2. Extends sys.path so that packages in that venv can be imported.
+    """
+    logger.debug("creating a new venv")
+    td = tempfile.TemporaryDirectory(prefix="cascade_runner_venv_")
+    # NOTE we create a venv instead of just plain directory, because some of the packages create files
+    # outside of site-packages. Thus we then install with --prefix, not with --target
+    run_command(Commands.venv_command(td.name))
+
+    # NOTE not sure if getsitepackages was intended for this -- if issues, attempt replacing
+    # with something like f"{td.name}/lib/python*/site-packages" + globbing
+    extra_sp = site.getsitepackages(prefixes=[td.name])
+    # NOTE this makes the explicit packages go first, in case of a different version
+    logger.debug(f"extending sys.path with {extra_sp}")
+    sys.path = extra_sp + sys.path
+    logger.debug(f"new sys.path: {sys.path}")
+
+    return td
+
+
 class PackagesEnv(AbstractContextManager):
     def __init__(self) -> None:
         self.td: tempfile.TemporaryDirectory | None = None
@@ -32,38 +78,32 @@ class PackagesEnv(AbstractContextManager):
         if not packages:
             return
         if self.td is None:
-            logger.debug("creating a new venv")
-            self.td = tempfile.TemporaryDirectory()
-            venv_command = ["uv", "venv", self.td.name]
-            # NOTE we create a venv instead of just plain directory, because some of the packages create files
-            # outside of site-packages. Thus we then install with --prefix, not with --target
-            subprocess.run(venv_command, check=True)
+            self.td = new_venv()
 
         logger.debug(
             f"installing {len(packages)} packages: {','.join(packages[:3])}{',...' if len(packages) > 3 else ''}"
         )
-        install_command = [
-            "uv",
-            "pip",
-            "install",
-            "--prefix",
-            self.td.name,
-            "--prerelease",
-            "allow",
-        ]
+        install_command = Commands.install_command(self.td.name)
         if os.environ.get("VENV_OFFLINE", "") == "YES":
             install_command += ["--offline"]
         if cache_dir := os.environ.get("VENV_CACHE", ""):
             install_command += ["--cache-dir", cache_dir]
         install_command.extend(set(packages))
-        subprocess.run(install_command, check=True)
-        # NOTE not sure if getsitepackages was intended for this -- if issues, attempt replacing
-        # with something like f"{self.td.name}/lib/python*/site-packages" + globbing
-        extra_sp = site.getsitepackages(prefixes=[self.td.name])
-        # NOTE this makes the explicit packages go first, in case of a different version
-        sys.path = extra_sp + sys.path
+        logger.debug(f"running install command: {' '.join(install_command)}")
+        run_command(install_command)
+
+        # NOTE we need this due to namespace packages:
+        # 1. task 1 installs ns.pkg1 in its venv
+        # 2. task 1 finishes, task 2 starts on the same worker
+        # 3. task 2 starts, installs ns.pkg2. However, importlib is in a state that ns is aware only of pkg1 submod
+        # Additionally, the caches are invalid anyway, because task 1's venv is already deleted
+        importlib.invalidate_caches()
+        # TODO some namespace packages may require a reimport because they dynamically build `__all__` -- eg earthkit
 
     def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
+        sys.path = [
+            p for p in sys.path if self.td is None or not p.startswith(self.td.name)
+        ]
         if self.td is not None:
             self.td.cleanup()
         return False

cascade/low/builders.py

@@ -6,10 +6,11 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import importlib
 import inspect
 import itertools
 from dataclasses import dataclass, field, replace
-from typing import Callable, Iterable, Iterator, Type, cast
+from typing import Any, Callable, Iterable, Iterator, Type, cast
 
 import pyrsistent
 from typing_extensions import Self
@@ -29,8 +30,17 @@ class TaskBuilder(TaskInstance):
     @classmethod
     def from_callable(cls, f: Callable, environment: list[str] | None = None) -> Self:
         def type2str(t: str | Type) -> str:
-            type_name: str = t if isinstance(t, str) else t.__name__
-            return "Any" if type_name == "_empty" else type_name
+            type_name: str
+            if isinstance(t, str):
+                type_name = t
+            elif isinstance(t, tuple):
+                # TODO properly break down etc
+                type_name = "tuple"
+            elif t.__module__ == "builtins":
+                type_name = t.__name__
+            else:
+                type_name = f"{t.__module__}.{t.__name__}"
+            return "Any" if type_name in ("_empty", "inspect._empty") else type_name
 
         sig = inspect.signature(f)
         input_schema = {
@@ -91,10 +101,14 @@ class TaskBuilder(TaskInstance):
 class JobBuilder:
     nodes: pyrsistent.PMap = field(default_factory=lambda: pyrsistent.m())
     edges: pyrsistent.PVector = field(default_factory=lambda: pyrsistent.v())
+    outputs: pyrsistent.PVector = field(default_factory=lambda: pyrsistent.v())
 
     def with_node(self, name: str, task: TaskInstance) -> Self:
         return replace(self, nodes=self.nodes.set(name, task))
 
+    def with_output(self, task: str, output: str = Node.DEFAULT_OUTPUT) -> Self:
+        return replace(self, outputs=self.outputs.append(DatasetId(task, output)))
+
     def with_edge(
         self, source: str, sink: str, into: str | int, frum: str = Node.DEFAULT_OUTPUT
     ) -> Self:
@@ -116,7 +130,24 @@ class JobBuilder:
             "marsParamList",
             "grib",
         }
-        _isinstance = lambda v, t: t == "Any" or t in skipped or isinstance(v, eval(t))
+
+        def getType(
+            fqn: str,
+        ) -> (
+            Any
+        ):  # NOTE: typing.Type return type is tempting but not true for builtin aliases
+            if fqn.startswith("tuple"):
+                # TODO recursive parsing of tuples etc!
+                return tuple
+            if "." in fqn:
+                mpath, name = fqn.rsplit(".", 1)
+                return getattr(importlib.import_module(mpath), name)
+            else:
+                return eval(fqn)
+
+        _isinstance = (
+            lambda v, t: t == "Any" or t in skipped or isinstance(v, getType(t))
+        )
 
         # static input types
         static_kw_errors: Iterable[str] = (
@@ -157,7 +188,9 @@ class JobBuilder:
                 lambda t1, t2: t2 == "Any"
                 or t1 == t2
                 or (t1, t2) in legits
-                or issubclass(eval(t1), eval(t2))
+                or t1
+                == "typing.Iterator"  # TODO replace with type extraction *and* check that this is multi-output
+                or issubclass(getType(t1), getType(t2))
             )
             if not _issubclass(output_param, input_param):
                 yield f"edge connects two incompatible nodes: {edge}"
@@ -169,6 +202,9 @@ class JobBuilder:
         # all inputs present
         # TODO
 
+        # all outputs created
+        # TODO
+
         errors = list(itertools.chain(static_kw_errors, edge_errors))
         if errors:
             return Either.error(errors)
@@ -177,5 +213,6 @@ class JobBuilder:
                 JobInstance(
                     tasks=cast(dict[str, TaskInstance], pyrsistent.thaw(self.nodes)),
                     edges=pyrsistent.thaw(self.edges),
+                    ext_outputs=pyrsistent.thaw(self.outputs),
                 )
             )

cascade/shm/api.py

@@ -6,13 +6,17 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import logging
 import os
+import socket
 from dataclasses import dataclass
 from enum import Enum, auto
 from typing import Protocol, Type, runtime_checkable
 
 from typing_extensions import Self
 
+logger = logging.getLogger(__name__)
+
 # TODO too much manual serde... either automate it based on dataclass field inspection, or just pickle it
 # (mind the server.recv/client.recv comment tho)
 # Also, consider switching from GetRequest, PurgeRequest, to DatasetRequest(get|purge|...)
@@ -244,15 +248,52 @@ def deser(data: bytes) -> Comm:
     return b2c[data[:1]].deser(data[1:])
 
 
-client_port_envvar = "CASCADE_SHM_PORT"
-
-
-def publish_client_port(port: int) -> None:
-    os.environ[client_port_envvar] = str(port)
-
-
-def get_client_port() -> int:
-    port = os.getenv(client_port_envvar)
-    if not port:
-        raise ValueError("missing port")
-    return int(port)
+client_socket_envvar = "CASCADE_SHM_SOCKET"
+
+
+def publish_socket_addr(sock: int | str) -> None:
+    if isinstance(sock, int):
+        ssock = f"port:{sock}"
+    else:
+        ssock = f"file:{sock}"
+    os.environ[client_socket_envvar] = ssock
+
+
+def get_socket_addr() -> tuple[socket.socket, int | str]:
+    ssock = os.getenv(client_socket_envvar)
+    if not ssock:
+        raise ValueError(f"missing sock addr in {client_socket_envvar}")
+    kind, addr = ssock.split(":", 1)
+    if kind == "port":
+        addr = int(addr)
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    elif kind == "file":
+        # TODO can we support SOCK_DGRAM too? Problem with response address
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    else:
+        raise NotImplementedError(kind)
+    return sock, addr
+
+
+def get_client_socket():
+    sock, addr = get_socket_addr()
+    if isinstance(addr, int):
+        sock.connect(("localhost", addr))
+    else:
+        sock.connect(addr)
+    return sock
+
+
+def get_server_socket():
+    sock, addr = get_socket_addr()
+    if isinstance(addr, int):
+        sock.bind(("0.0.0.0", addr))
+    else:
+        try:
+            os.unlink(addr)
+            logger.warning(f"unlinking at {addr}")
+        except FileNotFoundError:
+            pass
+        sock.bind(addr)
+        sock.listen(32)
+    return sock

cascade/shm/client.py

@@ -8,7 +8,7 @@
 
 import logging
 import multiprocessing.resource_tracker
-import socket
+import sys
 import time
 from multiprocessing.shared_memory import SharedMemory
 from typing import Callable, Type, TypeVar
@@ -17,14 +17,19 @@ import cascade.shm.api as api
 
 logger = logging.getLogger(__name__)
 
-# TODO eleminate in favour of track=False, once we are on python 3.13+
-is_unregister = True  # NOTE exposed for pytest control
-
 
 class ConflictError(Exception):
     pass
 
 
+if (sys.version_info.major, sys.version_info.minor) >= (3, 13):
+    is_unregister = False
+    shm_kwargs = {"track": False}
+else:
+    is_unregister = True
+    shm_kwargs = {}
+
+
 class AllocatedBuffer:
     def __init__(
         self,
@@ -34,8 +39,9 @@ class AllocatedBuffer:
         close_callback: Callable[[], None] | None,
         deser_fun: str,
     ):
+        self.shm: SharedMemory | None
         try:
-            self.shm: SharedMemory | None = SharedMemory(shmid, create=create, size=l)
+            self.shm = SharedMemory(shmid, create=create, size=l, **shm_kwargs)
         except FileExistsError:
             # NOTE this is quite wrong as instead of crashing, it would lead to undefined behaviour
             # However, as the systems we operate on don't seem to be reliable wrt cleanup/isolation,
@@ -43,13 +49,16 @@ class AllocatedBuffer:
             logger.error(
                 f"attempted opening {shmid=} but gotten FileExists. Will delete and retry"
             )
-            _shm = SharedMemory(shmid, create=False)
+            _shm = SharedMemory(shmid, create=False, **shm_kwargs)
+            _shm.close()
             _shm.unlink()
-            self.shm: SharedMemory | None = SharedMemory(shmid, create=create, size=l)
+            self.shm = SharedMemory(shmid, create=create, size=l, **shm_kwargs)
         self.l = l
         self.readonly = not create
         self.close_callback = close_callback
         self.deser_fun = deser_fun
+        if is_unregister:
+            multiprocessing.resource_tracker.unregister(self.shm._name, "shared_memory")  # type: ignore # _name
 
     def view(self) -> memoryview:
         if not self.shm:
@@ -60,14 +69,19 @@ class AllocatedBuffer:
         return mv
 
     def close(self) -> None:
-        if self.shm is not None:
+        if hasattr(self, "shm") and self.shm is not None:
             self.shm.close()
-            if is_unregister:
-                multiprocessing.resource_tracker.unregister(self.shm._name, "shared_memory")  # type: ignore # _name
             if self.close_callback:
                 self.close_callback()
             self.shm = None
 
+    def __del__(self) -> None:
+        if hasattr(self, "shm") and self.shm is not None:
+            try:
+                logger.error(f"missed close() call on {self.shm._name}")  # type: ignore # _name
+            except Exception as e:
+                logger.exception(f"failed to log due to {repr(e)}")
+
     # TODO context manager
 
 
@@ -80,11 +94,10 @@ def _send_command(comm: api.Comm, resp_class: Type[T], timeout_sec: float = 60.0
     # timeout_i and coeff determine rate of busy-waits: coeff=1 is additive, =2 is exponential
     # eventually this busy-waits will go away as we switch to event driven behaviour
     while timeout_sec > 0:
-        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        client_port = api.get_client_port()
-        sock.connect(("localhost", client_port))
+        sock = api.get_client_socket()
         logger.debug(f"sending message {comm}")
         sock.send(api.ser(comm))
+        # TODO rewrite to poller with timeout
         response_raw = sock.recv(1024)  # TODO or recv(4) + recv(int.from_bytes)?
         sock.close()
         response_com = api.deser(response_raw)
@@ -148,9 +161,9 @@ def status(key: str) -> api.DatasetStatus:
     return response.status
 
 
-def shutdown() -> None:
+def shutdown(timeout_sec: float = 2.0) -> None:
     comm = api.ShutdownCommand()
-    _send_command(comm, api.OkResponse)
+    _send_command(comm, api.OkResponse, timeout_sec)
 
 
 def ensure() -> None:
@@ -161,7 +174,7 @@ def ensure() -> None:
         try:
             _send_command(comm, api.OkResponse)
             logger.debug("shm server responds ok, leaving ensure loop")
-        except ConnectionRefusedError:
+        except (ConnectionRefusedError, FileNotFoundError):
             time.sleep(0.1)
             continue
         break

cascade/shm/dataset.py

@@ -277,7 +277,21 @@ class Manager:
         return ds.shmid, ds.size, rdid, ds.deser_fun, ""
 
     def purge(self, key: str, is_exit: bool = False) -> None:
-        # TODO the is_exit invocation is causing mess in logs, consider removal
+        if key not in self.datasets:
+            # NOTE known cases for this appearing:
+            # - a1/ a TaskSequence([t1, t2], publish={t2}) is sent by controller
+            # - a2/ t2 is computed and published by worker
+            # - a3/ controller sees t1 not required anymore => sends purge
+            # - a4/ as t1 was never materialized in shm => keyerror here
+            # - a/ this would be fixed by expaned dataset2host/worker model at the controller
+            # - b1/ purge request is sent by the data server
+            # - b2/ it is received by shm and purged, but ack fails on zmq
+            # - b3/ data server retries, but now the key is unknown
+            # - b/ this would be fixed by keeping track of tombstones
+            logger.warning(
+                f"key unknown to this shm instance: {key}, ignoring purge request"
+            )
+            return
         try:
             logger.debug(f"attempting purge-inquire of {key}")
             try:

cascade/shm/server.py

@@ -10,7 +10,7 @@ import logging
 import logging.config
 import signal
 import socket
-from typing import Any
+from typing import Any, cast
 
 import cascade.shm.api as api
 import cascade.shm.dataset as dataset
@@ -21,20 +21,35 @@ logger = logging.getLogger(__name__)
 class LocalServer:
     """Handles the socket communication, and the invocation of dataset.Manager which has the business logic"""
 
-    def __init__(self, port: int, shm_pref: str, capacity: int | None = None):
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        server = ("0.0.0.0", port)
-        self.sock.bind(server)
+    def __init__(self, shm_pref: str, capacity: int | None = None):
+        self.sock = api.get_server_socket()
+        logger.info(
+            f"shm server starting with {self.sock=} with {capacity=} and prefix {shm_pref}"
+        )
         self.manager = dataset.Manager(shm_pref, capacity)
         signal.signal(signal.SIGINT, self.atexit)
         signal.signal(signal.SIGTERM, self.atexit)
 
-    def receive(self) -> tuple[api.Comm, str]:
-        b, addr = self.sock.recvfrom(1024)  # TODO or recv(4) + recv(int.from_bytes)?
-        return api.deser(b), addr
+    def receive(self) -> tuple[api.Comm, str | socket.socket]:
+        # TODO recv(1024) or recv(4) + recv(int.from_bytes)?
+        if self.sock.type == socket.SOCK_DGRAM:
+            b, resp = self.sock.recvfrom(1024)
+        elif self.sock.type == socket.SOCK_STREAM:
+            resp, _addr = self.sock.accept()
+            b = resp.recv(1024)
+        else:
+            raise NotImplementedError(self.sock.type)
+        return api.deser(b), resp
 
-    def respond(self, comm: api.Comm, address: str) -> None:
-        self.sock.sendto(api.ser(comm), address)
+    def respond(self, comm: api.Comm, address: str | socket.socket) -> None:
+        m = api.ser(comm)
+        if self.sock.type == socket.SOCK_DGRAM:
+            self.sock.sendto(m, address)
+        elif self.sock.type == socket.SOCK_STREAM:
+            logger.debug(f"will send to {address} message {m}")
+            cast(socket.socket, address).send(m)
+        else:
+            raise NotImplementedError(self.sock.type)
 
     def atexit(self, signum: int, frame: Any) -> None:
         self.manager.atexit()
@@ -43,6 +58,7 @@ class LocalServer:
     def start(self):
         while True:
             payload, client = self.receive()
+            logger.debug(f"gotten {payload=}")
             try:
                 if isinstance(payload, api.AllocateRequest):
                     shmid, error = self.manager.add(
@@ -96,21 +112,18 @@ class LocalServer:
             except Exception as e:
                 logger.exception(f"failure during handling of {payload}")
                 response = api.OkResponse(error=repr(e))
+            logger.debug(f"sending {response=} to {client}")
             self.respond(response, client)
 
 
 def entrypoint(
-    port: int,
     capacity: int | None = None,
     logging_config: dict | None = None,
     shm_pref: str = "shm",
 ):
     if logging_config:
         logging.config.dictConfig(logging_config)
-    server = LocalServer(port, shm_pref, capacity)
-    logger.info(
-        f"shm server starting on {port=} with {capacity=} and prefix {shm_pref}"
-    )
+    server = LocalServer(shm_pref, capacity)
     try:
         server.start()
     except Exception as e:

earthkit/workflows/_version.py

@@ -1,2 +1,2 @@
 # Do not change! Do not track in version control!
-__version__ = "0.4.7"
+__version__ = "0.5.1"

earthkit/workflows/decorators.py

@@ -7,7 +7,7 @@
 # nor does it submit to any jurisdiction.
 
 from functools import wraps
-from typing import Callable, ParamSpec, TypeVar
+from typing import Any, Callable, Concatenate, ParamSpec, ParamSpecArgs, TypeVar
 
 from .fluent import Payload
 
@@ -15,16 +15,30 @@ P = ParamSpec("P")
 R = TypeVar("R")
 
 
-def as_payload(func: Callable[P, R]) -> Callable[P, Payload]:
-    """Wrap a function and return a payload object.
+def as_payload(func: Callable[Concatenate[ParamSpecArgs, P], R]):
+    """Wrap a function and return a Payload object.
 
-    Will pop metadata from kwargs and pass it to the payload.
+    Forces the function to be called with keyword arguments only, with args being passed
+    once the payload is executed from earlier Nodes.
+
+    Set `metadata` to pass metadata to the payload.
+
+    Examples
+    --------
+        ```python
+        @as_payload
+        def my_function(a, b, *, keyword):
+            pass
+
+        my_function(1, 2, keyword='test')  # Raises an error
+        my_function(b=2, keyword='test')  # OK, a will be passed from earlier nodes
+        my_function(keyword='test')  # OK, a and b will be passed from earlier nodes
+
+        ```
     """
-    from .fluent import Payload
 
-    @wraps(func)
-    def decorator(*args, **kwargs) -> Payload:
-        metadata = kwargs.pop("metadata", None)
-        return Payload(func, args, kwargs, metadata=metadata)
+    @wraps(func, assigned=["__name__", "__doc__"])
+    def decorator(*, metadata: dict[str, Any] | None = None, **kwargs) -> Payload:
+        return Payload(func, args=None, kwargs=kwargs, metadata=metadata)
 
     return decorator

earthkit/workflows/fluent.py

@@ -10,7 +10,16 @@ from __future__ import annotations
 
 import functools
 import hashlib
-from typing import Any, Callable, Hashable, Iterable, Optional, Sequence, TypeVar
+from typing import (
+    Any,
+    Callable,
+    Hashable,
+    Iterable,
+    Optional,
+    ParamSpec,
+    Sequence,
+    TypeVar,
+)
 
 import numpy as np
 import xarray as xr
@@ -20,8 +29,6 @@ from .graph import Graph
 from .graph import Node as BaseNode
 from .graph import Output
 
-ActionType = TypeVar("ActionType", bound="Action")
-
 
 class Payload:
     """Class for detailing function, args and kwargs to be computing in a graph node"""
@@ -68,7 +75,7 @@ class Payload:
     def __str__(self) -> str:
         return f"{self.name()}{self.args}{self.kwargs}:{self.metadata}"
 
-    def __eq__(self, other: Payload) -> bool:
+    def __eq__(self, other) -> bool:
         if not isinstance(other, Payload):
             return False
         return str(self) == str(other)
@@ -86,15 +93,16 @@ def custom_hash(string: str) -> str:
 Coord = tuple[str, list[Any]]
 Input = BaseNode | Output
 
-F = TypeVar("F", bound=Callable[..., Any])
+P = ParamSpec("P")
+R = TypeVar("R")
 
 
-def capture_payload_metadata(func: F) -> F:
+def capture_payload_metadata(func: Callable[P, R]) -> Callable[P, R]:
     """Wrap a function which returns a new action and insert
     given `payload_metadata`
     """
 
-    @functools.wraps(func)
+    # @functools.wraps(func)
     def decorator(*args, **kwargs):
         metadata = kwargs.pop("payload_metadata", {})
         result = func(*args, **kwargs)
@@ -162,7 +170,7 @@ class Node(BaseNode):
 
 class Action:
 
-    REGISTRY: dict[str, Action] = {}
+    REGISTRY: dict[str, type[Action]] = {}
 
     def __init__(self, nodes: xr.DataArray, yields: Optional[Coord] = None):
         if yields:
@@ -224,24 +232,17 @@ class Action:
                 f"Action class {obj} already has an attribute {name}, will not override"
             )
 
-        cls.REGISTRY[name] = obj  # type: ignore
+        cls.REGISTRY[name] = obj
 
     @classmethod
     def flush_registry(cls):
         """Flush the registry of all registered actions"""
         cls.REGISTRY = {}
 
-    def as_action(self, other: ActionType) -> ActionType:
+    def as_action(self, other) -> Action:
         """Parse action into another action class"""
         return other(self.nodes)
 
-    def __getattr__(self, attr):
-        if attr in Action.REGISTRY:
-            return RegisteredAction(
-                attr, Action.REGISTRY[attr], self
-            )  # When the attr is a registered action class
-        raise AttributeError(f"{self.__class__.__name__} has no attribute {attr!r}")
-
     def join(
         self,
         other_action: "Action",
@@ -803,6 +804,13 @@ class Action:
         if dim_name in self.nodes.coords and len(self.nodes.coords[dim_name]) == 1:
             self.nodes = self.nodes.squeeze(dim_name, drop=drop)
 
+    def __getattr__(self, attr):
+        if attr in Action.REGISTRY:
+            return RegisteredAction(
+                attr, Action.REGISTRY[attr], self
+            )  # When the attr is a registered action class
+        raise AttributeError(f"{self.__class__.__name__} has no attribute {attr!r}")
+
 
 class RegisteredAction:
     """Wrapper around registered actions"""

earthkit/workflows/mark.py

@@ -15,10 +15,9 @@ def add_execution_metadata(**kwargs) -> Callable[[F], F]:
     """Add execution metadata to a function."""
 
     def decorator(func: F) -> F:
-        if not hasattr(func, "_cascade"):
-            func._cascade = {}
-
-        func._cascade.update(kwargs)
+        kw = getattr(func, "_cascade", {}).copy()
+        kw.update(kwargs)
+        setattr(func, "_cascade", kw)
         return func
 
     return decorator

{earthkit_workflows-0.4.7.dist-info → earthkit_workflows-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: earthkit-workflows
-Version: 0.4.7
+Version: 0.5.1
 Summary: Earthkit Workflows is a Python library for declaring earthkit task DAGs, as well as scheduling and executing them on heterogeneous computing systems.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License-Expression: Apache-2.0

{earthkit_workflows-0.4.7.dist-info → earthkit_workflows-0.5.1.dist-info}/RECORD

@@ -10,7 +10,8 @@ cascade/benchmarks/job1.py,sha256=MOcZZYgf36MzHCjtby0lQyenM1ODUlagG8wtt2CbpnI,46
 cascade/benchmarks/matmul.py,sha256=5STuvPY6Q37E2pKRCde9dQjL5M6tx7tkES9cBLZ6eK4,1972
 cascade/benchmarks/plotting.py,sha256=vSz9HHbqZwMXHpBUS-In6xsXGgK7QIoQTTiYfSwYwZs,4428
 cascade/benchmarks/reporting.py,sha256=MejaM-eekbMYLAnuBxGv_t4dR1ODJs4Rpc0fiZSGjyw,5410
-cascade/benchmarks/util.py,sha256=ivnd4olB10_F2BCCJan4gzYiyZXMn3gA2iNbacbu464,9812
+cascade/benchmarks/tests.py,sha256=eeQE0YR4FKi5k9BMJaTcXjKF5eIu3xXJsHc099P0Jio,5537
+cascade/benchmarks/util.py,sha256=wP7lDI6v9ATIF96uagVB-23EiagCTVYJhUUy-_CfqQ8,9892
 cascade/controller/__init__.py,sha256=p4C2p3S_0nUGamP9Mi6cSa5bvpiWbI6sVWtGhFnNqjw,1278
 cascade/controller/act.py,sha256=WHIsk4H-Bbyl_DABX2VWhyKy_cNnp12x1nilatPCL8I,2981
 cascade/controller/core.py,sha256=NqvZ5g5GNphwOpzdXbCI0_fxIzzmO97_n2xZKswK72Q,3589
@@ -20,15 +21,15 @@ cascade/controller/report.py,sha256=rKGYmq4nIiDqKuP_C7YSwpEAUOPdjILlDcbKkdUt30s,
 cascade/executor/bridge.py,sha256=WDE-GM2Bv7nUk1-nV-otMGuaRYw1-Vmd7PWploXBp6Y,8267
 cascade/executor/comms.py,sha256=-9qrKwva6WXkHRQtzSnLFy5gB3bOWuxYJP5fL6Uavw8,8736
 cascade/executor/config.py,sha256=8azy_sXdvDGO0zTNqA0pdtkXsyihM4FQ4U1W_3Dhua0,1571
-cascade/executor/data_server.py,sha256=xLIbLkWn8PnJl4lMP8ADHa2S0EgPwr0-bH7_Sib_Y70,13701
-cascade/executor/executor.py,sha256=htHobVN0eAxofii2g29yN--qCkaQgA05MVKCgCW7Dhg,13644
+cascade/executor/data_server.py,sha256=TSFJdSR9PtKSvvLTosHt0ITQlqtGGAl5N_io6wtvL0A,13569
+cascade/executor/executor.py,sha256=OwLrhSLm4bIHsWdnjXlnQntxGIOHgrIPSSVZ5nbNWvQ,13686
 cascade/executor/msg.py,sha256=7HI0rKeCRaV1ONR4HWEa64nHbu-p6-QdBwJNitmst48,4340
 cascade/executor/platform.py,sha256=mRUauodvRle9rAbtFr5n9toKzIgt_pecNlhOjon4dvY,2348
 cascade/executor/serde.py,sha256=z6klTOZqW_BVGrbIRNz4FN0_XTfRiKBRQuvgsQIuyAo,2827
 cascade/executor/runner/__init__.py,sha256=30BM80ZyA7w3IrGiKKLSFuhRehbR2Mm99OJ8q5PJ63c,1547
 cascade/executor/runner/entrypoint.py,sha256=WyxOFGAYDQD_fXsM4H9_6xBrnAmQrCTUnljfcW6-BoM,7918
-cascade/executor/runner/memory.py,sha256=jkAV9T7-imciVcGvkV7OhRfosEpOQJU1OME7z-4ztAs,6371
-cascade/executor/runner/packages.py,sha256=zOMCKRX34j6vSIK741KV7kTFRgy3bZz_xeLrldslaL4,2633
+cascade/executor/runner/memory.py,sha256=VEOrYfFNGNBM7vMY05wjbX3L0U-RJZWpm_Ud4bMUR5g,6486
+cascade/executor/runner/packages.py,sha256=lic5ItjyDpcQVRBFOZssvnco9bmxWpq_JRFDeShVR8k,4150
 cascade/executor/runner/runner.py,sha256=zqpkvxdWLbwyUFaUbZmSj0KQEBNRpmF8gwVotiaamhc,4870
 cascade/gateway/__init__.py,sha256=1EzMKdLFXEucj0YWOlyVqLx4suOntitwM03T_rRubIk,829
 cascade/gateway/__main__.py,sha256=kmfklSeA7v5ie75SBHOql-eHuY6x4eTHlItMYqCQ1Pg,969
@@ -37,7 +38,7 @@ cascade/gateway/client.py,sha256=1p4Tvrf-BH0LQHOES5rY1z3JNIfmXcqWG2kYl4rpcE0,406
 cascade/gateway/router.py,sha256=9oTkqssb3dHF24TIaAn_7oQoNfm4qkOvriufbOJxnyE,11582
 cascade/gateway/server.py,sha256=BfUKpU2nCEB_zI4BdZU_9zHYHX1WoQaLARCTxMSP0Nk,3971
 cascade/low/__init__.py,sha256=5cw2taOGITK_gFbICftzK2YLdEAnLUY5OzblFzdHss4,769
-cascade/low/builders.py,sha256=_u5X8G_EF00hFt8Anv9AXo6yPf1O8MHDmqs2kKmREl0,7073
+cascade/low/builders.py,sha256=F2W47zIa8tfBxHvvRekNd8SV7l4HOPEisQujhh9gisQ,8428
 cascade/low/core.py,sha256=_3x4ka_pmCgZbfwFeyhq8S4M6wmh0s24VRCLhk5yQFM,6444
 cascade/low/dask.py,sha256=xToT_vyfkgUUxSFN7dS7qLttxzuBbBZfDylPzGg7sPg,3319
 cascade/low/execution_context.py,sha256=cdDJLYhreo4T7t4qXgFBosncubZpTrm0hELo7q4miqo,6640
@@ -52,18 +53,17 @@ cascade/scheduler/core.py,sha256=umORLC6SDeOyS4z8nQuVFkDukBJ96JfH4hdLSj6Km20,337
 cascade/scheduler/precompute.py,sha256=AhTn8RgnU4XuV_WAgbVXz9z0YRpNS6LCY1dJeHdTfCc,8709
 cascade/shm/__init__.py,sha256=R9QgGSnsl_YDjFjAUQkoleM_5yGM37ce9S8a4ReA1mE,3854
 cascade/shm/algorithms.py,sha256=SGxnJF4ovUaywTunMJWkG77l5DN-jXx7HgABt3sRJXM,2356
-cascade/shm/api.py,sha256=a_KrjyELsDms0Di0ThHsZe7MfmNEkekflmjXAQ1_Qws,6040
-cascade/shm/client.py,sha256=pnod_dmUROJZRtipCpoeCuAEuynW0IgSfgjrp21CH2M,5893
-cascade/shm/dataset.py,sha256=Z2ewpnW7mVDJB9GylIVoOWV0DYOF7FWLIXkIvV-Y7sI,12347
+cascade/shm/api.py,sha256=TFK0ioKJpJ2-rTxwk_O5BtB6AKjgwfM8CIl-VZaUIZo,7180
+cascade/shm/client.py,sha256=7rUG0bra7XTJRumywQ-Gos4pWeZoXpTZqseh36uNWFg,6312
+cascade/shm/dataset.py,sha256=QAALiWK0fyMLet9XFXmATm-c9gTuF77cifGbjP3WjXo,13155
 cascade/shm/disk.py,sha256=Fdl_pKOseaXroRp01OwqWVsdI-sSmiFizIFCdxBuMWM,2653
 cascade/shm/func.py,sha256=ZWikgnSLCmbSoW2LDRJwtjxdwTxkR00OUHAsIRQ-ChE,638
-cascade/shm/server.py,sha256=LnnNX0F6QJt5V_JLfmC3ZMHGNL5WpLY44wpB_pYDr7Y,5042
+cascade/shm/server.py,sha256=geWo2BuF8sa_BqY8akh6ardWFfKfDJktWujrdDHn624,5648
 earthkit/workflows/__init__.py,sha256=-p4anEn0YQbYWM2tbXb0Vc3wq4-m6kFhcNEgAVu5Jis,1948
-earthkit/workflows/_version.py,sha256=Vm-kUKLcx3Zcn5dopRQ_3Wdf6JGosDY_tCioAs7HFG0,72
-earthkit/workflows/decorators.py,sha256=DM4QAtQ2glUUcDecwPkXcdlu4dio7MvgpcdmU5LYvD8,937
-earthkit/workflows/fluent.py,sha256=IN_sqwr7W8wbwP7wTOklgnjVe34IUCmv1ku-DWVTCJc,30179
-earthkit/workflows/mark.py,sha256=PdsXmRfhw1SyyJ74mzFPsLRqMCdlYv556fFX4bqlh9Y,1319
-earthkit/workflows/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+earthkit/workflows/_version.py,sha256=gCSK2O7UArCP94MovPvZF039nZ9JlETEx89VakvCxQg,72
+earthkit/workflows/decorators.py,sha256=YK6AN-Ta9cAOX__DjZbn_vNYdpRL98N6dbF31E6Vu1c,1478
+earthkit/workflows/fluent.py,sha256=3CvZfdLjXCoGR0VJDTB8_PDFgR7n-UhGLdKo7E5zuvM,30161
+earthkit/workflows/mark.py,sha256=otgR6ar_9R7q5VRFD6RlLUROfjhyiaMIsgcleW2icKI,1322
 earthkit/workflows/taskgraph.py,sha256=RsT1Qlng1uPZSaSBNqE8vFsoI5J8DDcQl468YPX-kCY,4460
 earthkit/workflows/transformers.py,sha256=BsUUvnG-UyerT3XUYcHc1qJkSsLc0ZX3Zxqq70tJWLU,2105
 earthkit/workflows/utility.py,sha256=ygqn1s846WQbo7HGY46Z8N1AXrDFGwyygSgsv4YnGJ8,1344
@@ -89,8 +89,8 @@ earthkit/workflows/graph/split.py,sha256=t-Sji5eZb01QO1szqmDNTodDDALqdo-0R0x1ESs
 earthkit/workflows/graph/transform.py,sha256=BZ8n7ePUnuGgoHkMqZC3SLzifu4oq6q6t6vka0khFtg,3842
 earthkit/workflows/graph/visit.py,sha256=MP-aFSqOl7aqJY2i7QTgY4epqb6yM7_lK3ofvOqfahw,1755
 earthkit/workflows/plugins/__init__.py,sha256=nhMAC0eMLxoJamjqB5Ns0OWy0OuxEJ_YvaDFGEQITls,129
-earthkit_workflows-0.4.7.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
-earthkit_workflows-0.4.7.dist-info/METADATA,sha256=8hfeDm0i94yYA1_CJRFcGrfWE1r7MCzb-HyzvIc5tPs,1571
-earthkit_workflows-0.4.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-earthkit_workflows-0.4.7.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
-earthkit_workflows-0.4.7.dist-info/RECORD,,
+earthkit_workflows-0.5.1.dist-info/licenses/LICENSE,sha256=73MJ7twXMKnWwmzmrMiFwUeY7c6JTvxphVggeUq9Sq4,11381
+earthkit_workflows-0.5.1.dist-info/METADATA,sha256=r90HXtWZwIhQ1Em7bIlByUbkiT1YQDAa9nyOUvejago,1571
+earthkit_workflows-0.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+earthkit_workflows-0.5.1.dist-info/top_level.txt,sha256=oNrH3Km3hK5kDkTOiM-8G8OQglvZcy-gUKy7rlooWXs,17
+earthkit_workflows-0.5.1.dist-info/RECORD,,