earthkit-workflows 0.4.6__py3-none-any.whl → 0.5.0__py3-none-any.whl

cascade/benchmarks/tests.py

@@ -0,0 +1,173 @@
+# (C) Copyright 2025- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+"""Module for simplifying writing tests
+
+Similar to util, but not enough to unify
+
+It is capable, for a single given task, to spin an shm server, put all task's inputs into it, execute the task, store outputs in memory, and retrieve the result.
+See the `demo()` function at the very end
+"""
+
+import logging
+from contextlib import contextmanager
+from dataclasses import dataclass
+from time import perf_counter_ns
+from typing import Any, Callable
+
+import cloudpickle
+
+import cascade.executor.platform as platform
+import cascade.shm.api as shm_api
+import cascade.shm.client as shm_client
+from cascade.executor.comms import Listener as ZmqListener
+from cascade.executor.config import logging_config
+from cascade.executor.msg import BackboneAddress, DatasetPublished
+from cascade.executor.runner.memory import Memory, ds2shmid
+from cascade.executor.runner.packages import PackagesEnv
+from cascade.executor.runner.runner import ExecutionContext, run
+from cascade.low.builders import TaskBuilder
+from cascade.low.core import DatasetId
+from cascade.shm.server import entrypoint as shm_server
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def setup_shm(testId: str):
+    mp_ctx = platform.get_mp_ctx("executor-aux")
+    shm_socket = f"/tmp/tcShm-{testId}"
+    shm_api.publish_socket_addr(shm_socket)
+    shm_process = mp_ctx.Process(
+        target=shm_server,
+        kwargs={
+            "logging_config": logging_config,
+            "shm_pref": f"tc{testId}",
+        },
+    )
+    shm_process.start()
+    shm_client.ensure()
+    try:
+        yield
+    except Exception as e:
+        # NOTE we log like this in case shm shutdown freezes
+        logger.exception(f"gotten {repr(e)}, proceed with shm shutdown")
+        raise
+    finally:
+        shm_client.shutdown(timeout_sec=1.0)
+        shm_process.join(1)
+        if shm_process.is_alive():
+            shm_process.terminate()
+            shm_process.join(1)
+        if shm_process.is_alive():
+            shm_process.kill()
+            shm_process.join(1)
+
+
+def simple_runner(callback: BackboneAddress, executionContext: ExecutionContext):
+    tasks = list(executionContext.tasks.keys())
+    if len(tasks) != 1:
+        raise ValueError(f"expected 1 task, gotten {len(tasks)}")
+    taskId = tasks[0]
+    taskInstance = executionContext.tasks[taskId]
+    with Memory(callback, "testWorker") as memory, PackagesEnv() as pckg:
+        # for key, value in taskSequence.extra_env.items():
+        #    os.environ[key] = value
+
+        pckg.extend(taskInstance.definition.environment)
+        run(taskId, executionContext, memory)
+        memory.flush()
+
+
+@dataclass
+class CallableInstance:
+    func: Callable
+    kwargs: dict[str, Any]
+    args: list[tuple[int, Any]]
+    env: list[str]
+    exp_output: Any
+
+
+def callable2ctx(
+    callableInstance: CallableInstance, callback: BackboneAddress
+) -> ExecutionContext:
+    taskInstance = TaskBuilder.from_callable(
+        callableInstance.func, callableInstance.env
+    )
+    param_source = {}
+    params = [
+        (key, DatasetId("taskId", f"kwarg.{key}"), value)
+        for key, value in callableInstance.kwargs.items()
+    ] + [
+        (key, DatasetId("taskId", f"pos.{key}"), value)
+        for key, value in callableInstance.args
+    ]
+    for key, ds_key, value in params:
+        raw = cloudpickle.dumps(value)
+        L = len(raw)
+        buf = shm_client.allocate(ds2shmid(ds_key), L, "cloudpickle.loads")
+        buf.view()[:L] = raw
+        buf.close()
+        param_source[key] = (ds_key, "Any")
+
+    return ExecutionContext(
+        tasks={"taskId": taskInstance},
+        param_source={"taskId": param_source},
+        callback=callback,
+        publish={
+            DatasetId("taskId", output)
+            for output, _ in taskInstance.definition.output_schema
+        },
+    )
+
+
+def run_test(
+    callableInstance: CallableInstance, testId: str, max_runtime_sec: int
+) -> Any:
+    with setup_shm(testId):
+        addr = f"ipc:///tmp/tc{testId}"
+        listener = ZmqListener(addr)
+        ec_ctx = callable2ctx(callableInstance, addr)
+        mp_ctx = platform.get_mp_ctx("executor-aux")
+        runner = mp_ctx.Process(target=simple_runner, args=(addr, ec_ctx))
+        runner.start()
+        output = DatasetId("taskId", "0")
+
+        end = perf_counter_ns() + max_runtime_sec * int(1e9)
+        while perf_counter_ns() < end:
+            mess = listener.recv_messages()
+            if mess == [
+                DatasetPublished(origin="testWorker", ds=output, transmit_idx=None)
+            ]:
+                break
+            elif not mess:
+                continue
+            else:
+                raise ValueError(mess)
+
+        runner.join()
+        output_buf = shm_client.get(ds2shmid(output))
+        output_des = cloudpickle.loads(output_buf.view())
+        output_buf.close()
+    assert output_des == callableInstance.exp_output
+
+
+def demo():
+    def myfunc(l: int) -> float:
+        import numpy as np
+
+        return np.arange(l).sum()
+
+    ci = CallableInstance(
+        func=myfunc, kwargs={"l": 4}, args=[], env=["numpy"], exp_output=6
+    )
+    run_test(ci, "numpyTest1", 2)
+
+
+if __name__ == "__main__":
+    demo()

cascade/benchmarks/util.py

@@ -17,6 +17,7 @@ import subprocess
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from time import perf_counter_ns
+from typing import Any
 
 import orjson
 
@@ -28,7 +29,7 @@ from cascade.executor.comms import callback
 from cascade.executor.config import logging_config, logging_config_filehandler
 from cascade.executor.executor import Executor
 from cascade.executor.msg import BackboneAddress, ExecutorShutdown
-from cascade.low.core import JobInstance
+from cascade.low.core import DatasetId, JobInstance
 from cascade.low.func import msum
 from cascade.scheduler.precompute import precompute
 from earthkit.workflows.graph import Graph, deduplicate_nodes
@@ -159,7 +160,7 @@ def run_locally(
     portBase: int = 12345,
     log_base: str | None = None,
     report_address: str | None = None,
-) -> None:
+) -> dict[DatasetId, Any]:
     if log_base is not None:
         log_path = f"{log_base}.controller.txt"
         logging.config.dictConfig(logging_config_filehandler(log_path))
@@ -216,6 +217,7 @@ def run_locally(
         if os.environ.get("CASCADE_DEBUG_PRINT"):
             for key, value in result.outputs.items():
                 print(f"{key} => {value}")
+        return result.outputs
     except Exception:
         # NOTE we log this to get the stacktrace into the logfile
         logger.exception("controller failure, proceed with executor shutdown")

cascade/controller/report.py

@@ -27,6 +27,7 @@ JobId = str
 
 @dataclass
 class JobProgress:
+    started: bool
     completed: bool
     pct: (
         str | None
@@ -35,19 +36,20 @@ class JobProgress:
 
     @classmethod
     def failed(cls, failure: str) -> Self:
-        return cls(True, None, failure)
+        return cls(True, True, None, failure)
 
     @classmethod
     def progressed(cls, pct: float) -> Self:
         progress = "{:.2%}".format(pct)[:-1]
-        return cls(False, progress, None)
+        return cls(True, False, progress, None)
 
     @classmethod
     def succeeded(cls) -> Self:
-        return cls(True, None, None)
+        return cls(True, True, None, None)
 
 
-JobProgressStarted = JobProgress(False, "0.00", None)
+JobProgressStarted = JobProgress(True, False, "0.00", None)
+JobProgressEnqueued = JobProgress(False, False, None, None)
 
 
 @dataclass

cascade/executor/data_server.py

@@ -20,7 +20,6 @@ from concurrent.futures import Executor as PythonExecutor
 from concurrent.futures import Future, ThreadPoolExecutor, wait
 from time import time_ns
 
-import cascade.shm.api as shm_api
 import cascade.shm.client as shm_client
 from cascade.executor.comms import Listener, callback, send_data
 from cascade.executor.msg import (
@@ -48,7 +47,6 @@ class DataServer:
         maddress: BackboneAddress,
         daddress: BackboneAddress,
         host: str,
-        shm_port: int,
         logging_config: dict,
     ):
         logging.config.dictConfig(logging_config)
@@ -58,7 +56,6 @@ class DataServer:
         self.daddress = daddress
         self.dlistener = Listener(daddress)
         self.terminating = False
-        shm_api.publish_client_port(shm_port)
         self.cap = 2
         self.ds_proc_tp: PythonExecutor = ThreadPoolExecutor(max_workers=self.cap)
         self.futs_in_progress: dict[
@@ -305,8 +302,7 @@ def start_data_server(
     maddress: BackboneAddress,
     daddress: BackboneAddress,
     host: str,
-    shm_port: int,
     logging_config: dict,
 ):
-    server = DataServer(maddress, daddress, host, shm_port, logging_config)
+    server = DataServer(maddress, daddress, host, logging_config)
     server.recv_loop()

cascade/executor/executor.py

@@ -18,6 +18,7 @@ the tasks themselves.
 import atexit
 import logging
 import os
+import uuid
 from multiprocessing.process import BaseProcess
 from typing import Iterable
 
@@ -94,8 +95,8 @@ class Executor:
         self.sender = ReliableSender(self.mlistener.address, resend_grace_ms)
         self.sender.add_host("controller", controller_address)
         # TODO make the shm server params configurable
-        shm_port = portBase + 2
-        shm_api.publish_client_port(shm_port)
+        shm_port = f"/tmp/cascShmSock-{uuid.uuid4()}"  # portBase + 2
+        shm_api.publish_socket_addr(shm_port)
         ctx = platform.get_mp_ctx("executor-aux")
         if log_base:
             shm_logging = logging_config_filehandler(f"{log_base}.shm.txt")
@@ -104,12 +105,11 @@ class Executor:
         logger.debug("about to start an shm process")
         self.shm_process = ctx.Process(
             target=shm_server,
-            args=(
-                shm_port,
-                shm_vol_gb * (1024**3) if shm_vol_gb else None,
-                shm_logging,
-                f"sCasc{host}",
-            ),
+            kwargs={
+                "capacity": shm_vol_gb * (1024**3) if shm_vol_gb else None,
+                "logging_config": shm_logging,
+                "shm_pref": f"sCasc{host}",
+            },
         )
         self.shm_process.start()
         self.daddress = address_of(portBase + 1)
@@ -124,7 +124,6 @@ class Executor:
                 self.mlistener.address,
                 self.daddress,
                 self.host,
-                shm_port,
                 dsr_logging,
             ),
         )

cascade/executor/runner/memory.py

@@ -12,6 +12,7 @@ Interaction with shm
 
 import hashlib
 import logging
+import sys
 from contextlib import AbstractContextManager
 from typing import Any, Literal
 
@@ -119,27 +120,29 @@ class Memory(AbstractContextManager):
             self.local.pop(inputId)
 
         # NOTE poor man's gpu mem management -- currently torch only. Given the task sequence limitation,
-        # this may not be the best place to invoke. Additionally, we may want to check first whether
-        # the worker is gpu aware, etc
-        try:
-            import torch
-
-            if torch.cuda.is_available():
-                free, total = torch.cuda.mem_get_info()
-                logger.debug(f"cuda mem avail: {free/total:.2%}")
-                if free / total < 0.8:
-                    torch.cuda.empty_cache()
+        # this may not be the best place to invoke.
+        if (
+            "torch" in sys.modules
+        ):  # if no task on this worker imported torch, no need to flush
+            try:
+                import torch
+
+                if torch.cuda.is_available():
                     free, total = torch.cuda.mem_get_info()
-                    logger.debug(f"cuda mem avail post cache empty: {free/total:.2%}")
+                    logger.debug(f"cuda mem avail: {free/total:.2%}")
                     if free / total < 0.8:
-                        # NOTE this ofc makes low sense if there is any other application (like browser or ollama)
-                        # that the user may be running
-                        logger.warning("cuda mem avail low despite cache empty!")
-                        logger.debug(torch.cuda.memory_summary())
-        except ImportError:
-            return
-        except Exception:
-            logger.exception("failed to free cuda cache")
+                        torch.cuda.empty_cache()
+                        free, total = torch.cuda.mem_get_info()
+                        logger.debug(
+                            f"cuda mem avail post cache empty: {free/total:.2%}"
+                        )
+                        if free / total < 0.8:
+                            # NOTE this ofc makes low sense if there is any other application (like browser or ollama)
+                            # that the user may be running
+                            logger.warning("cuda mem avail low despite cache empty!")
+                            logger.debug(torch.cuda.memory_summary())
+            except Exception:
+                logger.exception("failed to free cuda cache")
 
     def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
         # this is required so that the Shm can be properly freed, otherwise you get 'pointers cannot be closed'

cascade/executor/runner/packages.py

@@ -12,6 +12,7 @@ Note that venv itself is left untouched after the run finishes -- we extend sys
 with a temporary directory and install in there
 """
 
+import importlib
 import logging
 import os
 import site
@@ -24,6 +25,51 @@ from typing import Literal
 logger = logging.getLogger(__name__)
 
 
+class Commands:
+    venv_command = lambda name: ["uv", "venv", name]
+    install_command = lambda name: [
+        "uv",
+        "pip",
+        "install",
+        "--prefix",
+        name,
+        "--prerelease",
+        "explicit",
+    ]
+
+
+def run_command(command: list[str]) -> None:
+    try:
+        result = subprocess.run(command, check=False, capture_output=True)
+    except FileNotFoundError as ex:
+        raise ValueError(f"command failure: {ex}")
+    if result.returncode != 0:
+        msg = f"command failed with {result.returncode}. Stderr: {result.stderr}, Stdout: {result.stdout}, Args: {result.args}"
+        logger.error(msg)
+        raise ValueError(msg)
+
+
+def new_venv() -> tempfile.TemporaryDirectory:
+    """1. Creates a new temporary directory with a venv inside.
+    2. Extends sys.path so that packages in that venv can be imported.
+    """
+    logger.debug("creating a new venv")
+    td = tempfile.TemporaryDirectory(prefix="cascade_runner_venv_")
+    # NOTE we create a venv instead of just plain directory, because some of the packages create files
+    # outside of site-packages. Thus we then install with --prefix, not with --target
+    run_command(Commands.venv_command(td.name))
+
+    # NOTE not sure if getsitepackages was intended for this -- if issues, attempt replacing
+    # with something like f"{td.name}/lib/python*/site-packages" + globbing
+    extra_sp = site.getsitepackages(prefixes=[td.name])
+    # NOTE this makes the explicit packages go first, in case of a different version
+    logger.debug(f"extending sys.path with {extra_sp}")
+    sys.path = extra_sp + sys.path
+    logger.debug(f"new sys.path: {sys.path}")
+
+    return td
+
+
 class PackagesEnv(AbstractContextManager):
     def __init__(self) -> None:
         self.td: tempfile.TemporaryDirectory | None = None
@@ -32,38 +78,32 @@ class PackagesEnv(AbstractContextManager):
         if not packages:
             return
         if self.td is None:
-            logger.debug("creating a new venv")
-            self.td = tempfile.TemporaryDirectory()
-            venv_command = ["uv", "venv", self.td.name]
-            # NOTE we create a venv instead of just plain directory, because some of the packages create files
-            # outside of site-packages. Thus we then install with --prefix, not with --target
-            subprocess.run(venv_command, check=True)
+            self.td = new_venv()
 
         logger.debug(
             f"installing {len(packages)} packages: {','.join(packages[:3])}{',...' if len(packages) > 3 else ''}"
         )
-        install_command = [
-            "uv",
-            "pip",
-            "install",
-            "--prefix",
-            self.td.name,
-            "--prerelease",
-            "allow",
-        ]
+        install_command = Commands.install_command(self.td.name)
         if os.environ.get("VENV_OFFLINE", "") == "YES":
             install_command += ["--offline"]
         if cache_dir := os.environ.get("VENV_CACHE", ""):
             install_command += ["--cache-dir", cache_dir]
         install_command.extend(set(packages))
-        subprocess.run(install_command, check=True)
-        # NOTE not sure if getsitepackages was intended for this -- if issues, attempt replacing
-        # with something like f"{self.td.name}/lib/python*/site-packages" + globbing
-        extra_sp = site.getsitepackages(prefixes=[self.td.name])
-        # NOTE this makes the explicit packages go first, in case of a different version
-        sys.path = extra_sp + sys.path
+        logger.debug(f"running install command: {' '.join(install_command)}")
+        run_command(install_command)
+
+        # NOTE we need this due to namespace packages:
+        # 1. task 1 installs ns.pkg1 in its venv
+        # 2. task 1 finishes, task 2 starts on the same worker
+        # 3. task 2 starts, installs ns.pkg2. However, importlib is in a state that ns is aware only of pkg1 submod
+        # Additionally, the caches are invalid anyway, because task 1's venv is already deleted
+        importlib.invalidate_caches()
+        # TODO some namespace packages may require a reimport because they dynamically build `__all__` -- eg earthkit
 
     def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]:
+        sys.path = [
+            p for p in sys.path if self.td is None or not p.startswith(self.td.name)
+        ]
         if self.td is not None:
             self.td.cleanup()
         return False

cascade/gateway/__main__.py

@@ -15,14 +15,17 @@ from cascade.gateway.server import serve
 
 
 def main(
-    url: str, log_base: str | None = None, troika_config: str | None = None
+    url: str,
+    log_base: str | None = None,
+    troika_config: str | None = None,
+    max_jobs: int | None = None,
 ) -> None:
     if log_base:
         log_path = f"{log_base}/gateway.txt"
         logging.config.dictConfig(logging_config_filehandler(log_path))
     else:
         logging.config.dictConfig(logging_config)
-    serve(url, log_base, troika_config)
+    serve(url, log_base, troika_config, max_jobs)
 
 
 if __name__ == "__main__":

cascade/gateway/api.py

@@ -62,8 +62,9 @@ class JobProgressRequest(CascadeGatewayAPI):
 
 
 class JobProgressResponse(CascadeGatewayAPI):
-    progresses: dict[JobId, JobProgress]
+    progresses: dict[JobId, JobProgress | None]
     datasets: dict[JobId, list[DatasetId]]
+    queue_length: int
     error: str | None  # top level error
 
 

cascade/gateway/router.py

@@ -15,6 +15,7 @@ import os
 import stat
 import subprocess
 import uuid
+from collections import OrderedDict
 from dataclasses import dataclass
 from typing import Iterable
 
@@ -22,7 +23,12 @@ import orjson
 import zmq
 
 import cascade.executor.platform as platform
-from cascade.controller.report import JobId, JobProgress, JobProgressStarted
+from cascade.controller.report import (
+    JobId,
+    JobProgress,
+    JobProgressEnqueued,
+    JobProgressStarted,
+)
 from cascade.executor.comms import get_context
 from cascade.gateway.api import JobSpec, TroikaSpec
 from cascade.low.core import DatasetId
@@ -202,16 +208,29 @@ def _spawn_subprocess(
 
 class JobRouter:
     def __init__(
-        self, poller: zmq.Poller, log_base: str | None, troika_config: str | None
+        self,
+        poller: zmq.Poller,
+        log_base: str | None,
+        troika_config: str | None,
+        max_jobs: int | None,
     ):
         self.poller = poller
         self.jobs: dict[str, Job] = {}
+        self.active_jobs = 0
+        self.max_jobs = max_jobs
+        self.jobs_queue: OrderedDict[JobId, JobSpec] = OrderedDict()
         self.procs: dict[str, subprocess.Popen] = {}
         self.log_base = log_base
         self.troika_config = troika_config
 
-    def spawn_job(self, job_spec: JobSpec) -> JobId:
-        job_id = next_uuid(self.jobs.keys(), lambda: str(uuid.uuid4()))
+    def maybe_spawn(self) -> None:
+        if not self.jobs_queue:
+            return
+        if self.max_jobs is not None and self.active_jobs >= self.max_jobs:
+            logger.debug(f"already running {self.active_jobs}, no spawn")
+            return
+
+        job_id, job_spec = self.jobs_queue.popitem(False)
         base_addr = f"tcp://{platform.get_bindabble_self()}"
         socket = get_context().socket(zmq.PULL)
         port = socket.bind_to_random_port(base_addr)
@@ -222,18 +241,37 @@ class JobRouter:
         self.procs[job_id] = _spawn_subprocess(
             job_spec, full_addr, job_id, self.log_base, self.troika_config
         )
+        self.active_jobs += 1
+        return job_id
+
+    def enqueue_job(self, job_spec: JobSpec) -> JobId:
+        job_id = next_uuid(
+            set(self.jobs.keys()).union(self.jobs_queue.keys()),
+            lambda: str(uuid.uuid4()),
+        )
+        self.jobs_queue[job_id] = job_spec
+        self.maybe_spawn()
         return job_id
 
     def progress_of(
         self, job_ids: Iterable[JobId]
-    ) -> tuple[dict[JobId, JobProgress], dict[JobId, list[DatasetId]]]:
+    ) -> tuple[dict[JobId, JobProgress], dict[JobId, list[DatasetId]], int]:
         if not job_ids:
-            job_ids = self.jobs.keys()
-        progresses = {job_id: self.jobs[job_id].progress for job_id in job_ids}
+            job_ids = set(self.jobs.keys()).union(self.jobs_queue.keys())
+        progresses = {}
+        for job_id in job_ids:
+            if job_id in self.jobs:
+                progresses[job_id] = self.jobs[job_id].progress
+            elif job_id in self.jobs_queue:
+                progresses[job_id] = JobProgressEnqueued
+            else:
+                progresses[job_id] = None
         datasets = {
-            job_id: list(self.jobs[job_id].results.keys()) for job_id in job_ids
+            job_id: list(self.jobs[job_id].results.keys())
+            for job_id in job_ids
+            if job_id in self.jobs
         }
-        return progresses, datasets
+        return progresses, datasets, len(self.jobs_queue)
 
     def get_result(self, job_id: JobId, dataset_id: DatasetId) -> bytes:
         return self.jobs[job_id].results[dataset_id]
@@ -246,6 +284,8 @@ class JobRouter:
         job = self.jobs[job_id]
         if progress.completed:
             self.poller.unregister(job.socket)
+            self.active_jobs -= 1
+            self.maybe_spawn()
         if progress.failure is not None and job.progress.failure is None:
             job.progress = progress
         elif job.last_seen >= timestamp or job.progress.failure is not None:

cascade/gateway/server.py

@@ -31,16 +31,19 @@ def handle_fe(socket: zmq.Socket, jobs: JobRouter) -> bool:
     rv: api.CascadeGatewayAPI
     if isinstance(m, api.SubmitJobRequest):
         try:
-            job_id = jobs.spawn_job(m.job)
+            job_id = jobs.enqueue_job(m.job)
             rv = api.SubmitJobResponse(job_id=job_id, error=None)
         except Exception as e:
             logger.exception(f"failed to spawn a job: {m}")
             rv = api.SubmitJobResponse(job_id=None, error=repr(e))
     elif isinstance(m, api.JobProgressRequest):
         try:
-            progresses, datasets = jobs.progress_of(m.job_ids)
+            progresses, datasets, queue_length = jobs.progress_of(m.job_ids)
             rv = api.JobProgressResponse(
-                progresses=progresses, datasets=datasets, error=None
+                progresses=progresses,
+                datasets=datasets,
+                error=None,
+                queue_length=queue_length,
             )
         except Exception as e:
             logger.exception(f"failed to get progress of: {m}")
@@ -80,7 +83,10 @@ def handle_controller(socket: zmq.Socket, jobs: JobRouter) -> None:
 
 
 def serve(
-    url: str, log_base: str | None = None, troika_config: str | None = None
+    url: str,
+    log_base: str | None = None,
+    troika_config: str | None = None,
+    max_jobs: int | None = None,
 ) -> None:
     ctx = get_context()
     poller = zmq.Poller()
@@ -88,7 +94,7 @@ def serve(
     fe = ctx.socket(zmq.REP)
     fe.bind(url)
     poller.register(fe, flags=zmq.POLLIN)
-    jobs = JobRouter(poller, log_base, troika_config)
+    jobs = JobRouter(poller, log_base, troika_config, max_jobs)
 
     logger.debug("entering recv loop")
     is_break = False