flowtask 5.12.5__tar.gz → 5.12.7__tar.gz

{flowtask-5.12.5 → flowtask-5.12.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.12.5
+Version: 5.12.7
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/conf.py

@@ -231,7 +231,7 @@ DEFAULT_EXECUTOR = config.get("DEFAULT_EXECUTOR", fallback="local")
 DOCKER_IMAGE = config.get("DOCKER_IMAGE", fallback="flowtask:latest")
 K8S_NAMESPACE = config.get("K8S_NAMESPACE", fallback="default")
 ALLOW_RESCHEDULE = config.getboolean("ALLOW_RESCHEDULE", fallback=False)
-SCHEDULER_STARTUP_JOB = config.getboolean("SCHEDULER_STARTUP_JOBS", fallback=False)
+SCHEDULER_STARTUP_JOB = config.getboolean("SCHEDULER_STARTUP_JOB", fallback=False)
 
 """
 Email Configuration

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/executors/docker.py

@@ -177,7 +177,7 @@ class DockerJobExecutor(AbstractJobExecutor):
             ExecutorConnectionError: If Docker daemon is unreachable.
             ExecutorError: If image pull or container creation fails.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         channel = f"FLOWTASK:EXEC:{execution_id}"
         container_name = self._container_name(program, task, execution_id)
 
@@ -244,7 +244,7 @@ class DockerJobExecutor(AbstractJobExecutor):
         Raises:
             ExecutorConnectionError: If Docker daemon is unreachable.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         container_name = self._container_name(program, task, execution_id)
         timeout = self._config.timeout
 

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/executors/k8s.py

@@ -214,7 +214,7 @@ class K8sJobExecutor(AbstractJobExecutor):
             ExecutorConnectionError: If the K8s cluster is unreachable.
             ExecutorError: If the Job cannot be created.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         channel = f"FLOWTASK:EXEC:{execution_id}"
         job_name = self._job_name(program, task, execution_id)
 
@@ -273,7 +273,7 @@ class K8sJobExecutor(AbstractJobExecutor):
         Raises:
             ExecutorConnectionError: If the K8s cluster is unreachable.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         job_name = self._job_name(program, task, execution_id)
         timeout = self._config.timeout
 

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/executors/local.py

@@ -132,7 +132,7 @@ class LocalJobExecutor(AbstractJobExecutor):
             program=program,
             **kwargs,
         )
-        execution_id = task_id  # always set by dispatch() / run() callers
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
 
         try:
             await task_obj.start()

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/executors/models.py

@@ -2,9 +2,10 @@
 #
 """Pydantic data models for the flowtask executor framework."""
 from __future__ import annotations
+import uuid
 from enum import Enum
 from typing import Any, Literal, Optional
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 
 class ExecutorType(str, Enum):
@@ -78,6 +79,19 @@ class ExecutionHandle(BaseModel):
     task_name: str
     channel: str
 
+    @field_validator("execution_id", mode="before")
+    @classmethod
+    def _coerce_execution_id(cls, value: Any) -> Any:
+        """Coerce UUID (or other non-str ids) into a string.
+
+        Callers frequently pass a ``uuid.UUID`` object as the execution id;
+        Pydantic's strict ``str`` validation would reject it. Normalise here so
+        the handle always carries a plain string.
+        """
+        if isinstance(value, uuid.UUID):
+            return str(value)
+        return value
+
 
 class TaskResult(BaseModel):
     """Result returned by run() when task execution completes.
@@ -95,3 +109,11 @@ class TaskResult(BaseModel):
     stats: Optional[Any] = None
     error: Optional[str] = None
     execution_id: Optional[str] = None
+
+    @field_validator("execution_id", mode="before")
+    @classmethod
+    def _coerce_execution_id(cls, value: Any) -> Any:
+        """Coerce UUID execution ids into a string (see ExecutionHandle)."""
+        if isinstance(value, uuid.UUID):
+            return str(value)
+        return value

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/executors/publish.py

@@ -118,7 +118,7 @@ class PublishJobExecutor(AbstractJobExecutor):
         Raises:
             ExecutorConnectionError: If Redis is unavailable.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         channel = f"FLOWTASK:EXEC:{execution_id}"
 
         await self._publisher.publish("dispatched", program, task, execution_id)
@@ -181,7 +181,7 @@ class PublishJobExecutor(AbstractJobExecutor):
         Raises:
             ExecutorConnectionError: If the initial Redis connection fails.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         result_channel = f"{RESULT_CHANNEL_PREFIX}:{execution_id}"
         timeout = self._config.timeout
 

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/executors/qworker.py

@@ -52,6 +52,45 @@ class QworkerJobExecutor(AbstractJobExecutor):
         """
         return "qworker"
 
+    def _sanitize_kwargs(self, program: str, task: str, kwargs: dict) -> dict:
+        """Drop kwargs that cannot be cloudpickled before queue dispatch.
+
+        The TaskWrapper is serialized with ``cloudpickle`` when handed to the
+        worker. Any kwarg holding a non-serializable object (e.g. a live uvloop
+        event loop, a DB/Redis pool, or the navconfig ``config`` singleton)
+        raises ``TypeError: no default __reduce__ due to non-trivial __cinit__``
+        and kills the whole dispatch. This guards the boundary: each kwarg is
+        probed individually, and any that fails is logged by name and removed so
+        the remaining payload still serializes.
+
+        Args:
+            program: Program slug (for log context).
+            task: Task identifier (for log context).
+            kwargs: Raw kwargs destined for the TaskWrapper constructor.
+
+        Returns:
+            A new dict containing only the serializable kwargs.
+        """
+        import cloudpickle  # lazy import
+
+        clean: dict = {}
+        for key, value in kwargs.items():
+            try:
+                cloudpickle.dumps(value)
+            except Exception as exc:  # noqa: BLE001 — probe, not control flow
+                self.logger.warning(
+                    "Dropping non-serializable kwarg '%s' (%s) from %s.%s "
+                    "dispatch — would break cloudpickle: %s",
+                    key,
+                    type(value).__name__,
+                    program,
+                    task,
+                    exc,
+                )
+                continue
+            clean[key] = value
+        return clean
+
     def _make_wrapper(
         self,
         program: str,
@@ -73,6 +112,7 @@ class QworkerJobExecutor(AbstractJobExecutor):
             A TaskWrapper instance ready for dispatch.
         """
         from qw.wrappers import TaskWrapper  # lazy import
+        kwargs = self._sanitize_kwargs(program, task, kwargs)
         wrapper = TaskWrapper(
             program=program,
             task=task,
@@ -164,7 +204,7 @@ class QworkerJobExecutor(AbstractJobExecutor):
         Returns:
             An ExecutionHandle with the queue message result.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         priority: Optional[str] = kwargs.pop(
             "priority", self._config.priority
         )
@@ -226,7 +266,7 @@ class QworkerJobExecutor(AbstractJobExecutor):
         Returns:
             TaskResult with the worker's return value.
         """
-        execution_id = task_id or str(uuid.uuid4())
+        execution_id = str(task_id) if task_id else str(uuid.uuid4())
         priority: Optional[str] = kwargs.pop(
             "priority", self._config.priority
         )

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/runner.py

@@ -47,6 +47,62 @@ def my_handler():
     for task in asyncio.all_tasks():
         task.cancel()
 
+
+def resolve_runner_route(
+    cli_executor: str | None,
+    no_worker: bool,
+    wait_for_result: bool,
+    default_executor: str | None,
+) -> tuple[str, str]:
+    """Decide the executor name and execution mode for a console task.
+
+    The Runner mirrors the scheduler's priority-driven router
+    (``flowtask/scheduler/functions.py``): here the *CLI flags* — not a
+    priority string — pick whether a task runs in-process or is handed to the
+    QWorker queue.  ``"local"`` is the router sentinel, NOT a force-in-process
+    selector.
+
+    Resolution (highest precedence first):
+
+    * ``cli_executor`` (``--executor=<name>``) → that executor, as-is
+      (``qworker``, ``docker``, ``k8s``, ``publish``, ``local``…).
+    * ``no_worker`` (``--no-worker``) → ``"local"`` (run in-process).
+    * otherwise → the QWorker, unless ``default_executor`` names a non-local
+      global backend (e.g. ``docker``/``k8s``), which is honoured instead.
+
+    Mode for the QWorker:
+
+    * default → ``"dispatch"`` (fire-and-forget, ``QClient.queue()``).
+    * ``wait_for_result`` (``--run``) → ``"run"`` (``QClient.run()``, waits).
+
+    Every non-QWorker executor (local/docker/k8s/…) always runs and waits, so
+    its mode is ``"run"``.
+
+    Args:
+        cli_executor: Value of the ``--executor`` flag, or ``None``.
+        no_worker: Whether ``--no-worker`` was passed.
+        wait_for_result: Whether ``--run`` was passed.
+        default_executor: The global ``DEFAULT_EXECUTOR`` conf value.
+
+    Returns:
+        A ``(executor_name, mode)`` tuple where ``mode`` is ``"run"`` or
+        ``"dispatch"``.
+    """
+    if cli_executor:
+        executor_name = cli_executor
+    elif no_worker:
+        executor_name = "local"
+    else:
+        global_default = default_executor or "local"
+        executor_name = "qworker" if global_default == "local" else global_default
+
+    if executor_name == "qworker":
+        mode = "run" if wait_for_result else "dispatch"
+    else:
+        mode = "run"
+
+    return executor_name, mode
+
 class TaskRunner:
     """
     TaskRunner.
@@ -217,26 +273,12 @@ class TaskRunner:
             # plain "local" sentinel, normal CLI dispatch routes to the worker;
             # an explicit non-local DEFAULT_EXECUTOR is honoured as the default
             # backend.
-            _cli_executor = getattr(self._options, "executor", None)
-            if _cli_executor:
-                executor_name = _cli_executor
-            elif self.no_worker:
-                executor_name = "local"
-            else:
-                _global_default = DEFAULT_EXECUTOR or "local"
-                executor_name = (
-                    "qworker" if _global_default == "local" else _global_default
-                )
-
-            # Mode: the QWorker fires-and-forgets via QClient.queue() by
-            # default; --run switches to QClient.run() and waits for the
-            # worker's result.  Every other executor (local/docker/k8s/…) runs
-            # and waits in-line.
-            _wait_for_result = getattr(self._options, "run", False)
-            if executor_name == "qworker":
-                mode = "run" if _wait_for_result else "dispatch"
-            else:
-                mode = "run"
+            executor_name, mode = resolve_runner_route(
+                cli_executor=getattr(self._options, "executor", None),
+                no_worker=self.no_worker,
+                wait_for_result=getattr(self._options, "run", False),
+                default_executor=DEFAULT_EXECUTOR,
+            )
 
             # Resolve via the executor name we just decided.  Setting it on the
             # options object (instead of relying on the legacy --no-worker /

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/scheduler/scheduler.py

@@ -439,7 +439,9 @@ class NavScheduler:
         return self._loop
 
     def set_test_job(self):
-        self.logger.debug("Scheduler: Adding a Test job")
+        self.logger.debug(
+            "Scheduler: Adding a Test job"
+        )
         run_date = datetime.now(self._timezone) + timedelta(minutes=1)
         # define Task:
         program = "navigator"
@@ -478,6 +480,7 @@ class NavScheduler:
                 f"[{ENVIRONMENT} - Scheduler] Error getting Jobs: {error!s}"
             )
         # Add a Job for testing purposes.
+        print('HERE > ', SCHEDULER_STARTUP_JOB)
         if SCHEDULER_STARTUP_JOB is True:
             self.set_test_job()
         for job in jobs:

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/services/files/service.py

@@ -664,7 +664,7 @@ class FileService(BaseView):
                     program_slug = filedef.program_slug
                     try:
                         # loop = asyncio.new_event_loop()
-                        task_uuid = uuid.uuid4()
+                        task_uuid = str(uuid.uuid4())
                         if not filedef.params:
                             params = {}
                         else:

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/services/tasks/launcher.py

@@ -174,7 +174,7 @@ class TaskLauncher(BaseView):
                 args = params
             if data:
                 args = {**args, **data}
-            task_uuid = uuid.uuid4()
+            task_uuid = str(uuid.uuid4())
             print('Launching Task: ', program, task_id, args)
             uid, action, result = await launch_task(
                 program_slug=program,

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/services/tasks/task_manager.py

@@ -226,7 +226,7 @@ class TaskManager(DataView):
                     args = params
                 if data:
                     args = {**args, **data}
-                task_uuid = uuid.uuid4()
+                task_uuid = str(uuid.uuid4())
                 status, action, result = await launch_task(
                     program_slug=program,
                     task_id=task_id,

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/services/tasks/tasks.py

@@ -2,7 +2,6 @@ from typing import Union
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
 import pandas as pd
-from navconfig import config as ENV
 from navconfig.logging import logging
 from ...conf import DEBUG
 from ...exceptions import (
@@ -98,11 +97,15 @@ async def launch_task(
         action = "Dispatched"
         mode = "run"
 
-    # Build extra kwargs forwarded to the executor
+    # Build extra kwargs forwarded to the executor.
+    # NOTE: do NOT forward the live navconfig ``config`` (ENV) object here.
+    # The qworker executor cloudpickles the TaskWrapper, and ``config`` holds a
+    # live uvloop event loop (Cython type, non-trivial ``__cinit__``, no
+    # ``__reduce__``) which is unpicklable. The worker rebuilds its own
+    # ``self._env = config`` inside ``Task`` anyway, so the kwarg is never read.
     exec_kwargs = {
         "userid": userid,
         "priority": priority,
-        "ENV": ENV,
         "debug": DEBUG,
         **kwargs,
     }

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask/version.py

@@ -5,7 +5,7 @@ __description__ = (
     "Framework for running Tasks and from CLI and API for orchestation. "
     "Component-based Task builder/Runner for non-programmers."
 )
-__version__ = "5.12.5"
+__version__ = "5.12.7"
 __copyright__ = "Copyright (c) 2018-2025 Jesus Lara"
 __author__ = "Jesus Lara"
 __author_email__ = "jesuslarag@gmail.com"

{flowtask-5.12.5 → flowtask-5.12.7}/flowtask.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.12.5
+Version: 5.12.7
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0