flowtask 5.12.4__tar.gz → 5.12.6__tar.gz

{flowtask-5.12.4 → flowtask-5.12.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.12.4
+Version: 5.12.6
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -64,10 +64,9 @@ Requires-Dist: datatable==1.1.0
 Requires-Dist: imageio==2.37.0
 Requires-Dist: decorator>=5.1.1
 Requires-Dist: emoji==2.14.1
-Requires-Dist: caio==0.9.24
 Requires-Dist: Wand==0.6.13
 Requires-Dist: pylibdmtx==0.1.10
-Requires-Dist: aiofile>=3.9.0
+Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: filetype==1.2.0
 Requires-Dist: proxylists>=0.14.0
 Requires-Dist: aioftp==0.23.1

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/components/OpenWithBase.py

@@ -571,7 +571,7 @@ class OpenWithBase(FlowComponent):
                 encoding = self.encoding
             else:
                 count = 0
-                # migrate to aiofile
+                # migrate to aiofiles
                 bt = min(32, os.path.getsize(filename))
                 with open(filename, "rb") as f:
                     line = f.read(bt)

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/components/SearchQuery.py

@@ -132,6 +132,11 @@ class SearchQuery(SearchInterface, FlowComponent):
                     f"SearchQuery: 'query' is not valid JSON — {exc}"
                 ) from exc
 
+        # Apply masks to a dict query too (when 'query' is given as YAML maps,
+        # not a JSON string), so {start}/{end}-style masks work in both forms.
+        if isinstance(self.query, dict):
+            self.query = self._mask_obj(self.query)
+
         if not self.index:
             raise ComponentError("SearchQuery: 'index' is required.")
         if not self.query:
@@ -140,6 +145,21 @@ class SearchQuery(SearchInterface, FlowComponent):
         await self.open_search_client()
         return True
 
+    def _mask_obj(self, obj: Any) -> Any:
+        """Recursively apply ``mask_replacement`` to string values.
+
+        Walks a dict/list and substitutes mask placeholders (e.g. ``{start}``,
+        ``{end}``, ``{today}``) in every string value, so a query written as
+        YAML maps gets the same mask handling as a JSON-string query.
+        """
+        if isinstance(obj, dict):
+            return {k: self._mask_obj(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [self._mask_obj(v) for v in obj]
+        if isinstance(obj, str) and "{" in obj:
+            return self.mask_replacement(obj)
+        return obj
+
     async def run(self) -> pandas.DataFrame:
         """Execute the search and return a DataFrame of results.
 
@@ -162,42 +182,48 @@ class SearchQuery(SearchInterface, FlowComponent):
             self.backend,
         )
         max_docs = int(self.max_documents) if self.max_documents is not None else None
-        hits = await self.paginate(
-            self.index,
-            self.query,
-            mode=self.pagination,
-            size=self.size,
-            scroll=self.scroll,
-            max_documents=max_docs,
-            raw_hits=self.include_metadata,
-        )
-
-        if not hits:
-            self.logger.warning(
-                "SearchQuery: no documents returned from index=%r", self.index
+        try:
+            hits = await self.paginate(
+                self.index,
+                self.query,
+                mode=self.pagination,
+                size=self.size,
+                scroll=self.scroll,
+                max_documents=max_docs,
+                raw_hits=self.include_metadata,
             )
-            df = pandas.DataFrame()
-        elif self.include_metadata:
-            records: list[dict] = []
-            for h in hits:
-                row: dict = {
-                    "_id": h.get("_id", ""),
-                    "_index": h.get("_index", ""),
-                }
-                if h.get("_score") is not None:
-                    row["_score"] = h["_score"]
-                row.update(h.get("_source", {}))
-                records.append(row)
-            df = pandas.DataFrame(records)
-        else:
-            df = pandas.DataFrame(hits)
 
-        self.logger.info(
-            "SearchQuery: result shape %s",
-            df.shape if not df.empty else "(0, 0)",
-        )
-        self._result = df
-        return df
+            if not hits:
+                self.logger.warning(
+                    "SearchQuery: no documents returned from index=%r", self.index
+                )
+                df = pandas.DataFrame()
+            elif self.include_metadata:
+                records: list[dict] = []
+                for h in hits:
+                    row: dict = {
+                        "_id": h.get("_id", ""),
+                        "_index": h.get("_index", ""),
+                    }
+                    if h.get("_score") is not None:
+                        row["_score"] = h["_score"]
+                    row.update(h.get("_source", {}))
+                    records.append(row)
+                df = pandas.DataFrame(records)
+            else:
+                df = pandas.DataFrame(hits)
+
+            self.logger.info(
+                "SearchQuery: result shape %s",
+                df.shape if not df.empty else "(0, 0)",
+            )
+            self._result = df
+            return df
+        finally:
+            # Source component: the connection is no longer needed once all docs
+            # are fetched. Close here so the aiohttp session is always released
+            # (even on error / if the runner does not call close()). Idempotent.
+            await self.close_search_client()
 
     async def close(self) -> None:
         """Close the search client, releasing any scroll or PIT context."""

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/executors/qworker.py

@@ -52,6 +52,45 @@ class QworkerJobExecutor(AbstractJobExecutor):
         """
         return "qworker"
 
+    def _sanitize_kwargs(self, program: str, task: str, kwargs: dict) -> dict:
+        """Drop kwargs that cannot be cloudpickled before queue dispatch.
+
+        The TaskWrapper is serialized with ``cloudpickle`` when handed to the
+        worker. Any kwarg holding a non-serializable object (e.g. a live uvloop
+        event loop, a DB/Redis pool, or the navconfig ``config`` singleton)
+        raises ``TypeError: no default __reduce__ due to non-trivial __cinit__``
+        and kills the whole dispatch. This guards the boundary: each kwarg is
+        probed individually, and any that fails is logged by name and removed so
+        the remaining payload still serializes.
+
+        Args:
+            program: Program slug (for log context).
+            task: Task identifier (for log context).
+            kwargs: Raw kwargs destined for the TaskWrapper constructor.
+
+        Returns:
+            A new dict containing only the serializable kwargs.
+        """
+        import cloudpickle  # lazy import
+
+        clean: dict = {}
+        for key, value in kwargs.items():
+            try:
+                cloudpickle.dumps(value)
+            except Exception as exc:  # noqa: BLE001 — probe, not control flow
+                self.logger.warning(
+                    "Dropping non-serializable kwarg '%s' (%s) from %s.%s "
+                    "dispatch — would break cloudpickle: %s",
+                    key,
+                    type(value).__name__,
+                    program,
+                    task,
+                    exc,
+                )
+                continue
+            clean[key] = value
+        return clean
+
     def _make_wrapper(
         self,
         program: str,
@@ -73,6 +112,7 @@ class QworkerJobExecutor(AbstractJobExecutor):
             A TaskWrapper instance ready for dispatch.
         """
         from qw.wrappers import TaskWrapper  # lazy import
+        kwargs = self._sanitize_kwargs(program, task, kwargs)
         wrapper = TaskWrapper(
             program=program,
             task=task,

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/parsers/argparser.py

@@ -105,6 +105,18 @@ class ConfigParser(object):
             default=False,
             help="Put Task in Queue Worker",
         )
+        self.parser.add_argument(
+            "-r",
+            "--run",
+            dest="run",
+            action="store_true",
+            default=False,
+            help=(
+                "When dispatching to the QWorker, wait for the result "
+                "(QClient.run) instead of the default fire-and-forget "
+                "queue (QClient.queue)."
+            ),
+        )
         # ── Executor selection flags (FEAT-022) ───────────────────────────────
         self.parser.add_argument(
             "--executor",

flowtask-5.12.6/flowtask/parsers/base.pyx

@@ -0,0 +1,76 @@
+# cython: embedsignature=True, boundscheck=False, wraparound=False, initializedcheck=False
+# Copyright (C) 2018-present Jesus Lara
+#
+"""
+Parsing a Task File (abstract Method)
+"""
+cimport cython
+import errno
+from pathlib import PosixPath
+import aiofiles
+from ..exceptions import TaskError, TaskParseError, TaskDefinition, TaskNotFound
+
+
+cdef class TaskParser:
+    # cdef public object filename
+
+    def __init__(self, str file = None, str content = None):
+        self.filename = file
+        self.content = content
+        if file:
+            self.filename = PosixPath(file)
+
+    def task_exists(self):
+        return self.filename.exists()
+
+    def _is_eagain(self, exc):
+        """Return True when *exc* signals EAGAIN ('Resource temporarily
+        unavailable'), regardless of the concrete exception class.
+
+        Kept as defence-in-depth: a plain ``OSError`` carries ``errno`` set
+        to EAGAIN, while native-AIO backends may surface it as a
+        ``SystemError(11, ...)`` instead. Detect both shapes.
+        """
+        if getattr(exc, 'errno', None) == errno.EAGAIN:
+            return True
+        args = getattr(exc, 'args', ())
+        return bool(args) and args[0] == errno.EAGAIN
+
+    async def open_file(self):
+        cdef str content
+        try:
+            # aiofiles delegates to a thread-pool executor (no native libaio
+            # io_setup / fs.aio-max-nr budget), so it is immune to the EAGAIN
+            # context-exhaustion seen under high Qworker concurrency.
+            async with aiofiles.open(self.filename, 'r') as fp:
+                return await fp.read()
+        except (OSError, SystemError) as e:
+            # EAGAIN: fall back to a plain blocking read so the task can
+            # still be parsed.
+            if self._is_eagain(e):
+                try:
+                    with open(self.filename, 'r') as fp:
+                        return fp.read()
+                except Exception as ex:
+                    raise TaskNotFound(
+                        f'Task {self.filename!s} IO Error: {ex!s}'
+                    )
+            raise TaskError(
+                f'Task File: {self.filename!s} with Error: {e!s}'
+            )
+        except Exception as err:
+            raise TaskError(
+                f'Task File: {self.filename!s} with Error: {err!s}'
+            )
+
+    async def run(self):
+        cdef str content
+        content = None
+        if self.filename:
+            content = await self.open_file()
+        else:
+            content = self.content
+        return await self.parse(content)
+
+    async def parse(self, str content):
+        pass

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/runner.py

@@ -37,13 +37,8 @@ from .utils.stats import TaskMonitor
 from .parsers.argparser import ConfigParser
 from .exceptions import (
     TaskError,
-    TaskException,
     NotSupported,
-    TaskParseError,
-    ConfigError
 )
-from .tasks.task import Task
-from .tasks.command import Command
 
 
 def my_handler():
@@ -52,6 +47,62 @@ def my_handler():
     for task in asyncio.all_tasks():
         task.cancel()
 
+
+def resolve_runner_route(
+    cli_executor: str | None,
+    no_worker: bool,
+    wait_for_result: bool,
+    default_executor: str | None,
+) -> tuple[str, str]:
+    """Decide the executor name and execution mode for a console task.
+
+    The Runner mirrors the scheduler's priority-driven router
+    (``flowtask/scheduler/functions.py``): here the *CLI flags* — not a
+    priority string — pick whether a task runs in-process or is handed to the
+    QWorker queue.  ``"local"`` is the router sentinel, NOT a force-in-process
+    selector.
+
+    Resolution (highest precedence first):
+
+    * ``cli_executor`` (``--executor=<name>``) → that executor, as-is
+      (``qworker``, ``docker``, ``k8s``, ``publish``, ``local``…).
+    * ``no_worker`` (``--no-worker``) → ``"local"`` (run in-process).
+    * otherwise → the QWorker, unless ``default_executor`` names a non-local
+      global backend (e.g. ``docker``/``k8s``), which is honoured instead.
+
+    Mode for the QWorker:
+
+    * default → ``"dispatch"`` (fire-and-forget, ``QClient.queue()``).
+    * ``wait_for_result`` (``--run``) → ``"run"`` (``QClient.run()``, waits).
+
+    Every non-QWorker executor (local/docker/k8s/…) always runs and waits, so
+    its mode is ``"run"``.
+
+    Args:
+        cli_executor: Value of the ``--executor`` flag, or ``None``.
+        no_worker: Whether ``--no-worker`` was passed.
+        wait_for_result: Whether ``--run`` was passed.
+        default_executor: The global ``DEFAULT_EXECUTOR`` conf value.
+
+    Returns:
+        A ``(executor_name, mode)`` tuple where ``mode`` is ``"run"`` or
+        ``"dispatch"``.
+    """
+    if cli_executor:
+        executor_name = cli_executor
+    elif no_worker:
+        executor_name = "local"
+    else:
+        global_default = default_executor or "local"
+        executor_name = "qworker" if global_default == "local" else global_default
+
+    if executor_name == "qworker":
+        mode = "run" if wait_for_result else "dispatch"
+    else:
+        mode = "run"
+
+    return executor_name, mode
+
 class TaskRunner:
     """
     TaskRunner.
@@ -187,55 +238,56 @@ class TaskRunner:
             await self.stat.start()
         except Exception as err:
             raise TaskError(f"Task Runner: Error on TaskMonitor: {err}") from err
-        # create the task object:
-        if self.task_type == "command":
-            try:
-                self._task = Command(
-                    task_id=self.task_id,
-                    task=self._command,
-                    program=self._program,
-                    loop=self._loop,
-                    parser=self._argparser,
-                    **self._kwargs,
-                )
-            except Exception as err:
-                raise TaskException(f"{err!s}") from err
-        elif self.task_type == "task":
-            try:
-                self._task = Task(
-                    task_id=self.task_id,
-                    task=self._taskname,
-                    program=self._program,
-                    loop=self._loop,
-                    parser=self._argparser,
-                    storage=self._taskstorage,
-                    disable_notifications=self.no_notify,
-                    no_events=self.no_events,
-                    **self._kwargs,
-                )
-            except Exception as err:
-                raise TaskException(f"{err!s}") from err
-        # then, try to "start" the task:
+        # NOTE: do NOT instantiate or start a Task/Command here.  Since the
+        # executor-framework refactor (TASK-073), run() resolves an executor
+        # and that executor owns the entire task lifecycle (it builds its own
+        # Task and calls start()+run()).  Creating + start()ing a Task here as
+        # well caused the task to be parsed/started twice — duplicate
+        # "Starting Task" events, a duplicate local execution alongside the
+        # intended executor, and a leaked TaskMonitor sampler.  start() is now
+        # only responsible for resolving the task type and the monitor; run()
+        # performs the execution.
         logging.debug(
-            f"::: FlowTask: Running {self.task_type}: {self._task!r} with id: {self.task_id}"
+            f"::: FlowTask: Prepared {self.task_type}: {self._taskname} with id: {self.task_id}"
         )
-        try:
-            self._task.setStat(self.stat)
-            await self._task.start()
-        except (ConfigError, TaskParseError, TaskError):
-            raise
-        except Exception as err:
-            logging.error(err)
-            raise TaskException(f"{err!s}") from err
         return True
 
     async def run(self):
-        from flowtask.executors.resolver import resolve_executor, determine_execution_mode
+        from flowtask.executors.resolver import resolve_executor
+        from flowtask.conf import DEFAULT_EXECUTOR
         try:
-            # Build a minimal task definition for the resolver
-            task_def = {"executor": getattr(self._options, "executor", None) or "default"}
+            # ── Flag-driven executor router ──────────────────────────────────
+            # The Runner mirrors the scheduler's priority-driven router
+            # (flowtask/scheduler/functions.py): here the *CLI flags* — not a
+            # priority string — decide whether a console task runs in-process
+            # or is handed to the QWorker queue.  "local" is the router
+            # sentinel, NOT a force-in-process flag.
+            #
+            #   --no-worker            → local    (run in-process, wait)
+            #   --executor=<name>      → <name>   (explicit override: qworker,
+            #                                       docker, k8s, publish, local…)
+            #   (no flag)              → qworker  (default dispatch)
+            #
+            # DEFAULT_EXECUTOR only switches the *global* backend (e.g. a
+            # deployment that runs everything on docker/k8s).  When it is the
+            # plain "local" sentinel, normal CLI dispatch routes to the worker;
+            # an explicit non-local DEFAULT_EXECUTOR is honoured as the default
+            # backend.
+            executor_name, mode = resolve_runner_route(
+                cli_executor=getattr(self._options, "executor", None),
+                no_worker=self.no_worker,
+                wait_for_result=getattr(self._options, "run", False),
+                default_executor=DEFAULT_EXECUTOR,
+            )
+
+            # Resolve via the executor name we just decided.  Setting it on the
+            # options object (instead of relying on the legacy --no-worker /
+            # --queued handling inside resolve_executor) makes the choice
+            # explicit and suppresses the deprecation warnings, exactly like
+            # the scheduler does.
+            self._options.executor = executor_name
+            task_def = {"executor": executor_name}
             executor = resolve_executor(task_def, self._options)
-            mode = determine_execution_mode(self._options)
             # The executor builds a fresh Task WITHOUT the CLI parser, so options
             # that the Task only reads from the parser (run_only, ignore,
             # override_attributes, variables, conditions) would be silently lost

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/scheduler/functions.py

@@ -240,16 +240,39 @@ class TaskScheduler:
                 pass
 
             opts = _Opts()
-            opts.executor = self._executor_name
             opts.executor_image = None
             opts.executor_namespace = None
-            # Map legacy priority flags to no_worker / queued for the resolver.
             opts.no_worker = False
-            opts.queued = self.priority not in ("direct", None) and not self._executor_name
 
-            task_def: dict = {}
-            if self._executor_name:
-                task_def["executor"] = self._executor_name
+            # --- Routing decision -------------------------------------------------
+            # The default scheduler executor ("local" / None) is a priority-driven
+            # *router*, not a force-local flag:
+            #   * priority == "local"  -> "local"   (run in-scheduler)
+            #   * any other priority   -> "qworker" (dispatch via QWorker; the
+            #     qworker executor maps pub→publish, direct→run,
+            #     low/high/None/<list-key>→queue)
+            # priority == "local" is normally intercepted upstream in
+            # get_function() via launch_task and never reaches here, but the branch
+            # is kept explicit so the routing decision is self-contained.
+            #
+            # An explicit executor name (qworker/docker/k8s/…) overrides the router:
+            #   * "qworker" relies solely on QWorker dispatch
+            #   * anything else resolves to that executor as-is
+            #
+            # We set ``opts.executor`` directly (instead of the legacy ``queued``
+            # flag) so resolve_executor() picks the executor without emitting the
+            # ``--queued is deprecated`` warning.
+            if self._executor_name in (None, "local"):
+                opts.executor = "local" if self.priority == "local" else "qworker"
+            else:
+                opts.executor = self._executor_name
+
+            # Mode: the qworker executor dispatches fire-and-forget, except
+            # "direct" which waits for the worker result (QClient.run). Every other
+            # executor (local/docker/k8s/…) runs and waits.
+            opts.queued = opts.executor == "qworker"
+
+            task_def: dict = {"executor": opts.executor}
 
             executor = resolve_executor(task_def, opts)
 
@@ -259,6 +282,8 @@ class TaskScheduler:
             }
 
             mode = determine_execution_mode(opts)
+            if self.priority == "direct" and opts.executor == "qworker":
+                mode = "run"
 
             if mode == "dispatch":
                 self._scheduled = True

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/services/tasks/tasks.py

@@ -2,7 +2,6 @@ from typing import Union
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
 import pandas as pd
-from navconfig import config as ENV
 from navconfig.logging import logging
 from ...conf import DEBUG
 from ...exceptions import (
@@ -98,11 +97,15 @@ async def launch_task(
         action = "Dispatched"
         mode = "run"
 
-    # Build extra kwargs forwarded to the executor
+    # Build extra kwargs forwarded to the executor.
+    # NOTE: do NOT forward the live navconfig ``config`` (ENV) object here.
+    # The qworker executor cloudpickles the TaskWrapper, and ``config`` holds a
+    # live uvloop event loop (Cython type, non-trivial ``__cinit__``, no
+    # ``__reduce__``) which is unpicklable. The worker rebuilds its own
+    # ``self._env = config`` inside ``Task`` anyway, so the kwarg is never read.
     exec_kwargs = {
         "userid": userid,
         "priority": priority,
-        "ENV": ENV,
         "debug": DEBUG,
         **kwargs,
     }

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/tasks/abstract.py

@@ -5,6 +5,7 @@ import asyncio
 import uuid
 import random
 import locale
+import contextlib
 import traceback
 from collections.abc import Callable, Awaitable
 from abc import ABC, abstractmethod
@@ -478,4 +479,15 @@ class AbstractTask(ABC):
 
     async def close(self):
         self.set_timezone("UTC")  # forcing UTC at Task End.
+        # Always stop the TaskMonitor sampler, even when the task aborts early
+        # (e.g. a parse error in get_task() makes start() return False before
+        # run() ever stops it). Its _get_current_state() loop runs forever
+        # otherwise; once orphaned it is GC'd as "Task was destroyed but it is
+        # pending!", which asyncdb escalates into a noisy error cascade.
+        # stop() is idempotent, so this is a no-op on the success path where
+        # run() already stopped it.
+        stat = getattr(self, "stat", None)
+        if stat is not None and hasattr(stat, "stop"):
+            with contextlib.suppress(Exception):
+                await stat.stop()
         # TODO: closing Memcached-related connections

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask/version.py

@@ -5,7 +5,7 @@ __description__ = (
     "Framework for running Tasks and from CLI and API for orchestation. "
     "Component-based Task builder/Runner for non-programmers."
 )
-__version__ = "5.12.4"
+__version__ = "5.12.6"
 __copyright__ = "Copyright (c) 2018-2025 Jesus Lara"
 __author__ = "Jesus Lara"
 __author_email__ = "jesuslarag@gmail.com"

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.12.4
+Version: 5.12.6
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -64,10 +64,9 @@ Requires-Dist: datatable==1.1.0
 Requires-Dist: imageio==2.37.0
 Requires-Dist: decorator>=5.1.1
 Requires-Dist: emoji==2.14.1
-Requires-Dist: caio==0.9.24
 Requires-Dist: Wand==0.6.13
 Requires-Dist: pylibdmtx==0.1.10
-Requires-Dist: aiofile>=3.9.0
+Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: filetype==1.2.0
 Requires-Dist: proxylists>=0.14.0
 Requires-Dist: aioftp==0.23.1

{flowtask-5.12.4 → flowtask-5.12.6}/flowtask.egg-info/requires.txt

@@ -37,10 +37,9 @@ datatable==1.1.0
 imageio==2.37.0
 decorator>=5.1.1
 emoji==2.14.1
-caio==0.9.24
 Wand==0.6.13
 pylibdmtx==0.1.10
-aiofile>=3.9.0
+aiofiles>=24.1.0
 filetype==1.2.0
 proxylists>=0.14.0
 aioftp==0.23.1

{flowtask-5.12.4 → flowtask-5.12.6}/pyproject.toml

@@ -74,10 +74,9 @@ dependencies = [
     "imageio==2.37.0",
     "decorator>=5.1.1",
     "emoji==2.14.1",
-    "caio==0.9.24",
     "Wand==0.6.13",
     "pylibdmtx==0.1.10",
-    "aiofile>=3.9.0",
+    "aiofiles>=24.1.0",
     "filetype==1.2.0",
     "proxylists>=0.14.0",
     "aioftp==0.23.1",