conductor-python 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

conductor/client/automator/task_handler.py

@@ -4,10 +4,11 @@ import importlib
 import inspect
 import logging
 import os
+import pickle
 import signal
 import threading
 import time
-from multiprocessing import Process, freeze_support, Queue, set_start_method
+from multiprocessing import Process, freeze_support, Queue, set_start_method, get_start_method
 from sys import platform
 from typing import List, Optional, Any, Dict
 
@@ -37,12 +38,25 @@ _VALID_MP_START_METHODS = {"spawn", "fork", "forkserver"}
 _mp_fork_set = False
 if not _mp_fork_set:
     try:
-        # The prometheus_client library holds a module-level threading lock;
-        # forking while that lock is held causes a deadlock in child processes.
-        # Set CONDUCTOR_MP_START_METHOD=spawn to avoid this if you hit the
-        # deadlock.  Default is fork for backward compatibility (spawn requires
-        # all Process arguments to be picklable).
-        _default_method = "spawn" if platform == "win32" else "fork"
+        # Default start method: "spawn" on every platform.
+        #
+        # fork() is fundamentally unsafe in a process that holds native
+        # framework state or non-main threads:
+        #   - macOS: Apple frameworks (CFNetwork, Security, libdispatch) may
+        #     SIGSEGV in forked children, producing silently-restarting worker
+        #     processes with exitcode=-11. CPython switched its own macOS
+        #     default to spawn in 3.8 for the same reason (bpo-33725).
+        #   - all POSIX: forking while another thread holds a lock (e.g. the
+        #     prometheus_client module-level lock, or the TaskHandler monitor
+        #     thread restarting a worker) leaves that lock permanently held in
+        #     the child -> silent deadlock.
+        # Workers (including the @worker_task decorator path) are pickle-safe,
+        # so spawn works everywhere. Deployments that rely on fork's
+        # copy-on-write inheritance can opt back in with
+        # CONDUCTOR_MP_START_METHOD=fork (re-exposing the hazards above).
+        # NOTE: spawn requires the standard `if __name__ == "__main__":` guard
+        # in the entrypoint script.
+        _default_method = "spawn"
         _method = os.environ.get("CONDUCTOR_MP_START_METHOD", "").strip().lower() or _default_method
         if _method not in _VALID_MP_START_METHODS:
             logger.warning(
@@ -341,8 +355,26 @@ class TaskHandler:
         logger.info("Starting worker processes...")
         freeze_support()
         self._monitor_stop_event.clear()
-        self.__start_task_runner_processes()
-        self.__start_metrics_provider_process()
+        try:
+            self.__start_task_runner_processes()
+            self.__start_metrics_provider_process()
+        except (TypeError, pickle.PicklingError, AttributeError) as e:
+            # Under the 'spawn'/'forkserver' start methods, Process arguments
+            # are pickled at start(); unpicklable state fails here. Clean up
+            # everything already started - otherwise the non-daemon logger
+            # subprocess keeps the interpreter alive forever after the
+            # exception (observed as a hang after "cannot pickle
+            # '_thread.lock' object").
+            logger.error(
+                "Failed to start worker processes with start method %r: %s. "
+                "Workers must be defined at module level (not nested inside "
+                "functions), and configuration, metrics_settings and "
+                "event_listeners must be picklable. "
+                "See https://github.com/conductor-oss/conductor-python/issues/264",
+                get_start_method(allow_none=True), e,
+            )
+            self.stop_processes()
+            raise
         self.__start_monitor_thread()
         logger.info("Started all processes")
 
@@ -445,6 +477,21 @@ class TaskHandler:
                 worker = self.workers[i] if i < len(self.workers) else None
                 worker_name = worker.get_task_definition_name() if worker is not None else f"worker[{i}]"
                 logger.warning("Worker process exited (worker=%s, pid=%s, exitcode=%s)", worker_name, process.pid, exitcode)
+                if exitcode < 0:
+                    # Negative exitcode == killed by signal (e.g. -11 is
+                    # SIGSEGV). Under the 'fork' start method this is the
+                    # classic symptom of fork-unsafe native libraries in the
+                    # parent (Apple frameworks on macOS, numpy/Accelerate,
+                    # grpc, ...).
+                    logger.warning(
+                        "Worker process %s was killed by signal %s. If this "
+                        "repeats on every restart, the process is likely "
+                        "crashing at startup: use the 'spawn' start method "
+                        "(CONDUCTOR_MP_START_METHOD=spawn, the default) and "
+                        "set PYTHONFAULTHANDLER=1 to capture the crashing "
+                        "stack trace.",
+                        worker_name, -exitcode,
+                    )
                 if not self.restart_on_failure:
                     continue
                 self.__restart_worker_process(i)

conductor/client/worker/worker.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 import asyncio
 import atexit
 import dataclasses
+import importlib
 import inspect
 import logging
+import sys
 import threading
 import time
 import traceback
@@ -275,6 +277,79 @@ class BackgroundEventLoop:
         logger.debug("Background event loop stopped")
 
 
+class _ExecuteFunctionReference:
+    """Pickle surrogate for functions whose module-level name was rebound.
+
+    The ``@worker_task`` decorator registers the *original* function but rebinds
+    the module-level name to its wrapper. Standard pickle-by-reference then
+    fails with ``PicklingError: it's not the same object as module.name`` when
+    the ``spawn``/``forkserver`` start methods pickle Process arguments
+    (GitHub issues #264/#271).
+
+    Instead of the raw function we pickle ``(module, qualname, unwrap_depth)``
+    and re-resolve in the child process: import the module, walk the qualname,
+    then follow ``__wrapped__`` (set by ``functools.wraps``) ``unwrap_depth``
+    times to get back to the registered function.
+    """
+
+    __slots__ = ("module_name", "qualname", "unwrap_depth")
+
+    def __init__(self, module_name: str, qualname: str, unwrap_depth: int):
+        self.module_name = module_name
+        self.qualname = qualname
+        self.unwrap_depth = unwrap_depth
+
+    def resolve(self) -> Callable:
+        module = importlib.import_module(self.module_name)
+        obj: Any = module
+        for part in self.qualname.split("."):
+            obj = getattr(obj, part)
+        for _ in range(self.unwrap_depth):
+            obj = obj.__wrapped__
+        return obj
+
+
+_MAX_UNWRAP_DEPTH = 32
+
+
+def _importable_function_reference(func: Any) -> Optional[_ExecuteFunctionReference]:
+    """Return a pickle surrogate for *func* if it needs one, else None.
+
+    None is returned when normal pickle-by-reference works (module-level
+    function whose name still points at itself) or when the function is not
+    resolvable by import at all (lambdas, closures, bound methods); in the
+    latter case pickling fails with the standard, descriptive pickle error.
+    """
+    if not inspect.isfunction(func):
+        return None
+    module_name = getattr(func, "__module__", None)
+    qualname = getattr(func, "__qualname__", None)
+    if not module_name or not qualname or "<locals>" in qualname or "<lambda>" in qualname:
+        return None
+    module = sys.modules.get(module_name)
+    if module is None:
+        return None
+    obj: Any = module
+    try:
+        for part in qualname.split("."):
+            obj = getattr(obj, part)
+    except AttributeError:
+        return None
+    if obj is func:
+        # Plain module-level function: default pickling works, no surrogate.
+        return None
+    # The name was rebound (typically by @worker_task). Walk the wrapper chain
+    # to find the registered function and record how many hops it takes.
+    depth = 0
+    current = obj
+    while hasattr(current, "__wrapped__") and depth < _MAX_UNWRAP_DEPTH:
+        current = current.__wrapped__
+        depth += 1
+        if current is func:
+            return _ExecuteFunctionReference(module_name, qualname, depth)
+    return None
+
+
 def is_callable_input_parameter_a_task(callable: ExecuteTaskFunction, object_type: Any) -> bool:
     parameters = inspect.signature(callable).parameters
     if len(parameters) != 1:
@@ -306,7 +381,11 @@ class Worker(WorkerInterface):
                  register_schema: bool = False
                  ) -> Self:
         super().__init__(task_definition_name)
-        self.api_client = ApiClient()
+        # Lazily constructed (see the api_client property): an eager ApiClient
+        # holds an httpx.Client and threading locks, which made Worker
+        # unpicklable under the 'spawn' start method and initialized TLS state
+        # in the parent process before fork() (unsafe on macOS).
+        self._api_client = None
         if poll_interval is None:
             self.poll_interval = DEFAULT_POLLING_INTERVAL
         else:
@@ -335,6 +414,46 @@ class Worker(WorkerInterface):
         # Add thread lock for safe concurrent access to _pending_async_tasks
         self._pending_tasks_lock = threading.Lock()
 
+    @property
+    def api_client(self) -> ApiClient:
+        """ApiClient used for output serialization, created on first use.
+
+        Lazy so that Worker instances remain picklable (required by the
+        'spawn'/'forkserver' multiprocessing start methods) and so the parent
+        process never initializes HTTP/TLS state on behalf of workers.
+        """
+        if self._api_client is None:
+            self._api_client = ApiClient()
+        return self._api_client
+
+    @api_client.setter
+    def api_client(self, value: ApiClient) -> None:
+        self._api_client = value
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        # Process-local runtime state must not cross process boundaries: the
+        # 'spawn'/'forkserver' start methods pickle Process args (issues
+        # #264/#271: "cannot pickle '_thread.lock' object"). Everything below
+        # is rebuilt lazily in the child process.
+        state["_api_client"] = None
+        state["_background_loop"] = None
+        state["_pending_async_tasks"] = {}
+        state["_pending_tasks_lock"] = None
+        ref = _importable_function_reference(state.get("_execute_function"))
+        if ref is not None:
+            state["_execute_function"] = ref
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        if self._pending_tasks_lock is None:
+            self._pending_tasks_lock = threading.Lock()
+        if isinstance(self._execute_function, _ExecuteFunctionReference):
+            # Assign through the property setter so the signature-derived
+            # flags are recomputed against the resolved function.
+            self.execute_function = self._execute_function.resolve()
+
     def execute(self, task: Task) -> TaskResult:
         task_input = {}
         task_output = None

{conductor_python-1.4.0.dist-info → conductor_python-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: conductor-python
-Version: 1.4.0
+Version: 1.5.0
 Summary: Python SDK for working with https://github.com/conductor-oss/conductor
 License: Apache-2.0
 Author: Orkes

{conductor_python-1.4.0.dist-info → conductor_python-1.5.0.dist-info}/RECORD

@@ -9,7 +9,7 @@ conductor/client/automator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 conductor/client/automator/async_task_runner.py,sha256=VtG-3300GJK3klCX-WuxjUfkcpVcwayzi5tTUqAxiWw,50394
 conductor/client/automator/json_schema_generator.py,sha256=Rgi74jBctJXgqv3SQl9gi062tQvIQRrFwVXIIztig6E,9718
 conductor/client/automator/lease_tracker.py,sha256=0mOB_3G-RV8QlCtQFUTCRW8rb1VVDw0dAaa4v1bUTX4,8541
-conductor/client/automator/task_handler.py,sha256=IrlNCPMcTiFN44kfQXM2M2AZ2-qveiz_0bmmBUTFkGo,29876
+conductor/client/automator/task_handler.py,sha256=5b2EhVyQwDF_Y-GQ4UsdUh4Inf_0nVj-dTlgNqfyKXs,32576
 conductor/client/automator/task_runner.py,sha256=yrBYjAJy71a_GA8BEIpqQCwAMpOVwYn0YJE4o1f5YnY,58255
 conductor/client/automator/utils.py,sha256=IenXCf_LLORSCD06XBOE_IbVu4yaOiELxH46-xNe7dE,7593
 conductor/client/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -185,7 +185,7 @@ conductor/client/telemetry/model/metric_label.py,sha256=NVC9GWsDMMBSIK38AjA4W4ih
 conductor/client/telemetry/model/metric_name.py,sha256=7_WEMWMhv_zu8yKYFJ-r6gqKtHj5brwq79UJYsfv2RY,1361
 conductor/client/worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 conductor/client/worker/exception.py,sha256=-QzD8EdJoRBLaVzNOOIdTpvQWm4XvLeTTZxgPz3dtYc,120
-conductor/client/worker/worker.py,sha256=E6OSwrcoxzK1zxMp4jv8P_ktGttTZzGWQn4VIsR9-40,24525
+conductor/client/worker/worker.py,sha256=rpR7bkzr--a6xJEWRG7qzpGbQxa4-gedvr8sCet0-xA,29412
 conductor/client/worker/worker_config.py,sha256=zrG2xgzNkQfic_3HrPAQwsB9YSybU48IDBrkE1JxsA8,15274
 conductor/client/worker/worker_interface.py,sha256=79ow6eqVOh4ogl51vXYKJYPlXpEi_jTWK-sMLEauRuE,5970
 conductor/client/worker/worker_loader.py,sha256=egEXSyHsh714NZEFcR7A3OiGvUezloGPHu8xjaDlJOo,10552
@@ -243,6 +243,6 @@ conductor/client/workflow/task/timeout_policy.py,sha256=JzoZGtCQ4scykATG1oXksHTS
 conductor/client/workflow/task/wait_for_webhook_task.py,sha256=irmcSu-sKoR8AHff_KCsQYTn2RIwXdize9bck0Jc4Uc,1546
 conductor/client/workflow/task/wait_task.py,sha256=yHfuFf7N0HhNMP4LHU6_sw2AOVDAcnCb-6cS9sRoeW0,1618
 conductor/client/workflow_client.py,sha256=sr2c9oYvm7unCKsuuYaJB_NIdfqz2SnyS1_DK11KV1M,4821
-conductor_python-1.4.0.dist-info/METADATA,sha256=xe5BM9OsyiIhjG2beJ0w7m5dSys8Tbcuia7qguhk4Rg,27225
-conductor_python-1.4.0.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
-conductor_python-1.4.0.dist-info/RECORD,,
+conductor_python-1.5.0.dist-info/METADATA,sha256=MmYaPlTtxV_r-5V0JbRk0EyPCqQOWWKiAUsBKWOJvwA,27225
+conductor_python-1.5.0.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
+conductor_python-1.5.0.dist-info/RECORD,,