conductor-python 1.3.11__py3-none-any.whl → 1.5.0__py3-none-any.whl

conductor/client/__init__.py

@@ -3,7 +3,7 @@
 from conductor.client.configuration.configuration import Configuration
 from conductor.client.automator.task_handler import TaskHandler
 from conductor.client.automator.task_runner import TaskRunner
-from conductor.client.orkes_clients import OrkesClients
+from conductor.client.orkes_clients import OrkesClients, ConductorClients
 from conductor.client.workflow.conductor_workflow import ConductorWorkflow
 from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor
 from conductor.client.worker.worker_task import worker_task
@@ -18,6 +18,7 @@ __all__ = [
     "TaskHandler",
     "TaskRunner",
     "OrkesClients",
+    "ConductorClients",
     "ConductorWorkflow",
     "WorkflowExecutor",
     "worker_task",

conductor/client/automator/async_task_runner.py

@@ -27,7 +27,7 @@ from conductor.client.http.models.schema_def import SchemaDef, SchemaType
 from conductor.client.http.rest import AuthorizationException, ApiException
 from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient
 from conductor.client.orkes.orkes_schema_client import OrkesSchemaClient
-from conductor.client.telemetry.metrics_collector import MetricsCollector
+from conductor.client.telemetry.metrics_factory import create_metrics_collector
 from conductor.client.worker.worker_interface import WorkerInterface
 from conductor.client.worker.worker_config import resolve_worker_config, get_worker_config_oneline
 from conductor.client.worker.exception import NonRetryableException
@@ -88,7 +88,7 @@ class AsyncTaskRunner:
 
         self.metrics_collector = None
         if metrics_settings is not None:
-            self.metrics_collector = MetricsCollector(
+            self.metrics_collector = create_metrics_collector(
                 metrics_settings
             )
             # Register metrics collector as event listener
@@ -863,6 +863,7 @@ class AsyncTaskRunner:
             if attempt > 0:
                 # Exponential backoff: [10s, 20s, 30s] before retry
                 await asyncio.sleep(attempt * 10)
+            update_start = time.time()
             try:
                 if self._use_update_v2:
                     next_task = await self.async_task_client.update_task_v2(body=task_result)
@@ -873,6 +874,10 @@ class AsyncTaskRunner:
                         task_definition_name,
                         next_task.task_id if next_task else None
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return next_task
                 else:
                     await self.async_task_client.update_task(body=task_result)
@@ -882,6 +887,10 @@ class AsyncTaskRunner:
                         task_result.workflow_instance_id,
                         task_definition_name,
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return None
             except ApiException as e:
                 if e.status in (404, 405) and self._use_update_v2:
@@ -895,12 +904,19 @@ class AsyncTaskRunner:
                     # Retry immediately with v1
                     try:
                         await self.async_task_client.update_task(body=task_result)
+                        if self.metrics_collector is not None:
+                            self.metrics_collector.record_task_update_time(
+                                task_definition_name, time.time() - update_start, status="SUCCESS"
+                            )
                         return None
                     except Exception as fallback_e:
                         last_exception = fallback_e
                         continue
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )
@@ -917,6 +933,9 @@ class AsyncTaskRunner:
             except Exception as e:
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )

conductor/client/automator/task_handler.py

@@ -4,10 +4,11 @@ import importlib
 import inspect
 import logging
 import os
+import pickle
 import signal
 import threading
 import time
-from multiprocessing import Process, freeze_support, Queue, set_start_method
+from multiprocessing import Process, freeze_support, Queue, set_start_method, get_start_method
 from sys import platform
 from typing import List, Optional, Any, Dict
 
@@ -18,7 +19,7 @@ from conductor.client.configuration.settings.metrics_settings import MetricsSett
 from conductor.client.event.task_runner_events import TaskRunnerEvent
 from conductor.client.event.sync_event_dispatcher import SyncEventDispatcher
 from conductor.client.event.sync_listener_register import register_task_runner_listener
-from conductor.client.telemetry.metrics_collector import MetricsCollector
+from conductor.client.telemetry.metrics_collector_base import MetricsCollectorBase
 from conductor.client.telemetry.model.metric_documentation import MetricDocumentation
 from conductor.client.telemetry.model.metric_label import MetricLabel
 from conductor.client.telemetry.model.metric_name import MetricName
@@ -33,13 +34,37 @@ logger = logging.getLogger(
 )
 
 _decorated_functions = {}
+_VALID_MP_START_METHODS = {"spawn", "fork", "forkserver"}
 _mp_fork_set = False
 if not _mp_fork_set:
     try:
-        if platform == "win32":
-            set_start_method("spawn")
-        else:
-            set_start_method("fork")
+        # Default start method: "spawn" on every platform.
+        #
+        # fork() is fundamentally unsafe in a process that holds native
+        # framework state or non-main threads:
+        #   - macOS: Apple frameworks (CFNetwork, Security, libdispatch) may
+        #     SIGSEGV in forked children, producing silently-restarting worker
+        #     processes with exitcode=-11. CPython switched its own macOS
+        #     default to spawn in 3.8 for the same reason (bpo-33725).
+        #   - all POSIX: forking while another thread holds a lock (e.g. the
+        #     prometheus_client module-level lock, or the TaskHandler monitor
+        #     thread restarting a worker) leaves that lock permanently held in
+        #     the child -> silent deadlock.
+        # Workers (including the @worker_task decorator path) are pickle-safe,
+        # so spawn works everywhere. Deployments that rely on fork's
+        # copy-on-write inheritance can opt back in with
+        # CONDUCTOR_MP_START_METHOD=fork (re-exposing the hazards above).
+        # NOTE: spawn requires the standard `if __name__ == "__main__":` guard
+        # in the entrypoint script.
+        _default_method = "spawn"
+        _method = os.environ.get("CONDUCTOR_MP_START_METHOD", "").strip().lower() or _default_method
+        if _method not in _VALID_MP_START_METHODS:
+            logger.warning(
+                "Ignoring invalid CONDUCTOR_MP_START_METHOD=%r; falling back to %r",
+                _method, _default_method,
+            )
+            _method = _default_method
+        set_start_method(_method)
         _mp_fork_set = True
     except Exception as e:
         logger.info("error when setting multiprocessing.set_start_method - maybe the context is set %s", e.args)
@@ -221,11 +246,11 @@ class TaskHandler:
         self._configuration = configuration
         self._metrics_settings = metrics_settings
 
-        # Set prometheus multiprocess directory BEFORE any worker processes start
-        # This must be done before prometheus_client is imported in worker processes
+        # Set PROMETHEUS_MULTIPROC_DIR BEFORE any worker processes start.
+        # MetricsSettings resolves the subdirectory eagerly at construction.
         if metrics_settings is not None:
-            os.environ["PROMETHEUS_MULTIPROC_DIR"] = metrics_settings.directory
-            logger.info(f"Set PROMETHEUS_MULTIPROC_DIR={metrics_settings.directory}")
+            os.environ["PROMETHEUS_MULTIPROC_DIR"] = metrics_settings.metrics_directory
+            logger.info(f"Set PROMETHEUS_MULTIPROC_DIR={metrics_settings.metrics_directory}")
 
         # Store event listeners to pass to each worker process
         self.event_listeners = event_listeners or []
@@ -330,8 +355,26 @@ class TaskHandler:
         logger.info("Starting worker processes...")
         freeze_support()
         self._monitor_stop_event.clear()
-        self.__start_task_runner_processes()
-        self.__start_metrics_provider_process()
+        try:
+            self.__start_task_runner_processes()
+            self.__start_metrics_provider_process()
+        except (TypeError, pickle.PicklingError, AttributeError) as e:
+            # Under the 'spawn'/'forkserver' start methods, Process arguments
+            # are pickled at start(); unpicklable state fails here. Clean up
+            # everything already started - otherwise the non-daemon logger
+            # subprocess keeps the interpreter alive forever after the
+            # exception (observed as a hang after "cannot pickle
+            # '_thread.lock' object").
+            logger.error(
+                "Failed to start worker processes with start method %r: %s. "
+                "Workers must be defined at module level (not nested inside "
+                "functions), and configuration, metrics_settings and "
+                "event_listeners must be picklable. "
+                "See https://github.com/conductor-oss/conductor-python/issues/264",
+                get_start_method(allow_none=True), e,
+            )
+            self.stop_processes()
+            raise
         self.__start_monitor_thread()
         logger.info("Started all processes")
 
@@ -345,7 +388,7 @@ class TaskHandler:
             self.metrics_provider_process = None
             return
         self.metrics_provider_process = Process(
-            target=MetricsCollector.provide_metrics,
+            target=MetricsCollectorBase.provide_metrics,
             args=(metrics_settings,)
         )
         logger.info("Created MetricsProvider process")
@@ -434,6 +477,21 @@ class TaskHandler:
                 worker = self.workers[i] if i < len(self.workers) else None
                 worker_name = worker.get_task_definition_name() if worker is not None else f"worker[{i}]"
                 logger.warning("Worker process exited (worker=%s, pid=%s, exitcode=%s)", worker_name, process.pid, exitcode)
+                if exitcode < 0:
+                    # Negative exitcode == killed by signal (e.g. -11 is
+                    # SIGSEGV). Under the 'fork' start method this is the
+                    # classic symptom of fork-unsafe native libraries in the
+                    # parent (Apple frameworks on macOS, numpy/Accelerate,
+                    # grpc, ...).
+                    logger.warning(
+                        "Worker process %s was killed by signal %s. If this "
+                        "repeats on every restart, the process is likely "
+                        "crashing at startup: use the 'spawn' start method "
+                        "(CONDUCTOR_MP_START_METHOD=spawn, the default) and "
+                        "set PYTHONFAULTHANDLER=1 to capture the crashing "
+                        "stack trace.",
+                        worker_name, -exitcode,
+                    )
                 if not self.restart_on_failure:
                     continue
                 self.__restart_worker_process(i)

conductor/client/automator/task_runner.py

@@ -30,7 +30,7 @@ from conductor.client.http.models.schema_def import SchemaDef, SchemaType
 from conductor.client.http.rest import AuthorizationException, ApiException
 from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient
 from conductor.client.orkes.orkes_schema_client import OrkesSchemaClient
-from conductor.client.telemetry.metrics_collector import MetricsCollector
+from conductor.client.telemetry.metrics_factory import create_metrics_collector
 from conductor.client.worker.worker import ASYNC_TASK_RUNNING
 from conductor.client.worker.worker_interface import WorkerInterface
 from conductor.client.worker.worker_config import resolve_worker_config, get_worker_config_oneline
@@ -69,7 +69,7 @@ class TaskRunner:
 
         self.metrics_collector = None
         if metrics_settings is not None:
-            self.metrics_collector = MetricsCollector(
+            self.metrics_collector = create_metrics_collector(
                 metrics_settings
             )
             # Register metrics collector as event listener
@@ -972,6 +972,7 @@ class TaskRunner:
             if attempt > 0:
                 # Exponential backoff: [10s, 20s, 30s] before retry
                 time.sleep(attempt * 10)
+            update_start = time.time()
             try:
                 if self._use_update_v2:
                     next_task = self.task_client.update_task_v2(body=task_result)
@@ -982,6 +983,10 @@ class TaskRunner:
                         task_definition_name,
                         next_task.task_id if next_task else None
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return next_task
                 else:
                     self.task_client.update_task(body=task_result)
@@ -991,6 +996,10 @@ class TaskRunner:
                         task_result.workflow_instance_id,
                         task_definition_name,
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return None
             except ApiException as e:
                 if e.status in (404, 405) and self._use_update_v2:
@@ -1004,12 +1013,19 @@ class TaskRunner:
                     # Retry immediately with v1
                     try:
                         self.task_client.update_task(body=task_result)
+                        if self.metrics_collector is not None:
+                            self.metrics_collector.record_task_update_time(
+                                task_definition_name, time.time() - update_start, status="SUCCESS"
+                            )
                         return None
                     except Exception as fallback_e:
                         last_exception = fallback_e
                         continue
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )
@@ -1044,6 +1060,9 @@ class TaskRunner:
             except Exception as e:
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )

conductor/client/configuration/settings/metrics_settings.py

@@ -14,6 +14,16 @@ logger = logging.getLogger(
 )
 
 
+CANONICAL_SUBDIR = "canonical"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    value = os.environ.get(name, "")
+    if not value:
+        return default
+    return value.strip().lower() in ("true", "1", "yes")
+
+
 def get_default_temporary_folder() -> str:
     return f"{Path.home()!s}/tmp/"
 
@@ -24,12 +34,23 @@ class MetricsSettings:
             directory: Optional[str] = None,
             file_name: str = "metrics.log",
             update_interval: float = 0.1,
-            http_port: Optional[int] = None):
+            http_port: Optional[int] = None,
+            clean_directory: bool = False,
+            clean_dead_pids: bool = False):
         """
         Configure metrics collection settings.
 
+        The ``WORKER_CANONICAL_METRICS`` env var is read at construction time
+        to decide whether ``.db`` files go in *directory* (legacy) or in a
+        ``canonical/`` subdirectory (canonical mode).  Set the env var before
+        creating this object.
+
         Args:
-            directory: Directory for storing multiprocess metrics .db files
+            directory: Base directory for storing multiprocess metrics .db files.
+                      Legacy metrics use this directory directly (unchanged from
+                      prior releases).  Canonical metrics use a ``canonical/``
+                      subdirectory so that switching implementations never
+                      produces stale metric names.
             file_name: Name of the metrics output file (only used when http_port is None)
             update_interval: How often to update metrics (in seconds)
             http_port: Optional HTTP port to expose metrics endpoint for Prometheus scraping.
@@ -40,6 +61,13 @@ class MetricsSettings:
                       If None:
                       - Metrics will be written to file at {directory}/{file_name}
                       - No HTTP server will be started
+            clean_directory: If True, remove all prometheus_client .db files from
+                      the metrics directory when the collector is created.  Only
+                      safe when no other live process shares the same directory.
+                      Defaults to False.
+            clean_dead_pids: If True, remove .db files whose owning PID no
+                      longer exists.  Safer than ``clean_directory`` in shared
+                      environments.  Defaults to False.
         """
         if directory is None:
             directory = get_default_temporary_folder()
@@ -47,6 +75,30 @@ class MetricsSettings:
         self.file_name = file_name
         self.update_interval = update_interval
         self.http_port = http_port
+        self.clean_directory = clean_directory
+        self.clean_dead_pids = clean_dead_pids
+        self._subdir: str = (
+            CANONICAL_SUBDIR
+            if _env_bool("WORKER_CANONICAL_METRICS", False)
+            else ""
+        )
+
+    @property
+    def is_canonical(self) -> bool:
+        return self._subdir == CANONICAL_SUBDIR
+
+    @property
+    def metrics_directory(self) -> str:
+        """Full path where .db files live (base directory + optional subdir).
+
+        Legacy leaves the subdirectory empty (base directory unchanged from
+        prior releases); canonical sets it to ``"canonical"`` to avoid stale
+        metric-name collisions.  Resolved eagerly at construction time from
+        the ``WORKER_CANONICAL_METRICS`` env var.
+        """
+        if self._subdir:
+            return os.path.join(self.directory, self._subdir)
+        return self.directory
 
     def __set_dir(self, dir: str) -> None:
         if not os.path.isdir(dir):
@@ -57,3 +109,43 @@ class MetricsSettings:
                     "Failed to create metrics temporary folder, reason: %s", e)
 
         self.directory = dir
+
+    def _clean_stale_db_files(self) -> None:
+        """Remove all prometheus_client multiprocess .db files."""
+        import glob
+        pattern = os.path.join(self.metrics_directory, "*.db")
+        for path in glob.glob(pattern):
+            try:
+                os.remove(path)
+            except Exception as e:
+                logger.debug("Could not remove stale metrics db file %s: %s", path, e)
+
+    def _clean_dead_pid_files(self) -> None:
+        """Remove .db files whose owning PID no longer exists."""
+        import glob
+        import re
+        pattern = os.path.join(self.metrics_directory, "*.db")
+        for path in glob.glob(pattern):
+            match = re.search(r'_(\d+)\.db$', os.path.basename(path))
+            if not match:
+                continue
+            pid = int(match.group(1))
+            try:
+                os.kill(pid, 0)
+            except ProcessLookupError:
+                # ESRCH: no process owns this PID -> safe to remove its db file.
+                try:
+                    os.remove(path)
+                    logger.debug("Removed dead-pid metrics file %s (pid %d)", path, pid)
+                except OSError as e:
+                    logger.debug("Could not remove dead-pid metrics db file %s: %s", path, e)
+            except OSError as e:
+                # Any non-ESRCH probe failure (commonly EPERM: the process is
+                # alive but owned by another user) -> keep the file; deleting it
+                # could corrupt a live worker's metrics in a shared directory.
+                logger.debug(
+                    "Keeping metrics db file %s; pid %d probe returned a "
+                    "non-ProcessLookupError OSError (process likely alive): %s",
+                    path, pid, e,
+                )
+                continue

conductor/client/http/api/workflow_resource_api.py

@@ -2329,6 +2329,7 @@ class WorkflowResourceApi(object):
         all_params.append('_return_http_data_only')
         all_params.append('_preload_content')
         all_params.append('_request_timeout')
+        all_params.append('metric_context')
 
         params = locals()
         for key, val in six.iteritems(params['kwargs']):
@@ -2383,7 +2384,8 @@ class WorkflowResourceApi(object):
             _return_http_data_only=params.get('_return_http_data_only'),
             _preload_content=params.get('_preload_content', True),
             _request_timeout=params.get('_request_timeout'),
-            collection_formats=collection_formats)
+            collection_formats=collection_formats,
+            metric_context=params.get('metric_context'))
 
     def start_workflow1(self, body, name, **kwargs):  # noqa: E501
         """Start a new workflow. Returns the ID of the workflow instance that can be later used for tracking  # noqa: E501
@@ -2434,6 +2436,7 @@ class WorkflowResourceApi(object):
         all_params.append('_return_http_data_only')
         all_params.append('_preload_content')
         all_params.append('_request_timeout')
+        all_params.append('metric_context')
 
         params = locals()
         for key, val in six.iteritems(params['kwargs']):
@@ -2500,7 +2503,8 @@ class WorkflowResourceApi(object):
             _return_http_data_only=params.get('_return_http_data_only'),
             _preload_content=params.get('_preload_content', True),
             _request_timeout=params.get('_request_timeout'),
-            collection_formats=collection_formats)
+            collection_formats=collection_formats,
+            metric_context=params.get('metric_context'))
 
     def terminate1(self, workflow_id, **kwargs):  # noqa: E501
         """