conductor-python 1.3.11__py3-none-any.whl → 1.4.0__py3-none-any.whl

conductor/client/__init__.py

@@ -3,7 +3,7 @@
 from conductor.client.configuration.configuration import Configuration
 from conductor.client.automator.task_handler import TaskHandler
 from conductor.client.automator.task_runner import TaskRunner
-from conductor.client.orkes_clients import OrkesClients
+from conductor.client.orkes_clients import OrkesClients, ConductorClients
 from conductor.client.workflow.conductor_workflow import ConductorWorkflow
 from conductor.client.workflow.executor.workflow_executor import WorkflowExecutor
 from conductor.client.worker.worker_task import worker_task
@@ -18,6 +18,7 @@ __all__ = [
     "TaskHandler",
     "TaskRunner",
     "OrkesClients",
+    "ConductorClients",
     "ConductorWorkflow",
     "WorkflowExecutor",
     "worker_task",

conductor/client/automator/async_task_runner.py

@@ -27,7 +27,7 @@ from conductor.client.http.models.schema_def import SchemaDef, SchemaType
 from conductor.client.http.rest import AuthorizationException, ApiException
 from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient
 from conductor.client.orkes.orkes_schema_client import OrkesSchemaClient
-from conductor.client.telemetry.metrics_collector import MetricsCollector
+from conductor.client.telemetry.metrics_factory import create_metrics_collector
 from conductor.client.worker.worker_interface import WorkerInterface
 from conductor.client.worker.worker_config import resolve_worker_config, get_worker_config_oneline
 from conductor.client.worker.exception import NonRetryableException
@@ -88,7 +88,7 @@ class AsyncTaskRunner:
 
         self.metrics_collector = None
         if metrics_settings is not None:
-            self.metrics_collector = MetricsCollector(
+            self.metrics_collector = create_metrics_collector(
                 metrics_settings
             )
             # Register metrics collector as event listener
@@ -863,6 +863,7 @@ class AsyncTaskRunner:
             if attempt > 0:
                 # Exponential backoff: [10s, 20s, 30s] before retry
                 await asyncio.sleep(attempt * 10)
+            update_start = time.time()
             try:
                 if self._use_update_v2:
                     next_task = await self.async_task_client.update_task_v2(body=task_result)
@@ -873,6 +874,10 @@ class AsyncTaskRunner:
                         task_definition_name,
                         next_task.task_id if next_task else None
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return next_task
                 else:
                     await self.async_task_client.update_task(body=task_result)
@@ -882,6 +887,10 @@ class AsyncTaskRunner:
                         task_result.workflow_instance_id,
                         task_definition_name,
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return None
             except ApiException as e:
                 if e.status in (404, 405) and self._use_update_v2:
@@ -895,12 +904,19 @@ class AsyncTaskRunner:
                     # Retry immediately with v1
                     try:
                         await self.async_task_client.update_task(body=task_result)
+                        if self.metrics_collector is not None:
+                            self.metrics_collector.record_task_update_time(
+                                task_definition_name, time.time() - update_start, status="SUCCESS"
+                            )
                         return None
                     except Exception as fallback_e:
                         last_exception = fallback_e
                         continue
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )
@@ -917,6 +933,9 @@ class AsyncTaskRunner:
             except Exception as e:
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )

conductor/client/automator/task_handler.py

@@ -18,7 +18,7 @@ from conductor.client.configuration.settings.metrics_settings import MetricsSett
 from conductor.client.event.task_runner_events import TaskRunnerEvent
 from conductor.client.event.sync_event_dispatcher import SyncEventDispatcher
 from conductor.client.event.sync_listener_register import register_task_runner_listener
-from conductor.client.telemetry.metrics_collector import MetricsCollector
+from conductor.client.telemetry.metrics_collector_base import MetricsCollectorBase
 from conductor.client.telemetry.model.metric_documentation import MetricDocumentation
 from conductor.client.telemetry.model.metric_label import MetricLabel
 from conductor.client.telemetry.model.metric_name import MetricName
@@ -33,13 +33,24 @@ logger = logging.getLogger(
 )
 
 _decorated_functions = {}
+_VALID_MP_START_METHODS = {"spawn", "fork", "forkserver"}
 _mp_fork_set = False
 if not _mp_fork_set:
     try:
-        if platform == "win32":
-            set_start_method("spawn")
-        else:
-            set_start_method("fork")
+        # The prometheus_client library holds a module-level threading lock;
+        # forking while that lock is held causes a deadlock in child processes.
+        # Set CONDUCTOR_MP_START_METHOD=spawn to avoid this if you hit the
+        # deadlock.  Default is fork for backward compatibility (spawn requires
+        # all Process arguments to be picklable).
+        _default_method = "spawn" if platform == "win32" else "fork"
+        _method = os.environ.get("CONDUCTOR_MP_START_METHOD", "").strip().lower() or _default_method
+        if _method not in _VALID_MP_START_METHODS:
+            logger.warning(
+                "Ignoring invalid CONDUCTOR_MP_START_METHOD=%r; falling back to %r",
+                _method, _default_method,
+            )
+            _method = _default_method
+        set_start_method(_method)
         _mp_fork_set = True
     except Exception as e:
         logger.info("error when setting multiprocessing.set_start_method - maybe the context is set %s", e.args)
@@ -221,11 +232,11 @@ class TaskHandler:
         self._configuration = configuration
         self._metrics_settings = metrics_settings
 
-        # Set prometheus multiprocess directory BEFORE any worker processes start
-        # This must be done before prometheus_client is imported in worker processes
+        # Set PROMETHEUS_MULTIPROC_DIR BEFORE any worker processes start.
+        # MetricsSettings resolves the subdirectory eagerly at construction.
         if metrics_settings is not None:
-            os.environ["PROMETHEUS_MULTIPROC_DIR"] = metrics_settings.directory
-            logger.info(f"Set PROMETHEUS_MULTIPROC_DIR={metrics_settings.directory}")
+            os.environ["PROMETHEUS_MULTIPROC_DIR"] = metrics_settings.metrics_directory
+            logger.info(f"Set PROMETHEUS_MULTIPROC_DIR={metrics_settings.metrics_directory}")
 
         # Store event listeners to pass to each worker process
         self.event_listeners = event_listeners or []
@@ -345,7 +356,7 @@ class TaskHandler:
             self.metrics_provider_process = None
             return
         self.metrics_provider_process = Process(
-            target=MetricsCollector.provide_metrics,
+            target=MetricsCollectorBase.provide_metrics,
             args=(metrics_settings,)
         )
         logger.info("Created MetricsProvider process")

conductor/client/automator/task_runner.py

@@ -30,7 +30,7 @@ from conductor.client.http.models.schema_def import SchemaDef, SchemaType
 from conductor.client.http.rest import AuthorizationException, ApiException
 from conductor.client.orkes.orkes_metadata_client import OrkesMetadataClient
 from conductor.client.orkes.orkes_schema_client import OrkesSchemaClient
-from conductor.client.telemetry.metrics_collector import MetricsCollector
+from conductor.client.telemetry.metrics_factory import create_metrics_collector
 from conductor.client.worker.worker import ASYNC_TASK_RUNNING
 from conductor.client.worker.worker_interface import WorkerInterface
 from conductor.client.worker.worker_config import resolve_worker_config, get_worker_config_oneline
@@ -69,7 +69,7 @@ class TaskRunner:
 
         self.metrics_collector = None
         if metrics_settings is not None:
-            self.metrics_collector = MetricsCollector(
+            self.metrics_collector = create_metrics_collector(
                 metrics_settings
             )
             # Register metrics collector as event listener
@@ -972,6 +972,7 @@ class TaskRunner:
             if attempt > 0:
                 # Exponential backoff: [10s, 20s, 30s] before retry
                 time.sleep(attempt * 10)
+            update_start = time.time()
             try:
                 if self._use_update_v2:
                     next_task = self.task_client.update_task_v2(body=task_result)
@@ -982,6 +983,10 @@ class TaskRunner:
                         task_definition_name,
                         next_task.task_id if next_task else None
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return next_task
                 else:
                     self.task_client.update_task(body=task_result)
@@ -991,6 +996,10 @@ class TaskRunner:
                         task_result.workflow_instance_id,
                         task_definition_name,
                     )
+                    if self.metrics_collector is not None:
+                        self.metrics_collector.record_task_update_time(
+                            task_definition_name, time.time() - update_start, status="SUCCESS"
+                        )
                     return None
             except ApiException as e:
                 if e.status in (404, 405) and self._use_update_v2:
@@ -1004,12 +1013,19 @@ class TaskRunner:
                     # Retry immediately with v1
                     try:
                         self.task_client.update_task(body=task_result)
+                        if self.metrics_collector is not None:
+                            self.metrics_collector.record_task_update_time(
+                                task_definition_name, time.time() - update_start, status="SUCCESS"
+                            )
                         return None
                     except Exception as fallback_e:
                         last_exception = fallback_e
                         continue
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )
@@ -1044,6 +1060,9 @@ class TaskRunner:
             except Exception as e:
                 last_exception = e
                 if self.metrics_collector is not None:
+                    self.metrics_collector.record_task_update_time(
+                        task_definition_name, time.time() - update_start, status="FAILURE"
+                    )
                     self.metrics_collector.increment_task_update_error(
                         task_definition_name, type(e)
                     )

conductor/client/configuration/settings/metrics_settings.py

@@ -14,6 +14,16 @@ logger = logging.getLogger(
 )
 
 
+CANONICAL_SUBDIR = "canonical"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    value = os.environ.get(name, "")
+    if not value:
+        return default
+    return value.strip().lower() in ("true", "1", "yes")
+
+
 def get_default_temporary_folder() -> str:
     return f"{Path.home()!s}/tmp/"
 
@@ -24,12 +34,23 @@ class MetricsSettings:
             directory: Optional[str] = None,
             file_name: str = "metrics.log",
             update_interval: float = 0.1,
-            http_port: Optional[int] = None):
+            http_port: Optional[int] = None,
+            clean_directory: bool = False,
+            clean_dead_pids: bool = False):
         """
         Configure metrics collection settings.
 
+        The ``WORKER_CANONICAL_METRICS`` env var is read at construction time
+        to decide whether ``.db`` files go in *directory* (legacy) or in a
+        ``canonical/`` subdirectory (canonical mode).  Set the env var before
+        creating this object.
+
         Args:
-            directory: Directory for storing multiprocess metrics .db files
+            directory: Base directory for storing multiprocess metrics .db files.
+                      Legacy metrics use this directory directly (unchanged from
+                      prior releases).  Canonical metrics use a ``canonical/``
+                      subdirectory so that switching implementations never
+                      produces stale metric names.
             file_name: Name of the metrics output file (only used when http_port is None)
             update_interval: How often to update metrics (in seconds)
             http_port: Optional HTTP port to expose metrics endpoint for Prometheus scraping.
@@ -40,6 +61,13 @@ class MetricsSettings:
                       If None:
                       - Metrics will be written to file at {directory}/{file_name}
                       - No HTTP server will be started
+            clean_directory: If True, remove all prometheus_client .db files from
+                      the metrics directory when the collector is created.  Only
+                      safe when no other live process shares the same directory.
+                      Defaults to False.
+            clean_dead_pids: If True, remove .db files whose owning PID no
+                      longer exists.  Safer than ``clean_directory`` in shared
+                      environments.  Defaults to False.
         """
         if directory is None:
             directory = get_default_temporary_folder()
@@ -47,6 +75,30 @@ class MetricsSettings:
         self.file_name = file_name
         self.update_interval = update_interval
         self.http_port = http_port
+        self.clean_directory = clean_directory
+        self.clean_dead_pids = clean_dead_pids
+        self._subdir: str = (
+            CANONICAL_SUBDIR
+            if _env_bool("WORKER_CANONICAL_METRICS", False)
+            else ""
+        )
+
+    @property
+    def is_canonical(self) -> bool:
+        return self._subdir == CANONICAL_SUBDIR
+
+    @property
+    def metrics_directory(self) -> str:
+        """Full path where .db files live (base directory + optional subdir).
+
+        Legacy leaves the subdirectory empty (base directory unchanged from
+        prior releases); canonical sets it to ``"canonical"`` to avoid stale
+        metric-name collisions.  Resolved eagerly at construction time from
+        the ``WORKER_CANONICAL_METRICS`` env var.
+        """
+        if self._subdir:
+            return os.path.join(self.directory, self._subdir)
+        return self.directory
 
     def __set_dir(self, dir: str) -> None:
         if not os.path.isdir(dir):
@@ -57,3 +109,43 @@ class MetricsSettings:
                     "Failed to create metrics temporary folder, reason: %s", e)
 
         self.directory = dir
+
+    def _clean_stale_db_files(self) -> None:
+        """Remove all prometheus_client multiprocess .db files."""
+        import glob
+        pattern = os.path.join(self.metrics_directory, "*.db")
+        for path in glob.glob(pattern):
+            try:
+                os.remove(path)
+            except Exception as e:
+                logger.debug("Could not remove stale metrics db file %s: %s", path, e)
+
+    def _clean_dead_pid_files(self) -> None:
+        """Remove .db files whose owning PID no longer exists."""
+        import glob
+        import re
+        pattern = os.path.join(self.metrics_directory, "*.db")
+        for path in glob.glob(pattern):
+            match = re.search(r'_(\d+)\.db$', os.path.basename(path))
+            if not match:
+                continue
+            pid = int(match.group(1))
+            try:
+                os.kill(pid, 0)
+            except ProcessLookupError:
+                # ESRCH: no process owns this PID -> safe to remove its db file.
+                try:
+                    os.remove(path)
+                    logger.debug("Removed dead-pid metrics file %s (pid %d)", path, pid)
+                except OSError as e:
+                    logger.debug("Could not remove dead-pid metrics db file %s: %s", path, e)
+            except OSError as e:
+                # Any non-ESRCH probe failure (commonly EPERM: the process is
+                # alive but owned by another user) -> keep the file; deleting it
+                # could corrupt a live worker's metrics in a shared directory.
+                logger.debug(
+                    "Keeping metrics db file %s; pid %d probe returned a "
+                    "non-ProcessLookupError OSError (process likely alive): %s",
+                    path, pid, e,
+                )
+                continue

conductor/client/http/api/workflow_resource_api.py

@@ -2329,6 +2329,7 @@ class WorkflowResourceApi(object):
         all_params.append('_return_http_data_only')
         all_params.append('_preload_content')
         all_params.append('_request_timeout')
+        all_params.append('metric_context')
 
         params = locals()
         for key, val in six.iteritems(params['kwargs']):
@@ -2383,7 +2384,8 @@ class WorkflowResourceApi(object):
             _return_http_data_only=params.get('_return_http_data_only'),
             _preload_content=params.get('_preload_content', True),
             _request_timeout=params.get('_request_timeout'),
-            collection_formats=collection_formats)
+            collection_formats=collection_formats,
+            metric_context=params.get('metric_context'))
 
     def start_workflow1(self, body, name, **kwargs):  # noqa: E501
         """Start a new workflow. Returns the ID of the workflow instance that can be later used for tracking  # noqa: E501
@@ -2434,6 +2436,7 @@ class WorkflowResourceApi(object):
         all_params.append('_return_http_data_only')
         all_params.append('_preload_content')
         all_params.append('_request_timeout')
+        all_params.append('metric_context')
 
         params = locals()
         for key, val in six.iteritems(params['kwargs']):
@@ -2500,7 +2503,8 @@ class WorkflowResourceApi(object):
             _return_http_data_only=params.get('_return_http_data_only'),
             _preload_content=params.get('_preload_content', True),
             _request_timeout=params.get('_request_timeout'),
-            collection_formats=collection_formats)
+            collection_formats=collection_formats,
+            metric_context=params.get('metric_context'))
 
     def terminate1(self, workflow_id, **kwargs):  # noqa: E501
         """

conductor/client/http/api_client.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import base64
 import datetime
 import logging
@@ -6,7 +8,7 @@ import os
 import re
 import tempfile
 import time
-from typing import Dict
+from typing import Dict, Optional, TYPE_CHECKING
 import uuid
 
 import six
@@ -20,13 +22,15 @@ from conductor.client.http import rest
 from conductor.client.http.rest import AuthorizationException
 from conductor.client.http.thread import AwaitableThread
 
+if TYPE_CHECKING:
+    from conductor.client.telemetry.metrics_collector_base import MetricsCollectorBase
+
 logger = logging.getLogger(
     Configuration.get_logging_formatted_name(
         __name__
     )
 )
 
-
 class ApiClient(object):
     PRIMITIVE_TYPES = (float, bool, bytes, six.text_type) + six.integer_types
     NATIVE_TYPES_MAPPING = {
@@ -46,7 +50,7 @@ class ApiClient(object):
             header_name=None,
             header_value=None,
             cookie=None,
-            metrics_collector=None
+            metrics_collector: Optional[MetricsCollectorBase] = None
     ):
         if configuration is None:
             configuration = Configuration()
@@ -75,14 +79,15 @@ class ApiClient(object):
             query_params=None, header_params=None, body=None, post_params=None,
             files=None, response_type=None, auth_settings=None,
             _return_http_data_only=None, collection_formats=None,
-            _preload_content=True, _request_timeout=None):
+            _preload_content=True, _request_timeout=None, metric_context=None):
         try:
             return self.__call_api_no_retry(
                 resource_path=resource_path, method=method, path_params=path_params,
                 query_params=query_params, header_params=header_params, body=body, post_params=post_params,
                 files=files, response_type=response_type, auth_settings=auth_settings,
                 _return_http_data_only=_return_http_data_only, collection_formats=collection_formats,
-                _preload_content=_preload_content, _request_timeout=_request_timeout
+                _preload_content=_preload_content, _request_timeout=_request_timeout,
+                metric_context=metric_context
             )
         except AuthorizationException as ae:
             if ae.token_expired or ae.invalid_token:
@@ -99,7 +104,8 @@ class ApiClient(object):
                         query_params=query_params, header_params=header_params, body=body, post_params=post_params,
                         files=files, response_type=response_type, auth_settings=auth_settings,
                         _return_http_data_only=_return_http_data_only, collection_formats=collection_formats,
-                        _preload_content=_preload_content, _request_timeout=_request_timeout
+                        _preload_content=_preload_content, _request_timeout=_request_timeout,
+                        metric_context=metric_context
                     )
                 else:
                     logger.error('Failed to renew authentication token. Please check your credentials.')
@@ -110,7 +116,7 @@ class ApiClient(object):
             query_params=None, header_params=None, body=None, post_params=None,
             files=None, response_type=None, auth_settings=None,
             _return_http_data_only=None, collection_formats=None,
-            _preload_content=True, _request_timeout=None):
+            _preload_content=True, _request_timeout=None, metric_context=None):
 
         config = self.configuration
 
@@ -124,6 +130,9 @@ class ApiClient(object):
             header_params = dict(self.parameters_to_tuples(header_params,
                                                            collection_formats))
 
+        # Save the URI template before path param substitution for metrics
+        metric_uri = resource_path
+
         # path parameters
         if path_params:
             path_params = self.sanitize_for_serialization(path_params)
@@ -171,7 +180,9 @@ class ApiClient(object):
             method, url, query_params=query_params, headers=header_params,
             post_params=post_params, body=body,
             _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+            _request_timeout=_request_timeout,
+            metric_uri=metric_uri,
+            metric_context=metric_context)
 
         return_data = response_data
         if _preload_content:
@@ -326,7 +337,7 @@ class ApiClient(object):
                  body=None, post_params=None, files=None,
                  response_type=None, auth_settings=None, async_req=None,
                  _return_http_data_only=None, collection_formats=None,
-                 _preload_content=True, _request_timeout=None):
+                 _preload_content=True, _request_timeout=None, metric_context=None):
         """Makes the HTTP request (synchronous) and returns deserialized data.
 
         To make an async request, set the async_req parameter.
@@ -369,7 +380,8 @@ class ApiClient(object):
                                    body, post_params, files,
                                    response_type, auth_settings,
                                    _return_http_data_only, collection_formats,
-                                   _preload_content, _request_timeout)
+                                   _preload_content, _request_timeout,
+                                   metric_context)
         thread = AwaitableThread(
             target=self.__call_api,
             args=(
@@ -378,15 +390,36 @@ class ApiClient(object):
                 body, post_params, files,
                 response_type, auth_settings,
                 _return_http_data_only, collection_formats,
-                _preload_content, _request_timeout
+                _preload_content, _request_timeout,
+                metric_context
             )
         )
         thread.start()
         return thread
 
+    def _safe_record_api_request_time(self, method, uri, status, time_spent,
+                                      metric_uri):
+        """Record the api-request metric without ever raising.
+
+        Metrics are best-effort instrumentation; a failure here must not
+        propagate and take down the actual API call.
+        """
+        if self.metrics_collector is None:
+            return
+        try:
+            self.metrics_collector.record_api_request_time(
+                method=method,
+                uri=uri,
+                status=status,
+                time_spent=time_spent,
+                metric_uri=metric_uri,
+            )
+        except Exception as e:
+            logger.debug(f'Failed to record api request metric (ignored): {e}')
+
     def request(self, method, url, query_params=None, headers=None,
                 post_params=None, body=None, _preload_content=True,
-                _request_timeout=None):
+                _request_timeout=None, metric_uri=None, metric_context=None):
         """Makes the HTTP request using RESTClient."""
         # Extract URI path from URL (remove query params and domain)
         try:
@@ -461,16 +494,28 @@ class ApiClient(object):
             # Extract status code from response
             status_code = str(response.status) if hasattr(response, 'status') else "200"
 
-            # Record metrics
+            # Record metrics. A metrics failure must never turn a successful
+            # request into a failed one, so the emit is isolated.
             if self.metrics_collector is not None:
                 elapsed_time = time.time() - start_time
-                self.metrics_collector.record_api_request_time(
-                    method=method,
-                    uri=uri,
-                    status=status_code,
-                    time_spent=elapsed_time
+                self._safe_record_api_request_time(
+                    method, uri, status_code, elapsed_time, metric_uri,
                 )
 
+                # Record workflow input payload size when the caller supplies
+                # workflow metadata (only the start-workflow paths do). The size
+                # was computed once in rest.py and lives on the response.
+                if metric_context is not None:
+                    try:
+                        size = getattr(response, 'request_body_size', 0)
+                        self.metrics_collector.record_workflow_input_payload_size(
+                            metric_context["workflow_name"],
+                            metric_context["version"],
+                            size,
+                        )
+                    except Exception:
+                        pass
+
             return response
 
         except Exception as e:
@@ -482,14 +527,12 @@ class ApiClient(object):
             else:
                 status_code = "error"
 
-            # Record metrics for failed requests
+            # Record metrics for failed requests. Guard the emit so a metrics
+            # error can't mask the original request exception below.
             if self.metrics_collector is not None:
                 elapsed_time = time.time() - start_time
-                self.metrics_collector.record_api_request_time(
-                    method=method,
-                    uri=uri,
-                    status=status_code,
-                    time_spent=elapsed_time
+                self._safe_record_api_request_time(
+                    method, uri, status_code, elapsed_time, metric_uri,
                 )
 
             # Re-raise the exception