scalable-pypeline 1.2.3__py2.py3-none-any.whl → 2.0.2__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.3'
+__version__ = "2.0.2"

pypeline/barrier.py

@@ -0,0 +1,34 @@
+import time
+
+import redis
+
+
+class LockingParallelBarrier:
+    def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
+        # Connect to Redis using the provided URL
+        self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
+        self.task_key = task_key
+        self.lock_key = lock_key
+
+    def acquire_lock(self, timeout=5):
+        """Acquire a lock using Redis."""
+        while True:
+            if self.redis.set(self.lock_key, "locked", nx=True, ex=timeout):
+                return True
+            time.sleep(0.1)
+
+    def release_lock(self):
+        """Release the lock in Redis."""
+        self.redis.delete(self.lock_key)
+
+    def set_task_count(self, count):
+        """Initialize the task counter in Redis."""
+        self.redis.set(self.task_key, count)
+
+    def decrement_task_count(self):
+        """Decrement the task counter in Redis."""
+        return self.redis.decr(self.task_key)
+
+    def get_task_count(self):
+        """Get the current value of the task counter."""
+        return int(self.redis.get(self.task_key) or 0)

pypeline/composition.py

@@ -0,0 +1,349 @@
+from __future__ import annotations
+
+import copy
+import json
+import time
+import typing
+from uuid import uuid4
+
+from dramatiq.broker import get_broker
+from dramatiq.results import ResultMissing
+from db_medley.redis_conf import RedisConnector
+from redis.exceptions import RedisError
+
+from pypeline.barrier import LockingParallelBarrier
+from pypeline.constants import DEFAULT_RESULT_TTL
+from pypeline.dramatiq import REDIS_URL
+
+from dramatiq.message import Message
+
+
+class parallel_pipeline:
+    """Chain actors together, passing the result of one actor to the
+    next one in line.
+
+    Parameters:
+      children(typing.List[typing.List[Message]]): A sequence of messages or
+        pipelines.  Child pipelines are flattened into the resulting
+        pipeline.
+      broker(Broker): The broker to run the pipeline on.  Defaults to
+        the current global broker.
+    """
+
+    messages: list[Message]
+
+    def __init__(self, messages: typing.List[typing.List[Message]], broker=None):
+        self.broker = broker or get_broker()
+        self.messages = messages
+        self.execution_id = str(uuid4())
+        execution_graph = []
+
+        for message_group in self.messages:
+            sub_execution_group = []
+            group_completion_uuid = str(uuid4())
+            for m in message_group:
+                m.kwargs["event"]["execution_id"] = self.execution_id
+                m.options["group_completion_uuid"] = group_completion_uuid
+                message_dict = copy.deepcopy(m.asdict())
+                sub_execution_group.append(message_dict)
+            # Last item in the group is the id of the group to be executed
+            execution_graph.append(sub_execution_group)
+
+        self.execution_graph = execution_graph
+
+        for message_group in self.messages:
+            for m in message_group:
+                m.options["execution_graph"] = execution_graph
+
+    def __len__(self):
+        """Returns the length of the parallel_pipeline."""
+        count = 0
+        for message_group in self.messages:
+            count = count + len(message_group)
+
+        return count
+
+    def __str__(self):  # pragma: no cover
+        """Return a string representation of the parallel_pipeline.
+
+        This representation shows the order of execution for each group of messages.
+        """
+        result = []
+
+        for i, message_group in enumerate(self.messages):
+            group_str = f"Group {i + 1}: [\n"
+            for j, message in enumerate(message_group):
+                message_str = f"  Message {j + 1}: {message.actor_name}\n"
+                group_str += message_str
+            group_str += "]\n"
+            result.append(group_str)
+
+        return "".join(result)
+
+    @property
+    def completed(self):
+        return self.completed_count == len(self)
+
+    @property
+    def completed_count(self):
+        count = 0
+
+        for message_group in self.messages:
+            for message in message_group:
+                try:
+                    message.get_result()
+                    count = count + 1
+                except ResultMissing:
+                    pass
+        return count
+
+    def run(self, *, delay=None):
+        """Run this parallel_pipeline.
+
+        Parameters:
+          delay(int): The minimum amount of time, in milliseconds, the
+            parallel_pipeline should be delayed by. If both parallel_pipeline's delay and
+            first message's delay are provided, the bigger value will be
+            used.
+
+        Returns:
+          parallel_pipeline: Itself.
+        """
+        starting_group = self.messages[0]
+
+        completion_uuid = starting_group[0].options["group_completion_uuid"]
+        locking_parallel_barrier = LockingParallelBarrier(
+            REDIS_URL, task_key=completion_uuid, lock_key=f"{completion_uuid}-lock"
+        )
+        locking_parallel_barrier.set_task_count(len(starting_group))
+
+        for m in starting_group:
+            self.broker.enqueue(m, delay=delay)
+
+        return self
+
+    def get_result(self, *, block=False, timeout=None):
+        """Get the result of this pipeline.
+
+        Pipeline results are represented by the result of the last
+        message in the chain.
+
+        Parameters:
+          block(bool): Whether or not to block until a result is set.
+          timeout(int): The maximum amount of time, in ms, to wait for
+            a result when block is True.  Defaults to 10 seconds.
+
+        Raises:
+          ResultMissing: When block is False and the result isn't set.
+          ResultTimeout: When waiting for a result times out.
+
+        Returns:
+          object: The result.
+        """
+        last_message = self.messages[-1][-1]
+
+        backend = self.broker.get_results_backend()
+        return last_message.get_result(backend=backend, block=block, timeout=timeout)
+
+    def get_results(self, *, block=False, timeout=None):
+        """Get the results of each job in the pipeline.
+
+        Parameters:
+          block(bool): Whether or not to block until a result is set.
+          timeout(int): The maximum amount of time, in ms, to wait for
+            a result when block is True.  Defaults to 10 seconds.
+
+        Raises:
+          ResultMissing: When block is False and the result isn't set.
+          ResultTimeout: When waiting for a result times out.
+
+        Returns:
+          A result generator.
+        """
+        deadline = None
+        if timeout:
+            deadline = time.monotonic() + timeout / 1000
+
+        for message_group in self.messages:
+            for message in message_group:
+                if deadline:
+                    timeout = max(0, int((deadline - time.monotonic()) * 1000))
+
+                backend = self.broker.get_results_backend()
+                yield {
+                    message.actor_name: message.get_result(
+                        backend=backend, block=block, timeout=timeout
+                    )
+                }
+
+    def to_json(self) -> str:
+        """Convert the execution graph to a JSON string representation.
+
+        This method serializes the execution graph of the pipeline into a JSON string.
+        This serialized form can be used to save the pipeline state or share it across different systems,
+        enabling the retrieval of a pipeline "run" for obtaining its results at a later time.
+
+        :return: A JSON string representing the execution graph.
+        :rtype: str
+        """
+        return json.dumps(self.execution_graph)
+
+    @classmethod
+    def from_json(cls, json_data: str) -> parallel_pipeline:
+        """Create a ParallelPipeline object from a JSON string representation of the execution graph.
+
+        This class method deserializes a JSON string into a list of messages, each representing
+        a task or operation in the pipeline. The method reconstructs the execution graph using
+        the `dramatiq.message.Message` objects and returns an instance of the `parallel_pipeline` class.
+
+        :param json_data: A JSON string containing the serialized execution graph.
+        :type json_data: str
+        :return: An instance of `parallel_pipeline` reconstructed from the JSON data.
+        :rtype: parallel_pipeline
+        """
+        execution_graph = json.loads(json_data)
+
+        messages = []
+
+        for message_group in execution_graph:
+            temp_group = []
+            for message in message_group:
+                temp_group.append(Message(**message))
+            messages.append(temp_group)
+
+        return cls(messages)
+
+
+class PipelineResult:
+    """
+    A class to manage and retrieve the results of a parallel pipeline execution.
+
+    The `PipelineResult` class provides methods for creating a result entry in a Redis database,
+    loading pipeline data from Redis, and retrieving the status and results of the pipeline execution.
+
+    Attributes:
+        pipeline (parallel_pipeline): The pipeline object representing the execution graph.
+        execution_id (str): A unique identifier for the execution of the pipeline.
+        redis_key (str): The key used to store and retrieve pipeline data from Redis.
+        redis_conn: A Redis connection object used to interact with the Redis database.
+        result_ttl (int): Time-to-live (TTL) for the result entry in Redis, in seconds.
+    """
+
+    def __init__(self, execution_id: str, result_ttl: int = DEFAULT_RESULT_TTL):
+        """
+        Initialize a PipelineResult object with an execution ID and optional result TTL.
+
+        :param execution_id: A unique identifier for the pipeline execution.
+        :type execution_id: str
+        :param result_ttl: The time-to-live (TTL) for the result entry in Redis. Defaults to DEFAULT_RESULT_TTL.
+        :type result_ttl: int
+        """
+        self.pipeline: parallel_pipeline = None
+        self.execution_id = execution_id
+        self.redis_key = f"{execution_id}-results-key"
+        self.redis_conn = RedisConnector().get_connection()
+        self.result_ttl = result_ttl
+
+    def create_result_entry(self, pipeline_json_str: str):
+        """
+        Store the serialized pipeline data in Redis with a specified TTL.
+
+        This method saves the JSON string representation of the pipeline in the Redis database
+        using the execution ID as the key. The entry is stored with a time-to-live (TTL) defined by `result_ttl`.
+
+        :param pipeline_json_str: A JSON string representing the pipeline execution graph.
+        :type pipeline_json_str: str
+        :raises ValueError: If the provided pipeline data is None or an empty string.
+        :raises RedisError: If there is an issue connecting to Redis or setting the value.
+        """
+        if not pipeline_json_str:
+            raise ValueError("No pipeline data passed to create result store")
+
+        try:
+            self.redis_conn.setex(self.redis_key, self.result_ttl, pipeline_json_str)
+        except RedisError as e:
+            raise RuntimeError(f"Failed to store pipeline data in Redis: {e}")
+
+    def load(self):
+        """
+        Load the pipeline data from Redis and reconstruct the pipeline object.
+
+        This method retrieves the JSON string stored in Redis and deserializes it
+        into a `parallel_pipeline` object, enabling access to the pipeline's execution details.
+
+        :raises RedisError: If there is an issue connecting to Redis or retrieving the data.
+        """
+        try:
+            pipeline_data = self.redis_conn.get(self.redis_key)
+            if pipeline_data:
+                self.pipeline = parallel_pipeline.from_json(pipeline_data)
+            else:
+                self.pipeline = None
+        except RedisError as e:
+            raise RuntimeError(f"Failed to load pipeline data from Redis: {e}")
+
+    @property
+    def status(self) -> str:
+        """
+        Get the current status of the pipeline execution.
+
+        This property checks the completion status of the pipeline and returns its current state.
+
+        :return: The status of the pipeline execution, which can be "complete", "pending", or "unavailable".
+        :rtype: str
+        """
+        if not self.pipeline:
+            return "unavailable"
+        return "complete" if self.pipeline.completed else "pending"
+
+    def get_results(self) -> dict:
+        """
+        Retrieve all results from the pipeline execution with unique actor identifiers.
+
+        This method aggregates results from the pipeline and ensures that each actor's result
+        has a unique identifier by appending a numeric suffix to duplicate actor names.
+
+        :return: A dictionary containing all results from the pipeline execution, keyed by unique actor identifiers.
+        :rtype: dict
+        """
+        if not self.pipeline:
+            return {}
+
+        results = {}
+        for result in self.pipeline.get_results():
+            for actor, res in result.items():
+                unique_actor = self._get_unique_actor_name(actor, results)
+                results[unique_actor] = res
+        return results
+
+    def get_result(self):
+        """
+        Retrieve a single result from the pipeline execution.
+
+        This method returns the result of a single execution step from the pipeline, if available.
+
+        :return: The result of a single execution step from the pipeline, or None if no pipeline is loaded.
+        """
+        if self.pipeline:
+            return self.pipeline.get_result()
+
+    def _get_unique_actor_name(self, actor: str, results: dict) -> str:
+        """
+        Generate a unique actor name by appending a numeric suffix if necessary.
+
+        :param actor: The base name of the actor.
+        :type actor: str
+        :param results: The current dictionary of results to check for uniqueness.
+        :type results: dict
+        :return: A unique actor name.
+        :rtype: str
+        """
+        if actor not in results:
+            return actor
+
+        suffix = 0
+        new_actor = f"{actor}-{suffix}"
+        while new_actor in results:
+            suffix += 1
+            new_actor = f"{actor}-{suffix}"
+        return new_actor

pypeline/constants.py

@@ -1,105 +1,72 @@
-""" Sermos Constants
+""" Pypeline Constants
 """
 import os
-from urllib.parse import urljoin
 
-API_PATH_V1 = '/api/v1'
+# Pypeline configuration defaults
+PYPELINE_YAML_PATH = os.environ.get("PYPELINE_YAML_PATH", "pypeline.yaml")
+PYPELINE_CLIENT_PKG_NAME = os.environ.get("PYPELINE_CLIENT_PKG_NAME", None)
+WORKER_NAME = os.environ.get("WORKER_NAME", None)
+API_ACCESS_KEY = os.environ.get("API_ACCESS_KEY", None)
+DEFAULT_BROKER_CALLABLE = os.environ.get(
+    "DEFAULT_BROKER_CLS", "pypeline.dramatiq:configure_default_broker"
+)
 
-DEFAULT_RESULT_TTL = 86400  # seconds (1 day)
-DEFAULT_TASK_TTL = 60  # seconds (1 minute)
-DEFAULT_MAX_RETRY = 3
-DEFAULT_REGULATOR_TASK = 'pypeline.celery.task_chain_regulator'
-DEFAULT_SUCCESS_TASK = 'pypeline.celery.pipeline_success'
+# Pypeline broker connections
+RABBIT_URL = os.environ.get("RABBIT_URL", "amqp://admin:password@127.0.0.1:5672")
+REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
 
-CHAIN_SUCCESS_MSG = 'Chain built successfully ...'
-CHAIN_FAILURE_MSG = 'Chain failed to build ...'
+# Pypeline task defaults
+PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL = int(
+    os.getenv("DRAMATIQ_PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL", "86400000")
+)
+DEFAULT_RESULT_TTL = int(os.getenv("DEFAULT_RESULT_TTL", 86400))  # seconds (1 day)
+DEFAULT_TASK_TTL = int(os.getenv("DEFAULT_TASK_TTL", 600))  # seconds (10 minutes)
+DEFAULT_TASK_MAX_RETRY = int(os.getenv("DEFAULT_TASK_MAX_RETRY", 3))
+DEFAULT_TASK_MIN_BACKOFF = int(os.getenv("DEFAULT_TASK_MIN_BACKOFF", 15))  # seconds
+DEFAULT_TASK_MAX_BACKOFF = int(
+    os.getenv("DEFAULT_TASK_MAX_BACKOFF", 3600)
+)  # seconds (1 hour)
 
-PIPELINE_RUN_WRAPPER_CACHE_KEY = 'sermos_{}_{}'  # pipeline_id + execution_id
-PIPELINE_RESULT_CACHE_KEY = 'sermos_result_{}'  # execution_id
 
-# Pipeline configurations and scheduled task configuration are cached in Redis
-# temporarily (default to CONFIG_REFRESH_RATE (in seconds)).
-# Each pipeline config/schedule config is specific to an individual deployment,
-# however, we cache only with the pipeline_id here because the usage of this
-# cache key is restricted to the redis instance associated with the deployment.
-PIPELINE_CONFIG_CACHE_KEY = 'sermos_pipeline_config_{}'  # pipeline_id
-SCHEDULE_CONFIG_CACHE_KEY = 'sermos_schedule_config'
-CONFIG_REFRESH_RATE = int(os.environ.get('CONFIG_REFRESH_RATE', 30))  # seconds
-
-# TODO where on earth is this crazy time format coming from?
-SCHEDULE_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
-
-AUTH_LOCK_KEY = os.environ.get('AUTH_LOCK_KEY', 'sermos-auth-lock')
-AUTH_LOCK_DURATION = int(os.environ.get('AUTH_LOCK_DURATION', 30))
-
-STORED_MODEL_KEY = '{}_{}{}'
-
-# Yaml path is relative to package
-SERMOS_YAML_PATH = os.environ.get('SERMOS_YAML_PATH', 'sermos.yaml')
-SERMOS_ACCESS_KEY = os.environ.get('SERMOS_ACCESS_KEY', None)
-SERMOS_CLIENT_PKG_NAME = os.environ.get('SERMOS_CLIENT_PKG_NAME', None)
-SERMOS_DEPLOYMENT_ID = os.environ.get('SERMOS_DEPLOYMENT_ID', 'local')
-LOCAL_DEPLOYMENT_VALUE = os.environ.get('LOCAL_DEPLOYMENT_VALUE', 'local')
-DEFAULT_BASE_URL = os.environ.get('SERMOS_BASE_URL', 'https://console.sermos.ai')
-if DEFAULT_BASE_URL != 'local':
-    DEFAULT_BASE_URL += '/api/v1/'
-DEPLOYMENTS_URL = "{}deployments/{}"
-DEPLOYMENTS_DEPLOY_URL = "{}deployments/{}/deploy"
-DEPLOYMENTS_SERVICES_URL = "{}deployments/{}/services"
-DEPLOYMENTS_SERVICE_URL = "{}deployments/{}/services/{}"
-DEFAULT_AUTH_URL = urljoin(DEFAULT_BASE_URL, 'auth')
-USING_SERMOS_CLOUD = DEFAULT_BASE_URL != LOCAL_DEPLOYMENT_VALUE
-DEFAULT_CONFIG_RETRIEVAL_PAGE_SIZE = 25
-WORKFLOW_PROCESSOR_DEFAULT_QUEUE = 'celery'
+MS_IN_SECONDS = 1000
+API_PATH_V1 = "/api/v1"
 
 # Default 'responses' dictionary when decorating endpoints with @api.doc()
 # Extend as necessary.
 API_DOC_RESPONSES = {
-    200: {
-        'code': 200,
-        'description': 'Successful response.'
-    },
-    400: {
-        'code': 400,
-        'description': 'Malformed request. Verify payload is correct.'
-    },
+    200: {"code": 200, "description": "Successful response."},
+    400: {"code": 400, "description": "Malformed request. Verify payload is correct."},
     401: {
-        'code': 401,
-        'description':
-        'Unauthorized. Verify your API Key (`accesskey`) header.'
-    }
+        "code": 401,
+        "description": "Unauthorized. Verify your API Key (`accesskey`) header.",
+    },
 }
 
 # Default 'params' dictionary when decorating endpoints with @api.doc()
 # Extend as necessary.
 API_DOC_PARAMS = {
-    'accesskey': {
-        'in': 'header',
-        'name': 'accesskey',
-        'description': 'Your API Consumer\'s `accesskey`',
-        'type': 'string',
-        'required': True
+    "accesskey": {
+        "in": "header",
+        "name": "accesskey",
+        "description": "Your API Consumer's `accesskey`",
+        "type": "string",
+        "required": False,
     }
 }
 
 DEFAULT_OPENAPI_CONFIG = (
-    ('SWAGGER_UI_DOC_EXPANSION',
-     'list'), ('API_DOCUMENTATION_TITLE',
-               'Sermos API Specs'), ('API_DOCUMENTATION_DESCRIPTION',
-                                     'Available API Endpoints'),
-    ('OPENAPI_VERSION', '3.0.2'), ('OPENAPI_URL_PREFIX',
-                                   '/api/v1'), ('OPENAPI_SWAGGER_APP_NAME',
-                                                'Sermos - API Reference'),
-    ('OPENAPI_SWAGGER_UI_PATH',
-     '/docs'), ('OPENAPI_SWAGGER_BASE_TEMPLATE',
-                'swagger/swagger_ui.html'), ('OPENAPI_SWAGGER_URL', '/docs'),
-    ('OPENAPI_SWAGGER_UI_URL',
-     'https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/'),
-    ('EXPLAIN_TEMPLATE_LOADING', False))
-
-def create_model_key(model_prefix: str,
-                     model_version: str,
-                     model_postfix: str = ''):
-    """ Ensures we're consistently creating the keys for storing/retrieving.
-    """
-    return STORED_MODEL_KEY.format(model_prefix, model_version, model_postfix)
+    ("SWAGGER_UI_DOC_EXPANSION", "list"),
+    ("API_DOCUMENTATION_TITLE", "Pypeline API Specs"),
+    ("API_DOCUMENTATION_DESCRIPTION", "Available API Endpoints"),
+    ("OPENAPI_VERSION", "3.0.2"),
+    ("OPENAPI_URL_PREFIX", "/api/v1"),
+    ("OPENAPI_SWAGGER_APP_NAME", "Pypeline - API Reference"),
+    ("OPENAPI_SWAGGER_UI_PATH", "/docs"),
+    ("OPENAPI_SWAGGER_BASE_TEMPLATE", "swagger/swagger_ui.html"),
+    ("OPENAPI_SWAGGER_URL", "/docs"),
+    (
+        "OPENAPI_SWAGGER_UI_URL",
+        "https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/",
+    ),
+    ("EXPLAIN_TEMPLATE_LOADING", False),
+)