scalable-pypeline 1.1.5__tar.gz → 1.2.1__tar.gz

{scalable-pypeline-1.1.5/scalable_pypeline.egg-info → scalable-pypeline-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 1.1.5
+Version: 1.2.1
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable-pypeline-1.2.1/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = '1.2.1'

{scalable-pypeline-1.1.5 → scalable-pypeline-1.2.1}/pypeline/celery.py

@@ -2,24 +2,19 @@
 """
 import os
 
-from pypeline.constants import DEFAULT_RETRY_TASK_MAX_TTL, DEFAULT_MAX_RETRY
-
 if os.environ.get('USE_GEVENT', "False").lower() == 'true':
     from gevent import monkey
     monkey.patch_all()
 
-import random
-import time
-
-from celery_dyrygent.tasks import register_workflow_processor
-
 import sys
 import logging
+from pypeline.pipeline.chained_task import ChainedTask
+from celery_dyrygent.tasks import register_workflow_processor
 from typing import List
 from celery import Celery
 from pypeline.logging_config import setup_logging
 from pypeline.utils.module_utils import SermosModuleLoader
-from pypeline.utils.task_utils import PipelineGenerator, PipelineResult, \
+from pypeline.utils.task_utils import PipelineResult, \
     get_service_config_for_worker
 from pypeline.extensions import sermos_config, sermos_client_version
 from pypeline import __version__
@@ -37,46 +32,6 @@ setup_logging(app_version=__version__,
               overload_elasticsearch=OVERLOAD_ES,
               establish_logging_config=True)
 
-
-def pipeline_retry(event: dict):
-    """ Handle pipeline retry and deadletter logic.
-    """
-    access_key = event.get('access_key', None)
-    pipeline_id = event.get('pipeline_id', None)
-    execution_id = event.get('execution_id', None)
-    if pipeline_id is None or execution_id is None:
-        logger.error(f"Unable to retry pipeline {pipeline_id} / "
-                     f"execution {execution_id}.")
-        return False
-
-    # generate_chain() will return `None` if the pipeline has exceeded
-    # max retry count or other erorrs happen.
-    gen = PipelineGenerator(pipeline_id=pipeline_id,
-                            access_key=access_key,
-                            execution_id=execution_id,
-                            queue=event.get('queue', None),
-                            default_task_ttl=event.get('default_task_ttl',
-                                                       None),
-                            add_retry=event.get('add_retry', False),
-                            chain_payload=event.get('chain_payload', None))
-
-    if gen.good_to_go:
-        chain = gen.generate_chain()
-        if chain is not None:
-            # Exponential backoff
-            exponential_backoff = min((3 ** gen.pipeline_wrapper.retry_count) +
-                                      (random.randint(0, 1000) / 1000),
-                                      DEFAULT_RETRY_TASK_MAX_TTL)
-            logger.debug(f"Exponential backoff sleep {exponential_backoff}")
-            time.sleep(exponential_backoff)
-            # Kick it off again.
-            chain.apply_async()
-
-    logger.warning(f"Pipeline retry was invoked for {pipeline_id} "
-                   f"({execution_id})")
-    return True
-
-
 def task_chain_regulator(*args, **kwargs):
     """ Utility task to ensure celery properly waits between groups in a chain.
 
@@ -114,8 +69,6 @@ class GenerateCeleryTasks(SermosModuleLoader):
         """ Sermos provides default tasks that all workers should know about.
         """
         return [{
-            'handler': 'pypeline.celery.pipeline_retry'
-        }, {
             'handler': 'pypeline.celery.task_chain_regulator'
         }, {
             'handler': 'pypeline.celery.pipeline_success'
@@ -147,6 +100,14 @@ class GenerateCeleryTasks(SermosModuleLoader):
         if not service:
             return
         for task in service.get('registeredTasks', []):
+            pipeline_meta = None
+            for pipeline_key, pipeline in sermos_config['pipelines'].items():
+                pipeline_config = pipeline["config"]
+                pipeline_tasks = [t["handler"] for t in pipeline_config["taskDefinitions"].values()]
+                if task["handler"] in pipeline_tasks:
+                    pipeline_meta = pipeline_config["metadata"]
+                    break
+
             try:
                 worker_path = task['handler']  # Required, no default
 
@@ -155,7 +116,17 @@ class GenerateCeleryTasks(SermosModuleLoader):
                 # Decorate the method as a celery task along with a default
                 # queue if provided in config. Set ChainedTask as the base
                 # which allows chained tasks to pass kwargs correctly.
-                tmp_handler = self.celery.task(tmp_handler)
+                if pipeline_meta and pipeline_meta["maxRetry"] > 0:
+                    tmp_handler = self.celery.task(
+                        tmp_handler,
+                        autoretry_for=(Exception,),
+                        max_retries=pipeline_meta["maxRetry"],
+                        retry_backoff=pipeline_meta["retryBackoff"],
+                        retry_jitter=pipeline_meta["retryJitter"],
+                        retry_backoff_max=pipeline_meta["retryBackoffMax"]
+                    )
+                else:
+                    tmp_handler = self.celery.task(tmp_handler)
             except Exception as e:
                 logger.warning(f"Unable to add a task to celery: {e}")
         # Sermos provides default tasks that all workers should know about, add
@@ -173,40 +144,6 @@ def configure_celery(celery: Celery):
     REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
     CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', REDIS_URL)
     CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', REDIS_URL)
-    TaskBase = celery.Task
-
-    class ChainedTask(TaskBase):
-        """ A Celery Task that is used as the _base_ for all dynamically
-        generated tasks (by GenerateCeleryTasks().generate()). This injects
-        `event` into every task's signature, which allows pipelines to pass
-        event information easily through a chain.
-        """
-        abstract = True
-        autoretry_for = (Exception,)
-        max_retries = DEFAULT_MAX_RETRY
-        retry_backoff = True
-        retry_jitter = True
-
-        def __call__(self, *args, **kwargs):
-            """ Allow the return value of one task to update the kwargs of a
-                subsequent task if it's a dictionary. Important to the function
-                of a pipeline to allow event information to flow easily.
-            """
-            # Inject app context
-            if len(args) == 1 and isinstance(args[0], dict):
-                kwargs.update(args[0])
-                args = ()
-
-            # Event holds information used in PipelineRunWrapper and
-            # other areas.
-            if 'event' not in kwargs.keys():
-                kwargs['event'] = {}
-            # This is a special worker from dyrygent that orchestrates our
-            # pipelines.  It provides a patch in fix for celery's poor
-            # implementation of Canvas work-flows
-            if self.__name__ == 'workflow_processor':
-                kwargs.pop('event', None)
-            return super(ChainedTask, self).__call__(*args, **kwargs)
 
     celery.Task = ChainedTask
 
@@ -230,7 +167,6 @@ def configure_celery(celery: Celery):
     # as the result backend, as Rabbit has horrible support as backend.
     celery.conf.result_backend = CELERY_RESULT_BACKEND
     celery.conf.task_ignore_result = False  # Must not ignore for Chords
-    celery.conf.task_acks_late = False  # Check per worker
     celery.conf.result_expires = int(
         os.environ.get('CELERY_RESULT_EXPIRES', 10800))  # 3 hours by default
     celery.conf.broker_pool_limit = int(os.environ.get('BROKER_POOL_LIMIT',

{scalable-pypeline-1.1.5 → scalable-pypeline-1.2.1}/pypeline/constants.py

@@ -7,11 +7,9 @@ API_PATH_V1 = '/api/v1'
 
 DEFAULT_RESULT_TTL = 86400  # seconds (1 day)
 DEFAULT_TASK_TTL = 60  # seconds (1 minute)
-DEFAULT_RETRY_TASK_MAX_TTL = 300
-DEFAULT_MAX_RETRY = 10
+DEFAULT_MAX_RETRY = 3
 DEFAULT_REGULATOR_TASK = 'pypeline.celery.task_chain_regulator'
 DEFAULT_SUCCESS_TASK = 'pypeline.celery.pipeline_success'
-DEFAULT_RETRY_TASK = 'pypeline.celery.pipeline_retry'
 
 CHAIN_SUCCESS_MSG = 'Chain built successfully ...'
 CHAIN_FAILURE_MSG = 'Chain failed to build ...'
@@ -52,6 +50,8 @@ DEPLOYMENTS_SERVICE_URL = "{}deployments/{}/services/{}"
 DEFAULT_AUTH_URL = urljoin(DEFAULT_BASE_URL, 'auth')
 USING_SERMOS_CLOUD = DEFAULT_BASE_URL != LOCAL_DEPLOYMENT_VALUE
 DEFAULT_CONFIG_RETRIEVAL_PAGE_SIZE = 25
+WORKFLOW_PROCESSOR_QUEUE = os.environ.get('WORKFLOW_PROCESSOR_QUEUE', 'default')
+
 # Default 'responses' dictionary when decorating endpoints with @api.doc()
 # Extend as necessary.
 API_DOC_RESPONSES = {

{scalable-pypeline-1.1.5 → scalable-pypeline-1.2.1}/pypeline/flask/api/pipelines.py

@@ -9,7 +9,8 @@ from flask_smorest import Blueprint
 from flask.views import MethodView
 from marshmallow import Schema, fields
 from marshmallow.exceptions import ValidationError
-from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1,\
+    WORKFLOW_PROCESSOR_QUEUE
 from pypeline.flask.decorators import require_accesskey
 from pypeline.flask.api.utils import chain_helper
 from pypeline.utils.task_utils import PipelineResult
@@ -140,7 +141,6 @@ class PipelineInvoke(MethodView):
     def post(self, payload: dict, pipeline_id: str):
         """ Invoke a pipeline by it's ID; optionally provide pipeline arguments.
         """
-
         access_key = request.headers.get('accesskey')
         pipeline_config = retrieve_latest_pipeline_config(
             pipeline_id=pipeline_id, access_key=access_key)
@@ -164,9 +164,22 @@ class PipelineInvoke(MethodView):
                 abort(400, message=gen.loading_message)
 
             chain: _chain = gen.chain
-            wf: Workflow = Workflow()
+            wf: Workflow = Workflow({"queue": WORKFLOW_PROCESSOR_QUEUE})
             wf.add_celery_canvas(chain)
             wf.apply_async()
+
+            celery_task_status = []
+            for node in wf.nodes:
+                celery_task = dict(
+                    name=wf.nodes[node].signature.name,
+                    task_id=node,
+                    status="RUNNING",
+                    retries=0
+                )
+                celery_task_status.append(celery_task)
+
+            gen.pipeline_wrapper.celery_task_status = celery_task_status
+            gen.pipeline_wrapper.save_to_cache()
             retval['status'] = 'success'
             retval['execution_id'] = gen.execution_id
             # Initialize the cached result

{scalable-pypeline-1.1.5 → scalable-pypeline-1.2.1}/pypeline/flask/api/utils.py

@@ -2,7 +2,8 @@
 """
 import logging
 from typing import Union
-from pypeline.utils.task_utils import PipelineGenerator
+
+from pypeline.pipeline.generator import PipelineGenerator
 
 logger = logging.getLogger(__name__)
 
@@ -10,7 +11,6 @@ logger = logging.getLogger(__name__)
 def chain_helper(pipeline_id: str,
                  access_key: Union[str, None] = None,
                  chain_payload: Union[dict, None] = None,
-                 add_retry: bool = True,
                  queue: Union[str, None] = None,
                  default_task_ttl: int = None):
     """ Helper method to generate a pipeline chain *with* error handling.
@@ -25,7 +25,6 @@ def chain_helper(pipeline_id: str,
                             access_key=access_key,
                             queue=queue,
                             default_task_ttl=default_task_ttl,
-                            add_retry=add_retry,
                             chain_payload=chain_payload)
     if gen.good_to_go:
         # Generate our 'chain', which is the grouping of celery constructs that

scalable-pypeline-1.2.1/pypeline/pipeline/chained_task.py

@@ -0,0 +1,70 @@
+import logging
+from celery import Task
+from pypeline.utils.task_utils import PipelineRunWrapper
+
+
+logger = logging.getLogger(__name__)
+
+
+class ChainedTask(Task):
+    """ A Celery Task that is used as the _base_ for all dynamically
+    generated tasks (by GenerateCeleryTasks().generate()). This injects
+    `event` into every task's signature, which allows pipelines to pass
+    event information easily through a chain.
+    """
+    abstract = True
+
+    def __call__(self, *args, **kwargs):
+        """ Allow the return value of one task to update the kwargs of a
+            subsequent task if it's a dictionary. Important to the function
+            of a pipeline to allow event information to flow easily.
+        """
+        # Inject app context
+        if len(args) == 1 and isinstance(args[0], dict):
+            kwargs.update(args[0])
+            args = ()
+
+        # Event holds information used in PipelineRunWrapper and
+        # other areas.
+        if 'event' not in kwargs.keys():
+            kwargs['event'] = {}
+        # This is a special worker from dyrygent that orchestrates our
+        # pipelines.  It provides a patch in fix for celery's poor
+        # implementation of Canvas work-flows
+        if self.__name__ == 'workflow_processor':
+            kwargs.pop('event', None)
+        return super(ChainedTask, self).__call__(*args, **kwargs)
+
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        if "event" in kwargs and "pipeline_id" in kwargs["event"]:
+            try:
+                pipeline_run_wrapper: PipelineRunWrapper = \
+                    PipelineRunWrapper.from_event(kwargs["event"])
+                current_task_status = pipeline_run_wrapper.get_task_celery_status(task_id)
+            except Exception:
+                logger.exception("Unable to retreive Pipeline Run Wrapper")
+                return
+
+            if current_task_status:
+                current_task_status["status"] = status
+            try:
+                pipeline_run_wrapper.save_to_cache()
+            except Exception:
+                logger.exception(f"Failed to update celery task status for task {task_id}")
+
+    def on_retry(self, exc, task_id, args, kwargs, einfo):
+        if "event" in kwargs and "pipeline_id" in kwargs["event"]:
+            try:
+                pipeline_run_wrapper: PipelineRunWrapper = \
+                    PipelineRunWrapper.from_event(kwargs["event"])
+                current_task_status = pipeline_run_wrapper.get_task_celery_status(task_id)
+            except Exception:
+                logger.exception("Unable to retreive Pipeline Run Wrapper")
+                return
+
+            if current_task_status:
+                current_task_status["retries"] = current_task_status["retries"] + 1
+            try:
+                pipeline_run_wrapper.save_to_cache()
+            except Exception:
+                logger.exception(f"Failed to update celery task status for task {task_id}")

scalable-pypeline-1.2.1/pypeline/pipeline/generator.py

@@ -0,0 +1,254 @@
+""" Utilities for running and managing tasks inside pipelines.
+"""
+import logging
+
+from celery import signature, chord, chain
+from pypeline.utils.graph_utils import get_chainable_tasks
+from pypeline.utils.config_utils import retrieve_latest_pipeline_config
+from pypeline.utils.task_utils import PipelineRunWrapper, get_task_signature
+from pypeline.constants import DEFAULT_TASK_TTL, DEFAULT_MAX_RETRY, \
+    DEFAULT_REGULATOR_TASK, CHAIN_FAILURE_MSG, CHAIN_SUCCESS_MSG, \
+    DEFAULT_SUCCESS_TASK
+from pypeline.pipeline_config_schema import PipelineConfigValidator
+
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineGenerator(object):
+    """ Allows an API endpoint to generate a functional pipeline based on the
+        requested pipeline id. Allows API to then issue the tasks asynchronously
+        to initiate the pipeline. Thereafter, celery will monitor status and
+        handle success/failure modes so the API web worker can return
+        immediately.
+
+        The primary purpose is to unpack the pipeline config, create the
+        requisite cached entities to track pipeline progress, and apply the
+        chained pipeline tasks asynchronously so Celery can take over.
+
+        Usage:
+            gen = PipelineGenerator(pipeline_id)
+            chain = gen.generate_chain()
+            chain.on_error(custom_error_task.s())  # Optional add error handling
+            chain.delay()
+    """
+    def __init__(self,
+                 pipeline_id: str,
+                 access_key: str = None,
+                 execution_id: str = None,
+                 queue: str = None,
+                 default_task_ttl: int = None,
+                 regulator_queue: str = None,
+                 regulator_task: str = None,
+                 success_queue: str = None,
+                 success_task: str = None,
+                 default_max_retry: int = None,
+                 retry_backoff: int = None,
+                 retry_jitter: bool = None,
+                 retry_backoff_max: int = None,
+                 chain_payload: dict = None):
+        super().__init__()
+        self.pipeline_id = pipeline_id
+        self.access_key = access_key
+
+        pipeline_config_api_resp = retrieve_latest_pipeline_config(
+            pipeline_id=self.pipeline_id, access_key=self.access_key)
+
+        if pipeline_config_api_resp is None:
+            raise ValueError("Unable to load Pipeline Configuration for "
+                             f"pipeline id: {self.pipeline_id} ...")
+
+        # The only part of the API response used for any 'pipeline config'
+        # is the `config` key. The API nests it under `config` to preserve
+        # ability to add additional detail at a later date.
+        self.pipeline_config = pipeline_config_api_resp.get('config', {})
+        schema_version = pipeline_config_api_resp.get('schemaVersion')
+        PipelineConfigValidator(config_dict=self.pipeline_config,
+                                schema_version=schema_version)
+
+        self.execution_id = execution_id  # UUID string
+        self.good_to_go = False  # Indicates initialization/loading success
+        self.loading_message = None  # Allows access to success/error messages
+        self.is_retry = False if self.execution_id is None else True
+
+        self.default_max_retry = default_max_retry \
+            if default_max_retry is not None else \
+            self.pipeline_config['metadata'].get('maxRetry', DEFAULT_MAX_RETRY)
+        self.retry_backoff = retry_backoff \
+            if retry_backoff is not None else \
+            self.pipeline_config['metadata'].get('retryBackoff', 3)
+        self.retry_backoff_max = retry_backoff \
+            if retry_backoff_max is not None else \
+            self.pipeline_config['metadata'].get('retryBackoffMax', 600)
+        self.retry_jitter = retry_jitter \
+            if retry_jitter is not None else \
+            self.pipeline_config['metadata'].get('retryJitter', False)
+
+        # Queue on which to place tasks by default and default TTL per task
+        # These can be overridden in PipelineConfig.config['taskDefinitions']
+        self.queue = queue \
+            if queue is not None \
+            else self.pipeline_config['metadata']['queue']
+        self.default_task_ttl = default_task_ttl \
+            if default_task_ttl is not None else \
+            self.pipeline_config['metadata'].get('maxTtl', DEFAULT_TASK_TTL)
+
+        # See docstring in self._get_regulator()
+        self.regulator_queue = regulator_queue \
+            if regulator_queue is not None \
+            else self.pipeline_config['metadata']['queue']
+        self.regulator_task = regulator_task\
+            if regulator_task is not None else DEFAULT_REGULATOR_TASK
+
+        # See docstring in self._get_success_task()
+        self.success_queue = success_queue \
+            if success_queue is not None \
+            else self.pipeline_config['metadata']['queue']
+        self.success_task = success_task\
+            if success_task is not None else DEFAULT_SUCCESS_TASK
+
+        # Optional data to pass to each step in chain
+        self.chain_payload = chain_payload\
+            if chain_payload is not None else {}
+
+        self.pipeline_wrapper = None  # Allows access to the PipelineRunWrapper
+        self.chain = None  # Must be intentionally built with generate_chain()
+
+        try:
+            # Generate our wrapper for this pipeline_id / execution_id
+            self.pipeline_wrapper = PipelineRunWrapper(
+                pipeline_id=self.pipeline_id,
+                pipeline_config=self.pipeline_config,
+                execution_id=self.execution_id,
+                max_ttl=self.default_task_ttl,
+                max_retry=self.default_max_retry,
+                chain_payload=self.chain_payload)
+
+            # Loads pipeline config from remote or cache if it's already there
+            # `is_retry` will be True for any PipelineGenerator instantiated
+            # with an execution_id. This flag helps the wrapper increment the
+            # retry count and determine if this should be deadlettered.
+            # This step also saves the valid/initialized run wrapper to cache.
+            self.pipeline_wrapper.load(is_retry=self.is_retry)
+
+            # Set all variables that were established from the run wrapper
+            # initialization. Notably, default_task_ttl can be overloaded
+            # if the pipeline config has an explicit maxTtl set in metadata.
+            self.good_to_go = self.pipeline_wrapper.good_to_go
+            self.loading_message = self.pipeline_wrapper.loading_message
+            self.execution_id = self.pipeline_wrapper.execution_id
+
+        except Exception as e:
+            fail_msg = "Failed to load Pipeline for id {} ... {}".format(
+                self.pipeline_id, e)
+            self.loading_message = fail_msg
+            logger.error(fail_msg)
+            raise e
+
+    def _get_regulator(self):
+        """ Create a chain regulator celery task signature.
+
+            For a chain(), if each element is a group() then celery does not
+            properly adhere to the chain elements occurring sequentially. If you
+            insert a task that is not a group() in between, though, then the
+            chain operates as expected.
+        """
+        return signature(self.regulator_task,
+                         queue=self.regulator_queue,
+                         immutable=True)
+
+    def _get_success_task(self):
+        """ A final 'success' task that's added to the end of every pipeline.
+
+            This stores the 'success' state in the cached result. Users can
+            set other values by using TaskRunner().save_result()
+        """
+        return get_task_signature(task_path=self.success_task,
+                                  queue=self.success_queue,
+                                  pipeline_id=self.pipeline_id,
+                                  execution_id=self.execution_id)
+
+    def _get_signature(self, node):
+        """ Create a celery task signature based on a graph node.
+        """
+        metadata = self.pipeline_config['metadata']
+        node_config = self.pipeline_config['taskDefinitions'][node]
+
+        # Node config takes precedence, pipeline metadata as default
+        queue = node_config.get('queue', metadata['queue'])
+        max_ttl = node_config.get('maxTtl', metadata.get('maxTtl', None))
+
+        # Ensures task signatures include requisite information to retrieve
+        # PipelineRunWrapper from cache using the pipeline id, and execution id.
+        # We set immutable=True to ensure each client task can be defined
+        # with this specific signature (event)
+        # http://docs.celeryproject.org/en/master/userguide/canvas.html#immutability
+        return get_task_signature(task_path=node_config.get('handler'),
+                                  queue=queue,
+                                  access_key=self.access_key,
+                                  pipeline_id=self.pipeline_id,
+                                  execution_id=self.execution_id,
+                                  max_ttl=max_ttl,
+                                  immutable=True,
+                                  task_config=node_config)
+
+    def generate_chain(self):
+        """ Generate the full pipeline chain.
+        """
+        logger.debug(f'Starting Pipeline {self.pipeline_id}')
+
+        if not self.good_to_go:
+            logger.info("Chain deemed to be not good to go.")
+            if self.loading_message is None:
+                self.loading_message = CHAIN_FAILURE_MSG
+            return None
+
+        try:
+            # Create the task chain such that all concurrent tasks are grouped
+            # and all high level node groups are run serially
+            G = self.pipeline_wrapper.execution_graph
+
+            total_tasks = 0
+            pipeline_chain = []
+            chainable_tasks = get_chainable_tasks(G, None, [])
+
+            # Current chord+chain solution based on
+            # https://stackoverflow.com/questions/15123772/celery-chaining-groups-and-subtasks-out-of-order-execution
+            # Look also at last comment from Nov 7, 2017 here
+            # https://github.com/celery/celery/issues/3597
+            # Big outstanding bug in Celery related to failures in chords that
+            # results in really nasty log output. See
+            # https://github.com/celery/celery/issues/4834
+            for i, node_group in enumerate(chainable_tasks):
+                total_tasks += len(node_group)
+                this_group = []
+                for node in node_group:
+                    node_signature = self._get_signature(node)
+                    this_group.append(node_signature)
+
+                if len(this_group) <= 1:
+                    this_group.append(self._get_regulator())
+
+                the_chord = chord(header=this_group,
+                                  body=self._get_regulator())
+
+                pipeline_chain.append(the_chord)
+
+            # Add a 'finished/success' task to the end of all pipelines
+            pipeline_chain.append(
+                chord(header=self._get_success_task(),
+                      body=self._get_regulator()))
+
+            the_chain = chain(*pipeline_chain)
+
+            self.loading_message = CHAIN_SUCCESS_MSG
+
+            self.chain = the_chain
+        except Exception as e:
+            self.loading_message = CHAIN_FAILURE_MSG + " {}".format(e)
+            logger.exception(e)
+            the_chain = None
+
+        self.chain = the_chain
+
+        return the_chain