scalable-pypeline 1.2.3__py2.py3-none-any.whl → 2.0.1__py2.py3-none-any.whl

pypeline/utils/pipeline_utils.py

@@ -0,0 +1,161 @@
+import logging
+import typing
+import networkx as nx
+from dramatiq import get_broker, Message
+from pypeline.composition import parallel_pipeline
+from pypeline.dramatiq import LazyActor, get_callable, register_lazy_actor
+from pypeline.utils.config_utils import retrieve_latest_pipeline_config
+
+T = typing.TypeVar("T")  # T can be any type
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_execution_graph(
+    config: dict,
+    adjacency_key: str = "dagAdjacency",
+    task_definitions_key: str = "taskDefinitions",
+) -> nx.DiGraph:
+    """Generate a directed graph based on a pipeline config's adjacency list
+    and task definitions.
+
+    `dagAdjacency` is a dictionary containing all nodes and downstream
+    nodes.
+
+    `taskDefinitions` is a dictionary containing metadata required for
+    each node such as the worker, model version, etc. This metadata is
+    attached to each node so it can be retrieved directly from the graph.
+    """
+    G = nx.DiGraph()
+
+    # Get our adjacency list and task definitions
+    adjacency_dict = config.get(adjacency_key, {})
+    task_definitions = config.get(task_definitions_key, {})
+    if len(adjacency_dict.keys()) == 0:
+        logger.warning(
+            "Adjacency definition `{}` was not found ...".format(adjacency_key)
+        )
+
+    # Build the graph
+    for node in adjacency_dict.keys():
+        adjacent_nodes = adjacency_dict[node]
+
+        # If no adjacent nodes, then this is a terminal node
+        if len(adjacent_nodes) == 0:
+            G.add_node(node, attr_dict=task_definitions.get(node, {}))
+            continue
+
+        # Otherwise, we'll add an edge from this node to all adjacent nodes
+        # and add the task defnition metadata to the edge
+        G.add_edges_from(
+            [(node, n, task_definitions.get(n, {})) for n in adjacent_nodes]
+        )
+    return G
+
+
+def process_non_none_value(value: T) -> None:
+    """
+    Processes a value that must not be None.
+
+    The function checks if the provided value is None and raises a ValueError if it is.
+    If the value is not None, it proceeds to process and print the value.
+
+    :param value: The value to process. Can be of any type, but must not be None.
+
+    :raises ValueError: If the value is None.
+
+    Example:
+    >>> process_non_none_value(42)
+    Processing value: 42
+
+    >>> process_non_none_value("hello")
+    Processing value: hello
+
+    >>> process_non_none_value(None)
+    Traceback (most recent call last):
+        ...
+    ValueError: None value is not allowed
+    """
+    if value is None:
+        raise ValueError("None value is not allowed")
+    # Process the value
+    print(f"Processing value: {value}")
+
+
+def topological_sort_with_parallelism(
+    graph: nx.DiGraph, executable_nodes=None
+) -> typing.List[typing.List[T]]:
+    """
+    Recurse over the graph to find an optimal execution strategy for processing nodes in an order where
+    no node shall be processed before all of its predecessors have been processed. The function handles
+    parallel execution by identifying nodes that can be processed in parallel at each step. If the graph
+    contains a cycle, the function will not be able to generate an execution plan and will raise an exception.
+
+    :param graph: A directed acyclic graph (DiGraph) from NetworkX.
+    :param executable_nodes: A list of lists where each inner list contains nodes that can be executed
+                              in parallel at each step. This parameter is used for recursion.
+    :return: A list of lists where each inner list contains nodes that can be executed in parallel at each step.
+
+    >>> g = nx.DiGraph()
+    >>> g.add_edges_from([(1, 2), (1, 3), (2, 4), (3, 4)])
+    >>> topological_sort_with_parallelism(g)
+    [[1], [2, 3], [4]]
+
+    >>> g = nx.DiGraph()
+    >>> g.add_edges_from([(1, 2), (2, 3), (3, 4)])
+    >>> topological_sort_with_parallelism(g)
+    [[1], [2], [3], [4]]
+
+    >>> g = nx.DiGraph()
+    >>> g.add_edges_from([(1, 2), (2, 3), (3, 1)])
+    >>> topological_sort_with_parallelism(g)
+    Traceback (most recent call last):
+        ...
+    NetworkXUnfeasible: Graph contains a cycle, cannot compute a topological sort.
+    """
+    nodes = list(nx.topological_sort(graph))
+    round_executable_nodes = [n for n in nodes if graph.in_degree(n) == 0]
+    graph.remove_nodes_from(round_executable_nodes)
+
+    if len(round_executable_nodes) == 0:
+        return executable_nodes
+    else:
+        executable_nodes = [] if executable_nodes is None else executable_nodes
+        executable_nodes.append(round_executable_nodes)
+        return topological_sort_with_parallelism(graph, executable_nodes)
+
+
+def dag_generator(pipeline_id: str, *args, **kwargs):
+    pipeline_config = retrieve_latest_pipeline_config(pipeline_id=pipeline_id)["config"]
+    graph = get_execution_graph(pipeline_config)
+    optimal_execution_graph = topological_sort_with_parallelism(graph.copy())
+    broker = get_broker()
+
+    registered_actors: typing.Dict[str, LazyActor] = {}
+    broker.actors.clear()
+
+    messages: typing.List[typing.List[Message]] = []
+
+    task_definitions = pipeline_config["taskDefinitions"]
+    for task_group in optimal_execution_graph:
+        message_group = []
+        for task in task_group:
+            module_path = task_definitions[task]["handler"]
+            tmp_handler = get_callable(module_path)
+            lazy_actor = register_lazy_actor(
+                broker, tmp_handler, pipeline_config["metadata"]
+            )
+            registered_actors[task] = lazy_actor
+            if args and not kwargs:
+                message_group.append(registered_actors[task].message(*args))
+            elif kwargs and not args:
+                message_group.append(registered_actors[task].message(**kwargs))
+            elif args and kwargs:
+                message_group.append(registered_actors[task].message(*args, **kwargs))
+            else:
+                message_group.append(registered_actors[task].message())
+        messages.append(message_group)
+    p = parallel_pipeline(messages)
+
+    return p

scalable_pypeline-2.0.1.dist-info/METADATA

@@ -0,0 +1,217 @@
+Metadata-Version: 2.1
+Name: scalable-pypeline
+Version: 2.0.1
+Summary: PypeLine - Python pipelines for the Real World
+Home-page: https://gitlab.com/bravos2/pypeline
+Author: Bravos Power Corporation
+License: Apache License 2.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: PyYAML (<7,>=6.0.1)
+Requires-Dist: click (==8.0.4)
+Requires-Dist: marshmallow (<4,>=3.2.1)
+Requires-Dist: redis (<5,>=4.5.4)
+Requires-Dist: db-medley[redis] (<2,>=1.0.2)
+Requires-Dist: croniter (<2,>=1.0.15)
+Provides-Extra: build
+Requires-Dist: wheel ; extra == 'build'
+Requires-Dist: twine ; extra == 'build'
+Provides-Extra: dev
+Requires-Dist: blackd ; extra == 'dev'
+Provides-Extra: flask
+Requires-Dist: Werkzeug (==2.0.3) ; extra == 'flask'
+Requires-Dist: itsdangerous (==2.0.1) ; extra == 'flask'
+Requires-Dist: Flask (<2,>=1.1.2) ; extra == 'flask'
+Requires-Dist: flask-smorest (<0.29,>=0.23.0) ; extra == 'flask'
+Requires-Dist: Jinja2 (==3.0.3) ; extra == 'flask'
+Provides-Extra: test
+Requires-Dist: pytest-cov (<3,>=2.6.1) ; extra == 'test'
+Requires-Dist: tox (<4,>=3.14.1) ; extra == 'test'
+Requires-Dist: mock (<2,>=1) ; extra == 'test'
+Requires-Dist: moto (<4,>=1.3.16) ; extra == 'test'
+Requires-Dist: responses (<0.11,>=0.10.16) ; extra == 'test'
+Requires-Dist: fakeredis (<3,>=2.10.3) ; extra == 'test'
+Requires-Dist: importlib-metadata (<5,>=4.12) ; extra == 'test'
+Provides-Extra: web
+Requires-Dist: gunicorn ; extra == 'web'
+Requires-Dist: gevent (<22,>=21.12.0) ; extra == 'web'
+Provides-Extra: workers
+Requires-Dist: networkx (>=2.4) ; extra == 'workers'
+Requires-Dist: dramatiq[rabbitmq] (<2,>=1.17.0) ; extra == 'workers'
+Requires-Dist: apscheduler (<4,>=3.10.4) ; extra == 'workers'
+
+```
+______ __   ________  _____  _     _____  _   _  _____ 
+| ___ \\ \ / /| ___ \|  ___|| |   |_   _|| \ | ||  ___|
+| |_/ / \ V / | |_/ /| |__  | |     | |  |  \| || |__  
+|  __/   \ /  |  __/ |  __| | |     | |  | . ` ||  __| 
+| |      | |  | |    | |___ | |_____| |_ | |\  || |___ 
+\_|      \_/  \_|    \____/ \_____/\___/ \_| \_/\____/                                 
+```
+
+## Overview
+
+PypeLine is a versatile open-source library designed to streamline the management of data workflows and APIs. With PypeLine, you can efficiently schedule cron jobs, execute complex Directed Acyclical Graph (DAG) pipelines, and set up a Flask API complete with OpenAPI documentation.
+
+#### Key Features
+- Cron Job Scheduling: Easily schedule recurring tasks with flexible cron job functionality, ensuring that your processes run reliably at specified intervals.
+- DAG Pipelines: Define and execute DAGs to manage complex data workflows with dependencies. PypeLine handles the execution order and parallelism, ensuring that each task runs in the correct sequence.
+- Flask API with OpenAPI: Quickly configure a RESTful API using Flask, with built-in support for OpenAPI documentation, allowing for clear, standardized documentation of your endpoints.
+
+## Requirements
+
+- RabbitMQ
+- Redis
+- Docker (optional for dev)
+
+## Getting Started
+
+Install PypeLines:
+
+```commandline
+pip install scalable-pypeline[flask,web,workers]>=1.2.3
+```
+
+Configure your Flask project (app.py)
+
+```python
+from flask import Flask
+from pypeline.flask import FlaskPypeline
+from pypeline_demo.api import bp
+from pypeline_demo.config import Config
+from pypeline_demo.extensions import dramatiq
+
+
+
+def create_app():
+    app = Flask(__name__)
+
+    dramatiq.init_app(app)
+
+    # Initialize your app with a configuration
+    app.config.from_object(Config)
+
+    pypeline = FlaskPypeline()
+    pypeline.init_app(app, init_api=True)
+
+    # Register API blueprints you wish 
+    app.extensions["pypeline_core_api"].register_blueprint(bp)
+    # Register application blueprints to application
+    app.register_blueprint(bp)
+
+    return app
+
+
+if __name__ == "__main__":
+    app = create_app()
+    app.run(port=5001)
+```
+
+Configure Dramatiq extension (extensions.py)
+
+```python
+from pypeline.dramatiq import Dramatiq
+
+
+dramatiq = Dramatiq()
+```
+
+Setup your yaml configuration for pypelines (pypeline.yaml)
+
+```yaml
+serviceConfig:
+    - name: pipeline-worker
+      registeredTasks:
+          - handler: pypeline_demo.pipeline.a
+          - handler: pypeline_demo.pipeline.b
+          - handler: pypeline_demo.pipeline.c
+          - handler: pypeline_demo.scheduled_tasks.cron_task
+
+pipelines:
+    demo_pipeline:
+        name: Demo Pipeline
+        description: Pipeline to show examples of DAG Adjacency
+        schemaVersion: 1
+        config:
+            dagAdjacency:
+                a:
+                    - b
+                    - c
+            metadata:
+                maxRetry: 1
+                retryBackoff: 180
+                retryBackoffMax: 300
+                retryJitter: true
+                maxTtl: 10800
+                queue: new-queue
+            taskDefinitions:
+                a:
+                    handler: pypeline_demo.pipeline.a
+                b:
+                    handler:  pypeline_demo.pipeline.b
+                c:
+                    handler:  pypeline_demo.pipeline.c
+scheduledTasks:
+    cron-task:
+        name: Example cron task
+        enabled: true
+        config:
+            task: pypeline_demo.scheduled_tasks.cron_task
+            queue: new-queue
+            schedule:
+                minute: '*'
+                hour: '*'
+                dayOfWeek: '*'
+                dayOfMonth: '*'
+                monthOfYear: '*'
+        schemaVersion: 1
+```
+
+Setup your modules to be executed by yaml (pipeline.py && scheduled_tasks.py)
+
+```python
+import time
+
+
+def a(event):
+    print("A")
+
+
+def b(event):
+    print("B")
+    time.sleep(10)
+
+
+def c(event):
+    print("C")
+```
+
+```python
+def cron_task():
+    print("HI")
+```
+
+Configure your environment variables (demo.env)
+
+```env
+SERMOS_BASE_URL=local
+PYPELINE_CLIENT_PKG_NAME=pypeline_demo
+REDIS_URL=redis://:password@localhost:6379/0
+RABBITMQ_URL=amqp://admin:password@localhost:5672
+```
+
+Start Rabbit & Redis as your message broker and backend results storage.  We use `docker compose` for this.
+
+## DEMO PROJECT COMING SOON!
+
+
+## Testing
+
+If you are developing pypeline and want to test this package,
+install the test dependencies:
+
+    $ pip install -e .[test]
+
+Now, run the tests:
+
+    $ tox

scalable_pypeline-2.0.1.dist-info/RECORD

@@ -0,0 +1,27 @@
+pypeline/__init__.py,sha256=wAxkK8w13vqoF47A8iqWdSlIgRRXmZiQ0R4wePZfzhs,22
+pypeline/barrier.py,sha256=dLDaprH5NB-C7MQjZqPpBBhMjmO0VV_kTonlgweznHc,1096
+pypeline/composition.py,sha256=s0p_KdD-UheyFe2yPG8GJ94txq9C8fPDU58UcLfal6Q,12955
+pypeline/constants.py,sha256=vi4UZz1xd0ZeIuelp4QgCQsMlIHW65-lVB8l_iA8kBE,2578
+pypeline/dramatiq.py,sha256=jr9WORqtusC1gnbvF59CNXny8ORk2Lmlbmf1qsbiLXo,14799
+pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
+pypeline/middleware.py,sha256=6vWNCoRVqnASJ40CAOEpe0JcYRpB6BTkLz8E51q4z2Y,2756
+pypeline/pipeline_config_schema.py,sha256=DQ_RMucnA0AyrndlW6lkb0orGromcO6C9GgLHyG6lJ0,8013
+pypeline/pypeline_yaml.py,sha256=Og08sUKwOjq7JYPnkg-NIcGbHravYCkC5Arz22rZEtA,16981
+pypeline/schedule_config_schema.py,sha256=vtZV-5wpGcAiYcXxdBPRkrjsbR6x_9E-1PC2elrKKbE,3611
+pypeline/flask/__init__.py,sha256=AdljRh0lMiS8ExgDmgzObwVs8jW7hqQuf83Ml8kn8GQ,491
+pypeline/flask/decorators.py,sha256=ki6jkjZwbDbCWuj7ET7N-ncZwrASp4Fy7257WIYiAAQ,1102
+pypeline/flask/flask_pypeline.py,sha256=Uqyu3PnSP3DoVZUJPqV9chjT4xdRgvcL3OMXxkbdTEg,5490
+pypeline/flask/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/flask/api/pipelines.py,sha256=sPvEoNwmnJPSA96lZHYS2fwKqZlVyE2OSjUmOPFi91o,7267
+pypeline/flask/api/schedules.py,sha256=31lwoFlGv-S-2ahGUCnD5YbmKws8yddj6_PEzzdBi9s,1321
+pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/utils/config_utils.py,sha256=rAIATyoW7kGETZ_Z2DqiXtGd7bJp5uPfcLtfNPOYsNs,2167
+pypeline/utils/module_utils.py,sha256=boEP9IYr4p_ick7HlVUfIxOYHQlEmo7dgvDBCQc-C28,2914
+pypeline/utils/pipeline_utils.py,sha256=dewzkMajs7uyPHyHjJfISA9pc2-1J5A99Hm4XqNw5qM,6031
+tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scalable_pypeline-2.0.1.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.0.1.dist-info/METADATA,sha256=4ozytitQKStqd6jqYr1ALmorfspjit-OTgi1ye_jvpk,6239
+scalable_pypeline-2.0.1.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.0.1.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.0.1.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.0.1.dist-info/RECORD,,

scalable_pypeline-2.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[flask.commands]
+cron-scheduler = pypeline.dramatiq:cron_scheduler
+pypeline-worker = pypeline.dramatiq:pypeline_worker

tests/fixtures/__init__.py

@@ -1 +0,0 @@
-from tests.fixtures.s3_fixtures import *

pypeline/celery.py

@@ -1,206 +0,0 @@
-""" Configure and instantiate Celery
-"""
-import os
-
-if os.environ.get('USE_GEVENT', "False").lower() == 'true':
-    from gevent import monkey
-    monkey.patch_all()
-
-import sys
-import logging
-from pypeline.pipeline.chained_task import ChainedTask
-from celery_dyrygent.tasks import register_workflow_processor
-from typing import List
-from celery import Celery
-from pypeline.logging_config import setup_logging
-from pypeline.utils.module_utils import SermosModuleLoader
-from pypeline.utils.task_utils import PipelineResult, \
-    get_service_config_for_worker
-from pypeline.extensions import sermos_config, sermos_client_version
-from pypeline import __version__
-
-logger = logging.getLogger('celery')
-ENABLE_TOOLS = str(os.environ.get('ENABLE_TOOLS', 'false')).lower() == 'true'
-CELERY_TASKS_ACK_LATE = str(os.environ.get('CELERY_TASKS_ACK_LATE', 'false')).lower() == 'true'
-LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
-OVERLOAD_ES = os.environ.get('ENV', 'production').lower() == 'production'
-PIPELINE_CHORD_COMPRESSION = os.environ.get('PIPELINE_CHORD_COMPRESSION', None)
-
-setup_logging(app_version=__version__,
-              client_version=sermos_client_version,
-              default_level=LOG_LEVEL,
-              overload_elasticsearch=OVERLOAD_ES,
-              establish_logging_config=True)
-
-def task_chain_regulator(*args, **kwargs):
-    """ Utility task to ensure celery properly waits between groups in a chain.
-
-        For a chain(), if each element is a group() then celery does not
-        properly adhere to the chain elements occurring sequentially. If you
-        insert a task that is not a group() in between, though, then the
-        chain operates as expected.
-    """
-    return True
-
-
-def pipeline_success(event: dict):
-    """ Utility task to ensure celery properly waits between groups in a chain.
-
-        For a chain(), if each element is a group() then celery does not
-        properly adhere to the chain elements occurring sequentially. If you
-        insert a task that is not a group() in between, though, then the
-        chain operates as expected.
-    """
-    pr = PipelineResult(event['execution_id'])
-    pr.load()
-    pr.save(status='success')
-
-
-class GenerateCeleryTasks(SermosModuleLoader):
-    """ Use the sermos.yaml configuration to turn customer methods into
-        decorated celery tasks that are available for work/pipelines
-    """
-    def __init__(self, config: dict, celery_instance: Celery):
-        super(GenerateCeleryTasks, self).__init__()
-        self.config = config if config else {}
-        self.celery = celery_instance
-
-    def _get_default_tasks(self) -> List[dict]:
-        """ Sermos provides default tasks that all workers should know about.
-        """
-        return [{
-            'handler': 'pypeline.celery.task_chain_regulator'
-        }, {
-            'handler': 'pypeline.celery.pipeline_success'
-        }]
-
-    def generate(self):
-        """ Loads methods based on sermos config file and decorates them as
-            celery tasks.
-
-            Customer's methods:
-            --------------------------------
-            def demo_task(*args, **kwargs):
-                return True
-
-            Turns into the equivallent of:
-            --------------------------------
-            @celery.task(queue='queue-name')
-            def demo_task(*args, **kwargs):t
-                return True
-        """
-        # Set in k8s deployment as an environment variable when Sermos Cloud
-        # generates the final secrets.yaml file. The name comes from the user's
-        # sermos.yaml file based on serviceConfig[].name. Each 'worker' will
-        # have a single name and each individually registers tasks through its
-        # registeredTasks list. This allows each worker to only attempt
-        # bootstrapping those tasks that are relevant to the worker and not, for
-        # example, attempt to import a package that's not used by this worker
-        service = get_service_config_for_worker(self.config)
-        if not service:
-            return
-        for task in service.get('registeredTasks', []):
-            pipeline_meta = None
-            for pipeline_key, pipeline in sermos_config['pipelines'].items():
-                pipeline_config = pipeline["config"]
-                pipeline_tasks = [t["handler"] for t in pipeline_config["taskDefinitions"].values()]
-                if task["handler"] in pipeline_tasks:
-                    pipeline_meta = pipeline_config["metadata"]
-                    break
-
-            try:
-                worker_path = task['handler']  # Required, no default
-
-                tmp_handler = self.get_callable(worker_path)
-
-                # Decorate the method as a celery task along with a default
-                # queue if provided in config. Set ChainedTask as the base
-                # which allows chained tasks to pass kwargs correctly.
-                if pipeline_meta and pipeline_meta["maxRetry"] > 0:
-                    tmp_handler = self.celery.task(
-                        tmp_handler,
-                        autoretry_for=(Exception,),
-                        max_retries=pipeline_meta["maxRetry"],
-                        retry_backoff=pipeline_meta["retryBackoff"],
-                        retry_jitter=pipeline_meta["retryJitter"],
-                        retry_backoff_max=pipeline_meta["retryBackoffMax"]
-                    )
-                else:
-                    tmp_handler = self.celery.task(tmp_handler)
-            except Exception as e:
-                logger.warning(f"Unable to add a task to celery: {e}")
-        # Sermos provides default tasks that all workers should know about, add
-        # them here.
-        for task in self._get_default_tasks():
-            tmp_handler = self.get_callable(task['handler'])
-            tmp_handler = self.celery.task(tmp_handler)
-
-
-def configure_celery(celery: Celery):
-    """ Configure Sermos-compatible Celery instance. Primarily this means
-    compatibility with Pipelines and Scheduled Tasks through injecting the
-    event kwarg. Also sets prebaked defaults that can be overloaded by user.
-    """
-    REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/0')
-    CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', REDIS_URL)
-    CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', REDIS_URL)
-
-    celery.Task = ChainedTask
-
-    # Configure the broker and tasks
-    celery.conf.broker_url = CELERY_BROKER_URL
-
-    # Use our custom database scheduler for dynamic celery beat updates.
-    celery.conf.beat_scheduler =\
-        'pypeline.celery_beat:SermosScheduler'
-
-    # Reasonable defaults, override as necessary
-    celery.conf.worker_redirect_stdouts = True
-    celery.conf.worker_redirect_stdouts_level = LOG_LEVEL
-    celery.conf.worker_hijack_root_logger = False
-
-    if PIPELINE_CHORD_COMPRESSION:
-      celery.conf.task_compression = PIPELINE_CHORD_COMPRESSION
-
-    # NOTE: The broker URL may not be the best result backend. For example,
-    # When using Rabbit as the broker (recommended), you should use Redis
-    # as the result backend, as Rabbit has horrible support as backend.
-    celery.conf.result_backend = CELERY_RESULT_BACKEND
-    celery.conf.task_ignore_result = False  # Must not ignore for Chords
-    celery.conf.result_expires = int(
-        os.environ.get('CELERY_RESULT_EXPIRES', 10800))  # 3 hours by default
-    celery.conf.broker_pool_limit = int(os.environ.get('BROKER_POOL_LIMIT',
-                                                       10))
-    celery.conf.worker_max_tasks_per_child = int(
-        os.environ.get('MAX_TASKS_PER_CHILD', 100))
-    celery.conf.task_soft_time_limit =\
-        int(os.environ.get('TASK_TIMEOUT_SECONDS', 3600))
-    celery.conf.task_time_limit =\
-        int(os.environ.get('TASK_TIMEOUT_SECONDS', 3600)) + 10  # Cleanup buffer
-    celery.conf.task_acks_late = CELERY_TASKS_ACK_LATE
-    celery.conf.task_serializer = 'json'
-    celery.conf.result_serializer = 'json'
-    celery.conf.accept_content = ['json']
-    # Required config options for some brokers we use frequently.
-    transport_options = {}
-    celery.conf.broker_transport_options = transport_options
-
-    # Sermos generally has long-running tasks (relatively speaking), so
-    # limit number of jobs a worker can reserve. This may not be true for
-    # all tasks, so configure this on a per application basis. In the event
-    # mutltiple task kinds exist in an application (short and long), see
-    # http://docs.celeryproject.org/en/latest/userguide/optimizing.html#optimizing-prefetch-limit
-    # for some guidance on combining multiple workers and routing tasks.
-    # TODO make configurable from env
-    celery.conf.worker_prefetch_multiplier = 1
-
-    # Add our application's workers & any other tasks to be made
-    # available
-    register_workflow_processor(celery)
-    try:
-        GenerateCeleryTasks(sermos_config, celery).generate()
-    except Exception as e:
-        logger.error(f"Unable to dynamically generate celery tasks: {e}")
-        sys.exit(1)
-
-    return celery