scalable-pypeline 1.2.3__py2.py3-none-any.whl → 2.0.2__py2.py3-none-any.whl

pypeline/flask/flask_pypeline.py

@@ -0,0 +1,156 @@
+""" Pypeline implementation as a Flask extension
+"""
+import os
+
+if os.getenv("USE_GEVENT", "false").lower() == "true":
+    import gevent.monkey
+
+    gevent.monkey.patch_all()
+
+import logging
+from flask import Flask
+from werkzeug.middleware.proxy_fix import ProxyFix
+from flask_smorest import Api, Blueprint
+from pypeline.extensions import pypeline_config
+from pypeline.constants import DEFAULT_OPENAPI_CONFIG
+from pypeline import __version__
+
+logger = logging.getLogger(__name__)
+
+
+class FlaskPypeline:
+    """Pypeline Flask extension."""
+
+    def __init__(self, app: Flask = None):
+        """Class init"""
+        self.app = app
+        self.pypeline_config = pypeline_config if pypeline_config is not None else {}
+
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app: Flask, init_api: bool = False):
+        """Pypeline bootstrapping process.
+
+        Application config variables to set include:
+
+            pypeline_client_version (default: v?.?.?)
+
+        Optional, if `init_api` is True:
+
+            API_DOCUMENTATION_TITLE
+            API_DOCUMENTATION_DESCRIPTION
+            OPENAPI_VERSION
+            OPENAPI_URL_PREFIX
+            OPENAPI_SWAGGER_APP_NAME
+            OPENAPI_SWAGGER_UI_PATH
+            OPENAPI_SWAGGER_BASE_TEMPLATE
+            OPENAPI_SWAGGER_URL
+            OPENAPI_SWAGGER_UI_URL
+            SWAGGER_UI_DOC_EXPANSION
+            EXPLAIN_TEMPLATE_LOADING
+
+        Args:
+            app (Flask): Flask Application to initialize.
+            init_api (bool): If `True`, Pypeline will initialize its
+                core APIs (including Pipelines, Scheduled Tasks, etc.) and
+                provide a pre-configured OpenAPI Spec/Swagger UI interface
+                available at the route defined in your application's config
+                under `OPENAPI_URL_PREFIX` (default `/api`). Refer to
+                [flask-smorest](https://flask-smorest.readthedocs.io/en/latest/openapi.html)
+                documentation for additional configuration options.
+        """
+        # Ensure there's a PYPELINE_CLIENT_VERSION on app config
+        app.config.setdefault(
+            "PYPELINE_CLIENT_VERSION",
+            app.config.get("PYPELINE_CLIENT_VERSION", "v?.?.?"),
+        )
+
+        app.wsgi_app = ProxyFix(app.wsgi_app)
+        app.url_map.strict_slashes = False
+
+        # Create and register the pypeline blueprint
+        bp = Blueprint(
+            "pypeline",
+            __name__,
+            template_folder="../templates",
+            static_folder="../static",
+            url_prefix="/pypeline",
+        )
+        app.register_blueprint(bp)
+
+        # Bootstrap api if app requests
+        if init_api is True:
+            self._bootstrap_api(app)
+
+    def _bootstrap_api(self, app: Flask):
+        """If initializing the API, we will create the core Pypeline API paths
+        and initialize the default Swagger documentation.
+        """
+        # Set sensible defaults for Swagger docs. Provided `app` will
+        # take precedent.
+        for swagger_config in DEFAULT_OPENAPI_CONFIG:
+            app.config.setdefault(
+                swagger_config[0], app.config.get(swagger_config[0], swagger_config[1])
+            )
+
+        # Attempt to override with values from client's pypeline.yaml if
+        # they are available. This will add new tags and new docs if
+        # defined and add to the core Pypeline API docs.
+        api_config = self.pypeline_config.get("apiConfig", {})
+        api_docs = api_config.get("apiDocumentation", {})
+
+        custom_tags = api_config.get("prefixDescriptions", [])
+
+        app.config["PYPELINE_CLIENT_VERSION"] = (
+            api_docs.get("version", None)
+            if api_docs.get("version", None) is not None
+            else app.config["PYPELINE_CLIENT_VERSION"]
+        )
+
+        app.config["API_DOCUMENTATION_TITLE"] = (
+            api_docs.get("title", None)
+            if api_docs.get("title", None) is not None
+            else app.config["API_DOCUMENTATION_TITLE"]
+        )
+
+        app.config["API_DOCUMENTATION_DESCRIPTION"] = (
+            api_docs.get("description", None)
+            if api_docs.get("description", None) is not None
+            else app.config["API_DOCUMENTATION_DESCRIPTION"]
+        )
+
+        tags = [
+            {"name": "Pipelines", "description": "Operations related to Pipelines"},
+            {"name": "Schedules", "description": "Operations related to Schedules"},
+        ] + custom_tags
+
+        # Set up  the initializing spec kwargs for API
+        spec_kwargs = {
+            "title": app.config["API_DOCUMENTATION_TITLE"],
+            "version": f"Pypeline: {__version__} - "
+            f"Client: {app.config['PYPELINE_CLIENT_VERSION']}",
+            "description": app.config["API_DOCUMENTATION_DESCRIPTION"],
+            "tags": tags,
+        }
+        try:
+            api = Api()
+            api.init_app(app, spec_kwargs=spec_kwargs)
+
+            self._register_api_namespaces(api)
+
+        except Exception as e:
+            api = None
+            logging.exception(f"Unable to initialize API ... {e}")
+
+        app.extensions.setdefault("pypeline_core_api", api)
+
+    @staticmethod
+    def _register_api_namespaces(api: Api):
+        """Register Default API namespaces"""
+        from pypeline.flask.api.pipelines import bp as pipelinesbp
+
+        api.register_blueprint(pipelinesbp)
+        from pypeline.flask.api.schedules import bp as schedulesbp
+
+        api.register_blueprint(schedulesbp)

pypeline/middleware.py

@@ -0,0 +1,61 @@
+import copy
+import os
+
+from dramatiq.middleware import Middleware
+
+from pypeline.barrier import LockingParallelBarrier
+from pypeline.constants import PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+
+
+class ParallelPipeline(Middleware):
+    def __init__(self, redis_url):
+        self.redis_url = redis_url
+
+    def after_process_message(self, broker, message, *, result=None, exception=None):
+        from dramatiq.message import Message
+
+        if exception is None:
+            group_completion_uuid = message.options.get("group_completion_uuid")
+            if group_completion_uuid:
+                locking_parallel_barrier = LockingParallelBarrier(
+                    self.redis_url,
+                    task_key=group_completion_uuid,
+                    lock_key=f"{group_completion_uuid}-lock",
+                )
+                try:
+                    locking_parallel_barrier.acquire_lock(
+                        timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                    )
+                    remaining_tasks = locking_parallel_barrier.decrement_task_count()
+                finally:
+                    locking_parallel_barrier.release_lock()
+
+                if remaining_tasks <= 0:
+                    execution_graph = message.options.get("execution_graph")
+
+                    for i in range(len(execution_graph)):
+                        message_group = execution_graph[i]
+
+                        # Check if the current group matches the group_completion_uuid
+                        if (
+                            message_group[0]["options"]["group_completion_uuid"]
+                            == group_completion_uuid
+                        ):
+                            # Check if there is a next group
+                            if i + 1 < len(execution_graph):
+                                next_group = execution_graph[i + 1]
+
+                                completion_uuid = next_group[0]["options"][
+                                    "group_completion_uuid"
+                                ]
+                                locking_parallel_barrier = LockingParallelBarrier(
+                                    self.redis_url,
+                                    task_key=completion_uuid,
+                                    lock_key=f"{completion_uuid}-lock",
+                                )
+                                locking_parallel_barrier.set_task_count(len(next_group))
+                                for next_message in next_group:
+                                    next_message["options"][
+                                        "execution_graph"
+                                    ] = copy.deepcopy(execution_graph)
+                                    broker.enqueue(Message(**next_message))

pypeline/pipeline_config_schema.py

@@ -1,7 +1,4 @@
 """ Schemas for Pipelines
-
-TODO: Add validation that all specified nodes in DAG have corresponding
-node in taskDefinitions
 """
 import yaml
 from marshmallow import Schema, fields, EXCLUDE, validates_schema
@@ -9,144 +6,155 @@ from marshmallow.exceptions import ValidationError
 
 
 class ExcludeUnknownSchema(Schema):
-    """ Remove unknown keys from loaded dictionary
-    """
+    """Remove unknown keys from loaded dictionary"""
+
     class Meta:
-        """ Exclude unknown properties.
-        """
+        """Exclude unknown properties."""
+
         unknown = EXCLUDE
 
 
 class MetadataSchema(Schema):
-    """ Schema for a pipeline's metadata object.
-    """
-    queue = fields.String(required=True,
-                          description="Default queue for all pipeline tasks.",
-                          example="default-queue-name")
-    processorQueue = fields.String(
-        required=False,
-        description="Default processor queue for all pipeline tasks.",
-        example="default-processor-queue-name",
-        default="celery"
+    """Schema for a pipeline's metadata object."""
+
+    queue = fields.String(
+        required=True,
+        description="Default queue for all pipeline tasks.",
+        example="default-queue-name",
     )
     maxRetry = fields.Integer(
         required=False,
         description="A number. Maximum number of retries before giving up. "
-                    "A value of None means task will retry forever. "
-                    "By default, this option is set to 3.",
+        "A value of None means task will retry forever. "
+        "By default, this option is set to 3.",
         default=3,
-        example=3)
+        example=3,
+    )
 
-    maxTtl = fields.Integer(required=False,
-                            description="The soft time limit, in seconds, "
-                                        "for this task. When not set the "
-                                        "workers default is used.  The hard "
-                                        "time limit will be derived from this"
-                                        "field, by adding 10 seconds.",
-                            default=60,
-                            example=60)
+    maxTtl = fields.Integer(
+        required=False,
+        description="The soft time limit, in seconds, "
+        "for this task. When not set the "
+        "workers default is used.  The hard "
+        "time limit will be derived from this"
+        "field, by adding 10 seconds.",
+        default=60,
+        example=60,
+    )
 
     retryBackoff = fields.Integer(
         required=False,
         description="A number. If this option is set , it is used as a delay"
-                    " factor. For example, if this option is set to 3, the"
-                    " first retry will delay 3 seconds, the second will delay"
-                    "  6 seconds, the third will delay 12 seconds, the fourth"
-                    " will delay 24 seconds, and so on. By default, this"
-                    " option is set to False, and autoretries will not"
-                    "  be delayed.",
+        " factor. For example, if this option is set to 3, the"
+        " first retry will delay 3 seconds, the second will delay"
+        "  6 seconds, the third will delay 12 seconds, the fourth"
+        " will delay 24 seconds, and so on. By default, this"
+        " option is set to False, and autoretries will not"
+        "  be delayed.",
         default=3,
-        example=3)
+        example=3,
+    )
 
     retryJitter = fields.Boolean(
         required=False,
         description="A boolean. Jitter is used to introduce randomness into "
-                    "exponential backoff delays, to prevent all tasks in the "
-                    "queue from being executed simultaneously. If this option "
-                    "is set to True, the delay value calculated by "
-                    "retry_backoff is treated as a maximum, and the actual "
-                    "delay value will be a random number between zero and that "
-                    "maximum. By default, this option is set to True.",
+        "exponential backoff delays, to prevent all tasks in the "
+        "queue from being executed simultaneously. If this option "
+        "is set to True, the delay value calculated by "
+        "retry_backoff is treated as a maximum, and the actual "
+        "delay value will be a random number between zero and that "
+        "maximum. By default, this option is set to True.",
         default=False,
-        example=True)
+        example=True,
+    )
 
     retryBackoffMax = fields.Integer(
         required=False,
         description="A boolean. Jitter is used to introduce randomness into "
-                    "exponential backoff delays, to prevent all tasks in the "
-                    "queue from being executed simultaneously. If this option "
-                    "is set to True, the delay value calculated by "
-                    "retry_backoff is treated as a maximum, and the actual "
-                    "delay value will be a random number between zero and "
-                    "that maximum. By default, this option is set to True.",
+        "exponential backoff delays, to prevent all tasks in the "
+        "queue from being executed simultaneously. If this option "
+        "is set to True, the delay value calculated by "
+        "retry_backoff is treated as a maximum, and the actual "
+        "delay value will be a random number between zero and "
+        "that maximum. By default, this option is set to True.",
         default=600,
-        example=600)
+        example=600,
+    )
 
 
 class TaskDefinitionsSchema(ExcludeUnknownSchema):
-    """ Schema for a single task's configuration
-    """
-    handler = fields.String(required=True,
-                            description="Path to the worker task definition",
-                            example="client.workers.my_task")
+    """Schema for a single task's configuration"""
+
+    handler = fields.String(
+        required=True,
+        description="Path to the worker task definition",
+        example="client.workers.my_task",
+    )
 
-    maxTtl = fields.Integer(required=False,
-                            description="Max TTL for a task in seconds.",
-                            default=60,
-                            example=60)
+    maxTtl = fields.Integer(
+        required=False,
+        description="Max TTL for a task in seconds.",
+        default=60,
+        example=60,
+    )
 
-    queue = fields.String(required=False,
-                          description="Non-default queue for this task.",
-                          example="custom-queue-name")
+    queue = fields.String(
+        required=False,
+        description="Non-default queue for this task.",
+        example="custom-queue-name",
+    )
 
 
 class PipelineConfigSchemaV1(Schema):
-    """ Overall pipeline configuration schema
-    """
+    """Overall pipeline configuration schema"""
+
     metadata = fields.Nested(
         MetadataSchema,
         required=True,
-        description="Metadata and configuration information for this pipeline."
+        description="Metadata and configuration information for this pipeline.",
     )
     dagAdjacency = fields.Dict(
         keys=fields.String(
             required=True,
-            description=
-            "Task's node name. *MUST* match key in taskDefinitions dict.",
-            example="node_a"),
+            description="Task's node name. *MUST* match key in taskDefinitions dict.",
+            example="node_a",
+        ),
         values=fields.List(
             fields.String(
                 required=True,
-                description=
-                "Task's node name. *Must* match key in taskDefinitions dict.")
+                description="Task's node name. *Must* match key in taskDefinitions dict.",
+            )
         ),
         required=True,
-        description="The DAG Adjacency definition.")
+        description="The DAG Adjacency definition.",
+    )
     taskDefinitions = fields.Dict(
         keys=fields.String(
             required=True,
-            description=
-            "Task's node name. *Must* match related key in dagAdjacency.",
-            example="node_a"),
+            description="Task's node name. *Must* match related key in dagAdjacency.",
+            example="node_a",
+        ),
         values=fields.Nested(
             TaskDefinitionsSchema,
             required=True,
             description="Definition of each task in the pipeline.",
-            example={
-                'handler': 'abc.task',
-                'maxRetry': 1
-            }),
+            example={"handler": "abc.task", "maxRetry": 1},
+        ),
         required=True,
-        description="Configuration for each node defined in DAG.")
+        description="Configuration for each node defined in DAG.",
+    )
 
 
 class BasePipelineSchema(ExcludeUnknownSchema):
     __schema_version__ = None
 
     name = fields.String(required=True, description="Pipeline name")
-    description = fields.String(required=False, missing=None,
-                                description="Description of the pipeline.",
-                                example="A valuable pipeline.")
+    description = fields.String(
+        required=False,
+        missing=None,
+        description="Description of the pipeline.",
+        example="A valuable pipeline.",
+    )
     schemaVersion = fields.Integer(required=True)
     config = fields.Dict(required=True)
 
@@ -171,9 +179,9 @@ class BasePipelineSchema(ExcludeUnknownSchema):
 
     @validates_schema
     def validate_pipeline(self, data, **kwargs):
-        schema_version = data['schemaVersion']
+        schema_version = data["schemaVersion"]
         PipelineSchema = BasePipelineSchema.get_by_version(schema_version)
-        schema = PipelineSchema(exclude=['name', 'description'])
+        schema = PipelineSchema(exclude=["name", "description"])
         schema.load(data)
 
 
@@ -186,7 +194,7 @@ class PipelineSchemaV1(BasePipelineSchema):
     config = fields.Nested(
         PipelineConfigSchemaV1,
         required=True,
-        description="Metadata and configuration information for this pipeline."
+        description="Metadata and configuration information for this pipeline.",
     )
 
     def validate_pipeline(self, data, **kwargs):
@@ -197,15 +205,20 @@ class PipelineSchemaV1(BasePipelineSchema):
 
 
 class PipelineConfigValidator(object):
-    """ Validate a pipeline configuration.
+    """Validate a pipeline configuration.
 
-        This is stored as a string in the database under `PipelineConfig.config`
-        in order to keep it easy for custom features to be added over time.
-        This model represents the required / valid features so we can
-        programmatically validate when saving, updating, viewing.
+    This is stored as a string in the database under `PipelineConfig.config`
+    in order to keep it easy for custom features to be added over time.
+    This model represents the required / valid features so we can
+    programmatically validate when saving, updating, viewing.
     """
-    def __init__(self, config_dict: dict = None, config_yaml: str = None,
-                 schema_version: int = None):
+
+    def __init__(
+        self,
+        config_dict: dict = None,
+        config_yaml: str = None,
+        schema_version: int = None,
+    ):
         super().__init__()
 
         # We validate this as a dictionary. Turn into dictionary if provided
@@ -226,7 +239,7 @@ class PipelineConfigValidator(object):
         try:
             # https://github.com/marshmallow-code/marshmallow/issues/377
             # See issue above when migrating to marshmallow 3
-            pcs = PipelineSchema._declared_fields['config'].schema
+            pcs = PipelineSchema._declared_fields["config"].schema
             self.validated_config = pcs.load(self.config)
             self.is_valid = True
         except ValidationError as e: