scalable-pypeline 1.2.3__py2.py3-none-any.whl → 2.0.2__py2.py3-none-any.whl

pypeline/flask/api/pipelines.py

@@ -2,36 +2,33 @@
 """
 import logging
 
-from celery.canvas import _chain
-from celery_dyrygent.workflows import Workflow
-from flask import jsonify, request, abort
+from flask import jsonify
 from flask_smorest import Blueprint
 from flask.views import MethodView
 from marshmallow import Schema, fields
 from marshmallow.exceptions import ValidationError
-from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1, \
-    WORKFLOW_PROCESSOR_DEFAULT_QUEUE
+from webargs.flaskparser import abort
+
+from pypeline.composition import PipelineResult
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1
+from pypeline.utils.pipeline_utils import dag_generator
 from pypeline.flask.decorators import require_accesskey
-from pypeline.flask.api.utils import chain_helper
-from pypeline.utils.task_utils import PipelineResult
 from pypeline.utils.config_utils import retrieve_latest_pipeline_config
 from pypeline.pipeline_config_schema import BasePipelineSchema, PipelineSchemaV1
 
 logger = logging.getLogger(__name__)
-bp = Blueprint('pipelines', __name__, url_prefix=API_PATH_V1 + '/pipelines')
+bp = Blueprint("pipelines", __name__, url_prefix=API_PATH_V1 + "/pipelines")
 
 
 class InvokePipelineSchema(Schema):
-    """ Incoming schema for invoking a pipeline
-    """
+    """Incoming schema for invoking a pipeline"""
+
     chain_payload = fields.Raw(
-        description='Payload contains whatever arguments the pipeline expects '
-        'to be passed to each node in the graph.',
-        example={
-            'document_id': '123',
-            'send_alert': True
-        },
-        required=False)
+        description="Payload contains whatever arguments the pipeline expects "
+        "to be passed to each node in the graph.",
+        example={"document_id": "123", "send_alert": True},
+        required=False,
+    )
 
 
 class InvokePipelineResponseSchema(Schema):
@@ -49,20 +46,19 @@ class GetPipelineResultResponseSchema(Schema):
     status_message = fields.String()
 
 
-@bp.route('/')
+@bp.route("/")
 class Pipelines(MethodView):
-    """ Operations against all pipelines.
-    """
+    """Operations against all pipelines."""
+
     @require_accesskey
-    @bp.doc(responses=API_DOC_RESPONSES,
-            parameters=[API_DOC_PARAMS['accesskey']],
-            tags=['Pipelines'])
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[API_DOC_PARAMS["accesskey"]],
+        tags=["Pipelines"],
+    )
     def get(self):
-        """ Retrieve list of available pipelines.
-        """
-        access_key = request.headers.get('accesskey')
-        pipeline_config_api_resp = retrieve_latest_pipeline_config(
-            access_key=access_key)
+        """Retrieve list of available pipelines."""
+        pipeline_config_api_resp = retrieve_latest_pipeline_config()
 
         if pipeline_config_api_resp is None:
             abort(404)
@@ -70,41 +66,41 @@ class Pipelines(MethodView):
         try:
             pipelines = []
             for p in pipeline_config_api_resp:
-                PipelineSchema = \
-                    BasePipelineSchema.get_by_version(p['schemaVersion'])
+                PipelineSchema = BasePipelineSchema.get_by_version(p["schemaVersion"])
                 pipeline_config = PipelineSchema().load(p)
                 pipelines.append(pipeline_config)
         except ValidationError as e:
             msg = f"Invalid pipeline configuration: {e}"
-            return jsonify({'message': msg}), 202
+            return jsonify({"message": msg}), 202
 
         return jsonify(pipelines)
 
 
-@bp.route('/<string:pipeline_id>')
+@bp.route("/<string:pipeline_id>")
 class PipelineInfo(MethodView):
-    """ Operations against a single pipeline
-    """
+    """Operations against a single pipeline"""
+
     @require_accesskey
-    @bp.doc(responses=API_DOC_RESPONSES,
-            parameters=[
-                API_DOC_PARAMS['accesskey'], {
-                    'in': 'path',
-                    'name': 'pipeline_id',
-                    'description':
-                    'pipeline_id for which to retrieve metrics.',
-                    'type': 'string',
-                    'example': 'my_pipeline',
-                    'required': True
-                }
-            ],
-            tags=['Pipelines'])
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[
+            API_DOC_PARAMS["accesskey"],
+            {
+                "in": "path",
+                "name": "pipeline_id",
+                "description": "pipeline_id for which to retrieve metrics.",
+                "type": "string",
+                "example": "my_pipeline",
+                "required": True,
+            },
+        ],
+        tags=["Pipelines"],
+    )
     def get(self, pipeline_id: str):
-        """ Retrieve details about a specific pipeline.
-        """
-        access_key = request.headers.get('accesskey')
+        """Retrieve details about a specific pipeline."""
         pipeline_config_api_resp = retrieve_latest_pipeline_config(
-            pipeline_id=pipeline_id, access_key=access_key)
+            pipeline_id=pipeline_id
+        )
 
         if pipeline_config_api_resp is None:
             abort(404)
@@ -113,83 +109,48 @@ class PipelineInfo(MethodView):
             pipeline_config = PipelineSchemaV1().load(pipeline_config_api_resp)
         except ValidationError as e:
             msg = f"Invalid pipeline configuration: {e}"
-            return jsonify({'message': msg}), 202
+            return jsonify({"message": msg}), 202
 
         return jsonify(pipeline_config)
 
 
-@bp.route('/invoke/<string:pipeline_id>')
+@bp.route("/invoke/<string:pipeline_id>")
 class PipelineInvoke(MethodView):
-    """ Operations involed with pipeline invocation
-    """
+    """Operations involed with pipeline invocation"""
+
     @require_accesskey
-    @bp.doc(responses=API_DOC_RESPONSES,
-            parameters=[
-                API_DOC_PARAMS['accesskey'], {
-                    'in': 'path',
-                    'name': 'pipeline_id',
-                    'description':
-                    'pipeline_id for which to retrieve metrics.',
-                    'type': 'string',
-                    'example': 'my_pipeline',
-                    'required': True
-                }
-            ],
-            tags=['Pipelines'])
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[
+            API_DOC_PARAMS["accesskey"],
+            {
+                "in": "path",
+                "name": "pipeline_id",
+                "description": "pipeline_id for which to retrieve metrics.",
+                "type": "string",
+                "example": "my_pipeline",
+                "required": True,
+            },
+        ],
+        tags=["Pipelines"],
+    )
     @bp.arguments(InvokePipelineSchema)
     @bp.response(InvokePipelineResponseSchema)
     def post(self, payload: dict, pipeline_id: str):
-        """ Invoke a pipeline by it's ID; optionally provide pipeline arguments.
-        """
-        access_key = request.headers.get('accesskey')
-        pipeline_config = retrieve_latest_pipeline_config(
-            pipeline_id=pipeline_id, access_key=access_key)
+        """Invoke a pipeline by it's ID; optionally provide pipeline arguments."""
+        pipeline_config = retrieve_latest_pipeline_config(pipeline_id=pipeline_id)
 
         if pipeline_config is None:
             return abort(404)
 
-        retval = {'pipeline_id': pipeline_id, 'status': ''}
+        retval = {"pipeline_id": pipeline_id, "status": "starting"}
         try:
-            # TODO - ideally we can validate the payload *at this stage*
-            # before the chain is ever invoked so we can handle issues
-            # without kicking off work.
-            payload = payload['chain_payload']\
-                if 'chain_payload' in payload else {}
-
-            gen = chain_helper(pipeline_id=pipeline_id,
-                               access_key=access_key,
-                               chain_payload=payload)
-
-            if gen.chain is None:
-                abort(400, message=gen.loading_message)
-
-            chain: _chain = gen.chain
-            work_flow_processor_queue = (
-                pipeline_config["config"]["metadata"]
-                .get("processorQueue", None) or
-                WORKFLOW_PROCESSOR_DEFAULT_QUEUE)
-            wf: Workflow = Workflow({"queue":  work_flow_processor_queue})
-            wf.add_celery_canvas(chain)
-            wf.apply_async()
-
-            celery_task_status = []
-            for node in wf.nodes:
-                celery_task = dict(
-                    name=wf.nodes[node].signature.name,
-                    task_id=node,
-                    status="RUNNING",
-                    retries=0
-                )
-                celery_task_status.append(celery_task)
-
-            gen.pipeline_wrapper.celery_task_status = celery_task_status
-            gen.pipeline_wrapper.save_to_cache()
-            retval['status'] = 'success'
-            retval['execution_id'] = gen.execution_id
-            # Initialize the cached result
-            pr = PipelineResult(gen.execution_id, status='pending')
-            pr.save()
-
+            payload = payload["chain_payload"] if "chain_payload" in payload else {}
+            pipeline = dag_generator(pipeline_id=pipeline_id, event=payload)
+            pipeline.run()
+            pipeline_result = PipelineResult(pipeline.execution_id)
+            pipeline_result.create_result_entry(pipeline.to_json())
+            retval["execution_id"] = pipeline.execution_id
         except Exception as e:
             msg = "Failed to invoke pipeline ... {}".format(pipeline_id)
             logger.error(msg)
@@ -201,61 +162,61 @@ class PipelineInvoke(MethodView):
 
 results_responses = API_DOC_RESPONSES.copy()
 results_responses[202] = {
-    'code': 202,
-    'description': 'Pipeline is still running. Try again later.'
+    "code": 202,
+    "description": "Pipeline is still running. Try again later.",
 }
 results_responses[204] = {
-    'code': 204,
-    'description': 'The execution results have expired. Re-run pipeline.'
+    "code": 204,
+    "description": "The execution results have expired. Re-run pipeline.",
 }
 
 
-@bp.route('/results/<string:execution_id>')
+@bp.route("/results/<string:execution_id>")
 class PipelineResults(MethodView):
-    """ Operations with respect to pipeline results
-    """
+    """Operations with respect to pipeline results"""
+
     @require_accesskey
-    @bp.doc(responses=results_responses,
-            parameters=[
-                API_DOC_PARAMS['accesskey'], {
-                    'in': 'path',
-                    'name': 'execution_id',
-                    'description':
-                    'execution_id for which to retrieve results',
-                    'type': 'string',
-                    'example': '4c595cca-9bf1-4150-8c34-6b43faf276c8',
-                    'required': True
-                }
-            ],
-            tags=['Pipelines'])
+    @bp.doc(
+        responses=results_responses,
+        parameters=[
+            API_DOC_PARAMS["accesskey"],
+            {
+                "in": "path",
+                "name": "execution_id",
+                "description": "execution_id for which to retrieve results",
+                "type": "string",
+                "example": "4c595cca-9bf1-4150-8c34-6b43faf276c8",
+                "required": True,
+            },
+        ],
+        tags=["Pipelines"],
+    )
     @bp.response(GetPipelineResultResponseSchema)
     def get(self, execution_id: str):
-        """ Retrieve results of a pipeline's execution based on execution_id
-
-            NOTE: Cached results expire after a time window so are not available
-            forever.
+        """Retrieve results of a pipeline's execution based on execution_id
 
-            TODO: Need to add response marshalling/schema here.
+        NOTE: Cached results expire after a time window so are not available
+        forever.
         """
         try:
             pr = PipelineResult(execution_id)
             pr.load()
-            retval = pr.to_dict()
-            if pr.status == 'unavailable':
-                retval['status_message'] = 'Results expired. Re-run pipeline.'
-                return retval, 204
+            retval = {"execution_id": execution_id, "status": pr.status}
+            if pr.status == "unavailable":
+                retval["status_message"] = "Results expired. Re-run pipeline."
+                return retval, 200
 
-            if pr.status == 'pending':
-                retval['status_message'] = 'Results pending. Check again soon.'
+            if pr.status == "pending":
+                retval["status_message"] = "Results pending. Check again soon."
                 return retval, 202
 
             else:
-                retval['status_message'] = 'Results available.'
+                retval["status_message"] = "Results available."
+                retval["results"] = pr.get_results()
                 return retval, 200
 
         except Exception as e:
-            msg = "Failed to retrieve results for execution id: {}".format(
-                execution_id)
+            msg = "Failed to retrieve results for execution id: {}".format(execution_id)
             logger.error(msg)
             logger.exception(f"{e}")
             abort(500, message=msg)

pypeline/flask/api/schedules.py

@@ -8,33 +8,31 @@ from marshmallow import Schema, fields
 from flask_smorest import Blueprint
 from flask import abort
 from marshmallow.exceptions import ValidationError
-from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS,\
-    API_PATH_V1
-from pypeline.utils.config_utils import retrieve_latest_schedule_config, \
-    update_schedule_config
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1
+from pypeline.utils.config_utils import retrieve_latest_schedule_config
 from pypeline.schedule_config_schema import BaseScheduleSchema
 from pypeline.flask.decorators import require_accesskey
 
 logger = logging.getLogger(__name__)
 
-bp = Blueprint('schedules', __name__, url_prefix=API_PATH_V1 + '/schedules')
+bp = Blueprint("schedules", __name__, url_prefix=API_PATH_V1 + "/schedules")
 
 
-@bp.route('/')
+@bp.route("/")
 class Schedules(MethodView):
-    """ Operations related to schedules
-    """
+    """Operations related to schedules"""
+
     @require_accesskey
-    @bp.doc(responses=API_DOC_RESPONSES,
-            parameters=[API_DOC_PARAMS['accesskey']],
-            tags=['Schedules'])
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[API_DOC_PARAMS["accesskey"]],
+        tags=["Schedules"],
+    )
     def get(self):
-        """ Retrieve list of available schedule entries.
-        """
-        access_key = request.headers.get('accesskey')
+        """Retrieve list of available schedule entries."""
+        access_key = request.headers.get("accesskey")
         try:
-            schedule_config = retrieve_latest_schedule_config(
-                access_key=access_key)
+            schedule_config = retrieve_latest_schedule_config()
         except ValidationError:
             abort(400, message="Invalid schedule found ...")
 
@@ -42,26 +40,3 @@ class Schedules(MethodView):
             abort(404)
 
         return jsonify(schedule_config)
-
-    @require_accesskey
-    @bp.doc(responses=API_DOC_RESPONSES,
-            parameters=[API_DOC_PARAMS['accesskey']],
-            tags=['Schedules'])
-    @bp.arguments(BaseScheduleSchema)
-    def post(self, payload: dict):
-        """ Update a deployment's schedules. Primarily used to update dynamic
-        keys such as last run at and total run count. This does not allow
-        overloading schedules, only updating select keys on known schedule
-        entries (as in, this is not destructive).
-        """
-        access_key = request.headers.get('accesskey')
-        try:
-            success = update_schedule_config(new_schedule_config=payload,
-                                             access_key=access_key)
-        except ValidationError as e:
-            abort(400, message=e)
-
-        if not success:
-            abort(500)
-
-        return jsonify({'message': 'Schedule update successful ...'})

pypeline/flask/decorators.py

@@ -4,71 +4,36 @@ import os
 import logging
 from http import HTTPStatus
 from functools import wraps
-import requests
-from rhodb.redis_conf import RedisConnector
 from flask import request
 from flask import abort
-from pypeline.constants import DEFAULT_AUTH_URL, AUTH_LOCK_KEY, \
-    AUTH_LOCK_DURATION, USING_SERMOS_CLOUD
 
 logger = logging.getLogger(__name__)
-redis_conn = RedisConnector().get_connection()
-
-
-def validate_access_key(access_key: str = None):
-    """ Verify whether an Access Key is valid according to Sermos Cloud.
-
-    If deploying in 'local' mode, no validation is done. To deploy in local
-    mode, set DEFAULT_BASE_URL=local in your environment.
-    """
-    # Always 'valid' in local mode
-    if not USING_SERMOS_CLOUD:
-        return True
-
-    # If get access key from either provided val or environment
-    # if None provided.
-    access_key = os.environ.get('SERMOS_ACCESS_KEY', access_key)
-
-    # Invalid if None, no need to ask.
-    if access_key is None:
-        return False
-
-    # Ask cache first
-    validated = redis_conn.get(AUTH_LOCK_KEY)
-    if validated is not None:
-        return True
-
-    # Ask Sermos Cloud (Note: Sermos Cloud's API expects `apikey`)
-    headers = {'apikey': access_key}
-    r = requests.post(DEFAULT_AUTH_URL, headers=headers, verify=True)
-
-    if r.status_code == 200:
-        redis_conn.setex(AUTH_LOCK_KEY, AUTH_LOCK_DURATION, '')
-        return True
-    return False
 
 
 def require_accesskey(fn):
-    """ Convenience decorator to add to a web route (typically an API)
-        when using Flask.
-
-        Usage::
-            from sermos import Blueprint, ApiServices
-            bp = Blueprint('api_routes', __name__, url_prefix='/api')
-
-            @bp.route('/my-api-route')
-            class ApiClass(MethodView):
-                @require_access_key
-                def post(self, payload: dict):
-                    return {}
+    """Convenience decorator to add to a web route (typically an API)
+    when using Flask.
+
+    Usage::
+        from sermos import Blueprint, ApiServices
+        bp = Blueprint('api_routes', __name__, url_prefix='/api')
+
+        @bp.route('/my-api-route')
+        class ApiClass(MethodView):
+            @require_access_key
+            def post(self, payload: dict):
+                return {}
     """
+
     @wraps(fn)
     def decorated_view(*args, **kwargs):
-        access_key = request.headers.get('accesskey')
+        access_key = request.headers.get("accesskey")
         if not access_key:
-            access_key = request.args.get('accesskey')
+            access_key = request.args.get("accesskey")
+
+        configured_access_key = os.environ.get("API_ACCESS_KEY", None)
 
-        if validate_access_key(access_key):
+        if access_key == configured_access_key:
             return fn(*args, **kwargs)
 
         abort(HTTPStatus.UNAUTHORIZED)