scalable-pypeline 2.1.31__py2.py3-none-any.whl

pypeline/flask/api/pipelines.py

@@ -0,0 +1,275 @@
+""" Pipeline APIs
+"""
+
+import importlib.metadata
+import logging
+from http import HTTPStatus
+
+from flask import jsonify
+from flask.views import MethodView
+from flask_smorest import Blueprint
+from marshmallow import Schema, fields
+from marshmallow.exceptions import ValidationError
+from webargs.flaskparser import abort
+from packaging import version
+from pypeline.pipelines.composition.parallel_pipeline_composition import PipelineResult
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1
+from pypeline.flask.decorators import require_accesskey
+from pypeline.pipeline_config_schema import BasePipelineSchema, PipelineSchemaV1
+from pypeline.pipeline_settings_schema import (
+    MissingSettingsException,
+    PipelineScenarioSchema,
+)
+from pypeline.pipelines.factory import dag_generator
+from pypeline.utils.config_utils import retrieve_latest_pipeline_config
+from pypeline.utils.schema_utils import get_clean_validation_messages
+
+logger = logging.getLogger(__name__)
+bp = Blueprint("pipelines", __name__, url_prefix=API_PATH_V1 + "/pipelines")
+
+
+try:
+    flask_smorest_version = importlib.metadata.version("flask-smorest")
+    flask_smorest_version_parsed = version.parse(flask_smorest_version)
+except importlib.metadata.PackageNotFoundError:
+    flask_smorest_version_parsed = None
+
+
+def get_response_decorator(bp, status_code, *args, **kwargs):
+    if flask_smorest_version is None:
+        # Handle the case where flask-smorest is not installed
+        raise ImportError("flask-smorest is not installed.")
+    elif flask_smorest_version_parsed < version.parse("0.29"):
+        # Adjust arguments for older versions if needed
+        return bp.response(*args, **kwargs)
+    else:
+        # Adjust arguments for newer versions if needed
+        return bp.response(status_code, *args, **kwargs)
+
+
+class InvokePipelineSchema(Schema):
+    """Incoming schema for invoking a pipeline"""
+
+    chain_payload = fields.Raw(
+        description="Payload contains whatever arguments the pipeline expects "
+        "to be passed to each node in the graph.",
+        example={"document_id": "123", "send_alert": True},
+        required=False,
+    )
+    scenarios = fields.List(
+        fields.Nested(PipelineScenarioSchema),
+        metadata={"description": "List of scenarios to run for a given pipeline"},
+        required=False,
+    )
+
+
+class InvokePipelineResponseSchema(Schema):
+    execution_id = fields.String()
+    pipeline_id = fields.String()
+    status = fields.String()
+
+
+class GetPipelineResultResponseSchema(Schema):
+    execution_id = fields.String()
+    result = fields.Raw()
+    result_ttl = fields.Integer()
+    results = fields.Raw()
+    status = fields.String()
+    status_message = fields.String()
+
+
+@bp.route("/")
+class Pipelines(MethodView):
+    """Operations against all pipelines."""
+
+    @require_accesskey
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[API_DOC_PARAMS["accesskey"]],
+        tags=["Pipelines"],
+    )
+    def get(self):
+        """Retrieve list of available pipelines."""
+        pipeline_config_api_resp = retrieve_latest_pipeline_config()
+
+        if pipeline_config_api_resp is None:
+            abort(404)
+
+        try:
+            pipelines = []
+            for p in pipeline_config_api_resp:
+                PipelineSchema = BasePipelineSchema.get_by_version(p["schemaVersion"])
+                pipeline_config = PipelineSchema().load(p)
+                pipelines.append(pipeline_config)
+        except ValidationError as e:
+            msg = f"Invalid pipeline configuration: {e}"
+            return jsonify({"message": msg}), 202
+
+        return jsonify(pipelines)
+
+
+@bp.route("/<string:pipeline_id>")
+class PipelineInfo(MethodView):
+    """Operations against a single pipeline"""
+
+    @require_accesskey
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[
+            API_DOC_PARAMS["accesskey"],
+            {
+                "in": "path",
+                "name": "pipeline_id",
+                "description": "pipeline_id for which to retrieve metrics.",
+                "type": "string",
+                "example": "my_pipeline",
+                "required": True,
+            },
+        ],
+        tags=["Pipelines"],
+    )
+    def get(self, pipeline_id: str):
+        """Retrieve details about a specific pipeline."""
+        pipeline_config_api_resp = retrieve_latest_pipeline_config(
+            pipeline_id=pipeline_id
+        )
+
+        if pipeline_config_api_resp is None:
+            abort(404)
+
+        try:
+            pipeline_config = PipelineSchemaV1().load(pipeline_config_api_resp)
+        except ValidationError as e:
+            msg = f"Invalid pipeline configuration: {e}"
+            return jsonify({"message": msg}), 202
+
+        return jsonify(pipeline_config)
+
+
+@bp.route("/invoke/<string:pipeline_id>")
+class PipelineInvoke(MethodView):
+    """Operations involed with pipeline invocation"""
+
+    @require_accesskey
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[
+            API_DOC_PARAMS["accesskey"],
+            {
+                "in": "path",
+                "name": "pipeline_id",
+                "description": "pipeline_id for which to retrieve metrics.",
+                "type": "string",
+                "example": "my_pipeline",
+                "required": True,
+            },
+        ],
+        tags=["Pipelines"],
+    )
+    @bp.arguments(InvokePipelineSchema)
+    @get_response_decorator(bp, "200", InvokePipelineResponseSchema)
+    def post(self, payload: dict, pipeline_id: str):
+        """Invoke a pipeline by it's ID; optionally provide pipeline arguments."""
+        pipeline_config = retrieve_latest_pipeline_config(pipeline_id=pipeline_id)
+
+        if pipeline_config is None:
+            return abort(404)
+
+        retval = {"pipeline_id": pipeline_id, "status": "starting"}
+        try:
+            chain_payload = payload.get("chain_payload", {})
+            scenarios = payload.get("scenarios", [])
+            if pipeline_config["schemaVersion"] == 1:
+                pipeline = dag_generator(
+                    pipeline_id=pipeline_id,
+                    event=chain_payload,
+                )
+            elif pipeline_config["schemaVersion"] == 2:
+                pipeline = dag_generator(
+                    pipeline_id=pipeline_id,
+                    scenarios=scenarios,
+                )
+                retval["scenarios"] = pipeline.scenarios
+            pipeline.run()
+            pipeline_result = PipelineResult(pipeline.execution_id)
+            pipeline_result.create_result_entry(pipeline.to_json())
+            retval["execution_id"] = pipeline.execution_id
+        except MissingSettingsException:
+            abort(
+                HTTPStatus.BAD_REQUEST,
+                message="Missing required settings in the request.",
+            )
+        except ValidationError as ve:
+            abort(
+                HTTPStatus.BAD_REQUEST,
+                message=get_clean_validation_messages(ve),
+            )
+        except Exception as e:
+            msg = "Failed to invoke pipeline ... {}".format(pipeline_id)
+            logger.error(msg)
+            logger.exception(f"{e}")
+            abort(500, message=msg)
+
+        return jsonify(retval)
+
+
+results_responses = API_DOC_RESPONSES.copy()
+results_responses[202] = {
+    "code": 202,
+    "description": "Pipeline is still running. Try again later.",
+}
+results_responses[204] = {
+    "code": 204,
+    "description": "The execution results have expired. Re-run pipeline.",
+}
+
+
+@bp.route("/results/<string:execution_id>")
+class PipelineResults(MethodView):
+    """Operations with respect to pipeline results"""
+
+    @require_accesskey
+    @bp.doc(
+        responses=results_responses,
+        parameters=[
+            API_DOC_PARAMS["accesskey"],
+            {
+                "in": "path",
+                "name": "execution_id",
+                "description": "execution_id for which to retrieve results",
+                "type": "string",
+                "example": "4c595cca-9bf1-4150-8c34-6b43faf276c8",
+                "required": True,
+            },
+        ],
+        tags=["Pipelines"],
+    )
+    @get_response_decorator(bp, "200", GetPipelineResultResponseSchema)
+    def get(self, execution_id: str):
+        """Retrieve results of a pipeline's execution based on execution_id
+
+        NOTE: Cached results expire after a time window so are not available
+        forever.
+        """
+        try:
+            pr = PipelineResult(execution_id)
+            pr.load()
+            retval = {"execution_id": execution_id, "status": pr.status}
+            if pr.status == "unavailable":
+                retval["status_message"] = "Results expired. Re-run pipeline."
+                return retval, 200
+
+            if pr.status == "pending":
+                retval["status_message"] = "Results pending. Check again soon."
+                return retval, 202
+
+            else:
+                retval["status_message"] = "Results available."
+                retval["results"] = pr.get_results()
+                return retval, 200
+
+        except Exception as e:
+            msg = "Failed to retrieve results for execution id: {}".format(execution_id)
+            logger.error(msg)
+            logger.exception(f"{e}")
+            abort(500, message=msg)

pypeline/flask/api/schedules.py

@@ -0,0 +1,40 @@
+""" API Endpoints for Scheduled Tasks
+"""
+
+import logging
+from flask import jsonify, request
+from flask.views import MethodView
+from flask_smorest import Blueprint
+from flask import abort
+from marshmallow.exceptions import ValidationError
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1
+from pypeline.utils.config_utils import retrieve_latest_schedule_config
+from pypeline.flask.decorators import require_accesskey
+
+logger = logging.getLogger(__name__)
+
+bp = Blueprint("schedules", __name__, url_prefix=API_PATH_V1 + "/schedules")
+
+
+@bp.route("/")
+class Schedules(MethodView):
+    """Operations related to schedules"""
+
+    @require_accesskey
+    @bp.doc(
+        responses=API_DOC_RESPONSES,
+        parameters=[API_DOC_PARAMS["accesskey"]],
+        tags=["Schedules"],
+    )
+    def get(self):
+        """Retrieve list of available schedule entries."""
+        access_key = request.headers.get("accesskey")
+        try:
+            schedule_config = retrieve_latest_schedule_config()
+        except ValidationError:
+            abort(400, message="Invalid schedule found ...")
+
+        if schedule_config is None:
+            abort(404)
+
+        return jsonify(schedule_config)

pypeline/flask/decorators.py

@@ -0,0 +1,41 @@
+""" Flask specific decorators (primarily for auth activities and app context)
+"""
+import os
+import logging
+from http import HTTPStatus
+from functools import wraps
+from flask import request
+from flask import abort
+
+logger = logging.getLogger(__name__)
+
+
+def require_accesskey(fn):
+    """Convenience decorator to add to a web route (typically an API)
+    when using Flask.
+
+    Usage::
+        from sermos import Blueprint, ApiServices
+        bp = Blueprint('api_routes', __name__, url_prefix='/api')
+
+        @bp.route('/my-api-route')
+        class ApiClass(MethodView):
+            @require_access_key
+            def post(self, payload: dict):
+                return {}
+    """
+
+    @wraps(fn)
+    def decorated_view(*args, **kwargs):
+        access_key = request.headers.get("accesskey")
+        if not access_key:
+            access_key = request.args.get("accesskey")
+
+        configured_access_key = os.environ.get("API_ACCESS_KEY", None)
+
+        if access_key == configured_access_key:
+            return fn(*args, **kwargs)
+
+        abort(HTTPStatus.UNAUTHORIZED)
+
+    return decorated_view

pypeline/flask/flask_pypeline.py

@@ -0,0 +1,156 @@
+""" Pypeline implementation as a Flask extension
+"""
+import os
+
+if os.getenv("USE_GEVENT", "false").lower() == "true":
+    import gevent.monkey
+
+    gevent.monkey.patch_all()
+
+import logging
+from flask import Flask
+from werkzeug.middleware.proxy_fix import ProxyFix
+from flask_smorest import Api, Blueprint
+from pypeline.extensions import pypeline_config
+from pypeline.constants import DEFAULT_OPENAPI_CONFIG
+from pypeline import __version__
+
+logger = logging.getLogger(__name__)
+
+
+class FlaskPypeline:
+    """Pypeline Flask extension."""
+
+    def __init__(self, app: Flask = None):
+        """Class init"""
+        self.app = app
+        self.pypeline_config = pypeline_config if pypeline_config is not None else {}
+
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app: Flask, init_api: bool = False):
+        """Pypeline bootstrapping process.
+
+        Application config variables to set include:
+
+            pypeline_client_version (default: v?.?.?)
+
+        Optional, if `init_api` is True:
+
+            API_DOCUMENTATION_TITLE
+            API_DOCUMENTATION_DESCRIPTION
+            OPENAPI_VERSION
+            OPENAPI_URL_PREFIX
+            OPENAPI_SWAGGER_APP_NAME
+            OPENAPI_SWAGGER_UI_PATH
+            OPENAPI_SWAGGER_BASE_TEMPLATE
+            OPENAPI_SWAGGER_URL
+            OPENAPI_SWAGGER_UI_URL
+            SWAGGER_UI_DOC_EXPANSION
+            EXPLAIN_TEMPLATE_LOADING
+
+        Args:
+            app (Flask): Flask Application to initialize.
+            init_api (bool): If `True`, Pypeline will initialize its
+                core APIs (including Pipelines, Scheduled Tasks, etc.) and
+                provide a pre-configured OpenAPI Spec/Swagger UI interface
+                available at the route defined in your application's config
+                under `OPENAPI_URL_PREFIX` (default `/api`). Refer to
+                [flask-smorest](https://flask-smorest.readthedocs.io/en/latest/openapi.html)
+                documentation for additional configuration options.
+        """
+        # Ensure there's a PYPELINE_CLIENT_VERSION on app config
+        app.config.setdefault(
+            "PYPELINE_CLIENT_VERSION",
+            app.config.get("PYPELINE_CLIENT_VERSION", "v?.?.?"),
+        )
+
+        app.wsgi_app = ProxyFix(app.wsgi_app)
+        app.url_map.strict_slashes = False
+
+        # Create and register the pypeline blueprint
+        bp = Blueprint(
+            "pypeline",
+            __name__,
+            template_folder="../templates",
+            static_folder="../static",
+            url_prefix="/pypeline",
+        )
+        app.register_blueprint(bp)
+
+        # Bootstrap api if app requests
+        if init_api is True:
+            self._bootstrap_api(app)
+
+    def _bootstrap_api(self, app: Flask):
+        """If initializing the API, we will create the core Pypeline API paths
+        and initialize the default Swagger documentation.
+        """
+        # Set sensible defaults for Swagger docs. Provided `app` will
+        # take precedent.
+        for swagger_config in DEFAULT_OPENAPI_CONFIG:
+            app.config.setdefault(
+                swagger_config[0], app.config.get(swagger_config[0], swagger_config[1])
+            )
+
+        # Attempt to override with values from client's pypeline.yaml if
+        # they are available. This will add new tags and new docs if
+        # defined and add to the core Pypeline API docs.
+        api_config = self.pypeline_config.get("apiConfig", {})
+        api_docs = api_config.get("apiDocumentation", {})
+
+        custom_tags = api_config.get("prefixDescriptions", [])
+
+        app.config["PYPELINE_CLIENT_VERSION"] = (
+            api_docs.get("version", None)
+            if api_docs.get("version", None) is not None
+            else app.config["PYPELINE_CLIENT_VERSION"]
+        )
+
+        app.config["API_DOCUMENTATION_TITLE"] = (
+            api_docs.get("title", None)
+            if api_docs.get("title", None) is not None
+            else app.config["API_DOCUMENTATION_TITLE"]
+        )
+
+        app.config["API_DOCUMENTATION_DESCRIPTION"] = (
+            api_docs.get("description", None)
+            if api_docs.get("description", None) is not None
+            else app.config["API_DOCUMENTATION_DESCRIPTION"]
+        )
+
+        tags = [
+            {"name": "Pipelines", "description": "Operations related to Pipelines"},
+            {"name": "Schedules", "description": "Operations related to Schedules"},
+        ] + custom_tags
+
+        # Set up  the initializing spec kwargs for API
+        spec_kwargs = {
+            "title": app.config["API_DOCUMENTATION_TITLE"],
+            "version": f"Pypeline: {__version__} - "
+            f"Client: {app.config['PYPELINE_CLIENT_VERSION']}",
+            "description": app.config["API_DOCUMENTATION_DESCRIPTION"],
+            "tags": tags,
+        }
+        try:
+            api = Api()
+            api.init_app(app, spec_kwargs=spec_kwargs)
+
+            self._register_api_namespaces(api)
+
+        except Exception as e:
+            api = None
+            logging.exception(f"Unable to initialize API ... {e}")
+
+        app.extensions.setdefault("pypeline_core_api", api)
+
+    @staticmethod
+    def _register_api_namespaces(api: Api):
+        """Register Default API namespaces"""
+        from pypeline.flask.api.pipelines import bp as pipelinesbp
+
+        api.register_blueprint(pipelinesbp)
+        from pypeline.flask.api.schedules import bp as schedulesbp
+
+        api.register_blueprint(schedulesbp)