scalable-pypeline 1.1.0__py2.py3-none-any.whl

pypeline/flask/api/pipelines.py

@@ -0,0 +1,245 @@
+""" Pipeline APIs
+"""
+import logging
+
+from celery.canvas import _chain
+from celery_dyrygent.workflows import Workflow
+from flask import jsonify, request
+from flask.views import MethodView
+from rho_web.smorest import Blueprint
+from rho_web.response import abort
+from marshmallow import Schema, fields
+from marshmallow.exceptions import ValidationError
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS, API_PATH_V1
+from pypeline.flask.decorators import require_accesskey
+from pypeline.flask.api.utils import chain_helper
+from pypeline.utils.task_utils import PipelineResult
+from pypeline.utils.config_utils import retrieve_latest_pipeline_config
+from pypeline.pipeline_config_schema import BasePipelineSchema, PipelineSchemaV1
+
+logger = logging.getLogger(__name__)
+bp = Blueprint('pipelines', __name__, url_prefix=API_PATH_V1 + '/pipelines')
+
+
+class InvokePipelineSchema(Schema):
+    """ Incoming schema for invoking a pipeline
+    """
+    chain_payload = fields.Raw(
+        description='Payload contains whatever arguments the pipeline expects '
+        'to be passed to each node in the graph.',
+        example={
+            'document_id': '123',
+            'send_alert': True
+        },
+        required=False)
+
+
+class InvokePipelineResponseSchema(Schema):
+    execution_id = fields.String()
+    pipeline_id = fields.String()
+    status = fields.String()
+
+
+class GetPipelineResultResponseSchema(Schema):
+    execution_id = fields.String()
+    result = fields.Raw()
+    result_ttl = fields.Integer()
+    results = fields.Raw()
+    status = fields.String()
+    status_message = fields.String()
+
+
+@bp.route('/')
+class Pipelines(MethodView):
+    """ Operations against all pipelines.
+    """
+    @require_accesskey
+    @bp.doc(responses=API_DOC_RESPONSES,
+            parameters=[API_DOC_PARAMS['accesskey']],
+            tags=['Pipelines'])
+    def get(self):
+        """ Retrieve list of available pipelines.
+        """
+        access_key = request.headers.get('accesskey')
+        pipeline_config_api_resp = retrieve_latest_pipeline_config(
+            access_key=access_key)
+
+        if pipeline_config_api_resp is None:
+            abort(404)
+
+        try:
+            pipelines = []
+            for p in pipeline_config_api_resp:
+                PipelineSchema = \
+                    BasePipelineSchema.get_by_version(p['schemaVersion'])
+                pipeline_config = PipelineSchema().load(p)
+                pipelines.append(pipeline_config)
+        except ValidationError as e:
+            msg = f"Invalid pipeline configuration: {e}"
+            return jsonify({'message': msg}), 202
+
+        return jsonify(pipelines)
+
+
+@bp.route('/<string:pipeline_id>')
+class PipelineInfo(MethodView):
+    """ Operations against a single pipeline
+    """
+    @require_accesskey
+    @bp.doc(responses=API_DOC_RESPONSES,
+            parameters=[
+                API_DOC_PARAMS['accesskey'], {
+                    'in': 'path',
+                    'name': 'pipeline_id',
+                    'description':
+                    'pipeline_id for which to retrieve metrics.',
+                    'type': 'string',
+                    'example': 'my_pipeline',
+                    'required': True
+                }
+            ],
+            tags=['Pipelines'])
+    def get(self, pipeline_id: str):
+        """ Retrieve details about a specific pipeline.
+        """
+        access_key = request.headers.get('accesskey')
+        pipeline_config_api_resp = retrieve_latest_pipeline_config(
+            pipeline_id=pipeline_id, access_key=access_key)
+
+        if pipeline_config_api_resp is None:
+            abort(404)
+
+        try:
+            pipeline_config = PipelineSchemaV1().load(pipeline_config_api_resp)
+        except ValidationError as e:
+            msg = f"Invalid pipeline configuration: {e}"
+            return jsonify({'message': msg}), 202
+
+        return jsonify(pipeline_config)
+
+
+@bp.route('/invoke/<string:pipeline_id>')
+class PipelineInvoke(MethodView):
+    """ Operations involed with pipeline invocation
+    """
+    @require_accesskey
+    @bp.doc(responses=API_DOC_RESPONSES,
+            parameters=[
+                API_DOC_PARAMS['accesskey'], {
+                    'in': 'path',
+                    'name': 'pipeline_id',
+                    'description':
+                    'pipeline_id for which to retrieve metrics.',
+                    'type': 'string',
+                    'example': 'my_pipeline',
+                    'required': True
+                }
+            ],
+            tags=['Pipelines'])
+    @bp.arguments(InvokePipelineSchema)
+    @bp.response(InvokePipelineResponseSchema)
+    def post(self, payload: dict, pipeline_id: str):
+        """ Invoke a pipeline by it's ID; optionally provide pipeline arguments.
+        """
+
+        access_key = request.headers.get('accesskey')
+        pipeline_config = retrieve_latest_pipeline_config(
+            pipeline_id=pipeline_id, access_key=access_key)
+
+        if pipeline_config is None:
+            return abort(404)
+
+        retval = {'pipeline_id': pipeline_id, 'status': ''}
+        try:
+            # TODO - ideally we can validate the payload *at this stage*
+            # before the chain is ever invoked so we can handle issues
+            # without kicking off work.
+            payload = payload['chain_payload']\
+                if 'chain_payload' in payload else {}
+
+            gen = chain_helper(pipeline_id=pipeline_id,
+                               access_key=access_key,
+                               chain_payload=payload)
+
+            if gen.chain is None:
+                abort(400, message=gen.loading_message)
+
+            chain: _chain = gen.chain
+            wf: Workflow = Workflow()
+            wf.add_celery_canvas(chain)
+            wf.apply_async()
+            retval['status'] = 'success'
+            retval['execution_id'] = gen.execution_id
+            # Initialize the cached result
+            pr = PipelineResult(gen.execution_id, status='pending')
+            pr.save()
+
+        except Exception as e:
+            msg = "Failed to invoke pipeline ... {}".format(pipeline_id)
+            logger.error(msg)
+            logger.exception(f"{e}")
+            abort(500, message=msg)
+
+        return jsonify(retval)
+
+
+results_responses = API_DOC_RESPONSES.copy()
+results_responses[202] = {
+    'code': 202,
+    'description': 'Pipeline is still running. Try again later.'
+}
+results_responses[204] = {
+    'code': 204,
+    'description': 'The execution results have expired. Re-run pipeline.'
+}
+
+
+@bp.route('/results/<string:execution_id>')
+class PipelineResults(MethodView):
+    """ Operations with respect to pipeline results
+    """
+    @require_accesskey
+    @bp.doc(responses=results_responses,
+            parameters=[
+                API_DOC_PARAMS['accesskey'], {
+                    'in': 'path',
+                    'name': 'execution_id',
+                    'description':
+                    'execution_id for which to retrieve results',
+                    'type': 'string',
+                    'example': '4c595cca-9bf1-4150-8c34-6b43faf276c8',
+                    'required': True
+                }
+            ],
+            tags=['Pipelines'])
+    @bp.response(GetPipelineResultResponseSchema)
+    def get(self, execution_id: str):
+        """ Retrieve results of a pipeline's execution based on execution_id
+
+            NOTE: Cached results expire after a time window so are not available
+            forever.
+
+            TODO: Need to add response marshalling/schema here.
+        """
+        try:
+            pr = PipelineResult(execution_id)
+            pr.load()
+            retval = pr.to_dict()
+            if pr.status == 'unavailable':
+                retval['status_message'] = 'Results expired. Re-run pipeline.'
+                return retval, 204
+
+            if pr.status == 'pending':
+                retval['status_message'] = 'Results pending. Check again soon.'
+                return retval, 202
+
+            else:
+                retval['status_message'] = 'Results available.'
+                return retval, 200
+
+        except Exception as e:
+            msg = "Failed to retrieve results for execution id: {}".format(
+                execution_id)
+            logger.error(msg)
+            logger.exception(f"{e}")
+            abort(500, message=msg)

pypeline/flask/api/schedules.py

@@ -0,0 +1,67 @@
+""" API Endpoints for Scheduled Tasks
+"""
+import os
+import logging
+from flask import jsonify, request
+from flask.views import MethodView
+from marshmallow import Schema, fields
+from rho_web.smorest import Blueprint
+from rho_web.response import abort
+from marshmallow.exceptions import ValidationError
+from pypeline.constants import API_DOC_RESPONSES, API_DOC_PARAMS,\
+    API_PATH_V1
+from pypeline.utils.config_utils import retrieve_latest_schedule_config, \
+    update_schedule_config
+from pypeline.schedule_config_schema import BaseScheduleSchema
+from pypeline.flask.decorators import require_accesskey
+
+logger = logging.getLogger(__name__)
+
+bp = Blueprint('schedules', __name__, url_prefix=API_PATH_V1 + '/schedules')
+
+
+@bp.route('/')
+class Schedules(MethodView):
+    """ Operations related to schedules
+    """
+    @require_accesskey
+    @bp.doc(responses=API_DOC_RESPONSES,
+            parameters=[API_DOC_PARAMS['accesskey']],
+            tags=['Schedules'])
+    def get(self):
+        """ Retrieve list of available schedule entries.
+        """
+        access_key = request.headers.get('accesskey')
+        try:
+            schedule_config = retrieve_latest_schedule_config(
+                access_key=access_key)
+        except ValidationError:
+            abort(400, message="Invalid schedule found ...")
+
+        if schedule_config is None:
+            abort(404)
+
+        return jsonify(schedule_config)
+
+    @require_accesskey
+    @bp.doc(responses=API_DOC_RESPONSES,
+            parameters=[API_DOC_PARAMS['accesskey']],
+            tags=['Schedules'])
+    @bp.arguments(BaseScheduleSchema)
+    def post(self, payload: dict):
+        """ Update a deployment's schedules. Primarily used to update dynamic
+        keys such as last run at and total run count. This does not allow
+        overloading schedules, only updating select keys on known schedule
+        entries (as in, this is not destructive).
+        """
+        access_key = request.headers.get('accesskey')
+        try:
+            success = update_schedule_config(new_schedule_config=payload,
+                                             access_key=access_key)
+        except ValidationError as e:
+            abort(400, message=e)
+
+        if not success:
+            abort(500)
+
+        return jsonify({'message': 'Schedule update successful ...'})

pypeline/flask/api/utils.py

@@ -0,0 +1,36 @@
+""" Utilities for Sermos APIs and interacting with Pipelines/Schedules
+"""
+import logging
+from typing import Union
+from pypeline.utils.task_utils import PipelineGenerator
+
+logger = logging.getLogger(__name__)
+
+
+def chain_helper(pipeline_id: str,
+                 access_key: Union[str, None] = None,
+                 chain_payload: Union[dict, None] = None,
+                 add_retry: bool = True,
+                 queue: Union[str, None] = None,
+                 default_task_ttl: int = None):
+    """ Helper method to generate a pipeline chain *with* error handling.
+
+        Usage:
+          my_chain = chain_helper('pipeline-name')
+          my_chain.delay()
+    """
+    # Get our pipeline. The PipelineGenerator will use the PipelineRunWrapper
+    # to cache this "run" of the pipeline.
+    gen = PipelineGenerator(pipeline_id,
+                            access_key=access_key,
+                            queue=queue,
+                            default_task_ttl=default_task_ttl,
+                            add_retry=add_retry,
+                            chain_payload=chain_payload)
+    if gen.good_to_go:
+        # Generate our 'chain', which is the grouping of celery constructs that
+        # allows our dag to run asynchronously and synchronously according to
+        # the adjacency list defined in our pipeline configuration.
+        gen.generate_chain()
+
+    return gen

pypeline/flask/decorators.py

@@ -0,0 +1,92 @@
+""" Flask specific decorators (primarily for auth activities and app context)
+"""
+import os
+import logging
+from http import HTTPStatus
+from functools import wraps
+import requests
+from rhodb.redis_conf import RedisConnector
+from flask import current_app, request
+from rho_web.response import abort
+from pypeline.flask import oidc
+from pypeline.constants import DEFAULT_AUTH_URL, AUTH_LOCK_KEY, \
+    AUTH_LOCK_DURATION, USING_SERMOS_CLOUD
+
+logger = logging.getLogger(__name__)
+redis_conn = RedisConnector().get_connection()
+
+
+def require_login(fn):
+    """ Utilize Flask RhoAuth to secure a route with @require_login decorator
+    """
+    secure_fn = oidc.require_login(fn)
+
+    @wraps(fn)
+    def decorated(*args, **kwargs):
+        if str(current_app.config['RHOAUTH_ENABLED']).lower() == 'true':
+            return secure_fn(*args, **kwargs)
+        return fn(*args, **kwargs)
+
+    return decorated
+
+
+def validate_access_key(access_key: str = None):
+    """ Verify whether an Access Key is valid according to Sermos Cloud.
+
+    If deploying in 'local' mode, no validation is done. To deploy in local
+    mode, set DEFAULT_BASE_URL=local in your environment.
+    """
+    # Always 'valid' in local mode
+    if not USING_SERMOS_CLOUD:
+        return True
+
+    # If get access key from either provided val or environment
+    # if None provided.
+    access_key = os.environ.get('SERMOS_ACCESS_KEY', access_key)
+
+    # Invalid if None, no need to ask.
+    if access_key is None:
+        return False
+
+    # Ask cache first
+    # TODO update to remove Redis as a dependency, merely an optional feature.
+    validated = redis_conn.get(AUTH_LOCK_KEY)
+    if validated is not None:
+        return True
+
+    # Ask Sermos Cloud (Note: Sermos Cloud's API expects `apikey`)
+    headers = {'apikey': access_key}
+    r = requests.post(DEFAULT_AUTH_URL, headers=headers, verify=True)
+
+    if r.status_code == 200:
+        redis_conn.setex(AUTH_LOCK_KEY, AUTH_LOCK_DURATION, '')
+        return True
+    return False
+
+
+def require_accesskey(fn):
+    """ Convenience decorator to add to a web route (typically an API)
+        when using Flask.
+
+        Usage::
+            from sermos import Blueprint, ApiServices
+            bp = Blueprint('api_routes', __name__, url_prefix='/api')
+
+            @bp.route('/my-api-route')
+            class ApiClass(MethodView):
+                @require_access_key
+                def post(self, payload: dict):
+                    return {}
+    """
+    @wraps(fn)
+    def decorated_view(*args, **kwargs):
+        access_key = request.headers.get('accesskey')
+        if not access_key:
+            access_key = request.args.get('accesskey')
+
+        if validate_access_key(access_key):
+            return fn(*args, **kwargs)
+
+        abort(HTTPStatus.UNAUTHORIZED)
+
+    return decorated_view

pypeline/flask/flask_sermos.py

@@ -0,0 +1,219 @@
+""" Sermos implementation as a Flask extension
+"""
+import os
+if os.getenv('USE_GEVENT', 'false').lower() == 'true':
+    import gevent.monkey
+    gevent.monkey.patch_all()
+
+import logging
+from functools import partial
+from flask import Flask
+from werkzeug.middleware.proxy_fix import ProxyFix
+from rho_web.smorest import Api, Blueprint
+from pypeline.utils.task_utils import TaskRunner, PipelineResult
+from pypeline.logging_config import setup_logging
+from pypeline.extensions import sermos_config
+from pypeline.flask import oidc
+from pypeline.constants import DEFAULT_OPENAPI_CONFIG, DEFAULT_RHOAUTH_CONFIG
+from pypeline import __version__
+
+logger = logging.getLogger(__name__)
+
+
+class FlaskSermos:
+    """ Sermos Flask extension.
+    """
+    def __init__(self, app: Flask = None):
+        """ Class init
+        """
+        self.app = app
+        self.sermos_config = sermos_config if sermos_config is not None else {}
+
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app: Flask, init_api: bool = False):
+        """ Sermos bootstrapping process.
+
+        Application config variables to set include:
+
+            SERMOS_CLIENT_VERSION (default: v?.?.?)
+            RHOAUTH_ENABLED (default: True)
+            SERMOS_HIJACK_ROOT_LOGGER (default: False)
+
+        Optional, if `init_api` is True:
+
+            API_DOCUMENTATION_TITLE
+            API_DOCUMENTATION_DESCRIPTION
+            OPENAPI_VERSION
+            OPENAPI_URL_PREFIX
+            OPENAPI_SWAGGER_APP_NAME
+            OPENAPI_SWAGGER_UI_PATH
+            OPENAPI_SWAGGER_BASE_TEMPLATE
+            OPENAPI_SWAGGER_URL
+            OPENAPI_SWAGGER_UI_URL
+            SWAGGER_UI_DOC_EXPANSION
+            EXPLAIN_TEMPLATE_LOADING
+
+        Args:
+            app (Flask): Flask Application to initialize.
+            init_api (bool): If `True`, Sermos will initialize its
+                core APIs (including Pipelines, Scheduled Tasks, etc.) and
+                provide a pre-configured OpenAPI Spec/Swagger UI interface
+                available at the route defined in your application's config
+                under `OPENAPI_URL_PREFIX` (default `/api`). Refer to
+                [flask-smorest](https://flask-smorest.readthedocs.io/en/latest/openapi.html)
+                documentation for additional configuration options.
+        """
+        # Ensure there's a SERMOS_CLIENT_VERSION on app config
+        app.config.setdefault(
+            'SERMOS_CLIENT_VERSION',
+            app.config.get("SERMOS_CLIENT_VERSION", "v?.?.?"))
+
+        # Bootstrap logging if app requests
+        self._bootstrap_logging(app)
+
+        app.wsgi_app = ProxyFix(app.wsgi_app)
+        app.url_map.strict_slashes = False
+
+        # Create and register the sermos blueprint
+        bp = Blueprint('sermos',
+                       __name__,
+                       template_folder='../templates',
+                       static_folder='../static',
+                       url_prefix='/sermos')
+        app.register_blueprint(bp)
+
+        # Bootstrap RhoAuth if app requests
+        self._bootstrap_rhoauth(app)
+
+        # Bootstrap api if app requests
+        if init_api is True:
+            self._bootstrap_api(app)
+
+    def _bootstrap_rhoauth(self, app: Flask):
+        """ If RHOAUTH_ENABLED is True, bootstrap the application with default
+            configuration, precedent given to the provide app.
+        """
+        # TODO Remove rhoauth by default and/or entirely in future release.
+        rhoauth_enabled = str(app.config.get('RHOAUTH_ENABLED',
+                                             True)).lower() == 'true'
+        app.config.setdefault('RHOAUTH_ENABLED', rhoauth_enabled)
+        if rhoauth_enabled is True:
+            # Set useful defaults so users don't need to request.
+            for rhoauth_config in DEFAULT_RHOAUTH_CONFIG:
+                app.config.setdefault(
+                    rhoauth_config[0],
+                    app.config.get(rhoauth_config[0], rhoauth_config[1]))
+
+        # Initialize open id connect for authentication
+        if rhoauth_enabled is True:
+            logger.info("Initializing using RhoAuth ...")
+            self.oidc = oidc
+            self.oidc.init_app(app)
+
+            @app.route('/logout')
+            def logout():
+                return self.oidc.logout("/")
+
+    @staticmethod
+    def _bootstrap_logging(app: Flask):
+        """ If application requests Sermos logging, enable here
+
+            Main reason to do this is to have clear versioning in each log
+            and to supress the elasticsearch log level by default, which is
+            extremely verbose if using elasticsearch-py. There is no dependency
+            on elasticsearch - we simply create the logger and upgrade level
+            because it's extremely annoying if you do need to use ES.
+        """
+        if app.config.get('SERMOS_HIJACK_ROOT_LOGGER', False):
+            overload_es = app.config.get('OVERLOAD_ES_LOGGING', True)
+            logger.info("Initializing using Sermos logging ...")
+            setup_logging(app_version=__version__,
+                          client_version=app.config['SERMOS_CLIENT_VERSION'],
+                          default_level=app.config.get("LOG_LEVEL", "INFO"),
+                          overload_elasticsearch=overload_es,
+                          establish_logging_config=True)
+
+    def _bootstrap_api(self, app: Flask):
+        """ If initializing the API, we will create the core Sermos API paths
+            and initialize the default Swagger documentation.
+        """
+        # Set sensible defaults for Swagger docs. Provided `app` will
+        # take precedent.
+        for swagger_config in DEFAULT_OPENAPI_CONFIG:
+            app.config.setdefault(
+                swagger_config[0],
+                app.config.get(swagger_config[0], swagger_config[1]))
+
+        # Attempt to override with values from client's sermos.yaml if
+        # they are available. This will add new tags and new docs if
+        # defined and add to the core Sermos API docs.
+        api_config = self.sermos_config.get('apiConfig', {})
+        api_docs = api_config.get('apiDocumentation', {})
+
+        custom_tags = api_config.get('prefixDescriptions', [])
+
+        app.config['SERMOS_CLIENT_VERSION'] = \
+            api_docs.get('version', None) \
+            if api_docs.get('version', None) is not None \
+            else app.config['SERMOS_CLIENT_VERSION']
+
+        app.config['API_DOCUMENTATION_TITLE'] = \
+            api_docs.get('title', None) \
+            if api_docs.get('title', None) is not None \
+            else app.config['API_DOCUMENTATION_TITLE']
+
+        app.config['API_DOCUMENTATION_DESCRIPTION'] = \
+            api_docs.get('description', None) \
+            if api_docs.get('description', None) is not None \
+            else app.config['API_DOCUMENTATION_DESCRIPTION']
+
+        # Set default Sermos Tags along with custom tags from sermos.yaml
+        tags = [{
+            'name': 'Pipelines',
+            'description': 'Operations related to Pipelines'
+        }, {
+            'name': 'Schedules',
+            'description': 'Operations related to Schedules'
+        }] + custom_tags
+
+        # Set up  the initializing spec kwargs for API
+        spec_kwargs = {
+            'title': app.config['API_DOCUMENTATION_TITLE'],
+            'version': f"Sermos: {__version__} - "
+            f"Client: {app.config['SERMOS_CLIENT_VERSION']}",
+            'description': app.config['API_DOCUMENTATION_DESCRIPTION'],
+            'tags': tags
+        }
+        try:
+            # Initialize the API documentation and ensure RhoAuth validates
+            # correct `role`
+            if str(app.config['RHOAUTH_ENABLED']).lower() == 'true':
+                decorator = partial(self.oidc.require_login,
+                                    roles=['sermos-client-api-docs'])
+                api = Api(auth_decorator=decorator)
+            else:
+                api = Api()
+
+            api.init_app(app, spec_kwargs=spec_kwargs)
+
+            # Register available Sermos API Namespaces
+            self._register_api_namespaces(api)
+
+        except Exception as e:
+            api = None
+            logging.exception(f"Unable to initialize API ... {e}")
+
+        # Register the Sermos Core API as an extension for use in Client App
+        app.extensions.setdefault('sermos_core_api', api)
+
+    @staticmethod
+    def _register_api_namespaces(api: Api):
+        """ Register Default API namespaces
+            TODO add metrics APIs
+        """
+        from pypeline.flask.api.pipelines import bp as pipelinesbp
+        api.register_blueprint(pipelinesbp)
+        from pypeline.flask.api.schedules import bp as schedulesbp
+        api.register_blueprint(schedulesbp)