scalable-pypeline 1.1.0__py2.py3-none-any.whl

pypeline/cli/deploy.py

@@ -0,0 +1,138 @@
+""" Command Line Utilities for Sermos Deployments
+"""
+import logging
+import click
+from pypeline.deploy import SermosDeploy
+
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+def deployment():
+    """ Deployment command group.
+    """
+
+
+@deployment.command()
+@click.option('--pkg-name', required=False, default=None)
+@click.option('--sermos-yaml', required=False, default=None)
+@click.option('--output-file', required=False, default=None)
+def validate(pkg_name: str = None,
+             sermos_yaml: str = None,
+             output_file: str = None):
+    """ Validate a compiled Sermos yaml is ready for deployment.
+
+        Arguments::
+
+            pkg-name (optional): Directory name for your Python package.
+                e.g. my_package_name If none provided, will check environment
+                for `SERMOS_CLIENT_PKG_NAME`. If not found, will exit.
+
+            sermos-yaml (optional): Path to find your `sermos.yaml`
+                configuration file. Defaults to `sermos.yaml`
+    """
+    # Instantiate SermosDeploy
+    sd = SermosDeploy(access_key='fake',
+                      pkg_name=pkg_name,
+                      sermos_yaml_filename=sermos_yaml)
+
+    # Validate deployment
+    sd.validate_deployment(output_file=output_file)
+    click.echo("Configuration is Valid and ready to Deploy.")
+
+
+@deployment.command()
+@click.option('--deployment-id', required=False, default=None)
+@click.option('--access-key', required=False, default=None)
+@click.option('--pkg-name', required=False, default=None)
+@click.option('--sermos-yaml', required=False, default=None)
+@click.option('--commit-hash', required=False, default=None)
+@click.option('--base-url', required=False, default=None)
+@click.option('--deploy-branch', required=False, default='main')
+def deploy(deployment_id: str = None,
+           access_key: str = None,
+           pkg_name: str = None,
+           sermos_yaml: str = None,
+           commit_hash: str = None,
+           base_url: str = None,
+           deploy_branch: str = 'main'):
+    """ Invoke a Sermos build for your application.
+
+        Arguments:
+
+            deployment-id (optional): UUID for Deployment. Find in your Sermos
+                Cloud Console. Will look under `SERMOS_DEPLOYMENT_ID` in
+                environment if not provided.
+
+            access-key (optional): Defaults to checking the environment for
+                `SERMOS_ACCESS_KEY`. If not found, will exit.
+
+            pkg-name (optional): Directory name for your Python package.
+                e.g. my_package_name If none provided, will check environment
+                for `SERMOS_CLIENT_PKG_NAME`. If not found, will exit.
+
+            sermos-yaml (optional): Path to find your `sermos.yaml`
+                configuration file. Defaults to `sermos.yaml`
+
+            commit-hash (optional): The specific commit hash of your git repo
+                to deploy. If not provided, then current HEAD as of invocation
+                will be used. This is the default usage, and is useful in the
+                case of a CI/CD pipeline such that the Sermos deployment is
+                invoked after your integration passes.
+
+            base-url (optional): Defaults to primary Sermos Cloud base URL.
+                    Only modify this if there is a specific, known reason to do so.
+
+            deploy-branch (optional): Defaults to 'main'. Only modify this
+                if there is a specific, known reason to do so.
+    """
+    # Instantiate SermosDeploy
+    sd = SermosDeploy(deployment_id=deployment_id,
+                      access_key=access_key,
+                      pkg_name=pkg_name,
+                      sermos_yaml_filename=sermos_yaml,
+                      commit_hash=commit_hash,
+                      base_url=base_url,
+                      deploy_branch=deploy_branch)
+
+    # Validate deployment
+    sd.validate_deployment()
+
+    # Invoke deployment
+    result = sd.invoke_deployment()
+    content = result.json()
+    if result.status_code < 300:
+        click.echo(content['data']['status'])
+    else:
+        logger.error(f"{content}")
+
+
+@deployment.command()
+@click.option('--deployment-id', required=False, default=None)
+@click.option('--access-key', required=False, default=None)
+@click.option('--base-url', required=False, default=None)
+def status(deployment_id: str = None,
+           access_key: str = None,
+           base_url: str = None):
+    """ Check on the status of a Sermos build.
+
+        Arguments:
+            deployment-id (optional): UUID for Deployment. Find in your Sermos
+                Cloud Console. If not provided, looks in environment under
+                `SERMOS_DEPLOYMENT_ID`
+            access-key (optional): Defaults to checking the environment for
+                `SERMOS_ACCESS_KEY`. If not found, will exit.
+            base-url (optional): Defaults to primary Sermos Cloud base URL.
+                    Only modify this if there is a specific, known reason to do so.
+    """
+    # Instantiate SermosDeploy
+    sd = SermosDeploy(deployment_id=deployment_id,
+                      access_key=access_key,
+                      base_url=base_url)
+
+    # Check deployment status
+    result = sd.get_deployment_status()
+    try:
+        click.echo(result['data']['results'])
+    except Exception as e:
+        logger.error(f"{result} / {e}")

pypeline/cloud.py

@@ -0,0 +1,80 @@
+""" Base class for interacting with Sermos Cloud API.
+"""
+import json
+import logging
+import requests
+from pypeline.utils.config_utils import get_access_key, get_deployment_id
+from pypeline.constants import DEFAULT_BASE_URL, DEPLOYMENTS_DEPLOY_URL,\
+    DEPLOYMENTS_SERVICES_URL
+
+logger = logging.getLogger(__name__)
+
+
+class SermosCloud():
+    """ Primary Sermos Cloud class for interacting with API.
+    """
+    def __init__(self,
+                 access_key: str = None,
+                 base_url: str = None,
+                 deployment_id: str = None):
+        """ Arguments:
+                access_key (optional): Access key, issued by Sermos, which is
+                    tied to a `Deployment`. Defaults to checking the environment
+                    for `SERMOS_ACCESS_KEY`. If not found, will exit.
+                base_url (optional): Defaults to primary Sermos Cloud API
+                    endpoint (https://cloud.sermos.ai/api/v1/).
+                    Only modify this if there is a specific, known reason to do so.
+                deployment_id: UUID for Deployment. Find in your Sermos
+                    Cloud Console.
+        """
+        super(SermosCloud, self).__init__()
+        self.access_key = get_access_key(access_key)
+        self.base_url = base_url if base_url\
+            else DEFAULT_BASE_URL
+        try:
+            self.deployment_id = get_deployment_id(deployment_id)
+        except KeyError:
+            self.deployment_id = None  # Not always required, so allow None ...
+
+        self.deploy_url = DEPLOYMENTS_DEPLOY_URL.format(
+            self.base_url, self.deployment_id)
+        self.services_url = DEPLOYMENTS_SERVICES_URL.format(
+            self.base_url, self.deployment_id)
+
+        # Note: Sermos Cloud's API expects `apikey`
+        self.headers = {
+            'Content-Type': 'application/json',
+            'apikey': self.access_key
+        }
+
+    def get(self, url: str, as_dict: bool = False):
+        """ Send a GET request to Sermos Cloud
+        """
+        r = requests.get(url, headers=self.headers)
+        if as_dict:
+            return r.json()
+        return r
+
+    def get_all(self, url: str, page: int = 0, page_size: int = 15):
+        """ Loop through all paginated results from a GET endpoint
+        """
+        new_results = True
+        results = []
+        while new_results:
+            this_url = f"{url}?page={page}&sort_order=DESC&page_size={page_size}"
+            r = requests.get(this_url, headers=self.headers).json()
+            new_results = r.get("data", {}).get("results", [])
+            results.extend(new_results)
+            page += 1
+
+        return {'data': {'results': results}, 'message': 'All Results'}
+
+    def post(self, url: str, payload: dict = None, as_dict: bool = False):
+        """ Send a POST request to Sermos Cloud
+        """
+        if payload is None:
+            payload = {}
+        r = requests.post(url, headers=self.headers, data=json.dumps(payload))
+        if as_dict:
+            return r.json()
+        return r

pypeline/constants.py

@@ -0,0 +1,139 @@
+""" Sermos Constants
+"""
+import os
+from urllib.parse import urljoin
+
+API_PATH_V1 = '/api/v1'
+
+DEFAULT_RESULT_TTL = 86400  # seconds (1 day)
+DEFAULT_TASK_TTL = 60  # seconds (1 minute)
+DEFAULT_RETRY_TASK_MAX_TTL = 300
+DEFAULT_MAX_RETRY = 10
+DEFAULT_REGULATOR_TASK = 'sermos.celery.task_chain_regulator'
+DEFAULT_SUCCESS_TASK = 'sermos.celery.pipeline_success'
+DEFAULT_RETRY_TASK = 'sermos.celery.pipeline_retry'
+
+CHAIN_SUCCESS_MSG = 'Chain built successfully ...'
+CHAIN_FAILURE_MSG = 'Chain failed to build ...'
+
+PIPELINE_RUN_WRAPPER_CACHE_KEY = 'sermos_{}_{}'  # pipeline_id + execution_id
+PIPELINE_RESULT_CACHE_KEY = 'sermos_result_{}'  # execution_id
+
+# Pipeline configurations and scheduled task configuration are cached in Redis
+# temporarily (default to CONFIG_REFRESH_RATE (in seconds)).
+# Each pipeline config/schedule config is specific to an individual deployment,
+# however, we cache only with the pipeline_id here because the usage of this
+# cache key is restricted to the redis instance associated with the deployment.
+PIPELINE_CONFIG_CACHE_KEY = 'sermos_pipeline_config_{}'  # pipeline_id
+SCHEDULE_CONFIG_CACHE_KEY = 'sermos_schedule_config'
+CONFIG_REFRESH_RATE = int(os.environ.get('CONFIG_REFRESH_RATE', 30))  # seconds
+
+# TODO where on earth is this crazy time format coming from?
+SCHEDULE_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+
+AUTH_LOCK_KEY = os.environ.get('AUTH_LOCK_KEY', 'sermos-auth-lock')
+AUTH_LOCK_DURATION = int(os.environ.get('AUTH_LOCK_DURATION', 30))
+
+STORED_MODEL_KEY = '{}_{}{}'
+
+# Yaml path is relative to package
+SERMOS_YAML_PATH = os.environ.get('SERMOS_YAML_PATH', 'sermos.yaml')
+SERMOS_ACCESS_KEY = os.environ.get('SERMOS_ACCESS_KEY', None)
+SERMOS_CLIENT_PKG_NAME = os.environ.get('SERMOS_CLIENT_PKG_NAME', None)
+SERMOS_DEPLOYMENT_ID = os.environ.get('SERMOS_DEPLOYMENT_ID', 'local')
+LOCAL_DEPLOYMENT_VALUE = os.environ.get('LOCAL_DEPLOYMENT_VALUE', 'local')
+DEFAULT_BASE_URL = os.environ.get('SERMOS_BASE_URL', 'https://console.sermos.ai')
+if DEFAULT_BASE_URL != 'local':
+    DEFAULT_BASE_URL += '/api/v1/'
+DEPLOYMENTS_URL = "{}deployments/{}"
+DEPLOYMENTS_DEPLOY_URL = "{}deployments/{}/deploy"
+DEPLOYMENTS_SERVICES_URL = "{}deployments/{}/services"
+DEPLOYMENTS_SERVICE_URL = "{}deployments/{}/services/{}"
+DEFAULT_AUTH_URL = urljoin(DEFAULT_BASE_URL, 'auth')
+USING_SERMOS_CLOUD = DEFAULT_BASE_URL != LOCAL_DEPLOYMENT_VALUE
+DEFAULT_CONFIG_RETRIEVAL_PAGE_SIZE = 25
+# Default 'responses' dictionary when decorating endpoints with @api.doc()
+# Extend as necessary.
+API_DOC_RESPONSES = {
+    200: {
+        'code': 200,
+        'description': 'Successful response.'
+    },
+    400: {
+        'code': 400,
+        'description': 'Malformed request. Verify payload is correct.'
+    },
+    401: {
+        'code': 401,
+        'description':
+        'Unauthorized. Verify your API Key (`accesskey`) header.'
+    }
+}
+
+# Default 'params' dictionary when decorating endpoints with @api.doc()
+# Extend as necessary.
+API_DOC_PARAMS = {
+    'accesskey': {
+        'in': 'header',
+        'name': 'accesskey',
+        'description': 'Your API Consumer\'s `accesskey`',
+        'type': 'string',
+        'required': True
+    }
+}
+
+DEFAULT_OPENAPI_CONFIG = (
+    ('SWAGGER_UI_DOC_EXPANSION',
+     'list'), ('API_DOCUMENTATION_TITLE',
+               'Sermos API Specs'), ('API_DOCUMENTATION_DESCRIPTION',
+                                     'Available API Endpoints'),
+    ('OPENAPI_VERSION', '3.0.2'), ('OPENAPI_URL_PREFIX',
+                                   '/api/v1'), ('OPENAPI_SWAGGER_APP_NAME',
+                                                'Sermos - API Reference'),
+    ('OPENAPI_SWAGGER_UI_PATH',
+     '/docs'), ('OPENAPI_SWAGGER_BASE_TEMPLATE',
+                'swagger/swagger_ui.html'), ('OPENAPI_SWAGGER_URL', '/docs'),
+    ('OPENAPI_SWAGGER_UI_URL',
+     'https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/'),
+    ('EXPLAIN_TEMPLATE_LOADING', False))
+
+# Rho Auth settings
+DEFAULT_RHOAUTH_CONFIG = (
+    ('RHOAUTH_OIDC_CLIENT_ID',
+     os.environ.get('RHOAUTH_OIDC_CLIENT_ID',
+                    os.environ.get('CLIENT_PKG_NAME', '').replace('_', '-'))),
+    ('RHOAUTH_OIDC_ISSUER',
+     os.environ.get('RHOAUTH_OIDC_ISSUER',
+                    'https://auth.rho.ai/auth/realms/sermos')),
+    ('RHOAUTH_OIDC_AUTH_ENDPOINT',
+     os.environ.get(
+         'RHOAUTH_OIDC_AUTH_ENDPOINT',
+         'https://auth.rho.ai/auth/realms/sermos/protocol/openid-connect/auth')
+     ),
+    ('RHOAUTH_OIDC_TOKEN_ENDPOINT',
+     os.environ.get(
+         'RHOAUTH_OIDC_TOKEN_ENDPOINT',
+         'https://auth.rho.ai/auth/realms/sermos/protocol/openid-connect/token'
+     )),
+    ('RHOAUTH_OIDC_JWKS_URI_ENDPOINT',
+     os.environ.get(
+         'RHOAUTH_OIDC_JWKS_URI_ENDPOINT',
+         'https://auth.rho.ai/auth/realms/sermos/protocol/openid-connect/certs'
+     )),
+    ('RHOAUTH_OIDC_END_SESSION_ENDPOINT',
+     os.environ.get(
+         'RHOAUTH_OIDC_END_SESSION_ENDPOINT',
+         'https://auth.rho.ai/auth/realms/sermos/protocol/openid-connect/logout'
+     )), ('RHOAUTH_OIDC_END_SESSION_REDIRECT_URI',
+          os.environ.get('RHOAUTH_OIDC_END_SESSION_REDIRECT_URI',
+                         '/')), ('OAUTHLIB_INSECURE_TRANSPORT',
+                                 os.environ.get('OAUTHLIB_INSECURE_TRANSPORT',
+                                                1)))
+
+
+def create_model_key(model_prefix: str,
+                     model_version: str,
+                     model_postfix: str = ''):
+    """ Ensures we're consistently creating the keys for storing/retrieving.
+    """
+    return STORED_MODEL_KEY.format(model_prefix, model_version, model_postfix)

pypeline/deploy.py

@@ -0,0 +1,167 @@
+""" Utilities for deploying applications to Sermos.
+
+    Example CLI Usage::
+
+        $ honcho run -e .env sermos deploy
+        $ honcho run -e .env sermos status
+
+    Example Programmatic Usage::
+
+        from sermos.deploy import SermosDeploy
+
+        sd = SermosDeploy(
+            os.environ.get("SERMOS_ACCESS_KEY", None),
+            pkg_name="sermos_demo_client"
+        )
+
+        # To Invoke
+        status = sd.invoke_deployment()
+        print(status)
+
+        # To Check Status:
+        status = sd.get_deployment_status()
+        print(status)
+
+"""
+import subprocess
+import base64
+import logging
+
+from pypeline.utils.module_utils import get_client_pkg_name
+from pypeline.sermos_yaml import load_sermos_config
+from pypeline.cloud import SermosCloud
+
+logger = logging.getLogger(__name__)
+
+
+class SermosDeploy(SermosCloud):
+    """ Primary Sermos Deployment class for invocation and status updates.
+    """
+    def __init__(self,
+                 deployment_id: str = None,
+                 access_key: str = None,
+                 pkg_name: str = None,
+                 sermos_yaml_filename: str = None,
+                 commit_hash: str = None,
+                 base_url: str = None,
+                 deploy_branch: str = 'main'):
+        """ Arguments:
+                deployment_id (optional): UUID for Deployment. Find in your
+                    Sermos Cloud Console.
+                access_key (optional): Access key, issued by Sermos, that
+                    dictates the environment into which this request will be
+                    deployed. Defaults to checking the environment for
+                    `SERMOS_ACCESS_KEY`. If not found, will exit.
+                pkg_name (optional): Directory name for your Python
+                    package. e.g. my_package_name . If none provided, will check
+                    environment for `SERMOS_CLIENT_PKG_NAME`. If not found,
+                    will exit.
+                sermos_yaml_filename (optional): Relative path to find your
+                    `sermos.yaml` configuration file. Defaults to `sermos.yaml`
+                    which should be found inside your `pkg_name`
+                commit_hash (optional): The specific commit hash of your git
+                    repo to deploy. If not provided, then current HEAD as of
+                    invocation will be used. This is the default usage, and is
+                    useful in the case of a CI/CD pipeline such that the Sermos
+                    deployment is invoked after your integration passes.
+                base_url (optional): Defaults to primary Sermos Cloud base URL.
+                    Only modify this if there is a specific, known reason to do so.
+                deploy_branch (optional): Defaults to 'main'. Only modify this
+                    if there is a specific, known reason to do so.
+
+        """
+        super().__init__(access_key, base_url, deployment_id)
+        self.pkg_name = get_client_pkg_name(pkg_name)
+        self.sermos_yaml_filename = sermos_yaml_filename
+        self.commit_hash = commit_hash
+        self.sermos_yaml = None  # Established later, only on `invoke`
+        self.encoded_sermos_yaml = None  # Established later, only on `invoke`
+        self.deploy_payload = None  # Established later, only on `invoke`
+        self.deploy_branch = deploy_branch
+
+    def _set_commit_hash(self):
+        """ Retrieve the commit hash of the current git state and set to
+            current deployment object.
+        """
+        if self.commit_hash is None:
+            self.commit_hash = subprocess.check_output(
+                ["git", "rev-parse", "--verify",
+                 "HEAD"]).strip().decode('utf-8')
+
+    def _set_encoded_sermos_yaml(self):
+        """ Provide the b64 encoded sermos.yaml file as part of request.
+            Primarily used to get the custom workers definitions, etc. so
+            the deployment endpoint can generate the values.yaml.
+        """
+        self.sermos_yaml = load_sermos_config(self.pkg_name,
+                                              self.sermos_yaml_filename,
+                                              as_dict=False)
+        self.encoded_sermos_yaml = base64.b64encode(
+            self.sermos_yaml.encode('utf-8')).decode('utf-8')
+
+    def _set_deploy_payload(self):
+        """ Set the deployment payload correctly.
+        """
+        self._set_commit_hash()
+        self._set_encoded_sermos_yaml()
+        self.deploy_payload = {
+            "sermos_yaml": self.encoded_sermos_yaml,
+            "commit_hash": self.commit_hash,
+            "deploy_branch": self.deploy_branch,
+            "client_package_name": self.pkg_name
+        }
+
+    def get_deployment_status(self):
+        """ Info on a specific deployment
+        """
+        resp = self.get(self.services_url)
+        services = resp.json().get('data', {}).get('results', [])
+        status_map = {
+            'data': {
+                'results': []
+            },
+            'message': 'Status of all Deployment Services'
+        }
+        for service in services:
+            status_map['data']['results'].append({
+                'service_id':
+                service['niceId'],
+                'name':
+                service['name'],
+                'status':
+                service['status']
+            })
+        return status_map
+
+    def validate_deployment(self, output_file: str = None):
+        """ Test rendering sermos.yaml and validate.
+        """
+        # Running this will raise an exception if something is invalid.
+        self._set_deploy_payload()
+
+        if output_file:
+            with open(output_file, 'w') as f:
+                f.write(self.sermos_yaml)
+
+        # Additional validation based on logical requirements (validation of
+        # the schemaa is handled by Marshmallow but there can be 'valid' entries
+        # that would result in a failed deployment, such as an invalid
+        # application under appConfig.appPath, for example)
+
+        return True
+
+    def invoke_deployment(self):
+        """ Invoke a Sermos AI Deployment
+
+            If no commit_hash was provided, use the "current" commit hash
+            of the client package during this invocation.
+
+            Required convention is that your client's python package
+            version number is specified in the file `my_package/__init__.py`
+            and is defined as a string assigned to the variable `__version__`,
+            e.g. `__version__ = '0.1.0'`
+        """
+        self._set_deploy_payload()
+
+        # Make request to your environment's endpoint
+        return self.post(self.deploy_url, self.deploy_payload)

pypeline/extensions.py

@@ -0,0 +1,16 @@
+""" Initialize most extensions used throughout application
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    # Client packages *should* provide a `sermos.yaml` file. This
+    # loads the configuration file with the provided name ofthe client
+    # package (e.g. sermos_demo_client)
+    from pypeline.sermos_yaml import load_client_config_and_version
+    sermos_config, sermos_client_version = load_client_config_and_version()
+except Exception as e:
+    sermos_config = None
+    sermos_client_version = None
+    logger.warning("Unable to load client Sermos config ... {}".format(e))

pypeline/flask/__init__.py

@@ -0,0 +1,28 @@
+""" Sermos' Flask Implementation and Tooling. Convenience imports here.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    from rho_web.smorest import Blueprint, Api
+    from rho_web.response import abort
+except Exception as e:
+    logger.error("Unable to import Web services (Blueprint, API, abort)"
+                 f" ... {e}")
+
+try:
+    from flask_rhoauth import OpenIDConnect
+except Exception as e:
+    oidc = None
+    OpenIDConnect = None
+    logging.info("Did not initialize oidc ... {}".format(e))
+else:
+    if OpenIDConnect is not None:
+        oidc = OpenIDConnect()
+
+try:
+    from pypeline.flask.flask_sermos import FlaskSermos
+except Exception as e:
+    logger.exception("Unable to import Sermos services (FlaskSermos)"
+                     f" ... {e}")