scalable-pypeline 1.1.5__py2.py3-none-any.whl → 1.2.1__py2.py3-none-any.whl

pypeline/sermos_yaml.py

@@ -3,59 +3,11 @@ managed deployments through Sermos.ai. If self-hosting, safely disregard this
 yaml format, no `sermos.yaml` is required for your application.
 
 If using, a basic file may look like::
-
-    imageConfig:
-        - name: base-image
-            installCommand: sermos-demo-client[core]
-        - name: public-api-image
-            repositoryUrl: myregistry/public-api-image
-
-    environmentVariables:
-        - name: GLOBAL_ENV_VAR
-            value: globally-available-env-var
-
     serviceConfig:
-        - name: base-worker
-          serviceType: celery-worker
-          imageName: base-image
-          queue: default-queue
-          registeredTasks:
-              - handler: sermos_demo_client.workers.demo_worker.demo_task
-
-
-    imageConfig:
-        - name: base-image
-            installCommand: sermos-demo-client
-            sourceSshUrl: git@gitlab.com:sermos/sermos-demo-client.git
-            baseImage: registry.gitlab.com/sermos/sermos-tools:0.3.0
-
-    serviceConfig:
-        - name: demo-api
-            serviceType: external
-            serviceId: ${SERVICE_ID_API}  # Rendered using `sermos deploy` if available in the environment.
-            imageName: base-image
-            command: gunicorn --log-level info -k gevent -b 0.0.0.0:5000 sermos_demo_client.app:create_app()
-            port: 5000
-            replicaCount: 1
-            cpuLimit: 0.5
-            memoryLimit: 0.5
-            environmentVariables:
-            - name: FLASK_SECRET_KEY
-                value: ${FLASK_SECRET_KEY}
         - name: sermos-worker
-            serviceType: celery-worker
-            serviceId: ${SERVICE_ID_WORKER}
-            imageName: base-image
-            command: celery -A sermos_demo_client.celery worker --without-gossip --without-mingle -c '1' -l INFO --queue default-task-queue
-            replicaCount: 1
-            cpuLimit: 0.5
-            memoryLimit: 0.5
             registeredTasks:
                 - handler: sermos_demo_client.workers.demo_worker.demo_worker_task
                 - handler: sermos_demo_client.workers.demo_worker.demo_model_task
-            environmentVariables:
-            - name: WORKER_NAME
-                value: sermos-worker
 
     pipelines:
         demo-pipeline:
@@ -124,77 +76,11 @@ class MissingSermosConfig(Exception):
     pass
 
 
-class InvalidImageConfig(Exception):
-    pass
-
-
 class ExcludeUnknownSchema(Schema):
     class Meta:
         unknown = EXCLUDE
 
 
-class EnvironmentVariableSchema(ExcludeUnknownSchema):
-    """ A single environment variables (singular)
-    """
-    name = fields.String(required=True,
-                         description="Environment variable name.",
-                         example="MY_ENV_VAR")
-    value = fields.String(required=True,
-                          description="Environment variable value.",
-                          example="my special value")
-
-
-class EnvironmentVariablesSchema(Schema):
-    """ Multiple environment variables (plural)
-    """
-    environmentVariables = fields.List(
-        fields.Nested(EnvironmentVariableSchema, required=True),
-        description="List of name/value environment variable pairs available "
-        "to the scope of this service.",
-        required=False)
-
-
-class ServiceRequestsSchema(Schema):
-
-    replicaCount = fields.Integer(
-        required=False,
-        description="Baseline (min) scale of this service to have available.",
-        default=1,
-        example=1)
-    cpuRequest = fields.Float(
-        required=False,
-        description="Requested CPUs to be available for each replica.",
-        default=0.5,
-        example="0.5")
-    memoryRequest = fields.Float(
-        required=False,
-        description="Requested memory (in GB) to be available for each replica.",
-        default=0.5,
-        example="0.5 (means half of 1 GB)")
-    ephemeralStorageRequest = fields.Float(
-        required=False,
-        description="Requested ephemeral storage (in GB) to "
-                    "be available for each replica.",
-        default=8,
-        example="2 (means 2 GB)")
-    cpuLimit = fields.Float(
-        required=False,
-        description="Maximum CPUs to be available for each replica.",
-        default=0.5,
-        example="0.5")
-    memoryLimit = fields.Float(
-        required=False,
-        description="Maximum memory (in GB) to be available for each replica.",
-        default=0.5,
-        example="0.5 (means half of 1 GB)")
-    ephemeralStorageLimit = fields.Float(
-        required=False,
-        description="Maximum ephemeral storage (in GB) to "
-                    "be available for each replica.",
-        default=8,
-        example="2 (means 2 GB)")
-
-
 class NameSchema(Schema):
     """ Validated name string field.
     """
@@ -216,98 +102,6 @@ class NameSchema(Schema):
         return item
 
 
-class SermosImageConfigSchema(NameSchema):
-    installCommand = fields.String(
-        required=False,
-        description="The pip install command to use when Sermos is "
-        "responsible for building your image.",
-        example="sermos-client-pkg[core,special_feature]")
-    sourceSshUrl = fields.String(
-        required=False,
-        description="The source code ssh url to use when Sermos is "
-        "responsible for building your image.",
-        example="git@github.com:myorg/sermos-client.git")
-    baseImage = fields.String(
-        required=False,
-        description="The Docker base image to use as the starting point when "
-        "Sermos is responsible for building your image",
-        example="rhoai/sermos:latest")
-    repositoryUrl = fields.String(
-        required=False,
-        description="Deprecated - Use imageUri instead.")
-    repositoryUser = fields.String(
-        required=False,
-        description="Deprecated - Use registryUser instead.",)
-    repositoryPassword = fields.String(
-        required=False,
-        description="Deprecated - Use registryPassword instead.")
-    imageUri = fields.String(
-        required=False,
-        description="The Docker image uri when using an image not built by "
-        "Sermos. Tag is optional, if excluded, `latest` is used.",
-        example="rhoai/custom-image ; rhoai/custom-image:v0.0.0")
-    registryDomain = fields.String(
-        required=False,
-        description="Registry domain for using a pre-built image.",
-        example="index.docker.io")
-    registryUser = fields.String(
-        required=False,
-        description="Optional registry username if provided registryDomain"
-        "is a private registry.",
-        example="rhoai")
-    registryPassword = fields.String(
-        required=False,
-        description="Optional registry password if provided registryDomain "
-        "is a private registry. NOTE: Strongly recommended to use environment "
-        "variable interpolation in your sermos.yaml file, do not commit "
-        "unencrypted secrets to a git repository. "
-        "e.g. repositoryPassword: ${DOCKER_REPOSITORY_PASSWORD}",
-        example="abc123")
-
-
-class SermosSharedConfigSchema(Schema):
-    """ Attributes shared across internal and external service types
-    """
-    command = fields.String(
-        required=False,
-        _required=True,
-        description="Command to be run as container CMD.",
-        example="gunicorn -b 0.0.0.0:5000 package.app:create_app()")
-    port = fields.Integer(
-        required=False,
-        _required=True,
-        description="Port (and targetPort) to direct traffic.",
-        example=5000)
-
-
-class SermosExternalConfigSchema(SermosSharedConfigSchema):
-    """ Attributes required for serviceType: external
-
-        Note: Schema lists these are not required in order for this to be used
-        as a mixin to SermosServiceConfigSchema. The validation is done
-        programmatically based on serviceType.
-    """
-    serviceId = fields.String(
-        required=False,
-        _required=True,
-        description="The serviceId provided by Sermos. Find in admin console.",
-        example="dry-gorge-8018")
-
-
-class SermosInternalSchema(SermosSharedConfigSchema):
-    """ Attributes required for serviceType: internal
-
-        Note: Schema lists these are not required in order for this to be used
-        as a mixin to SermosServiceConfigSchema. The validation is done
-        programmatically based on serviceType.
-    """
-    protocol = fields.String(required=False,
-                             _required=True,
-                             description="Protocol to use.",
-                             example="TCP",
-                             validate=OneOf(['TCP', 'UDP']))
-
-
 class SermosRegisteredTaskDetailConfigSchema(Schema):
     handler = fields.String(
         required=True,
@@ -321,18 +115,9 @@ class SermosRegisteredTaskDetailConfigSchema(Schema):
 
 
 class SermosCeleryWorkerConfigSchema(Schema):
-    """ Attributes required for serviceType: celery-worker
-
-        Note: Schema lists these are not required in order for this to be used
-        as a mixin to SermosServiceConfigSchema. The validation is done
-        programmatically based on serviceType.
+    """ Attributes for a celery worker.  This worker will run all of the
+        pipelines and scheduled tasks.
     """
-    command = fields.String(
-        required=False,
-        _required=True,
-        description="Command to be run as container CMD.",
-        example="celery -A mypkg.celery.celery worker --queue my-queue")
-
     registeredTasks = fields.List(
         fields.Nested(SermosRegisteredTaskDetailConfigSchema, required=True),
         required=False,
@@ -341,47 +126,17 @@ class SermosCeleryWorkerConfigSchema(Schema):
     )
 
 
-# Mapping of serviceType keys to their respective schema
-service_types = {
-    'external': SermosExternalConfigSchema,
-    'internal': SermosInternalSchema,
-    'celery-worker': SermosCeleryWorkerConfigSchema
-}
-
-
-class SermosServiceConfigSchema(ExcludeUnknownSchema, ServiceRequestsSchema,
-                                EnvironmentVariablesSchema,
-                                SermosExternalConfigSchema,
-                                SermosInternalSchema,
+class SermosServiceConfigSchema(ExcludeUnknownSchema,
                                 SermosCeleryWorkerConfigSchema, NameSchema):
-    """ Base service config object definition for workers and internal/external
-        services.
+    """ Base service config object definition for workers.
     """
-    imageName = fields.String(
-        required=True,
-        description="Specify the name of the base image to use for this "
-        "service.",
-        example="custom-worker-name")
-
-    serviceType = fields.String(required=True,
-                                description="Name of the worker.",
-                                example="useful_worker",
-                                validate=OneOf(service_types.keys()))
+    pass
 
 
-class SermosYamlSchema(ExcludeUnknownSchema, EnvironmentVariablesSchema):
+class SermosYamlSchema(ExcludeUnknownSchema):
     """ The primary `sermos.yaml` file schema. This defines all available
         properties in a valid Sermos configuration file.
     """
-
-    imageConfig = fields.List(fields.Nested(SermosImageConfigSchema,
-                                            required=True),
-                              required=True,
-                              description="List of available base images. At "
-                              "least one image must be defined. The 'name' "
-                              "of the image is used in each service defined "
-                              "in `serviceConfig`")
-
     serviceConfig = fields.List(
         fields.Nested(SermosServiceConfigSchema,
                       required=True,
@@ -414,17 +169,12 @@ class SermosYamlSchema(ExcludeUnknownSchema, EnvironmentVariablesSchema):
             Nested fields that are not required are not validated by Marshmallow
             by default. Do a single level down of validation for now.
 
-            Each serviceType has attributes that are required but are listed
-            as not required in the marshmallow schema. Validate here.
-
             imageConfig can provide *either* an install command for Sermos
             to use to build the image for customer *or* a Docker repository
             for Sermos to pull.
         """
         # Vaidate nested
         key_schema_pairs = (
-            ('imageConfig', SermosImageConfigSchema),
-            ('environmentVariables', EnvironmentVariableSchema),
             ('serviceConfig', SermosServiceConfigSchema),
         )
         for k_s in key_schema_pairs:
@@ -440,10 +190,8 @@ class SermosYamlSchema(ExcludeUnknownSchema, EnvironmentVariablesSchema):
         # required in order to use them as mixins for a generic service object,
         # however, they ARE required, so validate here using the custom
         # metadata property `_required`. Default to value of `required`.
-        service_image_names = []
         for service in data.get('serviceConfig'):
-            service_type = service['serviceType']
-            schema = service_types[service_type]
+            schema = SermosCeleryWorkerConfigSchema
             for field in schema().fields:
                 try:
                     if schema().fields[field].metadata.get(
@@ -452,43 +200,7 @@ class SermosYamlSchema(ExcludeUnknownSchema, EnvironmentVariablesSchema):
                         assert field in service
                 except AssertionError:
                     raise ValidationError(
-                        f"`{field}` missing in {service_type} definition.")
-
-            service_image_names.append(service['imageName'])
-
-        # Validate imageConfig
-        image_names = []
-        for image in data.get('imageConfig'):
-            image_names.append(image['name'])
-            try:
-                if image.get('installCommand', None) is not None:
-                    assert image.get('repositoryUrl', None) is None
-                if image.get('repositoryUrl', None) is not None:
-                    assert image.get('installCommand', None) is None
-            except AssertionError:
-                raise InvalidImageConfig(
-                    "Each imageConfig may have *either* an installCommand "
-                    "*or* a repositoryUrl but not both. Review "
-                    f"image `{image['name']}`")
-
-            if image.get('installCommand', None) is not None:
-                if image.get('sourceSshUrl') is None:
-                    raise InvalidImageConfig(
-                        "Each imageConfig must have a sourceSshUrl if "
-                        "installCommand is specified")
-                if image.get('baseImage') is None:
-                    raise InvalidImageConfig(
-                        "Each imageConfig must have a baseImage if "
-                        "installCommand is specified")
-
-        # Verify each imageName referenced in each service exists in imageConfig
-        try:
-            assert set(service_image_names).issubset(image_names)
-        except AssertionError:
-            raise InvalidImageConfig(
-                "Mismatched imageName in at least one "
-                f"service ({service_image_names}) compared to images available "
-                f"in imageConfig ({image_names})")
+                        f"`{field}` missing in worker definition.")
 
         # Validate unique pipeline ids
         if 'pipelines' in data:
@@ -612,11 +324,6 @@ def parse_config_file(sermos_yaml: str):
             .format(e.messages)
         logger.error(msg)
         raise InvalidSermosConfig(msg)
-    except InvalidImageConfig as e:
-        msg = "Invalid imageConfig configuration due to {}"\
-            .format(e)
-        logger.error(msg)
-        raise InvalidImageConfig(msg)
     except Exception as e:
         msg = "Invalid Sermos configuration, likely due to invalid "\
             "YAML formatting ..."
@@ -680,8 +387,6 @@ def load_sermos_config(pkg_name: str = None,
             sermos_config = parse_config_file(sermos_yaml)
     except InvalidSermosConfig as e:
         raise
-    except InvalidImageConfig as e:
-        raise
     except FileNotFoundError as e:
         msg = "Sermos config file could not be found at path {} ...".format(
             sermos_config_path)