scalable-pypeline 1.1.0__py2.py3-none-any.whl

pypeline/pipeline_config_schema.py

@@ -0,0 +1,197 @@
+""" Schemas for Pipelines
+
+TODO: Add validation that all specified nodes in DAG have corresponding
+node in taskDefinitions
+"""
+import yaml
+from marshmallow import Schema, fields, EXCLUDE, validates_schema
+from marshmallow.exceptions import ValidationError
+
+
+class ExcludeUnknownSchema(Schema):
+    """ Remove unknown keys from loaded dictionary
+    """
+    class Meta:
+        """ Exclude unknown properties.
+        """
+        unknown = EXCLUDE
+
+
+class MetadataSchema(Schema):
+    """ Schema for a pipeline's metadata object.
+    """
+    queue = fields.String(required=True,
+                          description="Default queue for all pipeline tasks.",
+                          example="default-queue-name")
+
+    maxRetry = fields.Integer(
+        required=False,
+        description="Max number of retries for a pipeline.",
+        default=3,
+        example=3)
+
+    maxTtl = fields.Integer(required=False,
+                            description="Max TTL for a pipeline in seconds.",
+                            default=60,
+                            example=60)
+
+
+class TaskDefinitionsSchema(ExcludeUnknownSchema):
+    """ Schema for a single task's configuration
+    """
+    handler = fields.String(required=True,
+                            description="Path to the worker task definition",
+                            example="client.workers.my_task")
+
+    maxTtl = fields.Integer(required=False,
+                            description="Max TTL for a task in seconds.",
+                            default=60,
+                            example=60)
+
+    queue = fields.String(required=False,
+                          description="Non-default queue for this task.",
+                          example="custom-queue-name")
+    # payload_args = fields.List(
+    #     fields.Dict(keys=fields.String(),
+    #                    values=fields.Nested(PayloadArgKwargSchema)))
+    # payload_kwargs = fields.List(
+    #     fields.Dict(keys=fields.String(),
+    #                    values=fields.Nested(PayloadArgKwargSchema)))
+    # model_version = fields.String()
+    # arbitrary other stuff passed to task?
+
+
+class PipelineConfigSchemaV1(Schema):
+    """ Overall pipeline configuration schema
+    """
+    metadata = fields.Nested(
+        MetadataSchema,
+        required=True,
+        description="Metadata and configuration information for this pipeline."
+    )
+    dagAdjacency = fields.Dict(
+        keys=fields.String(
+            required=True,
+            description=
+            "Task's node name. *MUST* match key in taskDefinitions dict.",
+            example="node_a"),
+        values=fields.List(
+            fields.String(
+                required=True,
+                description=
+                "Task's node name. *Must* match key in taskDefinitions dict.")
+        ),
+        required=True,
+        description="The DAG Adjacency definition.")
+    taskDefinitions = fields.Dict(
+        keys=fields.String(
+            required=True,
+            description=
+            "Task's node name. *Must* match related key in dagAdjacency.",
+            example="node_a"),
+        values=fields.Nested(
+            TaskDefinitionsSchema,
+            required=True,
+            description="Definition of each task in the pipeline.",
+            example={
+                'handler': 'abc.task',
+                'maxRetry': 1
+            }),
+        required=True,
+        description="Configuration for each node defined in DAG.")
+
+
+class BasePipelineSchema(ExcludeUnknownSchema):
+    __schema_version__ = None
+
+    name = fields.String(required=True, description="Pipeline name")
+    description = fields.String(required=False, missing=None,
+                                description="Description of the pipeline.",
+                                example="A valuable pipeline.")
+    schemaVersion = fields.Integer(required=True)
+    config = fields.Dict(required=True)
+
+    @classmethod
+    def get_by_version(cls, version):
+        for subclass in cls.__subclasses__():
+            if subclass.__schema_version__ == version:
+                return subclass
+
+        return None
+
+    @classmethod
+    def get_latest(cls):
+        max_version = 0
+        max_class = None
+        for subclass in cls.__subclasses__():
+            if subclass.__schema_version__ > max_version:
+                max_version = max_version
+                max_class = subclass
+
+        return max_class
+
+    @validates_schema
+    def validate_pipeline(self, data, **kwargs):
+        schema_version = data['schemaVersion']
+        PipelineSchema = BasePipelineSchema.get_by_version(schema_version)
+        schema = PipelineSchema(exclude=['name', 'description'])
+        schema.load(data)
+
+class PipelineSchemaV1(BasePipelineSchema):
+
+    __schema_version__ = 1
+    class Meta:
+        unknown = EXCLUDE
+
+    config = fields.Nested(
+        PipelineConfigSchemaV1,
+        required=True,
+        description="Metadata and configuration information for this pipeline."
+    )
+
+    def validate_pipeline(self, data, **kwargs):
+        # We need to add this function to avoid infinite recursion since
+        # the BasePipelineSchema class above uses the same method for
+        # validation
+        pass
+
+
+class PipelineConfigValidator(object):
+    """ Validate a pipeline configuration.
+
+        This is stored as a string in the database under `PipelineConfig.config`
+        in order to keep it easy for custom features to be added over time.
+        This model represents the required / valid features so we can
+        programmatically validate when saving, updating, viewing.
+    """
+    def __init__(self, config_dict: dict = None, config_yaml: str = None,
+                 schema_version: int = None):
+        super().__init__()
+
+        # We validate this as a dictionary. Turn into dictionary if provided
+        # as yaml.
+        if config_dict is not None:
+            self.config = config_dict
+        elif config_yaml is not None:
+            self.config = yaml.safe_load(config_yaml)
+
+        if schema_version is None:
+            PipelineSchema = BasePipelineSchema.get_latest()
+        else:
+            PipelineSchema = BasePipelineSchema.get_by_version(schema_version)
+            
+
+        self.is_valid = False
+        self.validated_config = {}
+        self.validation_errors = {}
+        try:
+            # https://github.com/marshmallow-code/marshmallow/issues/377
+            # See issue above when migrating to marshmallow 3
+            pcs = PipelineSchema._declared_fields['config'].schema
+            self.validated_config = pcs.load(self.config)
+            self.is_valid = True
+        except ValidationError as e:
+            self.validation_errors = e.messages
+            raise e
+        except Exception as e:
+            raise e

pypeline/schedule_config_schema.py

@@ -0,0 +1,210 @@
+""" Schemas for Schedule Configuration
+"""
+import re
+from celery.schedules import crontab_parser
+from croniter import croniter
+from marshmallow.validate import OneOf
+from marshmallow.exceptions import ValidationError
+from marshmallow import Schema, fields, EXCLUDE, pre_load, validates_schema
+
+
+class ExcludeUnknownSchema(Schema):
+    """ Remove unknown keys from loaded dictionary
+
+    # TODO this seems to be just ignoring and letting through vs excluding...
+    """
+    class Meta:
+        unknown = EXCLUDE
+
+
+class IntervalScheduleSchema(Schema):
+    every = fields.Integer(required=True)
+    period = fields.String(
+        required=True,
+        validate=OneOf(['microseconds', 'seconds', 'minutes', 'hours',
+                        'days']))
+
+
+class CrontabScheduleSchema(Schema):
+    minute = fields.String(required=True)
+    hour = fields.String(required=True)
+    dayOfWeek = fields.String(required=True)
+    dayOfMonth = fields.String(required=True)
+    monthOfYear = fields.String(required=True)
+
+    @validates_schema
+    def validate_values(self, data, **kwargs):
+        if data['minute'] is None or data['hour'] is None or \
+            data['dayOfWeek'] is None or data['dayOfMonth'] is None or\
+                data['monthOfYear'] is None:
+            raise ValidationError("Empty crontab value")
+
+        test_cron_expression = \
+            f"{data['minute']} {data['hour']} {data['dayOfMonth']} " \
+            f"{data['monthOfYear']} {data['dayOfWeek']}"
+
+        if not croniter.is_valid(test_cron_expression):
+            return ValidationError("Invalid crontab value")
+
+
+class Schedule(fields.Dict):
+    def _serialize(self, value, attr, obj, **kwargs):
+        return value
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        if data['scheduleType'] == 'crontab':
+            schema = CrontabScheduleSchema()
+        else:
+            schema = IntervalScheduleSchema()
+        return schema.load(value)
+
+
+class ScheduleConfigSchemaV1(ExcludeUnknownSchema):
+    """ Definition of a single schedule entry
+
+    TODO: Add validation based on schedule_type and the relevant optional fields
+    TODO: Add validation that each name is unique
+    """
+
+    scheduleType = fields.String(
+        required=True,
+        validate=OneOf(['interval', 'crontab']),
+        description="The Celery schedule type of this entry.",
+        example="interval",
+        data_key='scheduleType')
+
+    queue = fields.String(required=True,
+                          description="Name of queue on which to place task.",
+                          example="my-default-queue")
+    task = fields.String(required=True,
+                         description="Path to task to invoke.",
+                         example="my_app.module.method")
+    exchange = fields.String(
+        required=False,
+        description="Exchange for the task. Celery default "
+        "used if not set, which is recommended.",
+        example="tasks")
+    routing_key = fields.String(
+        required=False,
+        description="Routing key for the task. Celery "
+        "default used if not set, which is recommended.",
+        example="task.default",
+        data_key='routingKey')
+    expires = fields.Integer(
+        required=False,
+        description="Number of seconds after which task "
+        "expires if not executed. Default: no expiration.",
+        example=60)
+
+    schedule = Schedule(required=True)
+
+    @pre_load
+    def validate_string_fields(self, item, **kwargs):
+        """ Ensure string fields with no OneOf validation conform to patterns
+        """
+        if item is None:
+            raise ValidationError("NoneType provided, check input.")
+
+        validation_map = {
+            'name': r'^[\w\d\-\_\.\s]+$',
+            'queue': r'^[\w\d\-\_\.]+$',
+            'task': r'^[\w\d\-\_\.]+$',
+            'exchange': r'^[\w\d\-\_\.]+$',
+            'routing_key': r'^[\w\d\-\_\.]+$'
+        }
+        for field in validation_map:
+            if item.get(field, None) is None:
+                continue
+            if not bool(re.match(validation_map[field], item[field])):
+                raise ValidationError(
+                    f"Invalid {field}: `{item[field]}``. Must match pattern: "
+                    f"{validation_map[field]}")
+
+        if 'scheduleType' not in item:
+            raise ValidationError('Missing required field scheduleType')
+
+        if item['scheduleType'] == 'crontab':
+            cron_validation_map = {
+                'minute': crontab_parser(60),
+                'hour': crontab_parser(24),
+                'dayOfWeek': crontab_parser(7),
+                'dayOfMonth': crontab_parser(31, 1),
+                'monthOfYear': crontab_parser(12, 1)
+            }
+
+            for field in cron_validation_map:
+                try:
+                    cron_validation_map[field].parse(item['schedule'][field])
+                except:
+                    raise ValidationError(
+                        f"Invalid {field}: `{item['schedule'][field]}`. Must "
+                        "be valid crontab pattern.")
+
+        return item
+
+
+class BaseScheduleSchema(ExcludeUnknownSchema):
+    __schema_version__ = 0
+
+    name = fields.String(required=True,
+                         description="Name of schedule entry.",
+                         example="My Scheduled Task")
+    schemaVersion = fields.Integer(required=True)
+    config = fields.Dict(required=True)
+    enabled = fields.Boolean(required=True,
+                             description="Whether entry is enabled.",
+                             example=True)
+    # TODO Figure out where that wonky timestamp format is coming from and
+    # update this and in celery_beat.py.
+    lastRunAt = fields.DateTime(allow_none=True,
+                                missing=None,
+                                description="Timestamp of last run time.",
+                                example="Tue, 18 Aug 2020 01:36:06 GMT",
+                                data_key='lastRunAt')
+    totalRunCount = fields.Integer(
+        allow_none=True,
+        missing=0,
+        description="Count of number of executions.",
+        example=12345,
+        data_key='totalRunCount')
+
+    @classmethod
+    def get_by_version(cls, version):
+        for subclass in cls.__subclasses__():
+            if subclass.__schema_version__ == version:
+                return subclass
+
+        return None
+
+    @classmethod
+    def get_latest(cls):
+        max_version = 0
+        max_class = None
+        for subclass in cls.__subclasses__():
+            if subclass.__schema_version__ > max_version:
+                max_version = max_version
+                max_class = subclass
+
+        return max_class
+
+    @validates_schema
+    def validate_scheduled_tasks(self, data, **kwargs):
+        schema_version = data['schemaVersion']
+        TaskSchema = BaseScheduleSchema.get_by_version(schema_version)
+        schema = TaskSchema()
+        schema.load(data)
+
+
+class ScheduleSchemaV1(BaseScheduleSchema):
+    __schema_version__ = 1
+
+    config = fields.Nested(
+        ScheduleConfigSchemaV1,
+        required=True,
+        description="Configuration information for this schedule.")
+
+    def validate_scheduled_tasks(self, data, **kwargs):
+        # We need to add this function to avoid infinite recursion since
+        # the BaseScheduleSchema class above uses the same method for
+        # validation
+        pass