scalable-pypeline 1.2.3__py2.py3-none-any.whl → 2.0.2__py2.py3-none-any.whl

pypeline/celery_beat.py

@@ -1,254 +0,0 @@
-""" Custom Sermos Scheduler and Celery Entry classes used for dynamic beat.
-"""
-import datetime
-import os
-import logging
-from rhodb.redis_conf import RedisConnector
-from celery.beat import Scheduler, ScheduleEntry
-from celery import current_app
-from celery.utils.time import is_naive
-from celery.schedules import schedule as c_schedule, crontab as c_crontab
-from pypeline.utils.config_utils import retrieve_latest_schedule_config, \
-    update_schedule_config
-from pypeline.constants import CONFIG_REFRESH_RATE, SCHEDULE_DATE_FORMAT, \
-    USING_SERMOS_CLOUD
-
-logger = logging.getLogger(__name__)
-redis_conn = RedisConnector().get_connection()
-
-
-def convert_to_datetime(
-        datetime_str: str,
-        datetime_format: str = SCHEDULE_DATE_FORMAT) -> datetime.datetime:
-    """ Accept a string in the standard format and return a datetime object
-    """
-    return datetime.datetime.strptime(datetime_str, datetime_format)
-
-
-def instantiate_celery_schedule(schedule_entry: dict) -> c_schedule:
-    """ From a schedule entry and the full schedule from Sermos, create a
-        celery `schedule` object.
-    """
-    scheduleType = schedule_entry['config']['scheduleType']
-
-    if scheduleType == 'interval':
-        # Create a timedelta object
-        period = schedule_entry['config']['schedule']['period']
-        every = schedule_entry['config']['schedule']['every']
-        the_delta = datetime.timedelta(**{period: every})
-        # Instantiate the celery schedule object
-        return c_schedule(run_every=the_delta)
-
-    if scheduleType == 'crontab':
-        return c_crontab(
-            minute=schedule_entry['config']['schedule']['minute'],
-            hour=schedule_entry['config']['schedule']['hour'],
-            day_of_week=schedule_entry['config']['schedule']['dayOfWeek'],
-            day_of_month=schedule_entry['config']['schedule']['dayOfMonth'],
-            month_of_year=schedule_entry['config']['schedule']['monthOfYear'])
-
-    raise ValueError(f"Unsupported scheduleType ({scheduleType} ...")
-
-
-class SermosEntry(ScheduleEntry):
-    """ Create a beat entry with additional functionality for Sermos scheduler.
-
-        https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html
-    """
-    def __init__(self, schedule_entry: dict = None, **kwargs):
-        schedule_entry = schedule_entry if schedule_entry else {}
-        if schedule_entry:
-            # This event is being instantiated directly with the Sermos
-            # schedule entry
-            celery_schedule = instantiate_celery_schedule(schedule_entry)
-
-            # celery.beat.ScheduleEntry expects these keys in a dictionary
-            # called `options`. See
-            # https://docs.celeryproject.org/en/stable/userguide/calling.html
-            # In the case of Sermos, we require the queue in the
-            # ScheduleEntrySchema, others are all optional.
-            options = dict()
-            optional_keys = ('queue', 'exchange', 'routing_key', 'expires')
-            for key in optional_keys:
-                value = schedule_entry['config'].get(key, None)
-                if value is not None:
-                    options[key] = value
-
-            last_run_at = schedule_entry.get('lastRunAt')
-            if last_run_at is None:
-                last_run_at = current_app.now()
-                schedule_entry['lastRunAt'] = last_run_at
-            if isinstance(schedule_entry['lastRunAt'], str):
-                last_run_at = convert_to_datetime(
-                    schedule_entry['lastRunAt'])
-
-            # Verify times are accurate
-            orig = last_run_at
-            if not is_naive(last_run_at):
-                last_run_at = last_run_at.replace(tzinfo=None)
-            assert orig.hour == last_run_at.hour  # timezone sanity
-
-            if USING_SERMOS_CLOUD:
-                # We need to keep track of the id because this used to send
-                # updates to sermos cloud.  The name can't be concatenated
-                # with the sermos id or else it will be created as duplicate
-                # celery beat task.
-                name = schedule_entry['name']
-                self.sermos_id = schedule_entry['id']
-            else:
-                name = schedule_entry['name']
-
-            super().__init__(app=current_app._get_current_object(),
-                             name=name,
-                             task=schedule_entry['config']['task'],
-                             args=schedule_entry.get('args', None),
-                             kwargs=schedule_entry.get('kwargs', None),
-                             options=options,
-                             schedule=celery_schedule,
-                             last_run_at=last_run_at,
-                             total_run_count=schedule_entry.get(
-                                 'totalRunCount', 0))
-        else:
-            # This is a task issued directly by celery's scheduler so won't
-            # have the schedule_entry argument. Still not entirely clear why
-            # this is seen.  Pop the id before initializing the super class.
-            # Add it back after so we can keep sermos up to date w/ config.
-            if USING_SERMOS_CLOUD:
-                sermos_id = kwargs.pop('sermos_id')
-                super().__init__(**kwargs)
-                self.sermos_id = sermos_id
-            else:
-                super().__init__(**kwargs)
-
-        # Ensure all events have 'event' key - this is populated by ChainedTask
-        if 'event' not in self.kwargs.keys():
-            self.kwargs['event'] = {}
-
-
-class SermosScheduler(Scheduler):
-    """ Sermos' implementation of a Celery Scheduler. Leverages a Sermos
-    configuration server to provide the up-to-date schedule and provides to
-    this scheduler for in-memory tracking.
-    """
-    Entry = SermosEntry
-    _last_refresh = None  # Internal time keeper for Sermos syncing
-    _refresh_rate = CONFIG_REFRESH_RATE * 1000000  # Turn to microseconds
-    _schedule = None  # Holds latest Celery schedule with only enabled tasks
-    _schedule_full = None  # Holds latest schedule, regardless of enabled
-    _initial_read = True  # Set to False upon initial bootstrapping
-
-    def __init__(self, *args, **kwargs):
-        logger.info("Initializing SermosScheduler ...")
-        # This step ensures the latest schedule is pulled from Sermos/cache
-        # and bootstraps the local time checker we use.
-        self.set_under_schedule()
-        self._last_refresh = datetime.datetime.utcnow()
-
-        # Default 60 second max interval here so our schedule is always
-        # forced to be up to date.
-        max_interval = int(
-            os.environ.get('CELERY_BEAT_SYNC_MAX_INTERVAL',
-                           CONFIG_REFRESH_RATE))
-        kwargs['max_interval'] = max_interval
-
-        kwargs['schedule'] = self._schedule
-        Scheduler.__init__(self, *args, **kwargs)
-
-    def set_under_schedule(self):
-        """ Parse the latest schedule config and set self._schedule with parsed
-        schedule including only those that are enabled.
-        """
-        s = {}
-        s_full = []
-        s_full_orig = [s.copy() for s in self._schedule_full
-                       ] if self._schedule_full else []
-        latest_schedule = retrieve_latest_schedule_config()
-        for sched in latest_schedule:
-            s_full.append(sched)  # Append to full list regardless of enabled
-            if sched['enabled']:
-                s[sched['name']] = SermosEntry(sched)
-        self._schedule = s
-        self._schedule_full = s_full
-
-        # Report if schedule changed
-        if self._schedule_full != s_full_orig:
-            logger.info("SermosScheduler: Schedule updated ...")
-            logger.info(f"SermosScheduler: {self._schedule}")
-
-    def get_current_sermos_schedule(self):
-        """ Unpack Celery's current representation of the schedule into Sermos
-        format. This is used to send updates back to Sermos related to dynamic
-        properties such as last_run_at and total_run_count.
-        """
-
-        sched = {'schedules': []}
-        for entry_name, entry in self.schedule.items():
-            sched['schedules'].append({
-                'id': entry.sermos_id,
-                'lastRunAt': entry.last_run_at.isoformat(),
-                'totalRunCount': entry.total_run_count
-            })
-
-        return sched
-
-    def setup_schedule(self):
-        self.install_default_entries(self.data)
-        # Overload default behavior and instead bootstrap with our _schedule
-        # instead of app.conf.beat_schedule.
-        self.merge_inplace(self._schedule)
-
-    def should_refresh(self):
-        """ Determine if enough time has elapsed to perform a schedule refresh.
-
-        We turn everything into microseconds so we don't spam external services
-        intra-second as most of the time, more than one task exists in the
-        schedule and therefore we need to check the scheduler's `schedule`
-        on each task very rapidly when issuing tasks.
-        """
-        now = datetime.datetime.utcnow()
-        microseconds_since_last_refresh = float(
-            str((now - self._last_refresh).seconds) + "." +
-            str((now - self._last_refresh).microseconds)) * 1000000
-        res = bool(microseconds_since_last_refresh > self._refresh_rate)
-        if res is True:
-            self._last_refresh = now - datetime.timedelta(milliseconds=1)
-        return res
-
-    def sync(self):
-        """ Sync local schedule with Sermos and update Celery's representation
-        TODO check this vis-a-vis local vs cloud
-        """
-        if self.schedule and USING_SERMOS_CLOUD:
-            update_schedule_config(self.get_current_sermos_schedule())
-        self.set_under_schedule()  # Internal representation
-        self.merge_inplace(self._schedule)  # Celery representation
-
-    def get_schedule(self):
-        """ Overload default Scheduler get_schedule method to check for updates
-
-        Note: Celery uses a property function, e.g.:
-        https://www.tutorialsteacher.com/python/property-function
-        for getting/setting the schedule internally. We only override the
-        get_schedule method here.
-        """
-        update = False
-        if self._initial_read:
-            logger.info('SermosScheduler: Initial read ...')
-            update = True
-            self._initial_read = False
-        elif self.should_refresh():
-            logger.info('SermosScheduler: Refreshing schedule ...')
-            update = True
-
-        if update:
-            self.sync()
-
-        return self._schedule
-
-    def set_schedule(self, schedule):
-        """ Redefine Celery set_schedule method
-        """
-        self.data = schedule
-
-    # Redefine Celery schedule property()
-    schedule = property(get_schedule, set_schedule)

pypeline/flask/api/utils.py

@@ -1,35 +0,0 @@
-""" Utilities for Sermos APIs and interacting with Pipelines/Schedules
-"""
-import logging
-from typing import Union
-
-from pypeline.pipeline.generator import PipelineGenerator
-
-logger = logging.getLogger(__name__)
-
-
-def chain_helper(pipeline_id: str,
-                 access_key: Union[str, None] = None,
-                 chain_payload: Union[dict, None] = None,
-                 queue: Union[str, None] = None,
-                 default_task_ttl: int = None):
-    """ Helper method to generate a pipeline chain *with* error handling.
-
-        Usage:
-          my_chain = chain_helper('pipeline-name')
-          my_chain.delay()
-    """
-    # Get our pipeline. The PipelineGenerator will use the PipelineRunWrapper
-    # to cache this "run" of the pipeline.
-    gen = PipelineGenerator(pipeline_id,
-                            access_key=access_key,
-                            queue=queue,
-                            default_task_ttl=default_task_ttl,
-                            chain_payload=chain_payload)
-    if gen.good_to_go:
-        # Generate our 'chain', which is the grouping of celery constructs that
-        # allows our dag to run asynchronously and synchronously according to
-        # the adjacency list defined in our pipeline configuration.
-        gen.generate_chain()
-
-    return gen

pypeline/flask/flask_sermos.py

@@ -1,156 +0,0 @@
-""" Sermos implementation as a Flask extension
-"""
-import os
-if os.getenv('USE_GEVENT', 'false').lower() == 'true':
-    import gevent.monkey
-    gevent.monkey.patch_all()
-
-import logging
-from flask import Flask
-from werkzeug.middleware.proxy_fix import ProxyFix
-from flask_smorest import  Api, Blueprint
-from pypeline.extensions import sermos_config
-from pypeline.constants import DEFAULT_OPENAPI_CONFIG
-from pypeline import __version__
-
-logger = logging.getLogger(__name__)
-
-
-class FlaskSermos:
-    """ Sermos Flask extension.
-    """
-    def __init__(self, app: Flask = None):
-        """ Class init
-        """
-        self.app = app
-        self.sermos_config = sermos_config if sermos_config is not None else {}
-
-        if app is not None:
-            self.init_app(app)
-
-    def init_app(self, app: Flask, init_api: bool = False):
-        """ Sermos bootstrapping process.
-
-        Application config variables to set include:
-
-            SERMOS_CLIENT_VERSION (default: v?.?.?)
-            SERMOS_HIJACK_ROOT_LOGGER (default: False)
-
-        Optional, if `init_api` is True:
-
-            API_DOCUMENTATION_TITLE
-            API_DOCUMENTATION_DESCRIPTION
-            OPENAPI_VERSION
-            OPENAPI_URL_PREFIX
-            OPENAPI_SWAGGER_APP_NAME
-            OPENAPI_SWAGGER_UI_PATH
-            OPENAPI_SWAGGER_BASE_TEMPLATE
-            OPENAPI_SWAGGER_URL
-            OPENAPI_SWAGGER_UI_URL
-            SWAGGER_UI_DOC_EXPANSION
-            EXPLAIN_TEMPLATE_LOADING
-
-        Args:
-            app (Flask): Flask Application to initialize.
-            init_api (bool): If `True`, Sermos will initialize its
-                core APIs (including Pipelines, Scheduled Tasks, etc.) and
-                provide a pre-configured OpenAPI Spec/Swagger UI interface
-                available at the route defined in your application's config
-                under `OPENAPI_URL_PREFIX` (default `/api`). Refer to
-                [flask-smorest](https://flask-smorest.readthedocs.io/en/latest/openapi.html)
-                documentation for additional configuration options.
-        """
-        # Ensure there's a SERMOS_CLIENT_VERSION on app config
-        app.config.setdefault(
-            'SERMOS_CLIENT_VERSION',
-            app.config.get("SERMOS_CLIENT_VERSION", "v?.?.?"))
-
-        app.wsgi_app = ProxyFix(app.wsgi_app)
-        app.url_map.strict_slashes = False
-
-        # Create and register the sermos blueprint
-        bp = Blueprint('sermos',
-                       __name__,
-                       template_folder='../templates',
-                       static_folder='../static',
-                       url_prefix='/sermos')
-        app.register_blueprint(bp)
-
-        # Bootstrap api if app requests
-        if init_api is True:
-            self._bootstrap_api(app)
-
-    def _bootstrap_api(self, app: Flask):
-        """ If initializing the API, we will create the core Sermos API paths
-            and initialize the default Swagger documentation.
-        """
-        # Set sensible defaults for Swagger docs. Provided `app` will
-        # take precedent.
-        for swagger_config in DEFAULT_OPENAPI_CONFIG:
-            app.config.setdefault(
-                swagger_config[0],
-                app.config.get(swagger_config[0], swagger_config[1]))
-
-        # Attempt to override with values from client's sermos.yaml if
-        # they are available. This will add new tags and new docs if
-        # defined and add to the core Sermos API docs.
-        api_config = self.sermos_config.get('apiConfig', {})
-        api_docs = api_config.get('apiDocumentation', {})
-
-        custom_tags = api_config.get('prefixDescriptions', [])
-
-        app.config['SERMOS_CLIENT_VERSION'] = \
-            api_docs.get('version', None) \
-            if api_docs.get('version', None) is not None \
-            else app.config['SERMOS_CLIENT_VERSION']
-
-        app.config['API_DOCUMENTATION_TITLE'] = \
-            api_docs.get('title', None) \
-            if api_docs.get('title', None) is not None \
-            else app.config['API_DOCUMENTATION_TITLE']
-
-        app.config['API_DOCUMENTATION_DESCRIPTION'] = \
-            api_docs.get('description', None) \
-            if api_docs.get('description', None) is not None \
-            else app.config['API_DOCUMENTATION_DESCRIPTION']
-
-        # Set default Sermos Tags along with custom tags from sermos.yaml
-        tags = [{
-            'name': 'Pipelines',
-            'description': 'Operations related to Pipelines'
-        }, {
-            'name': 'Schedules',
-            'description': 'Operations related to Schedules'
-        }] + custom_tags
-
-        # Set up  the initializing spec kwargs for API
-        spec_kwargs = {
-            'title': app.config['API_DOCUMENTATION_TITLE'],
-            'version': f"Sermos: {__version__} - "
-            f"Client: {app.config['SERMOS_CLIENT_VERSION']}",
-            'description': app.config['API_DOCUMENTATION_DESCRIPTION'],
-            'tags': tags
-        }
-        try:
-            api = Api()
-            api.init_app(app, spec_kwargs=spec_kwargs)
-
-            # Register available Sermos API Namespaces
-            self._register_api_namespaces(api)
-
-        except Exception as e:
-            api = None
-            logging.exception(f"Unable to initialize API ... {e}")
-
-        # Register the Sermos Core API as an extension for use in Client App
-        app.extensions.setdefault('sermos_core_api', api)
-
-    @staticmethod
-    def _register_api_namespaces(api: Api):
-        """ Register Default API namespaces
-            TODO add metrics APIs
-        """
-        from pypeline.flask.api.pipelines import bp as pipelinesbp
-        api.register_blueprint(pipelinesbp)
-        from pypeline.flask.api.schedules import bp as schedulesbp
-        api.register_blueprint(schedulesbp)

pypeline/generators.py

@@ -1,196 +0,0 @@
-import os
-from boto3 import Session
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class KeyGenerator(object):
-    """ Common functions for key generators.
-    """
-    def __init__(self):
-        super(KeyGenerator, self).__init__()
-        self.hidden_files = ('.DS_Store', '.git', 'Icon', '.Dropbox')
-
-    def get_file_key(self, file_obj):
-        """ Required for each specific generator - how to extract key
-        """
-        return file_obj
-
-    def get_file_name(self, file_obj):
-        """ Required for each specific generator - how to extract file name
-        """
-        return file_obj
-
-    def get_file_size(self, base_path, file_obj):
-        """ Required for each specific generator - how to find file size (BYTES)
-        """
-        return 0
-
-    def get_final_path(self, base_path, file_name, return_full_path):
-        """ Required for each specific generator - create final file path that
-            is added to list.
-        """
-        if return_full_path:
-            return os.path.normpath(base_path + '/' + file_name)
-
-        return file_name
-
-    def list_iterator(self, all_files, base_path, limit=None, offset=None,
-                      size_limit=None, return_full_path=True,
-                      skip_common_hidden=True):
-        """ accept vars from everywhere to handle offset/limit/size logic
-        """
-        filtered_files = []
-        try:
-            # Compile list of all files within limit/offset if those exist
-            idx = -1
-            listed_files = 0
-            offset_reached = False
-            for f in all_files:
-                this_key = self.get_file_key(f)
-                this_filename = self.get_file_name(f)
-
-                if skip_common_hidden and this_filename in self.hidden_files:
-                    continue
-
-                idx += 1
-                if offset and idx >= int(offset):
-                    offset_reached = True
-
-                if (limit and listed_files >= int(limit))\
-                        or (offset and not offset_reached):
-                    continue
-
-                # Verify filesize. Having some issues with large PDFs (process
-                # simply killed). So allow option of skipping files above certain
-                # size in megabytes.
-                if size_limit is not None:
-                    size_in_bytes = self.get_file_size(base_path, f)
-                    if size_in_bytes > size_limit:
-                        continue
-
-                filtered_files.append(
-                    self.get_final_path(base_path, this_key, return_full_path)
-                )
-                listed_files += 1
-        except Exception as e:
-            logger.error("Unable to list objects: {0}".format(e))
-
-        return filtered_files
-
-
-class S3KeyGenerator(KeyGenerator):
-    """ Produce a list of object keys from S3.
-    """
-    def __init__(self, aws_access_key_id, aws_secret_access_key,
-                 aws_region='us-east-1'):
-        super(S3KeyGenerator, self).__init__()
-
-        session = Session(
-            aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key,
-            region_name=aws_region
-        )
-        self.s3 = session.client('s3')
-
-    def get_file_key(self, file_obj):
-        """ Get file key from s3 object """
-        return file_obj.get('Key', None)
-
-    def get_file_name(self, file_obj):
-        """ Get file name from s3 object """
-        if file_obj is not None:
-            key = file_obj.get('Key', None)
-            if key is not None:
-                return key.split('/')[-1]
-        return None
-
-    def get_file_size(self, base_path, file_obj):
-        """ Return file size of s3 object """
-        return file_obj.get('Size', 0)
-
-    # All files in bucket
-    # Range of files with an offset
-    def list_files(self, bucket, folder='', limit=None, offset=None,
-                   size_limit=None, return_full_path=True,
-                   skip_common_hidden=True):
-        """ Lists files inside an S3 bucket+folder
-
-            Note: This does not guarantee any sort of order. Boto+S3 does not
-            provide an interface for sorting results, so that would need
-            to happen in memory.
-
-            limit will include a maximum of 'limit' values
-            offset will start including values only after 'offset' keys
-            size_limit will not include files over a specific size (in bytes)
-            skip_common_hidden will exclude common hidden files
-            return_full_path will include 'bucket/' in key.
-        """
-        files = []
-
-        try:
-            file_data = self.s3.list_objects_v2(
-                Bucket=bucket, Delimiter='/', Prefix=folder)
-            files = self.list_iterator(
-                file_data['Contents'],
-                bucket,
-                limit=limit,
-                offset=offset,
-                size_limit=size_limit,
-                return_full_path=return_full_path,
-                skip_common_hidden=skip_common_hidden
-            )
-
-        except Exception as e:
-            logger.error("Unable to list objects: {0}".format(e))
-        return files
-
-
-class LocalKeyGenerator(KeyGenerator):
-    """ Generic generator to produce a list of file names from filesystem.
-    """
-    def __init__(self):
-        super(LocalKeyGenerator, self).__init__()
-
-    def get_file_key(self, file_obj):
-        """ Get file key from local object """
-        return file_obj
-
-    def get_file_name(self, file_obj):
-        """ Get file name from local object """
-        return file_obj
-
-    def get_file_size(self, base_path, file_obj):
-        """ Get file size from local object """
-        full_path = os.path.normpath(base_path + '/' + file_obj)
-        try:
-            return os.stat(full_path).st_size
-        except Exception as e:
-            logger.error("File {0} not found ...".format(full_path))
-        return 0
-
-    def list_files(self, folder_path, limit=None, offset=None,
-                   size_limit=None, return_full_path=True,
-                   skip_common_hidden=True):
-        """ Lists all file names inside a path.
-
-            skip_common_hidden will exclude common hidden files
-            return_full_path will include path in addition to filename
-        """
-        files = []
-        try:
-            file_data = os.listdir(folder_path)
-            files = self.list_iterator(
-                file_data,
-                folder_path,
-                limit=limit,
-                offset=offset,
-                size_limit=size_limit,
-                return_full_path=return_full_path,
-                skip_common_hidden=skip_common_hidden
-            )
-
-        except Exception as e:
-            logger.error("Unable to list objects: {0}".format(e))
-        return files