scalable-pypeline 2.1.31__py2.py3-none-any.whl

pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.1.31"

pypeline/barrier.py

@@ -0,0 +1,63 @@
+import time
+import redis
+from redis.sentinel import Sentinel
+from urllib.parse import urlparse
+from pypeline.constants import (
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
+
+
+class LockingParallelBarrier:
+    def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
+        # Connect to Redis using the provided URL
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(redis_url)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+                decode_responses=True,
+            )
+        else:
+            self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
+        self.task_key = task_key
+        self.lock_key = lock_key
+
+    def acquire_lock(self, timeout=5):
+        """Acquire a lock using Redis."""
+        while True:
+            if self.redis.set(self.lock_key, "locked", nx=True, ex=timeout):
+                return True
+            time.sleep(0.1)
+
+    def release_lock(self):
+        """Release the lock in Redis."""
+        self.redis.delete(self.lock_key)
+
+    def set_task_count(self, count):
+        """Initialize the task counter in Redis."""
+        self.redis.set(self.task_key, count)
+
+    def decrement_task_count(self):
+        """Decrement the task counter in Redis."""
+        return self.redis.decr(self.task_key)
+
+    def task_exists(self):
+        return self.redis.exists(self.task_key)
+
+    def get_task_count(self):
+        """Get the current value of the task counter."""
+        return int(self.redis.get(self.task_key) or 0)

pypeline/constants.py

@@ -0,0 +1,94 @@
+"""Pypeline Constants"""
+
+import os
+
+# Pypeline configuration defaults
+PYPELINE_YAML_PATH = os.environ.get("PYPELINE_YAML_PATH", "pypeline.yaml")
+PYPELINE_CLIENT_PKG_NAME = os.environ.get("PYPELINE_CLIENT_PKG_NAME", None)
+WORKER_NAME = os.environ.get("WORKER_NAME", None)
+API_ACCESS_KEY = os.environ.get("API_ACCESS_KEY", None)
+DEFAULT_BROKER_CALLABLE = os.environ.get(
+    "DEFAULT_BROKER_CLS", "pypeline.dramatiq:configure_default_broker"
+)
+
+# Pypeline broker connections
+RABBIT_URL = os.environ.get("RABBIT_URL", "amqp://admin:password@127.0.0.1:5672")
+REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+REDIS_SENTINEL_MASTER_NAME = os.environ.get("REDIS_SENTINEL_MASTER_NAME", None)
+
+# Pypeline task defaults
+PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL = int(
+    os.getenv("DRAMATIQ_PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL", "86400000")
+)
+DEFAULT_RESULT_TTL = int(os.getenv("DEFAULT_RESULT_TTL", 86400))  # seconds (1 day)
+DEFAULT_TASK_TTL = int(os.getenv("DEFAULT_TASK_TTL", 600))  # seconds (10 minutes)
+DEFAULT_TASK_MAX_RETRY = int(os.getenv("DEFAULT_TASK_MAX_RETRY", 3))
+DEFAULT_TASK_MIN_BACKOFF = int(os.getenv("DEFAULT_TASK_MIN_BACKOFF", 15))  # seconds
+DEFAULT_TASK_MAX_BACKOFF = int(
+    os.getenv("DEFAULT_TASK_MAX_BACKOFF", 3600)
+)  # seconds (1 hour)
+DEFAULT_BROKER_CONNECTION_HEARTBEAT = int(
+    os.getenv("DEFAULT_BROKER_CONNECTION_HEARTBEAT", 5)
+)
+DEFAULT_BROKER_CONNECTION_ATTEMPTS = int(
+    os.getenv("DEFAULT_BROKER_CONNECTION_ATTEMPTS", 5)
+)
+DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT = int(
+    os.getenv("DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT", 30)
+)
+DEFAULT_BROKER_HEARTBEAT_TIMEOUT = int(
+    os.getenv("DEFAULT_BROKER_HEARTBEAT_TIMEOUT", 300000)
+)
+DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT = int(
+    os.getenv("DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT", 1)
+)
+DEFAULT_REDIS_SOCKET_TIMEOUT = int(os.getenv("DEFAULT_REDIS_SOCKET_TIMEOUT", 2))
+DEFAULT_REDIS_RETRY_ON_TIMEOUT = bool(os.getenv("DEFAULT_REDIS_RETRY_ON_TIMEOUT", True))
+DEFAULT_REDIS_SOCKET_KEEPALIVE = bool(os.getenv("DEFAULT_REDIS_SOCKET_KEEPALIVE", True))
+DEFAULT_REDIS_HEALTH_CHECK_INTERVAL = int(
+    os.getenv("DEFAULT_REDIS_HEALTH_CHECK_INTERVAL", 30)
+)
+
+MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
+MS_IN_SECONDS = 1000
+API_PATH_V1 = "/api/v1"
+
+# Default 'responses' dictionary when decorating endpoints with @api.doc()
+# Extend as necessary.
+API_DOC_RESPONSES = {
+    200: {"code": 200, "description": "Successful response."},
+    400: {"code": 400, "description": "Malformed request. Verify payload is correct."},
+    401: {
+        "code": 401,
+        "description": "Unauthorized. Verify your API Key (`accesskey`) header.",
+    },
+}
+
+# Default 'params' dictionary when decorating endpoints with @api.doc()
+# Extend as necessary.
+API_DOC_PARAMS = {
+    "accesskey": {
+        "in": "header",
+        "name": "accesskey",
+        "description": "Your API Consumer's `accesskey`",
+        "type": "string",
+        "required": False,
+    }
+}
+
+DEFAULT_OPENAPI_CONFIG = (
+    ("SWAGGER_UI_DOC_EXPANSION", "list"),
+    ("API_DOCUMENTATION_TITLE", "Pypeline API Specs"),
+    ("API_DOCUMENTATION_DESCRIPTION", "Available API Endpoints"),
+    ("OPENAPI_VERSION", "3.0.2"),
+    ("OPENAPI_URL_PREFIX", "/api/v1"),
+    ("OPENAPI_SWAGGER_APP_NAME", "Pypeline - API Reference"),
+    ("OPENAPI_SWAGGER_UI_PATH", "/docs"),
+    ("OPENAPI_SWAGGER_BASE_TEMPLATE", "swagger/swagger_ui.html"),
+    ("OPENAPI_SWAGGER_URL", "/docs"),
+    (
+        "OPENAPI_SWAGGER_UI_URL",
+        "https://cdnjs.cloudflare.com/ajax/libs/swagger-ui/3.24.2/",
+    ),
+    ("EXPLAIN_TEMPLATE_LOADING", False),
+)

pypeline/dramatiq.py

@@ -0,0 +1,455 @@
+import copy
+import typing
+import pika
+import logging
+import os
+
+import click
+from urllib.parse import urlparse
+
+from dramatiq.brokers.redis import RedisBroker
+from redis.sentinel import Sentinel
+from pypeline.extensions import pypeline_config
+from warnings import warn
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.triggers.cron import CronTrigger
+from dramatiq import Broker, Middleware, set_broker, get_broker
+from dramatiq.brokers.rabbitmq import RabbitmqBroker
+from dramatiq.cli import (
+    CPUS,
+    HAS_WATCHDOG,
+    main as dramatiq_worker,
+    make_argument_parser as dramatiq_argument_parser,
+    import_object,
+)
+from dramatiq.middleware import default_middleware, CurrentMessage
+from dramatiq.results import Results
+from dramatiq.results.backends.redis import RedisBackend
+from flask import current_app, Flask
+from flask.cli import with_appcontext
+
+from pypeline.constants import (
+    REDIS_URL,
+    REDIS_SENTINEL_MASTER_NAME,
+    RABBIT_URL,
+    DEFAULT_BROKER_CALLABLE,
+    DEFAULT_BROKER_CONNECTION_HEARTBEAT,
+    DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+    DEFAULT_BROKER_CONNECTION_ATTEMPTS,
+    MESSAGE_BROKER,
+    DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
+from pypeline.pipelines.middleware.get_active_worker_id_middleware import (
+    GetActiveWorkerIdMiddleware,
+)
+from pypeline.pipelines.middleware.parallel_pipeline_middleware import ParallelPipeline
+from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
+from pypeline.utils.config_utils import (
+    retrieve_latest_schedule_config,
+    get_service_config_for_worker,
+    retrieve_executable_job_config,
+)
+from pypeline.utils.dramatiq_utils import (
+    guess_code_directory,
+    list_managed_actors,
+    register_lazy_actor,
+    LazyActor,
+)
+from pypeline.utils.graceful_shutdown_util import enable_graceful_shutdown
+from pypeline.utils.module_utils import get_callable
+from dramatiq.middleware import (
+    Retries,
+    Callbacks,
+    TimeLimit,
+    AgeLimit,
+    ShutdownNotifications,
+    Pipelines,
+)
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def configure_default_broker(broker: Broker = None):
+    reworked_defaults=[AgeLimit(), TimeLimit(), ShutdownNotifications(), Callbacks(), Pipelines(), Retries()]
+    redis_client = None
+    if REDIS_SENTINEL_MASTER_NAME is not None:
+        parsed_redis_url = urlparse(REDIS_URL)
+        redis_sentinel = Sentinel(
+            sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+        )
+        redis_client = redis_sentinel.master_for(
+            REDIS_SENTINEL_MASTER_NAME,
+            db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+            password=parsed_redis_url.password,
+            socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+            socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+            retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+            socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+            health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+        )
+    redis_backend = RedisBackend(client=redis_client, url=REDIS_URL)
+
+    if MESSAGE_BROKER == "RABBITMQ":
+        parsed_url = urlparse(RABBIT_URL)
+        credentials = pika.PlainCredentials(parsed_url.username, parsed_url.password)
+        broker = (
+            broker
+            if broker is not None
+            else RabbitmqBroker(
+                host=parsed_url.hostname,
+                port=parsed_url.port,
+                credentials=credentials,
+                heartbeat=DEFAULT_BROKER_CONNECTION_HEARTBEAT,
+                connection_attempts=DEFAULT_BROKER_CONNECTION_ATTEMPTS,
+                blocked_connection_timeout=DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+                middleware=reworked_defaults
+            )
+        )
+
+    elif MESSAGE_BROKER == "REDIS":
+        broker = (
+            broker
+            if broker is not None
+            else RedisBroker(
+                client=redis_client,
+                url=REDIS_URL,
+                heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+                middleware=reworked_defaults
+            )
+        )
+
+    broker.add_middleware(Results(backend=redis_backend))
+    broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
+    broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
+    broker.add_middleware(GetActiveWorkerIdMiddleware())
+    broker.add_middleware(CurrentMessage())
+    if (
+        os.getenv("RESTRICT_WORKER_SHUTDOWN_WHILE_JOBS_RUNNING", "false").lower()
+        == "true"
+    ):
+        enable_graceful_shutdown(broker=broker, redis_url=REDIS_URL)
+
+    register_actors_for_workers(broker)
+
+    set_broker(broker)
+
+
+def register_actors_for_workers(broker: Broker):
+    service = get_service_config_for_worker(pypeline_config)
+    scheduled_jobs_config = retrieve_latest_schedule_config()
+    executable_jobs_config = retrieve_executable_job_config()
+
+    if not service:
+        return
+
+    worker_registered_tasks = [
+        task_handler["handler"] for task_handler in service.get("registeredTasks")
+    ]
+
+    # Loop over the pipelines to get metadata and other information about the task for registration
+    for pipeline_key, pipeline in pypeline_config["pipelines"].items():
+        for task, task_handler_meta in pipeline["config"]["taskDefinitions"].items():
+            if pipeline["schemaVersion"] == 1:
+                # Check if any task in this pipeline is registered
+                task_handlers = [task_handler_meta["handler"]]
+            elif pipeline["schemaVersion"] == 2:
+                task_handlers = [t for t in task_handler_meta["handlers"]]
+
+            for task_handler in task_handlers:
+                if task_handler in worker_registered_tasks:
+                    server_type = task_handler_meta.get("serverType", None)
+
+                    try:
+                        pipeline_metadata = copy.deepcopy(
+                            pipeline["config"]["metadata"]
+                        )
+                        tmp_handler = get_callable(task_handler)
+                        if pipeline_metadata.get("maxRetry", 0) >= 0:
+                            pipeline_metadata["store_results"] = True
+                            _ = register_lazy_actor(
+                                broker, tmp_handler, pipeline_metadata, server_type
+                            )
+                    except Exception as e:
+                        logger.exception(
+                            f"Unable to add a task {task_handler} to dramatiq: {e}"
+                        )
+    # Loop over the scheduled jobs and create metadata and other information about the task for registration
+    for job in scheduled_jobs_config:
+        config = job["config"]
+        if config["task"] in worker_registered_tasks:
+            pipeline_meta = {"queue": config.get("queue", "default")}
+            try:
+                tmp_handler = get_callable(config["task"])
+                if pipeline_meta and pipeline_meta.get("maxRetry", 0) >= 0:
+                    pipeline_meta["store_results"] = True
+                    _ = register_lazy_actor(broker, tmp_handler, pipeline_meta, None)
+            except Exception as e:
+                logger.exception(f"Unable to add a task to dramatiq: {e}")
+
+    for job in executable_jobs_config or []:
+        config = job["config"]
+        if config["task"] in worker_registered_tasks:
+            pipeline_meta = {"queue": config.get("queue", "default")}
+            try:
+                tmp_handler = get_callable(config["task"])
+                if pipeline_meta and pipeline_meta.get("maxRetry", 0) >= 0:
+                    pipeline_meta["store_results"] = True
+                    _ = register_lazy_actor(broker, tmp_handler, pipeline_meta, None)
+            except Exception as e:
+                logger.exception(f"Unable to add a task to dramatiq: {e}")
+
+
+class Dramatiq:
+    """Flask extension bridging Dramatiq broker and Flask app.
+
+    Dramatiq API is eager. Broker initialisation precede actor declaration.
+    This breaks application factory pattern and other way to initialize
+    configuration after import.
+
+    This class enables lazy initialization of Dramatiq. Actual Dramatiq broker
+    is instanciated only once Flask app is created.
+
+    .. automethod:: actor
+    .. automethod:: init_app
+    """
+
+    def __init__(
+        self,
+        app: Flask = None,
+        name: str = "dramatiq",
+        config_prefix: str = None,
+        middleware: typing.List[Middleware] = None,
+    ):
+        """
+        :app: Flask application if created. See :meth:`init_app`.
+
+        :param broker_configuration_callable_module: In order to work in fork and spawn mode
+            we need to configure our broker using a callable function.  Default is specified as
+            "pypeline.flask_dramatiq:configure_default_broker".  This allows the user to
+            override if necessary.
+
+        :param name: Unique identifier for multi-broker app.
+
+        :param config_prefix: Flask configuration option prefix for this
+            broker. By default, it is derived from ``name`` parameter,
+            capitalized.
+
+        :param middleware: List of Dramatiq middleware instances to override
+             Dramatiq defaults.
+
+        Flask-Dramatiq always prepend a custom middleware to the middleware
+        stack that setup Flask context. This way, every middleware can use
+        Flask app context.
+
+        """
+        self.actors = []
+        self.app = None
+        self.config_prefix = config_prefix or name.upper() + "_BROKER"
+        self.name = name
+        self.broker = None
+        if middleware is None:
+            middleware = [m() for m in default_middleware]
+        self.middleware = middleware
+        if app:
+            self.init_app(app)
+
+    def __repr__(self) -> str:
+        return "<%s %s>" % (self.__class__.__name__, self.name)
+
+    def init_app(self, app: Flask):
+        """Initialize extension for one Flask application
+
+        This method triggers Dramatiq broker instantiation and effective actor
+        registration.
+
+        """
+        if self.app is not None:
+            warn(
+                "%s is used by more than one flask application. "
+                "Actor's context may be set incorrectly." % (self,),
+                stacklevel=2,
+            )
+        self.app = app
+        app.extensions["dramatiq-" + self.name] = self
+
+        module_name, broker_or_callable = import_object(DEFAULT_BROKER_CALLABLE)
+
+        # Callable function is expected to setBroker()
+        if callable(broker_or_callable):
+            logger.info(f"Configuring broker via {DEFAULT_BROKER_CALLABLE}")
+            broker_or_callable()
+        else:
+            raise TypeError("DEFAULT_BROKER_CALLABLE must point to a callable function")
+        self.broker = get_broker()
+        for actor in self.actors:
+            actor.register(broker=self.broker)
+
+    def actor(self, fn=None, **kw):
+        """Register a callable as Dramatiq actor.
+
+        This decorator lazily register a callable as a Dramatiq actor. The
+        actor can't be called before :meth:`init_app` is called.
+
+        :param kw: Keywords argument passed to :func:`dramatiq.actor`.
+
+        """
+        # Substitute dramatiq.actor decorator to return a lazy wrapper. This
+        # allows to register actors in extension before the broker is
+        # effectively configured by init_app.
+
+        def decorator(fn):
+            lazy_actor = LazyActor(self, fn, kw)
+            self.actors.append(lazy_actor)
+            if self.app:
+                lazy_actor.register(self.broker)
+            return lazy_actor
+
+        if fn:
+            return decorator(fn)
+        return decorator
+
+
+@click.command("cron-scheduler")
+def cron_scheduler():  # pragma: no cover
+    # Configure our broker that we will schedule registered tasks for
+    scheduler = BlockingScheduler()
+    module_name, broker_or_callable = import_object(DEFAULT_BROKER_CALLABLE)
+
+    # Callable function is expected to setBroker()
+    if callable(broker_or_callable):
+        logger.info(f"Configuring broker via {DEFAULT_BROKER_CALLABLE}")
+        broker_or_callable()
+    else:
+        raise TypeError("DEFAULT_BROKER_CALLABLE must point to a callable function")
+
+    broker = get_broker()
+    jobs = retrieve_latest_schedule_config()
+
+    for job in jobs:
+        if job["enabled"]:
+            config = job["config"]
+            worker_path = config["task"]
+            tmp_handler = get_callable(worker_path)
+            pipeline_meta = {"queue": config.get("queue", "default")}
+            actor = register_lazy_actor(broker, tmp_handler, pipeline_meta)
+            schedule = config["schedule"]
+            scheduler.add_job(
+                actor.send,
+                CronTrigger.from_crontab(
+                    f"{schedule['minute']} {schedule['hour']} {schedule['dayOfMonth']} {schedule['monthOfYear']} {schedule['dayOfWeek']}"
+                ),
+            )
+
+    try:
+        scheduler.start()
+    except KeyboardInterrupt:
+        scheduler.shutdown()
+
+
+@click.command("pypeline-worker")
+@click.argument("broker_name", default="dramatiq")
+@click.option(
+    "-v", "--verbose", default=0, count=True, help="turn on verbose log output"
+)
+@click.option(
+    "-p",
+    "--processes",
+    default=CPUS,
+    metavar="PROCESSES",
+    show_default=True,
+    help="the number of worker processes to run",
+)
+@click.option(
+    "-t",
+    "--threads",
+    default=8,
+    metavar="THREADS",
+    show_default=True,
+    help="the number of worker treads per processes",
+)
+@click.option(
+    "-Q",
+    "--queues",
+    type=str,
+    default=None,
+    metavar="QUEUES",
+    show_default=True,
+    help="listen to a subset of queues, comma separated",
+)
+@click.option(
+    "--use-spawn",
+    type=bool,
+    default=False,
+    metavar="USE_SPAWN",
+    show_default=True,
+    help="start processes by spawning (default: fork on unix, spawn on windows)",
+)
+@with_appcontext
+def pypeline_worker(
+    verbose, processes, threads, queues, broker_name, use_spawn
+):  # pragma: no cover
+    """Run dramatiq workers.
+
+    Setup Dramatiq with broker and task modules from Flask app.
+
+    \b
+    examples:
+      # Run dramatiq with 1 thread per process.
+      $ flask worker --threads 1
+
+    \b
+      # Listen only to the "foo" and "bar" queues.
+      $ flask worker -Q foo,bar
+
+    \b
+      # Consuming from a specific broker
+      $ flask worker mybroker
+    """
+    # Plugin for flask.commands entrypoint.
+    #
+    # Wraps dramatiq worker CLI in a Flask command. This is private API of
+    # dramatiq.
+
+    def format_actor(actor):
+        return "%s@%s" % (actor.actor_name, actor.queue_name)
+
+    parser = dramatiq_argument_parser()
+
+    # Set worker broker globally.
+    needle = "dramatiq-" + broker_name
+    broker = current_app.extensions[needle].broker
+    set_broker(broker)
+
+    command = [
+        "--processes",
+        str(processes),
+        "--threads",
+        str(threads),
+        # Fall back to flask_dramatiq global broker
+        DEFAULT_BROKER_CALLABLE,
+    ]
+
+    if use_spawn:
+        command += ["--use-spawn"]
+
+    if current_app.config["DEBUG"]:
+        verbose = max(1, verbose)
+        if HAS_WATCHDOG:
+            command += ["--watch", guess_code_directory(broker)]
+
+    queues = queues.split(",") if queues else []
+    if queues:
+        command += ["--queues"] + queues
+    command += verbose * ["-v"]
+    args = parser.parse_args(command)
+    logger.info("Able to execute the following actors:")
+    for actor in list_managed_actors(broker, queues):
+        logger.info("    %s.", format_actor(actor))
+
+    dramatiq_worker(args)

pypeline/executable_job_config_schema.py

@@ -0,0 +1,35 @@
+from marshmallow import Schema, EXCLUDE, fields
+
+class ExecutableJobConfigSchema(Schema):
+    queue = fields.String(
+        required=True,
+        description="Name of queue on which to place task.",
+        example="my-default-queue",
+    )
+    task = fields.String(
+        required=True,
+        description="Path to task to invoke.",
+        example="my_app.module.method",
+    )
+
+class ExecutableJobSchema(Schema):
+    """Definition of a single schedule entry"""
+    class Meta:
+        unknown = EXCLUDE
+
+    name = fields.String(
+        required=True,
+        description="Name of schedule entry.",
+        example="My Scheduled Task",
+    )
+    schemaVersion = fields.Integer(required=True)
+    config = fields.Dict(required=True)
+    enabled = fields.Boolean(
+        required=True, description="Whether entry is enabled.", example=True
+    )
+    config = fields.Nested(
+        ExecutableJobConfigSchema,
+        required=True,
+        description="Configuration information for this job.",
+    )
+

pypeline/extensions.py

@@ -0,0 +1,17 @@
+""" Initialize most extensions used throughout application
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    # Client packages *should* provide a `pypeline.yaml` file. This
+    # loads the configuration file with the provided name of the client
+    # package (e.g. pypeline_demo)
+    from pypeline.pypeline_yaml import load_client_config_and_version
+
+    pypeline_config, pypeline_client_version = load_client_config_and_version()
+except Exception as e:
+    pypeline_config = None
+    pypeline_client_version = None
+    logger.warning("Unable to load client Pypeline config ... {}".format(e))

pypeline/flask/__init__.py

@@ -0,0 +1,16 @@
+""" Sermos' Flask Implementation and Tooling. Convenience imports here.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    from flask_smorest import Blueprint, Api
+    from flask import abort
+except Exception as e:
+    logger.error("Unable to import Web services (Blueprint, API, abort)" f" ... {e}")
+
+try:
+    from pypeline.flask.flask_pypeline import FlaskPypeline
+except Exception as e:
+    logger.exception("Unable to import Sermos services (FlaskPypeline)" f" ... {e}")