scalable-pypeline 2.1.12__tar.gz → 2.1.31__tar.gz

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: scalable-pypeline
-Version: 2.1.12
+Name: scalable_pypeline
+Version: 2.1.31
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

scalable_pypeline-2.1.31/pypeline/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.1.31"

scalable_pypeline-2.1.31/pypeline/barrier.py

@@ -0,0 +1,63 @@
+import time
+import redis
+from redis.sentinel import Sentinel
+from urllib.parse import urlparse
+from pypeline.constants import (
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
+
+
+class LockingParallelBarrier:
+    def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
+        # Connect to Redis using the provided URL
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(redis_url)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+                decode_responses=True,
+            )
+        else:
+            self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
+        self.task_key = task_key
+        self.lock_key = lock_key
+
+    def acquire_lock(self, timeout=5):
+        """Acquire a lock using Redis."""
+        while True:
+            if self.redis.set(self.lock_key, "locked", nx=True, ex=timeout):
+                return True
+            time.sleep(0.1)
+
+    def release_lock(self):
+        """Release the lock in Redis."""
+        self.redis.delete(self.lock_key)
+
+    def set_task_count(self, count):
+        """Initialize the task counter in Redis."""
+        self.redis.set(self.task_key, count)
+
+    def decrement_task_count(self):
+        """Decrement the task counter in Redis."""
+        return self.redis.decr(self.task_key)
+
+    def task_exists(self):
+        return self.redis.exists(self.task_key)
+
+    def get_task_count(self):
+        """Get the current value of the task counter."""
+        return int(self.redis.get(self.task_key) or 0)

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/constants.py

@@ -1,5 +1,4 @@
-""" Pypeline Constants
-"""
+"""Pypeline Constants"""
 
 import os
 
@@ -15,6 +14,7 @@ DEFAULT_BROKER_CALLABLE = os.environ.get(
 # Pypeline broker connections
 RABBIT_URL = os.environ.get("RABBIT_URL", "amqp://admin:password@127.0.0.1:5672")
 REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+REDIS_SENTINEL_MASTER_NAME = os.environ.get("REDIS_SENTINEL_MASTER_NAME", None)
 
 # Pypeline task defaults
 PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL = int(
@@ -39,8 +39,17 @@ DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT = int(
 DEFAULT_BROKER_HEARTBEAT_TIMEOUT = int(
     os.getenv("DEFAULT_BROKER_HEARTBEAT_TIMEOUT", 300000)
 )
-MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
+DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT = int(
+    os.getenv("DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT", 1)
+)
+DEFAULT_REDIS_SOCKET_TIMEOUT = int(os.getenv("DEFAULT_REDIS_SOCKET_TIMEOUT", 2))
+DEFAULT_REDIS_RETRY_ON_TIMEOUT = bool(os.getenv("DEFAULT_REDIS_RETRY_ON_TIMEOUT", True))
+DEFAULT_REDIS_SOCKET_KEEPALIVE = bool(os.getenv("DEFAULT_REDIS_SOCKET_KEEPALIVE", True))
+DEFAULT_REDIS_HEALTH_CHECK_INTERVAL = int(
+    os.getenv("DEFAULT_REDIS_HEALTH_CHECK_INTERVAL", 30)
+)
 
+MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
 MS_IN_SECONDS = 1000
 API_PATH_V1 = "/api/v1"
 

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/dramatiq.py

@@ -2,12 +2,13 @@ import copy
 import typing
 import pika
 import logging
+import os
 
 import click
 from urllib.parse import urlparse
 
 from dramatiq.brokers.redis import RedisBroker
-
+from redis.sentinel import Sentinel
 from pypeline.extensions import pypeline_config
 from warnings import warn
 from apscheduler.schedulers.blocking import BlockingScheduler
@@ -29,6 +30,7 @@ from flask.cli import with_appcontext
 
 from pypeline.constants import (
     REDIS_URL,
+    REDIS_SENTINEL_MASTER_NAME,
     RABBIT_URL,
     DEFAULT_BROKER_CALLABLE,
     DEFAULT_BROKER_CONNECTION_HEARTBEAT,
@@ -36,6 +38,11 @@ from pypeline.constants import (
     DEFAULT_BROKER_CONNECTION_ATTEMPTS,
     MESSAGE_BROKER,
     DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
 )
 from pypeline.pipelines.middleware.get_active_worker_id_middleware import (
     GetActiveWorkerIdMiddleware,
@@ -45,6 +52,7 @@ from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
 from pypeline.utils.config_utils import (
     retrieve_latest_schedule_config,
     get_service_config_for_worker,
+    retrieve_executable_job_config,
 )
 from pypeline.utils.dramatiq_utils import (
     guess_code_directory,
@@ -52,15 +60,40 @@ from pypeline.utils.dramatiq_utils import (
     register_lazy_actor,
     LazyActor,
 )
+from pypeline.utils.graceful_shutdown_util import enable_graceful_shutdown
 from pypeline.utils.module_utils import get_callable
-
+from dramatiq.middleware import (
+    Retries,
+    Callbacks,
+    TimeLimit,
+    AgeLimit,
+    ShutdownNotifications,
+    Pipelines,
+)
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 def configure_default_broker(broker: Broker = None):
-    redis_backend = RedisBackend(url=REDIS_URL)
+    reworked_defaults=[AgeLimit(), TimeLimit(), ShutdownNotifications(), Callbacks(), Pipelines(), Retries()]
+    redis_client = None
+    if REDIS_SENTINEL_MASTER_NAME is not None:
+        parsed_redis_url = urlparse(REDIS_URL)
+        redis_sentinel = Sentinel(
+            sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+        )
+        redis_client = redis_sentinel.master_for(
+            REDIS_SENTINEL_MASTER_NAME,
+            db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+            password=parsed_redis_url.password,
+            socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+            socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+            retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+            socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+            health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+        )
+    redis_backend = RedisBackend(client=redis_client, url=REDIS_URL)
 
     if MESSAGE_BROKER == "RABBITMQ":
         parsed_url = urlparse(RABBIT_URL)
@@ -75,6 +108,7 @@ def configure_default_broker(broker: Broker = None):
                 heartbeat=DEFAULT_BROKER_CONNECTION_HEARTBEAT,
                 connection_attempts=DEFAULT_BROKER_CONNECTION_ATTEMPTS,
                 blocked_connection_timeout=DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+                middleware=reworked_defaults
             )
         )
 
@@ -83,22 +117,33 @@ def configure_default_broker(broker: Broker = None):
             broker
             if broker is not None
             else RedisBroker(
-                url=REDIS_URL, heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT
+                client=redis_client,
+                url=REDIS_URL,
+                heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+                middleware=reworked_defaults
             )
         )
 
     broker.add_middleware(Results(backend=redis_backend))
     broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
     broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
-    broker.add_middleware(CurrentMessage())
     broker.add_middleware(GetActiveWorkerIdMiddleware())
+    broker.add_middleware(CurrentMessage())
+    if (
+        os.getenv("RESTRICT_WORKER_SHUTDOWN_WHILE_JOBS_RUNNING", "false").lower()
+        == "true"
+    ):
+        enable_graceful_shutdown(broker=broker, redis_url=REDIS_URL)
+
     register_actors_for_workers(broker)
+
     set_broker(broker)
 
 
 def register_actors_for_workers(broker: Broker):
     service = get_service_config_for_worker(pypeline_config)
     scheduled_jobs_config = retrieve_latest_schedule_config()
+    executable_jobs_config = retrieve_executable_job_config()
 
     if not service:
         return
@@ -147,6 +192,18 @@ def register_actors_for_workers(broker: Broker):
             except Exception as e:
                 logger.exception(f"Unable to add a task to dramatiq: {e}")
 
+    for job in executable_jobs_config or []:
+        config = job["config"]
+        if config["task"] in worker_registered_tasks:
+            pipeline_meta = {"queue": config.get("queue", "default")}
+            try:
+                tmp_handler = get_callable(config["task"])
+                if pipeline_meta and pipeline_meta.get("maxRetry", 0) >= 0:
+                    pipeline_meta["store_results"] = True
+                    _ = register_lazy_actor(broker, tmp_handler, pipeline_meta, None)
+            except Exception as e:
+                logger.exception(f"Unable to add a task to dramatiq: {e}")
+
 
 class Dramatiq:
     """Flask extension bridging Dramatiq broker and Flask app.

scalable_pypeline-2.1.31/pypeline/executable_job_config_schema.py

@@ -0,0 +1,35 @@
+from marshmallow import Schema, EXCLUDE, fields
+
+class ExecutableJobConfigSchema(Schema):
+    queue = fields.String(
+        required=True,
+        description="Name of queue on which to place task.",
+        example="my-default-queue",
+    )
+    task = fields.String(
+        required=True,
+        description="Path to task to invoke.",
+        example="my_app.module.method",
+    )
+
+class ExecutableJobSchema(Schema):
+    """Definition of a single schedule entry"""
+    class Meta:
+        unknown = EXCLUDE
+
+    name = fields.String(
+        required=True,
+        description="Name of schedule entry.",
+        example="My Scheduled Task",
+    )
+    schemaVersion = fields.Integer(required=True)
+    config = fields.Dict(required=True)
+    enabled = fields.Boolean(
+        required=True, description="Whether entry is enabled.", example=True
+    )
+    config = fields.Nested(
+        ExecutableJobConfigSchema,
+        required=True,
+        description="Configuration information for this job.",
+    )
+

scalable_pypeline-2.1.31/pypeline/job_runner.py

@@ -0,0 +1,205 @@
+import os
+import logging
+import argparse
+import threading
+import multiprocessing as mp
+
+# Prefer 'spawn' for user code using multiprocessing
+if mp.get_start_method(allow_none=True) != "spawn":
+    mp.set_start_method("spawn", force=True)
+
+# Avoid staging more than one message; must be set before Dramatiq import path runs
+os.environ.setdefault("dramatiq_queue_prefetch", "1")
+
+from dramatiq import Worker, get_broker, set_broker
+from dramatiq.middleware import Middleware
+
+
+try:
+    # If your project exposes a helper to configure the default broker, use it.
+    from pypeline.dramatiq import configure_default_broker  # adjust import if needed
+
+    broker = configure_default_broker() or get_broker()
+    set_broker(broker)
+except Exception:
+    # Fall back to whatever Dramatiq has as the active broker.
+    import pypeline.dramatiq  # noqa: F401 (ensure module side-effects run)
+
+    broker = get_broker()
+
+
+class OneAndDone(Middleware):
+    """
+    Signals when the first message starts ('got_work') and completes ('done').
+    If stop_on_failure=True, we'll also mark done after the first failure.
+    """
+
+    def __init__(
+        self,
+        got_work: threading.Event,
+        done: threading.Event,
+        *,
+        stop_on_failure: bool = False
+    ):
+        self.got_work = got_work
+        self.done = done
+        self.stop_on_failure = stop_on_failure
+
+    def before_process_message(self, broker, message):
+        # First time we see a message begin processing in this process
+        if not self.got_work.is_set():
+            self.got_work.set()
+
+    def after_process_message(self, broker, message, *, result=None, exception=None):
+        # On success (or also on failure if configured), finish this worker
+        if exception is None or self.stop_on_failure:
+            if not self.done.is_set():
+                self.done.set()
+
+
+def _graceful_stop(worker: Worker, log: logging.Logger):
+    try:
+        log.info("Stopping dramatiq worker...")
+        worker.stop()  # stop consumers; no new messages will start
+        worker.join()
+        log.info("Worker stopped.")
+    except Exception as e:
+        log.exception("Error stopping worker: %s", e)
+
+
+def _close_broker(log: logging.Logger):
+    try:
+        b = get_broker()
+        if b is not None and hasattr(b, "close"):
+            b.close()
+            log.info("Broker closed.")
+    except Exception as e:
+        log.exception("Error closing broker: %s", e)
+
+
+def job_runner(queues, idle_timeout_ms: int = 0, *, stop_on_failure: bool = False):
+    """
+    Start a single-thread Dramatiq worker. Behavior:
+      - Wait up to `idle_timeout_ms` for *a job to start* (time-to-first-job).
+      - Once a job begins, wait indefinitely for it to complete.
+      - After the first successful job completes (or first job, if stop_on_failure=True), stop and exit.
+
+    Args:
+        queues (list[str]): queues to listen to
+        idle_timeout_ms (int): <=0 => wait forever for first job; >0 => exit if no job starts in time
+        stop_on_failure (bool): if True, exit after first job even if it fails
+    """
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+    log = logging.getLogger("oneshot")
+
+    # Normalize timeout (treat non-positive as "infinite")
+    timeout_ms = (
+        int(idle_timeout_ms) if idle_timeout_ms and int(idle_timeout_ms) > 0 else 0
+    )
+    log.info(
+        "Launching worker with queues=%s, idle_timeout_ms=%s", queues, timeout_ms or "∞"
+    )
+
+    got_work = threading.Event()
+    done = threading.Event()
+    broker.add_middleware(OneAndDone(got_work, done, stop_on_failure=stop_on_failure))
+
+    worker = Worker(
+        broker,
+        worker_threads=1,  # strictly one at a time
+        queues=queues,
+        worker_timeout=1000,  # ms; how often the worker checks for stop
+    )
+
+    worker.start()
+
+    def controller():
+        log.debug("Controller thread started.")
+        try:
+            # Phase 1: Wait for *first job to start*
+            if timeout_ms > 0:
+                started = got_work.wait(timeout_ms / 1000.0)
+                if not started:
+                    log.info(
+                        "Idle timeout reached (%d ms); no jobs started. Stopping worker.",
+                        timeout_ms,
+                    )
+                    return
+            else:
+                got_work.wait()
+
+            log.info("First job started; waiting for it to finish...")
+            # Phase 2: Wait for the first job to complete (no timeout)
+            done.wait()
+            log.info("First job finished; shutting down.")
+        finally:
+            _graceful_stop(worker, log)
+            _close_broker(log)
+            # Hard-exit to ensure K8s Job is marked Succeeded promptly, no lingering threads.
+            os._exit(0)
+
+    t = threading.Thread(target=controller, name="oneshot-controller", daemon=False)
+    t.start()
+    t.join()  # Block until controller completes (which shuts everything down)
+
+
+def _parse_args(argv=None):
+    ap = argparse.ArgumentParser(description="Run a one-shot Dramatiq worker.")
+    ap.add_argument(
+        "-q",
+        "--queue",
+        action="append",
+        default=None,
+        help="Queue to listen to (repeatable). You can also pass a comma-separated list.",
+    )
+    ap.add_argument(
+        "--idle-timeout-ms",
+        type=int,
+        default=int(os.getenv("IDLE_TIMEOUT_MS", "0")),
+        help="Exit if no job starts within this time (<=0 = wait forever).",
+    )
+    ap.add_argument(
+        "--stop-on-failure",
+        action="store_true",
+        help="Exit after the first job even if it fails.",
+    )
+    return ap.parse_args(argv)
+
+
+def main(argv=None):
+    args = _parse_args(argv)
+
+    # Build queue list from flags or env, support comma-separated entries.
+    raw_entries = (
+        args.queue if args.queue else [os.getenv("JOB_QUEUE", "pipeline-queue")]
+    )
+    queues = []
+    for entry in raw_entries:
+        queues.extend([q.strip() for q in str(entry).split(",") if q and q.strip()])
+
+    if not queues:
+        raise SystemExit("No queues provided. Use -q ... or set JOB_QUEUE.")
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+    log = logging.getLogger("oneshot")
+
+    pid = os.getpid()
+    ppid = os.getppid()
+    log.info(
+        "Starting one-shot worker PID=%s, Parent PID=%s, queues=%s, idle_timeout_ms=%s, stop_on_failure=%s",
+        pid,
+        ppid,
+        queues,
+        args.idle_timeout_ms if args.idle_timeout_ms > 0 else "∞",
+        args.stop_on_failure,
+    )
+
+    job_runner(
+        queues,
+        idle_timeout_ms=args.idle_timeout_ms,
+        stop_on_failure=args.stop_on_failure,
+    )
+
+
+if __name__ == "__main__":
+    main()

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pipeline_config_schema.py

@@ -119,10 +119,9 @@ class TaskDefinitionsSchemaV1(ExcludeUnknownSchema):
 
     serverType = fields.String(
         required=False,
-        validate=OneOf(
-            ["xs", "s", "m", "l", "xl", "xxl", "xxxl", "cpu-xl"],
-            error="Invalid serverType. Available options: 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl', 'cpu-xl'.",
-        ),
+        description="Recommended presets are listed in enum; custom strings are allowed.",
+        example="m",
+        metadata={"enum": ["xs", "s", "m", "l", "xl", "xxl", "xxxl", "cpu-xl"]},  # docs only
     )
 
 
@@ -151,10 +150,9 @@ class TaskDefinitionsSchemaV2(ExcludeUnknownSchema):
 
     serverType = fields.String(
         required=False,
-        validate=OneOf(
-            ["xs", "s", "m", "l", "xl", "xxl", "xxxl", "cpu-xl"],
-            error="Invalid serverType. Available options: 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl', 'cpu-xl'.",
-        ),
+        description="Recommended presets are listed in enum; custom strings are allowed.",
+        example="m",
+        metadata={"enum": ["xs", "s", "m", "l", "xl", "xxl", "xxxl", "cpu-xl"]},  # docs only
     )
 
 

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pipeline_settings_schema.py

@@ -1,3 +1,5 @@
+from datetime import date
+
 from marshmallow import Schema, fields, validate, ValidationError, validates_schema, INCLUDE
 
 
@@ -34,6 +36,7 @@ def create_pipeline_settings_schema(pipeline_settings_schema_data):
             "datetime": fields.DateTime,
             "array": fields.List,
             "object": fields.Nested,
+            "date": fields.Date,
         }.get(data_type)
 
         if not field_type:
@@ -50,6 +53,7 @@ def create_pipeline_settings_schema(pipeline_settings_schema_data):
                 "float": fields.Float,
                 "boolean": fields.Boolean,
                 "datetime": fields.DateTime,
+                "date": fields.Date,
             }.get(element_type)
             if not field_args["cls_or_instance"]:
                 raise ValidationError(
@@ -119,6 +123,7 @@ def create_pipeline_settings_schema(pipeline_settings_schema_data):
             "float": fields.Float,
             "boolean": fields.Boolean,
             "datetime": fields.DateTime,
+            "date": fields.Date,
             "array": fields.List,
             "object": fields.Nested,
         }.get(data_type)
@@ -137,6 +142,7 @@ def create_pipeline_settings_schema(pipeline_settings_schema_data):
                 "float": fields.Float,
                 "boolean": fields.Boolean,
                 "datetime": fields.DateTime,
+                "date": fields.Date,
             }.get(element_type)
             if not field_args["cls_or_instance"]:
                 raise ValidationError(
@@ -229,7 +235,7 @@ class SettingSchema(Schema):
     dataType = fields.String(
         required=True,
         validate=validate.OneOf(
-            ["string", "int", "float", "boolean", "datetime", "array", "object"]
+            ["string", "int", "float", "boolean", "datetime", "array", "object", "date"]
         ),
         metadata={"description": "The underlying data type of the setting"},
     )
@@ -317,6 +323,13 @@ class SettingSchema(Schema):
                     raise ValidationError(
                         f"Option value `{value}` must be an ISO 8601 string for `datetime`."
                     )
+                elif data_type == "date":
+                    try:
+                        date.fromisoformat(value)
+                    except Exception:
+                        raise ValidationError(
+                            f"Option value `{value}` must be an ISO 8601 string for `date`."
+                        )
 
     @validates_schema
     def validate_search_endpoint(self, data, **kwargs):
@@ -452,6 +465,7 @@ class PipelineScenarioSchema(Schema):
             "produce alternative calculations and or results."
         },
     )
+    execution_id = fields.String(required=False, metadata={"description":"Execution id for a known scenario"})
 
 
 class PipelineScenariosSchema(Schema):

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pipelines/composition/parallel_pipeline_composition.py

@@ -5,12 +5,22 @@ import json
 import time
 import typing
 from uuid import uuid4
+from urllib.parse import urlparse
 
 from dramatiq.broker import get_broker
 from dramatiq.results import ResultMissing
 from db_medley.redis_conf import RedisConnector
 from redis.exceptions import RedisError
-
+from redis.sentinel import Sentinel
+from pypeline.constants import (
+    REDIS_URL,
+    REDIS_SENTINEL_MASTER_NAME,
+    DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_TIMEOUT,
+    DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+    DEFAULT_REDIS_SOCKET_KEEPALIVE,
+    DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+)
 from pypeline.barrier import LockingParallelBarrier
 from pypeline.constants import DEFAULT_RESULT_TTL
 from pypeline.dramatiq import REDIS_URL
@@ -240,9 +250,26 @@ class PipelineResult:
         self.pipeline: parallel_pipeline = None
         self.execution_id = execution_id
         self.redis_key = f"{execution_id}-results-key"
-        self.redis_conn = RedisConnector().get_connection()
         self.result_ttl = result_ttl
 
+        if REDIS_SENTINEL_MASTER_NAME is not None:
+            parsed_redis_url = urlparse(REDIS_URL)
+            redis_sentinel = Sentinel(
+                sentinels=[(parsed_redis_url.hostname, parsed_redis_url.port)],
+            )
+            self.redis_conn = redis_sentinel.master_for(
+                REDIS_SENTINEL_MASTER_NAME,
+                db=int(parsed_redis_url.path[1]) if parsed_redis_url.path else 0,
+                password=parsed_redis_url.password,
+                socket_connect_timeout=DEFAULT_REDIS_SOCKET_CONNECT_TIMEOUT,
+                socket_timeout=DEFAULT_REDIS_SOCKET_TIMEOUT,
+                retry_on_timeout=DEFAULT_REDIS_RETRY_ON_TIMEOUT,
+                socket_keepalive=DEFAULT_REDIS_SOCKET_KEEPALIVE,
+                health_check_interval=DEFAULT_REDIS_HEALTH_CHECK_INTERVAL,
+            )
+        else:
+            self.redis_conn = RedisConnector().get_connection()
+
     def create_result_entry(self, pipeline_json_str: str):
         """
         Store the serialized pipeline data in Redis with a specified TTL.

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pipelines/composition/pypeline_composition.py

@@ -86,6 +86,10 @@ class Pypeline:
                 )
                 message = lazy_actor.message()
                 message.options["pipeline"] = pipeline
+                if pipeline_config["metadata"].get("maxRetry", None) is not None:
+                    message.options["max_retries"] = pipeline_config["metadata"][
+                        "maxRetry"
+                    ]
                 message.options["task_replacements"] = copy(
                     scenario["taskReplacements"]
                 )
@@ -121,6 +125,8 @@ class Pypeline:
         )
         message = lazy_actor.message()
         message.options["pipeline"] = pipeline
+        if pipeline_config["metadata"].get("maxRetry", None) is not None:
+            message.options["max_retries"] = pipeline_config["metadata"]["maxRetry"]
         message.options["task_replacements"] = first_scenario_task_replacements
         message.options["execution_id"] = base_case_execution_id
         message.options["task_name"] = first_task

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pipelines/factory.py

@@ -70,13 +70,16 @@ def dag_generator(
             )
             registered_actors[task] = lazy_actor
             if args and not kwargs:
-                message_group.append(registered_actors[task].message(*args))
+                msg = registered_actors[task].message(*args)
             elif kwargs and not args:
-                message_group.append(registered_actors[task].message(**kwargs))
+                msg = registered_actors[task].message(**kwargs)
             elif args and kwargs:
-                message_group.append(registered_actors[task].message(*args, **kwargs))
+                msg = registered_actors[task].message(*args, **kwargs)
             else:
-                message_group.append(registered_actors[task].message())
+                msg = registered_actors[task].message()
+            msg.options["task_ttl"] = pipeline_config["metadata"]["maxTtl"]
+            message_group.append(msg)
+
         messages.append(message_group)
     p = parallel_pipeline(messages)
 

scalable_pypeline-2.1.31/pypeline/pipelines/middleware/graceful_shutdown_middleware.py

@@ -0,0 +1,50 @@
+import os
+import socket
+import logging
+import redis
+
+from dramatiq.middleware import Middleware
+from tenacity import retry, stop_after_attempt, wait_exponential, after_log
+
+logger = logging.getLogger(__name__)
+
+
+class GraceFulShutdownMiddleware(Middleware):
+    def __init__(self, redis_url, key_prefix="busy"):
+        self.redis = redis.Redis.from_url(redis_url)
+        self.hostname = socket.gethostname()
+        self.pid = os.getpid()
+        self.key_prefix = key_prefix
+        self.key = f"{self.key_prefix}:{self.hostname}-{self.pid}"
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=2, min=2, max=10),
+        after=after_log(logger, logging.WARNING),
+        reraise=True,
+    )
+    def _set_busy_flag(self, message_ttl):
+        self.redis.set(self.key, "1", ex=message_ttl)
+        logger.debug(f"[GracefulShutdownMiddleware] Set busy flag: {self.key}")
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=2, min=2, max=10),
+        after=after_log(logger, logging.WARNING),
+        reraise=True,
+    )
+    def _clear_busy_flag(self):
+        self.redis.delete(self.key)
+        logger.debug(f"[GracefulShutdownMiddleware] Cleared busy flag: {self.key}")
+
+    def before_process_message(self, broker, message):
+        try:
+            self._set_busy_flag(message_ttl=message.options["task_ttl"])
+        except Exception as e:
+            logger.error(f"[GracefulShutdownMiddleware] Failed to set busy flag: {e}")
+
+    def after_process_message(self, broker, message, *, result=None, exception=None):
+        try:
+            self._clear_busy_flag()
+        except Exception as e:
+            logger.error(f"[GracefulShutdownMiddleware] Failed to clear busy flag: {e}")

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pipelines/middleware/pypeline_middleware.py

@@ -24,6 +24,7 @@ class PypelineMiddleware(Middleware):
             return
 
         pipeline = message.options["pipeline"]
+        max_retries = message.options.get("max_retries", None)
         pipeline_config = pipeline["config"]
         task_replacements = message.options["task_replacements"]
         execution_id = message.options["execution_id"]
@@ -45,78 +46,74 @@ class PypelineMiddleware(Middleware):
 
         graph = get_execution_graph(pipeline_config)
         children_tasks = pipeline_config["dagAdjacency"].get(task_name, [])
-
         messages = []
         for child in children_tasks:
             child_ancestors = sorted(graph.predecessors(child))
+            child_ancestors_complete = {a: False for a in child_ancestors}
 
-            ancestor_tasks_complete = True
+            for scenario in message.options["scenarios"]:
+                if scenario["execution_id"] == execution_id:
+                    tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
 
             for ancestor in child_ancestors:
-                ancestor_task_key = f"{execution_id}-{ancestor}"
-
-                locking_parallel_barrier = LockingParallelBarrier(
-                    self.redis_url,
-                    task_key=ancestor_task_key,
-                    lock_key=f"{message.options['base_case_execution_id']}-lock",
-                )
-                try:
-                    locking_parallel_barrier.acquire_lock(
-                        timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                if ancestor in tasks_to_run_in_scenario:
+                    current_scenario_ancestor_task_key = f"{execution_id}-{ancestor}"
+                    locking_parallel_barrier = LockingParallelBarrier(
+                        self.redis_url,
+                        task_key=current_scenario_ancestor_task_key,
+                        lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
-
-                    if locking_parallel_barrier.task_exists():
-                        remaining_tasks = locking_parallel_barrier.get_task_count()
-                    else:
-                        remaining_tasks = None
-                finally:
-                    locking_parallel_barrier.release_lock()
-
-                # If the lock didn't exist for the current tasks execution id then it would indicate
-                # that this is the start of a new scenario.  Therefore we need to find the ancestor
-                # that is executed in the base case execution id and make sure it has completed
-                if remaining_tasks is None:
-                    ancestor_task_key = (
+                    try:
+                        locking_parallel_barrier.acquire_lock(
+                            timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                        )
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
+                    finally:
+                        locking_parallel_barrier.release_lock()
+                else:
+                    base_scenario_ancestor_task_key = (
                         f"{message.options['base_case_execution_id']}-{ancestor}"
                     )
-
                     locking_parallel_barrier = LockingParallelBarrier(
                         self.redis_url,
-                        task_key=ancestor_task_key,
+                        task_key=base_scenario_ancestor_task_key,
                         lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
                     try:
                         locking_parallel_barrier.acquire_lock(
                             timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
                         )
-
-                        if locking_parallel_barrier.task_exists():
-                            remaining_tasks = locking_parallel_barrier.get_task_count()
-                        else:
-                            raise Exception(
-                                f"Parent task {ancestor} barrier lock not found"
-                            )
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
                     finally:
                         locking_parallel_barrier.release_lock()
-                if remaining_tasks is None or remaining_tasks >= 1:
-                    ancestor_tasks_complete = False
-                    break
 
-            # If the child's ancestor tasks aren't complete move onto the next child to check
-            if not ancestor_tasks_complete:
-                break
+            if any(complete is False for complete in child_ancestors_complete.values()):
+                continue
 
-            # Handle situation where base case kicks off new scenario
             if (
                 message.options["base_case_execution_id"]
                 == message.options["execution_id"]
             ):
                 for scenario in message.options["scenarios"]:
-                    child_predecessors = list(graph.predecessors(child))
+                    child_ancestors = list(graph.predecessors(child))
+                    child_has_other_ancestors_in_scenario = False
+
+                    for ancestor in child_ancestors:
+                        if ancestor in scenario["tasksToRunInScenario"]:
+                            child_has_other_ancestors_in_scenario = True
+                            break
+
                     if (
                         child in scenario["tasksToRunInScenario"]
-                        and task_name in child_predecessors
+                        and task_name in child_ancestors
                         and task_name not in scenario["tasksToRunInScenario"]
+                        and not child_has_other_ancestors_in_scenario
                     ):
                         task_key = f"{scenario['execution_id']}-{child}"
                         locking_parallel_barrier = LockingParallelBarrier(
@@ -138,6 +135,8 @@ class PypelineMiddleware(Middleware):
                         )
                         scenario_message = lazy_actor.message()
                         scenario_message.options["pipeline"] = pipeline
+                        if max_retries is not None:
+                            scenario_message.options["max_retries"] = max_retries
                         scenario_message.options["task_replacements"] = (
                             task_replacements
                         )
@@ -161,7 +160,7 @@ class PypelineMiddleware(Middleware):
                             )
                         messages.append(scenario_message)
 
-            # Kick off child task for current scenario
+            # If we've made it here all ancestors of this child are complete, and it's time to run.
             task_key = f"{execution_id}-{child}"
             locking_parallel_barrier = LockingParallelBarrier(
                 self.redis_url,
@@ -182,6 +181,8 @@ class PypelineMiddleware(Middleware):
 
             child_message = lazy_actor.message()
             child_message.options["pipeline"] = pipeline
+            if max_retries is not None:
+                child_message.options["max_retries"] = max_retries
             child_message.options["task_replacements"] = task_replacements
             child_message.options["execution_id"] = execution_id
             child_message.options["task_name"] = child

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/pypeline_yaml.py

@@ -54,6 +54,8 @@ import yaml
 from yaml.loader import SafeLoader
 from marshmallow import Schema, fields, pre_load, EXCLUDE, INCLUDE, validates_schema
 from marshmallow.exceptions import ValidationError
+
+from pypeline.executable_job_config_schema import ExecutableJobSchema
 from pypeline.utils.module_utils import PypelineModuleLoader, normalized_pkg_name
 from pypeline.constants import PYPELINE_YAML_PATH, PYPELINE_CLIENT_PKG_NAME
 from pypeline.pipeline_config_schema import BasePipelineSchema
@@ -165,6 +167,14 @@ class PypelineYamlSchema(ExcludeUnknownSchema):
         required=False,
     )
 
+    executableJobs = fields.Dict(
+        keys=fields.String(),
+        values=fields.Nested(ExecutableJobSchema),
+        description="List of executable jobs",
+        required=False,
+        allow_none=True,
+    )
+
     def validate_errors(self, schema: Schema, value: dict):
         """Run Marshmallow validate() and raise if any errors"""
         schema = schema()

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/utils/config_utils.py

@@ -41,6 +41,19 @@ def retrieve_latest_schedule_config():
     return None
 
 
+def retrieve_executable_job_config():
+    pypeline_config = load_pypeline_config()
+
+    if not pypeline_config:
+        return None
+    if "executableJobs" in pypeline_config:
+        tasks = []
+        for task_id, config in pypeline_config["executableJobs"].items():
+            tasks.append(config)
+        return tasks
+    return None
+
+
 def get_service_config_for_worker(
     pypeline_config: dict, worker_name: str = None
 ) -> Union[dict, None]:

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/pypeline/utils/dramatiq_utils.py

@@ -113,11 +113,15 @@ class LazyActor(object):
         return getattr(self.actor, name)
 
     def register(self, broker):
-        self.actor = register_actor(
-            actor_name=f"{self.fn.__module__}.{self.fn.__name__}-{self.kw['queue_name']}",
-            broker=broker,
-            **self.kw,
-        )(ensure_return_value(default_value=True)(self.fn))
+        actor_name = f"{self.fn.__module__}.{self.fn.__name__}-{self.kw['queue_name']}"
+        if actor_name in broker.actors:
+            self.actor = broker.actors[actor_name]
+        else:
+            self.actor = register_actor(
+                actor_name=actor_name,
+                broker=broker,
+                **self.kw,
+            )(ensure_return_value(default_value=True)(self.fn))
 
     # Next is regular actor API.
     def send(self, *a, **kw):

scalable_pypeline-2.1.31/pypeline/utils/executable_job_util.py

@@ -0,0 +1,35 @@
+from dramatiq.broker import get_broker
+
+from pypeline.utils.config_utils import retrieve_executable_job_config
+from pypeline.utils.dramatiq_utils import register_lazy_actor, LazyActor
+from pypeline.utils.module_utils import get_callable
+
+
+def execute_job(fn, *args, **kwargs):
+    executable_jobs_config = retrieve_executable_job_config()
+
+    module_path = kwargs.get("module_path", None)
+
+    job = None
+
+    for j in executable_jobs_config or []:
+        if module_path and module_path == j["config"]["task"]:
+            job = j
+            break
+        elif fn.__name__ in j["config"]["task"]:
+            if job:
+                raise ValueError(
+                    f"Multiple matches found in yaml for {fn.__name__}, "
+                    f"Consider passing module_path as a kwarg to avoid ambiguity."
+                )
+            job = j
+
+    if job is None:
+        raise ValueError(f"No match found in yaml for {fn.__name__} function.")
+
+    pipeline_meta = {"queue": job["config"].get("queue", "default")}
+    tmp_handler = get_callable(job["config"]["task"])
+
+    actor: LazyActor = register_lazy_actor(get_broker(), tmp_handler, pipeline_meta, None)
+
+    return actor.send(*args, **kwargs)

scalable_pypeline-2.1.31/pypeline/utils/graceful_shutdown_util.py

@@ -0,0 +1,39 @@
+import threading
+import signal
+import os
+import redis
+import socket
+import sys
+import time
+import logging
+from pypeline.pipelines.middleware.graceful_shutdown_middleware import (
+    GraceFulShutdownMiddleware,
+)
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def enable_graceful_shutdown(broker, redis_url):
+    """Attach GracefulShutdownMiddleware and a SIGTERM handler to the current process."""
+    broker.add_middleware(GraceFulShutdownMiddleware(redis_url=redis_url))
+
+    if threading.current_thread().name == "MainThread":
+        key_prefix = "busy"
+        hostname = socket.gethostname()
+        pid = os.getpid()
+        busy_key = f"{key_prefix}:{hostname}-{pid}"
+        r = redis.Redis.from_url(redis_url)
+
+        def shutdown_handler(signum, frame):
+            logger.info(f"[Signal Handler] Received signal {signum}")
+            wait_counter = 0
+            while r.get(busy_key):
+                if wait_counter % 30 == 0:  # Only log every 30 checks
+                    logger.info(f"[Signal Handler] Busy ({busy_key}), waiting...")
+                time.sleep(1)
+                wait_counter += 1
+            logger.info(f"[Signal Handler] Done. Exiting.")
+            sys.exit(0)
+
+        signal.signal(signal.SIGTERM, shutdown_handler)

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/scalable_pypeline.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.12
+Version: 2.1.31
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/scalable_pypeline.egg-info/SOURCES.txt

@@ -8,7 +8,9 @@ pypeline/__init__.py
 pypeline/barrier.py
 pypeline/constants.py
 pypeline/dramatiq.py
+pypeline/executable_job_config_schema.py
 pypeline/extensions.py
+pypeline/job_runner.py
 pypeline/pipeline_config_schema.py
 pypeline/pipeline_settings_schema.py
 pypeline/pypeline_yaml.py
@@ -26,11 +28,14 @@ pypeline/pipelines/composition/parallel_pipeline_composition.py
 pypeline/pipelines/composition/pypeline_composition.py
 pypeline/pipelines/middleware/__init__.py
 pypeline/pipelines/middleware/get_active_worker_id_middleware.py
+pypeline/pipelines/middleware/graceful_shutdown_middleware.py
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py
 pypeline/pipelines/middleware/pypeline_middleware.py
 pypeline/utils/__init__.py
 pypeline/utils/config_utils.py
 pypeline/utils/dramatiq_utils.py
+pypeline/utils/executable_job_util.py
+pypeline/utils/graceful_shutdown_util.py
 pypeline/utils/module_utils.py
 pypeline/utils/pipeline_utils.py
 pypeline/utils/schema_utils.py

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/scalable_pypeline.egg-info/entry_points.txt

@@ -1,3 +1,6 @@
+[console_scripts]
+job-runner = pypeline.job_runner:main
+
 [flask.commands]
 cron-scheduler = pypeline.dramatiq:cron_scheduler
 pypeline-worker = pypeline.dramatiq:pypeline_worker

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/scalable_pypeline.egg-info/requires.txt

@@ -20,7 +20,7 @@ pytest-cov<3,>=2.6.1
 tox<4,>=3.14.1
 mock<2,>=1
 responses<0.11,>=0.10.16
-fakeredis<3,>=2.10.3
+fakeredis<2.31,>=2.10.3
 
 [web]
 gunicorn
@@ -30,3 +30,4 @@ gevent<22,>=21.12.0
 networkx>=2.4
 dramatiq[rabbitmq]==1.17.0
 apscheduler<4,>=3.10.4
+tenacity==8.0.1

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/setup.cfg

@@ -5,6 +5,9 @@ packages =
 [bdist_wheel]
 universal = 1
 
+[metadata]
+name = scalable_pypeline
+
 [egg_info]
 tag_build = 
 tag_date = 0

{scalable-pypeline-2.1.12 → scalable_pypeline-2.1.31}/setup.py

@@ -1,5 +1,4 @@
-""" PypeLine Library Setup
-"""
+"""PypeLine Library Setup"""
 
 import re
 import ast
@@ -159,7 +158,7 @@ with open("requirements.txt", "r") as f:
     install_requires = f.read().splitlines()
 
 setup(
-    name="scalable-pypeline",
+    name="scalable_pypeline",
     version=__version__,
     description="PypeLine - Python pipelines for the Real World",
     long_description=open("README.md", "r").read(),
@@ -183,6 +182,7 @@ setup(
             "networkx>=2.4",
             "dramatiq[rabbitmq]==1.17.0",
             "apscheduler>=3.10.4,<4",
+            "tenacity==8.0.1",
         ],
         "dev": ["black"],
         "test": [
@@ -190,13 +190,14 @@ setup(
             "tox>=3.14.1,<4",
             "mock>=1,<2",
             "responses>=0.10.16,<0.11",
-            "fakeredis>=2.10.3,<3",
+            "fakeredis>=2.10.3,<2.31",  # fakeredis version compatible with redis 4.x
         ],
     },
     entry_points={
         "flask.commands": [
             "pypeline-worker=pypeline.dramatiq:pypeline_worker",
             "cron-scheduler=pypeline.dramatiq:cron_scheduler",
-        ]
+        ],
+        "console_scripts": ["job-runner = pypeline.job_runner:main"],
     },
 )

scalable-pypeline-2.1.12/pypeline/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.1.12"

scalable-pypeline-2.1.12/pypeline/barrier.py

@@ -1,37 +0,0 @@
-import time
-
-import redis
-
-
-class LockingParallelBarrier:
-    def __init__(self, redis_url, task_key="task_counter", lock_key="task_lock"):
-        # Connect to Redis using the provided URL
-        self.redis = redis.StrictRedis.from_url(redis_url, decode_responses=True)
-        self.task_key = task_key
-        self.lock_key = lock_key
-
-    def acquire_lock(self, timeout=5):
-        """Acquire a lock using Redis."""
-        while True:
-            if self.redis.set(self.lock_key, "locked", nx=True, ex=timeout):
-                return True
-            time.sleep(0.1)
-
-    def release_lock(self):
-        """Release the lock in Redis."""
-        self.redis.delete(self.lock_key)
-
-    def set_task_count(self, count):
-        """Initialize the task counter in Redis."""
-        self.redis.set(self.task_key, count)
-
-    def decrement_task_count(self):
-        """Decrement the task counter in Redis."""
-        return self.redis.decr(self.task_key)
-
-    def task_exists(self):
-        return self.redis.exists(self.task_key)
-
-    def get_task_count(self):
-        """Get the current value of the task counter."""
-        return int(self.redis.get(self.task_key) or 0)