scalable-pypeline 2.1.27__py2.py3-none-any.whl → 2.1.29__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.27"
+__version__ = "2.1.29"

pypeline/constants.py

@@ -50,7 +50,6 @@ DEFAULT_REDIS_HEALTH_CHECK_INTERVAL = int(
 )
 
 MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
-
 MS_IN_SECONDS = 1000
 API_PATH_V1 = "/api/v1"
 

pypeline/dramatiq.py

@@ -62,13 +62,21 @@ from pypeline.utils.dramatiq_utils import (
 )
 from pypeline.utils.graceful_shutdown_util import enable_graceful_shutdown
 from pypeline.utils.module_utils import get_callable
-
+from dramatiq.middleware import (
+    Retries,
+    Callbacks,
+    TimeLimit,
+    AgeLimit,
+    ShutdownNotifications,
+    Pipelines,
+)
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 def configure_default_broker(broker: Broker = None):
+    reworked_defaults=[AgeLimit(), TimeLimit(), ShutdownNotifications(), Callbacks(), Pipelines(), Retries()]
     redis_client = None
     if REDIS_SENTINEL_MASTER_NAME is not None:
         parsed_redis_url = urlparse(REDIS_URL)
@@ -100,6 +108,7 @@ def configure_default_broker(broker: Broker = None):
                 heartbeat=DEFAULT_BROKER_CONNECTION_HEARTBEAT,
                 connection_attempts=DEFAULT_BROKER_CONNECTION_ATTEMPTS,
                 blocked_connection_timeout=DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+                middleware=reworked_defaults
             )
         )
 
@@ -111,14 +120,15 @@ def configure_default_broker(broker: Broker = None):
                 client=redis_client,
                 url=REDIS_URL,
                 heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+                middleware=reworked_defaults
             )
         )
 
     broker.add_middleware(Results(backend=redis_backend))
     broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
     broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
-    broker.add_middleware(CurrentMessage())
     broker.add_middleware(GetActiveWorkerIdMiddleware())
+    broker.add_middleware(CurrentMessage())
     if (
         os.getenv("RESTRICT_WORKER_SHUTDOWN_WHILE_JOBS_RUNNING", "false").lower()
         == "true"
@@ -126,6 +136,7 @@ def configure_default_broker(broker: Broker = None):
         enable_graceful_shutdown(broker=broker, redis_url=REDIS_URL)
 
     register_actors_for_workers(broker)
+
     set_broker(broker)
 
 

pypeline/job_runner.py

@@ -0,0 +1,123 @@
+import os
+import sys
+import signal
+import logging
+import argparse
+import threading
+import multiprocessing as mp
+
+# Prefer spawn for user code using multiprocessing
+if mp.get_start_method(allow_none=True) != "spawn":
+    mp.set_start_method("spawn", force=True)
+
+# Avoid staging more than one message; must be set before Dramatiq import path runs
+os.environ.setdefault("dramatiq_queue_prefetch", "1")
+
+from dramatiq import Worker, get_broker, set_broker  # noqa: E402
+try:
+    from pypeline.dramatiq import configure_default_broker  # adjust path/name if needed
+    broker = configure_default_broker() or get_broker()
+    set_broker(broker)
+except Exception:
+    import pypeline.dramatiq  # noqa: F401
+    broker = get_broker()
+
+from dramatiq.middleware import Middleware  # noqa: E402
+
+
+class OneAndDone(Middleware):
+    """Flip an event after the first successful message in THIS process."""
+    def __init__(self, done_event: threading.Event):
+        self.done = done_event
+
+    def after_process_message(self, broker, message, *, result=None, exception=None):
+        if exception is None and not self.done.is_set():
+            self.done.set()
+
+
+def job_runner(queues, idle_timeout_ms: int = 0):
+    """
+    Start a single-thread Dramatiq worker, process exactly one successful message, then exit.
+    - queues: list[str]
+    - idle_timeout_ms: 0 or <0 => wait forever; >0 => exit if nothing processed in time
+    """
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+    log = logging.getLogger("oneshot")
+
+    # Normalize timeout
+    timeout_ms = int(idle_timeout_ms) if idle_timeout_ms and int(idle_timeout_ms) > 0 else 0
+    log.info("Launching worker with queues=%s, idle_timeout_ms=%s", queues, timeout_ms or "∞")
+
+    done = threading.Event()
+    broker.add_middleware(OneAndDone(done))
+
+    worker = Worker(
+        broker,
+        worker_threads=1,
+        queues=queues,
+        worker_timeout=1000,  # ms; how often the worker checks for stop
+    )
+
+    worker.start()
+
+    def controller():
+        try:
+            if timeout_ms > 0:
+                finished = done.wait(timeout=timeout_ms / 1000.0)
+                if not finished:
+                    log.info("Idle timeout reached (%d ms); stopping worker.", timeout_ms)
+            else:
+                done.wait()
+
+            log.info("Stopping worker now.")
+            worker.stop()    # halts consumers; no new message will start
+            worker.join()
+        finally:
+            # Exit cleanly so K8s Job is marked Succeeded
+            sys.exit(0)
+
+    t = threading.Thread(target=controller, name="oneshot-controller", daemon=False)
+    t.start()
+
+    # Block main thread until controller finishes (which joins the worker)
+    t.join()
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Run a one-shot Dramatiq worker.")
+    parser.add_argument(
+        "-q", "--queue",
+        action="append",
+        default=None,
+        help="Queue to listen to (repeatable). You can also pass a comma-separated list."
+    )
+    parser.add_argument(
+        "--idle-timeout-ms",
+        type=int,
+        default=int(os.getenv("IDLE_TIMEOUT_MS", "0")),
+        help="Exit if no job arrives within this time (0 or negative = wait forever)."
+    )
+    args = parser.parse_args(argv)
+
+    # Build the queue list from flags or env, splitting on commas for each entry.
+    raw_entries = args.queue if args.queue else [os.getenv("JOB_QUEUE", "pipeline-queue")]
+    queues = []
+    for entry in raw_entries:
+        queues.extend([q.strip() for q in str(entry).split(",") if q and q.strip()])
+
+    if not queues:
+        raise SystemExit("No queues provided. Use -q ... or set JOB_QUEUE.")
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+    log = logging.getLogger("oneshot")
+
+    pid = os.getpid()
+    ppid = os.getppid()
+    log.info("Starting one-shot worker PID=%s, Parent PID=%s, queues=%s, idle_timeout_ms=%s",
+             pid, ppid, queues, args.idle_timeout_ms if args.idle_timeout_ms > 0 else "∞")
+
+    job_runner(queues, idle_timeout_ms=args.idle_timeout_ms)
+
+
+if __name__ == "__main__":
+    main()

pypeline/pipelines/middleware/pypeline_middleware.py

@@ -46,82 +46,74 @@ class PypelineMiddleware(Middleware):
 
         graph = get_execution_graph(pipeline_config)
         children_tasks = pipeline_config["dagAdjacency"].get(task_name, [])
-
         messages = []
         for child in children_tasks:
             child_ancestors = sorted(graph.predecessors(child))
+            child_ancestors_complete = {a: False for a in child_ancestors}
 
-            ancestor_tasks_complete = True
+            for scenario in message.options["scenarios"]:
+                if scenario["execution_id"] == execution_id:
+                    tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
 
             for ancestor in child_ancestors:
-                ancestor_task_key = f"{execution_id}-{ancestor}"
-
-                locking_parallel_barrier = LockingParallelBarrier(
-                    self.redis_url,
-                    task_key=ancestor_task_key,
-                    lock_key=f"{message.options['base_case_execution_id']}-lock",
-                )
-                try:
-                    locking_parallel_barrier.acquire_lock(
-                        timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                if ancestor in tasks_to_run_in_scenario:
+                    current_scenario_ancestor_task_key = f"{execution_id}-{ancestor}"
+                    locking_parallel_barrier = LockingParallelBarrier(
+                        self.redis_url,
+                        task_key=current_scenario_ancestor_task_key,
+                        lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
-
-                    if locking_parallel_barrier.task_exists():
-                        remaining_tasks = locking_parallel_barrier.get_task_count()
-                    else:
-                        remaining_tasks = None
-                finally:
-                    locking_parallel_barrier.release_lock()
-
-                # If the lock didn't exist for the current tasks execution id then it would indicate
-                # that this is the start of a new scenario.  Therefore we need to find the ancestor
-                # that is executed in the base case execution id and make sure it has completed
-                tasks_to_run_in_scenario = None
-
-                for scenario in message.options["scenarios"]:
-                    if scenario["execution_id"] == execution_id:
-                        tasks_to_run_in_scenario = scenario["tasksToRunInScenario"]
-
-                if ancestor not in tasks_to_run_in_scenario and remaining_tasks is None:
-                    ancestor_task_key = (
+                    try:
+                        locking_parallel_barrier.acquire_lock(
+                            timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
+                        )
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
+                    finally:
+                        locking_parallel_barrier.release_lock()
+                else:
+                    base_scenario_ancestor_task_key = (
                         f"{message.options['base_case_execution_id']}-{ancestor}"
                     )
-
                     locking_parallel_barrier = LockingParallelBarrier(
                         self.redis_url,
-                        task_key=ancestor_task_key,
+                        task_key=base_scenario_ancestor_task_key,
                         lock_key=f"{message.options['base_case_execution_id']}-lock",
                     )
                     try:
                         locking_parallel_barrier.acquire_lock(
                             timeout=PARALLEL_PIPELINE_CALLBACK_BARRIER_TTL
                         )
-
-                        if locking_parallel_barrier.task_exists():
-                            remaining_tasks = locking_parallel_barrier.get_task_count()
+                        if not locking_parallel_barrier.task_exists():
+                            child_ancestors_complete[ancestor] = False
+                        elif locking_parallel_barrier.get_task_count() <= 0:
+                            child_ancestors_complete[ancestor] = True
                     finally:
                         locking_parallel_barrier.release_lock()
 
-
-                if remaining_tasks is None or remaining_tasks >= 1:
-                    ancestor_tasks_complete = False
-                    break
-
-            # If the child's ancestor tasks aren't complete move onto the next child to check
-            if not ancestor_tasks_complete:
+            if any(complete is False for complete in child_ancestors_complete.values()):
                 continue
 
-            # Handle situation where base case kicks off new scenario.
             if (
                 message.options["base_case_execution_id"]
                 == message.options["execution_id"]
             ):
                 for scenario in message.options["scenarios"]:
-                    child_predecessors = list(graph.predecessors(child))
+                    child_ancestors = list(graph.predecessors(child))
+                    child_has_other_ancestors_in_scenario = False
+
+                    for ancestor in child_ancestors:
+                        if ancestor in scenario["tasksToRunInScenario"]:
+                            child_has_other_ancestors_in_scenario = True
+                            break
+
                     if (
                         child in scenario["tasksToRunInScenario"]
-                        and task_name in child_predecessors
+                        and task_name in child_ancestors
                         and task_name not in scenario["tasksToRunInScenario"]
+                        and not child_has_other_ancestors_in_scenario
                     ):
                         task_key = f"{scenario['execution_id']}-{child}"
                         locking_parallel_barrier = LockingParallelBarrier(
@@ -168,7 +160,7 @@ class PypelineMiddleware(Middleware):
                             )
                         messages.append(scenario_message)
 
-            # If we've made it here all ancestors of this child are complete and it's time to run.
+            # If we've made it here all ancestors of this child are complete, and it's time to run.
             task_key = f"{execution_id}-{child}"
             locking_parallel_barrier = LockingParallelBarrier(
                 self.redis_url,

{scalable_pypeline-2.1.27.dist-info → scalable_pypeline-2.1.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.27
+Version: 2.1.29
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.27.dist-info → scalable_pypeline-2.1.29.dist-info}/RECORD

@@ -1,9 +1,10 @@
-pypeline/__init__.py,sha256=nRu7ZwJqfHD5II0bBxcWlJ8czOdWw1bjjPVhymDs-NE,23
+pypeline/__init__.py,sha256=U4bLKPYS0d1L7N6hY7AxGvLuei8dvTxJ7xg7Ag0OAG0,23
 pypeline/barrier.py,sha256=ojSgbuZnGKpKiSBYXTV4CxG9j1Z01YdzBSORli4MnzI,2376
-pypeline/constants.py,sha256=7COt9jfmLDvCNAFeN6ddRpwdvv2LpbYOCIQs6dPXpOQ,3592
-pypeline/dramatiq.py,sha256=XPpgPgiOaEFK8zORx9eveJ45wzcUXMjVGryFKY5Xiwg,15527
+pypeline/constants.py,sha256=SSRWNCpJ-VjwZrwCzfE1bLXyD6_h9_bHBr4olG9Oab0,3591
+pypeline/dramatiq.py,sha256=NHwKrXG_BN7O9zjGt8y04DfynsHJmINzxPiJl4_0rZY,15862
 pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
+pypeline/job_runner.py,sha256=_8yp6s4pJBUoR_Eu0SYcIvdQFRsFn2pHeru1uHooyzw,4172
 pypeline/pipeline_config_schema.py,sha256=kRZcCMlk2FIITDzVrAfcSmHnxi1mIWmDzasTW0TnaAU,11169
 pypeline/pipeline_settings_schema.py,sha256=s_oqZ-TBiLdInSprR9k6myw1zykV2PpEbEfVaMfb5VY,21010
 pypeline/pypeline_yaml.py,sha256=hbOdwKDUg10wsZnwVaBt46FbpS3iuB3bLwVuYyXh4OY,17270
@@ -23,7 +24,7 @@ pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 pypeline/pipelines/middleware/get_active_worker_id_middleware.py,sha256=X4ZfRk3L8MD00DTsGHth7oOdy-W7LQV96T8vu5UC42A,755
 pypeline/pipelines/middleware/graceful_shutdown_middleware.py,sha256=k37zmFk9dOye05BoQP7KcB9MEQgvodI16kOJyYhRyAc,1764
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
-pypeline/pipelines/middleware/pypeline_middleware.py,sha256=tnQcewRCCaQaNMTx9Kz0gx47YZxBJCDW9UH_8cBLlwY,9317
+pypeline/pipelines/middleware/pypeline_middleware.py,sha256=FjREuPDdTfeYOVGVUjJgx8Szh6yu7g8OnHRc5N__448,9385
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pypeline/utils/config_utils.py,sha256=bblh8clRDDNQpQIDkBrtskZBo-csakoO1IJiaqVGyr8,2508
 pypeline/utils/dramatiq_utils.py,sha256=tbG3o5FD0zUOKtJJJECE2cM2ovDv3OLQ5CplJ9cXfM4,4001
@@ -33,9 +34,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.27.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.27.dist-info/METADATA,sha256=D-3zqFydd6j3FTAvnz1VSjQBobpx7XcR_Oz4J_-2BU8,5985
-scalable_pypeline-2.1.27.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.27.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.27.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.27.dist-info/RECORD,,
+scalable_pypeline-2.1.29.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.29.dist-info/METADATA,sha256=jBs9oLJgLvPN75iy2QqHHB4YBhQMZX3dhXEo63pdIy0,5985
+scalable_pypeline-2.1.29.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.29.dist-info/entry_points.txt,sha256=q5V8Qe0-5mrZ982FGbUaT2-bEURsiGjLNKEvpR6m7VU,176
+scalable_pypeline-2.1.29.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.29.dist-info/RECORD,,

{scalable_pypeline-2.1.27.dist-info → scalable_pypeline-2.1.29.dist-info}/entry_points.txt

@@ -1,3 +1,6 @@
+[console_scripts]
+job-runner = pypeline.job_runner:main
+
 [flask.commands]
 cron-scheduler = pypeline.dramatiq:cron_scheduler
 pypeline-worker = pypeline.dramatiq:pypeline_worker