scalable-pypeline 2.1.28__py2.py3-none-any.whl → 2.1.29__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.28"
+__version__ = "2.1.29"

pypeline/constants.py

@@ -50,7 +50,6 @@ DEFAULT_REDIS_HEALTH_CHECK_INTERVAL = int(
 )
 
 MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
-
 MS_IN_SECONDS = 1000
 API_PATH_V1 = "/api/v1"
 

pypeline/dramatiq.py

@@ -62,13 +62,21 @@ from pypeline.utils.dramatiq_utils import (
 )
 from pypeline.utils.graceful_shutdown_util import enable_graceful_shutdown
 from pypeline.utils.module_utils import get_callable
-
+from dramatiq.middleware import (
+    Retries,
+    Callbacks,
+    TimeLimit,
+    AgeLimit,
+    ShutdownNotifications,
+    Pipelines,
+)
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 def configure_default_broker(broker: Broker = None):
+    reworked_defaults=[AgeLimit(), TimeLimit(), ShutdownNotifications(), Callbacks(), Pipelines(), Retries()]
     redis_client = None
     if REDIS_SENTINEL_MASTER_NAME is not None:
         parsed_redis_url = urlparse(REDIS_URL)
@@ -100,6 +108,7 @@ def configure_default_broker(broker: Broker = None):
                 heartbeat=DEFAULT_BROKER_CONNECTION_HEARTBEAT,
                 connection_attempts=DEFAULT_BROKER_CONNECTION_ATTEMPTS,
                 blocked_connection_timeout=DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+                middleware=reworked_defaults
             )
         )
 
@@ -111,14 +120,15 @@ def configure_default_broker(broker: Broker = None):
                 client=redis_client,
                 url=REDIS_URL,
                 heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+                middleware=reworked_defaults
             )
         )
 
     broker.add_middleware(Results(backend=redis_backend))
     broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
     broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
-    broker.add_middleware(CurrentMessage())
     broker.add_middleware(GetActiveWorkerIdMiddleware())
+    broker.add_middleware(CurrentMessage())
     if (
         os.getenv("RESTRICT_WORKER_SHUTDOWN_WHILE_JOBS_RUNNING", "false").lower()
         == "true"
@@ -126,6 +136,7 @@ def configure_default_broker(broker: Broker = None):
         enable_graceful_shutdown(broker=broker, redis_url=REDIS_URL)
 
     register_actors_for_workers(broker)
+
     set_broker(broker)
 
 

pypeline/job_runner.py

@@ -0,0 +1,123 @@
+import os
+import sys
+import signal
+import logging
+import argparse
+import threading
+import multiprocessing as mp
+
+# Prefer spawn for user code using multiprocessing
+if mp.get_start_method(allow_none=True) != "spawn":
+    mp.set_start_method("spawn", force=True)
+
+# Avoid staging more than one message; must be set before Dramatiq import path runs
+os.environ.setdefault("dramatiq_queue_prefetch", "1")
+
+from dramatiq import Worker, get_broker, set_broker  # noqa: E402
+try:
+    from pypeline.dramatiq import configure_default_broker  # adjust path/name if needed
+    broker = configure_default_broker() or get_broker()
+    set_broker(broker)
+except Exception:
+    import pypeline.dramatiq  # noqa: F401
+    broker = get_broker()
+
+from dramatiq.middleware import Middleware  # noqa: E402
+
+
+class OneAndDone(Middleware):
+    """Flip an event after the first successful message in THIS process."""
+    def __init__(self, done_event: threading.Event):
+        self.done = done_event
+
+    def after_process_message(self, broker, message, *, result=None, exception=None):
+        if exception is None and not self.done.is_set():
+            self.done.set()
+
+
+def job_runner(queues, idle_timeout_ms: int = 0):
+    """
+    Start a single-thread Dramatiq worker, process exactly one successful message, then exit.
+    - queues: list[str]
+    - idle_timeout_ms: 0 or <0 => wait forever; >0 => exit if nothing processed in time
+    """
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+    log = logging.getLogger("oneshot")
+
+    # Normalize timeout
+    timeout_ms = int(idle_timeout_ms) if idle_timeout_ms and int(idle_timeout_ms) > 0 else 0
+    log.info("Launching worker with queues=%s, idle_timeout_ms=%s", queues, timeout_ms or "∞")
+
+    done = threading.Event()
+    broker.add_middleware(OneAndDone(done))
+
+    worker = Worker(
+        broker,
+        worker_threads=1,
+        queues=queues,
+        worker_timeout=1000,  # ms; how often the worker checks for stop
+    )
+
+    worker.start()
+
+    def controller():
+        try:
+            if timeout_ms > 0:
+                finished = done.wait(timeout=timeout_ms / 1000.0)
+                if not finished:
+                    log.info("Idle timeout reached (%d ms); stopping worker.", timeout_ms)
+            else:
+                done.wait()
+
+            log.info("Stopping worker now.")
+            worker.stop()    # halts consumers; no new message will start
+            worker.join()
+        finally:
+            # Exit cleanly so K8s Job is marked Succeeded
+            sys.exit(0)
+
+    t = threading.Thread(target=controller, name="oneshot-controller", daemon=False)
+    t.start()
+
+    # Block main thread until controller finishes (which joins the worker)
+    t.join()
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Run a one-shot Dramatiq worker.")
+    parser.add_argument(
+        "-q", "--queue",
+        action="append",
+        default=None,
+        help="Queue to listen to (repeatable). You can also pass a comma-separated list."
+    )
+    parser.add_argument(
+        "--idle-timeout-ms",
+        type=int,
+        default=int(os.getenv("IDLE_TIMEOUT_MS", "0")),
+        help="Exit if no job arrives within this time (0 or negative = wait forever)."
+    )
+    args = parser.parse_args(argv)
+
+    # Build the queue list from flags or env, splitting on commas for each entry.
+    raw_entries = args.queue if args.queue else [os.getenv("JOB_QUEUE", "pipeline-queue")]
+    queues = []
+    for entry in raw_entries:
+        queues.extend([q.strip() for q in str(entry).split(",") if q and q.strip()])
+
+    if not queues:
+        raise SystemExit("No queues provided. Use -q ... or set JOB_QUEUE.")
+
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s")
+    log = logging.getLogger("oneshot")
+
+    pid = os.getpid()
+    ppid = os.getppid()
+    log.info("Starting one-shot worker PID=%s, Parent PID=%s, queues=%s, idle_timeout_ms=%s",
+             pid, ppid, queues, args.idle_timeout_ms if args.idle_timeout_ms > 0 else "∞")
+
+    job_runner(queues, idle_timeout_ms=args.idle_timeout_ms)
+
+
+if __name__ == "__main__":
+    main()

{scalable_pypeline-2.1.28.dist-info → scalable_pypeline-2.1.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.28
+Version: 2.1.29
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.28.dist-info → scalable_pypeline-2.1.29.dist-info}/RECORD

@@ -1,9 +1,10 @@
-pypeline/__init__.py,sha256=uJ5T75rsS9yyIgD9aFR6H760hkKMyujptvhmniMPJSY,23
+pypeline/__init__.py,sha256=U4bLKPYS0d1L7N6hY7AxGvLuei8dvTxJ7xg7Ag0OAG0,23
 pypeline/barrier.py,sha256=ojSgbuZnGKpKiSBYXTV4CxG9j1Z01YdzBSORli4MnzI,2376
-pypeline/constants.py,sha256=7COt9jfmLDvCNAFeN6ddRpwdvv2LpbYOCIQs6dPXpOQ,3592
-pypeline/dramatiq.py,sha256=XPpgPgiOaEFK8zORx9eveJ45wzcUXMjVGryFKY5Xiwg,15527
+pypeline/constants.py,sha256=SSRWNCpJ-VjwZrwCzfE1bLXyD6_h9_bHBr4olG9Oab0,3591
+pypeline/dramatiq.py,sha256=NHwKrXG_BN7O9zjGt8y04DfynsHJmINzxPiJl4_0rZY,15862
 pypeline/executable_job_config_schema.py,sha256=P2Z8SO057Jgyt4I5oZxcbEi1iaZkLoAh7qp8PtuqcqU,1010
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
+pypeline/job_runner.py,sha256=_8yp6s4pJBUoR_Eu0SYcIvdQFRsFn2pHeru1uHooyzw,4172
 pypeline/pipeline_config_schema.py,sha256=kRZcCMlk2FIITDzVrAfcSmHnxi1mIWmDzasTW0TnaAU,11169
 pypeline/pipeline_settings_schema.py,sha256=s_oqZ-TBiLdInSprR9k6myw1zykV2PpEbEfVaMfb5VY,21010
 pypeline/pypeline_yaml.py,sha256=hbOdwKDUg10wsZnwVaBt46FbpS3iuB3bLwVuYyXh4OY,17270
@@ -33,9 +34,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.28.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.28.dist-info/METADATA,sha256=f0Ib-ADg8STVoxyrTo1PCy0KbOsGW9PB-jBeo5keGF8,5985
-scalable_pypeline-2.1.28.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.28.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.28.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.28.dist-info/RECORD,,
+scalable_pypeline-2.1.29.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.29.dist-info/METADATA,sha256=jBs9oLJgLvPN75iy2QqHHB4YBhQMZX3dhXEo63pdIy0,5985
+scalable_pypeline-2.1.29.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.29.dist-info/entry_points.txt,sha256=q5V8Qe0-5mrZ982FGbUaT2-bEURsiGjLNKEvpR6m7VU,176
+scalable_pypeline-2.1.29.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.29.dist-info/RECORD,,

{scalable_pypeline-2.1.28.dist-info → scalable_pypeline-2.1.29.dist-info}/entry_points.txt

@@ -1,3 +1,6 @@
+[console_scripts]
+job-runner = pypeline.job_runner:main
+
 [flask.commands]
 cron-scheduler = pypeline.dramatiq:cron_scheduler
 pypeline-worker = pypeline.dramatiq:pypeline_worker