scalable-pypeline 2.1.3__py2.py3-none-any.whl → 2.1.4__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.3"
+__version__ = "2.1.4"

pypeline/constants.py

@@ -36,6 +36,7 @@ DEFAULT_BROKER_CONNECTION_ATTEMPTS = int(
 DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT = int(
     os.getenv("DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT", 30)
 )
+MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
 
 MS_IN_SECONDS = 1000
 API_PATH_V1 = "/api/v1"

pypeline/dramatiq.py

@@ -5,6 +5,8 @@ import logging
 import click
 from urllib.parse import urlparse
 
+from dramatiq.brokers.redis import RedisBroker
+
 from pypeline.extensions import pypeline_config
 from warnings import warn
 from apscheduler.schedulers.blocking import BlockingScheduler
@@ -31,12 +33,10 @@ from pypeline.constants import (
     DEFAULT_BROKER_CONNECTION_HEARTBEAT,
     DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
     DEFAULT_BROKER_CONNECTION_ATTEMPTS,
+    MESSAGE_BROKER,
 )
 from pypeline.pipelines.middleware.parallel_pipeline_middleware import ParallelPipeline
 from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
-from pypeline.pipelines.middleware.deduplication_middleware import (
-    DeduplicationMiddleware,
-)
 from pypeline.utils.config_utils import (
     retrieve_latest_schedule_config,
     get_service_config_for_worker,
@@ -56,27 +56,32 @@ logger = logging.getLogger(__name__)
 
 def configure_default_broker(broker: Broker = None):
     redis_backend = RedisBackend(url=REDIS_URL)
-    parsed_url = urlparse(RABBIT_URL)
-    credentials = pika.PlainCredentials(parsed_url.username, parsed_url.password)
-    rabbit_broker = (
-        broker
-        if broker is not None
-        else RabbitmqBroker(
-            host=parsed_url.hostname,
-            port=parsed_url.port,
-            credentials=credentials,
-            heartbeat=DEFAULT_BROKER_CONNECTION_HEARTBEAT,
-            connection_attempts=DEFAULT_BROKER_CONNECTION_ATTEMPTS,
-            blocked_connection_timeout=DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+
+    if MESSAGE_BROKER == "RABBITMQ":
+        parsed_url = urlparse(RABBIT_URL)
+        credentials = pika.PlainCredentials(parsed_url.username, parsed_url.password)
+        broker = (
+            broker
+            if broker is not None
+            else RabbitmqBroker(
+                host=parsed_url.hostname,
+                port=parsed_url.port,
+                credentials=credentials,
+                heartbeat=DEFAULT_BROKER_CONNECTION_HEARTBEAT,
+                connection_attempts=DEFAULT_BROKER_CONNECTION_ATTEMPTS,
+                blocked_connection_timeout=DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
+            )
         )
-    )
-    rabbit_broker.add_middleware(Results(backend=redis_backend))
-    rabbit_broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
-    rabbit_broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
-    rabbit_broker.add_middleware(CurrentMessage())
-    register_actors_for_workers(rabbit_broker)
-    rabbit_broker.add_middleware(DeduplicationMiddleware(redis_url=REDIS_URL))
-    set_broker(rabbit_broker)
+
+    elif MESSAGE_BROKER == "REDIS":
+        broker = broker if broker is not None else RedisBroker(url=REDIS_URL)
+
+    broker.add_middleware(Results(backend=redis_backend))
+    broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
+    broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
+    broker.add_middleware(CurrentMessage())
+    register_actors_for_workers(broker)
+    set_broker(broker)
 
 
 def register_actors_for_workers(broker: Broker):

{scalable_pypeline-2.1.3.dist-info → scalable_pypeline-2.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.3
+Version: 2.1.4
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.3.dist-info → scalable_pypeline-2.1.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
-pypeline/__init__.py,sha256=-5z5R8xV0UToQjp9-3ipF_dBiBdRXtdotx4_h9ZJZT8,22
+pypeline/__init__.py,sha256=1yR20YsjyDpnFQgDIQmHfutaSsaW0F7mDqjloRVRIG8,22
 pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
-pypeline/constants.py,sha256=coiF8dMP25qIwoNYSnS7oy7hCd4-5yqPFmdPsN93Q1A,2892
-pypeline/dramatiq.py,sha256=TGwXWSInpCPFtYC5C-Omc2sEk5ecpsOG8xHH_mx9WTo,12451
+pypeline/constants.py,sha256=415-5fTJQXPO4by14T4BBC6hOn11m96XFiAHSh9Sfxo,2949
+pypeline/dramatiq.py,sha256=NyNwAw4iibWnS5GhTVQWxAOfBj3VXkfgSliilMa4ajg,12501
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
 pypeline/pipeline_config_schema.py,sha256=hK2_egtg-YFx_XJDs_NyrOTGKkel7W83X-G0sic52sM,10592
 pypeline/pipeline_settings_schema.py,sha256=84AuNFYsOUpoADsjEo_n9T6Ica-c21oK_V9s15I4lCg,20212
@@ -19,7 +19,6 @@ pypeline/pipelines/composition/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 pypeline/pipelines/composition/parallel_pipeline_composition.py,sha256=pTw9Xb9h4JnV4siFc3JStm5lB-i9djUADo3Kh5K3s7g,12976
 pypeline/pipelines/composition/pypeline_composition.py,sha256=ieTuQZ8zxTtvmPEkrWFbItjGtvO3JUotXcR-Jim2mss,7204
 pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pypeline/pipelines/middleware/deduplication_middleware.py,sha256=AxlvvTRbXfMtZXW5z7P72HfF3zVM_vKd5VvZeKToZcM,3655
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
 pypeline/pipelines/middleware/pypeline_middleware.py,sha256=kvt5A9OxDwpIo0PsH11Im62tH6VquUc6OFoZDw2Gxsk,8036
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -29,9 +28,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.3.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.3.dist-info/METADATA,sha256=h_svvRncrZllbvgkpsYcD_Wd-bIGWGAq_hC4Hj7kCGk,5926
-scalable_pypeline-2.1.3.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.3.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.3.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.3.dist-info/RECORD,,
+scalable_pypeline-2.1.4.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.4.dist-info/METADATA,sha256=FLViI0pH_Zoa13dTAv75RvV-FZgYXZZZR2WIpbB81lg,5926
+scalable_pypeline-2.1.4.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.4.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.4.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.4.dist-info/RECORD,,

pypeline/pipelines/middleware/deduplication_middleware.py

@@ -1,94 +0,0 @@
-import dramatiq
-import signal
-from dramatiq.middleware import Middleware
-from pypeline.barrier import LockingParallelBarrier
-from pypeline.constants import DEFAULT_TASK_TTL
-import logging
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class DeduplicationMiddleware(Middleware):
-    def __init__(self, redis_url="redis://localhost:6379/0"):
-        self.redis_url = redis_url
-        self.active_locks = {}
-
-    def before_process_message(self, broker, message):
-        task_id = message.message_id
-        task_key = f"dramatiq:task_counter:{task_id}"
-        lock_key = f"dramatiq:lock:{task_id}"
-        try:
-            # Try to acquire a lock for the task
-            locking_parallel_barrier = LockingParallelBarrier(
-                self.redis_url,
-                task_key=task_key,
-                lock_key=lock_key,
-            )
-            if (
-                locking_parallel_barrier.get_task_count() > 0
-                or not locking_parallel_barrier.acquire_lock(timeout=DEFAULT_TASK_TTL)
-            ):
-                logger.info(f"Found duplicate task {task_id}.  Skipping...")
-                raise dramatiq.middleware.SkipMessage(
-                    f"Task {task_id} is already being processed."
-                )
-
-            locking_parallel_barrier.set_task_count(1)
-            # Store the lock reference in the message and track it globally
-            message.options["dedupe_task_key"] = task_key
-            message.options["dedupe_lock_key"] = lock_key
-            self.active_locks[lock_key] = locking_parallel_barrier
-        except dramatiq.middleware.SkipMessage:
-            raise dramatiq.middleware.SkipMessage(
-                f"Task {task_id} is already being processed."
-            )
-        except Exception as e:
-            logger.exception(e)
-            raise e
-
-    def after_process_message(self, broker, message, *, result=None, exception=None):
-        """Releases lock for the message that just finished."""
-        dedupe_task_key = message.options.get("dedupe_task_key", None)
-        dedupe_lock_key = message.options.get("dedupe_lock_key", None)
-        if not dedupe_lock_key or not dedupe_task_key:
-            logger.warning(
-                "unexpected in after_process_message: dedupe task or lock key not in message"
-            )
-            return
-        if dedupe_lock_key in self.active_locks:
-            try:
-                lock = self.active_locks[dedupe_lock_key]
-                lock.decrement_task_count()
-                lock.release_lock()
-                del self.active_locks[dedupe_lock_key]
-            except Exception as e:
-                logger.info(
-                    f"Exception while trying to release lock {dedupe_lock_key}: {e}"
-                )
-                raise e
-        else:
-            lock = LockingParallelBarrier(
-                self.redis_url,
-                task_key=dedupe_task_key,
-                lock_key=dedupe_lock_key,
-            )
-            lock.decrement_task_count()
-            lock.release_lock()
-
-    def before_worker_shutdown(self, *args):
-        self.release_all_locks()
-
-    def before_worker_thread_shutdown(self, *args):
-        self.release_all_locks()
-
-    def release_all_locks(self, *args):
-        """Release all locks when the worker shuts down."""
-        for lock_key, lock in self.active_locks.items():
-            try:
-                lock.decrement_task_count()
-                lock.release_lock()
-            except Exception as e:
-                logger.info(f"Exception while trying to release lock {lock_key}: {e}")
-                raise e
-        self.active_locks.clear()