scalable-pypeline 2.1.5__py2.py3-none-any.whl → 2.1.6__py2.py3-none-any.whl

pypeline/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.5"
+__version__ = "2.1.6"

pypeline/constants.py

@@ -36,6 +36,7 @@ DEFAULT_BROKER_CONNECTION_ATTEMPTS = int(
 DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT = int(
     os.getenv("DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT", 30)
 )
+DEFAULT_BROKER_HEARTBEAT_TIMEOUT = int("DEFAULT_BROKER_HEARTBEAT_TIMEOUT", 300_000)
 MESSAGE_BROKER = os.getenv("MESSAGE_BROKER", "RABBITMQ")
 
 MS_IN_SECONDS = 1000

pypeline/dramatiq.py

@@ -34,6 +34,10 @@ from pypeline.constants import (
     DEFAULT_BROKER_BLOCKED_CONNECTION_TIMEOUT,
     DEFAULT_BROKER_CONNECTION_ATTEMPTS,
     MESSAGE_BROKER,
+    DEFAULT_BROKER_HEARTBEAT_TIMEOUT,
+)
+from pypeline.pipelines.middleware.get_active_worker_id_middleware import (
+    GetActiveWorkerIdMiddleware,
 )
 from pypeline.pipelines.middleware.parallel_pipeline_middleware import ParallelPipeline
 from pypeline.pipelines.middleware.pypeline_middleware import PypelineMiddleware
@@ -74,12 +78,19 @@ def configure_default_broker(broker: Broker = None):
         )
 
     elif MESSAGE_BROKER == "REDIS":
-        broker = broker if broker is not None else RedisBroker(url=REDIS_URL)
+        broker = (
+            broker
+            if broker is not None
+            else RedisBroker(
+                url=REDIS_URL, heartbeat_timeout=DEFAULT_BROKER_HEARTBEAT_TIMEOUT
+            )
+        )
 
     broker.add_middleware(Results(backend=redis_backend))
     broker.add_middleware(ParallelPipeline(redis_url=REDIS_URL))
     broker.add_middleware(PypelineMiddleware(redis_url=REDIS_URL))
     broker.add_middleware(CurrentMessage())
+    broker.add_middleware(GetActiveWorkerIdMiddleware())
     register_actors_for_workers(broker)
     set_broker(broker)
 

pypeline/pipelines/middleware/get_active_worker_id_middleware.py

@@ -0,0 +1,18 @@
+import contextvars
+from dramatiq import Middleware
+
+
+class GetActiveWorkerIdMiddleware(Middleware):
+    _ACTIVE_WORKER_ID: contextvars.ContextVar["Optional[Message[Any]]"] = (
+        contextvars.ContextVar("_ACTIVE_WORKER_ID", default=None)
+    )
+
+    @classmethod
+    def get_active_worker_id(cls):
+        return cls._ACTIVE_WORKER_ID.get()
+
+    def before_process_message(self, broker, message):
+        self._ACTIVE_WORKER_ID.set(broker.broker_id)
+
+    def after_process_message(self, broker, message, *, result=None, exception=None):
+        self._ACTIVE_WORKER_ID.set(None)

{scalable_pypeline-2.1.5.dist-info → scalable_pypeline-2.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scalable-pypeline
-Version: 2.1.5
+Version: 2.1.6
 Summary: PypeLine - Python pipelines for the Real World
 Home-page: https://gitlab.com/bravos2/pypeline
 Author: Bravos Power Corporation

{scalable_pypeline-2.1.5.dist-info → scalable_pypeline-2.1.6.dist-info}/RECORD

@@ -1,7 +1,7 @@
-pypeline/__init__.py,sha256=Ol8KeLnnX1kAXAFgJsZj4d_cZMypHQtaiyICJpuzp64,22
+pypeline/__init__.py,sha256=Z4RaxSmfOjvKssLeOljGX3Csb8I17J-Dc7pSVg3Hb2U,22
 pypeline/barrier.py,sha256=oO964l9qOCOibweOHyNivmAvufdXOke9nz2tdgclouo,1172
-pypeline/constants.py,sha256=415-5fTJQXPO4by14T4BBC6hOn11m96XFiAHSh9Sfxo,2949
-pypeline/dramatiq.py,sha256=NyNwAw4iibWnS5GhTVQWxAOfBj3VXkfgSliilMa4ajg,12501
+pypeline/constants.py,sha256=CUo4hHfM0__nMjQq8Sr6N6sW5AwJddvvJhKrcXlST24,3033
+pypeline/dramatiq.py,sha256=5whdOQjzknLjl9lYNj5-f2jRw5ysRPhQExi5NN4KeOY,12837
 pypeline/extensions.py,sha256=BzOTnXhNxap3N7uIUUh_hO6dDwx08Vc_RJDE93_K0Lo,610
 pypeline/pipeline_config_schema.py,sha256=hK2_egtg-YFx_XJDs_NyrOTGKkel7W83X-G0sic52sM,10592
 pypeline/pipeline_settings_schema.py,sha256=84AuNFYsOUpoADsjEo_n9T6Ica-c21oK_V9s15I4lCg,20212
@@ -19,6 +19,7 @@ pypeline/pipelines/composition/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 pypeline/pipelines/composition/parallel_pipeline_composition.py,sha256=pTw9Xb9h4JnV4siFc3JStm5lB-i9djUADo3Kh5K3s7g,12976
 pypeline/pipelines/composition/pypeline_composition.py,sha256=UBuDKEfRoIbL-9c-HH2ZTVbzfkwFSlNoFH-AVNqt0QE,7965
 pypeline/pipelines/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pypeline/pipelines/middleware/get_active_worker_id_middleware.py,sha256=g2lDUUoUtD8do-x_XqGurkiKY_eNv0MUYSs5EchRwi0,581
 pypeline/pipelines/middleware/parallel_pipeline_middleware.py,sha256=kTp6niYoe2nXIiN6EGRfdpxrJyioo0GPxDkfefbGlEk,2821
 pypeline/pipelines/middleware/pypeline_middleware.py,sha256=IXVqzcOlSJ43lsn-i298RkaeygB-PTJjsvdTDtpgfwg,8141
 pypeline/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -28,9 +29,9 @@ pypeline/utils/module_utils.py,sha256=-yEJIukDCoXnmlZVXB6Dww25tH6GdPE5SoFqv6pfdV
 pypeline/utils/pipeline_utils.py,sha256=kGP1QwCJikGC5QNRtzRXCDVewyRMpWIqERTNnxGLlSY,4795
 pypeline/utils/schema_utils.py,sha256=Fgl0y9Cuo_TZeEx_S3gaSVnLjn6467LTkjb2ek7Ms98,851
 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scalable_pypeline-2.1.5.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
-scalable_pypeline-2.1.5.dist-info/METADATA,sha256=Fsm58uF_UU13xtUbVLq1bGtxPkwIZRnOvwbpB766URM,5926
-scalable_pypeline-2.1.5.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-scalable_pypeline-2.1.5.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
-scalable_pypeline-2.1.5.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
-scalable_pypeline-2.1.5.dist-info/RECORD,,
+scalable_pypeline-2.1.6.dist-info/LICENSE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+scalable_pypeline-2.1.6.dist-info/METADATA,sha256=BOpe4jN8ylP1iOf8FJyvdgTAF0V66sCEhwaibPvf3_Q,5926
+scalable_pypeline-2.1.6.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+scalable_pypeline-2.1.6.dist-info/entry_points.txt,sha256=uWs10ODfHSBKo2Cx_QaUjPHQTpZ3e77j9VlAdRRmMyg,119
+scalable_pypeline-2.1.6.dist-info/top_level.txt,sha256=C7dpkEOc_-nnsAQb28BfQknjD6XHRyS9ZrvVeoIbV7s,15
+scalable_pypeline-2.1.6.dist-info/RECORD,,