mrok 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

mrok/http/master.py

@@ -2,6 +2,7 @@ import logging
 import os
 import signal
 import threading
+import time
 from collections.abc import Callable
 from pathlib import Path
 
@@ -11,6 +12,10 @@ from watchfiles.run import CombinedProcess, start_process
 
 logger = logging.getLogger("mrok.agent")
 
+MONITOR_THREAD_JOIN_TIMEOUT = 5
+MONITOR_THREAD_CHECK_DELAY = 1
+MONITOR_THREAD_ERROR_DELAY = 3
+
 
 def print_path(path):
     try:
@@ -29,7 +34,7 @@ class Master:
         self.start_fn = start_fn
         self.workers = workers
         self.reload = reload
-        self.worker_processes: list[CombinedProcess] = []
+        self.worker_processes: dict[int, CombinedProcess] = {}
         self.stop_event = threading.Event()
         self.watch_filter = PythonFilter(ignore_paths=None)
         self.watcher = watch(
@@ -39,6 +44,7 @@ class Master:
             yield_on_timeout=True,
         )
         self.setup_signals_handler()
+        self.monitor_thread = None
 
     def setup_signals_handler(self):
         for sig in (signal.SIGINT, signal.SIGTERM):
@@ -47,26 +53,50 @@ class Master:
     def handle_signal(self, *args, **kwargs):
         self.stop_event.set()
 
+    def start_worker(self, worker_id: int):
+        """Start a single worker process"""
+        p = start_process(
+            self.start_fn,
+            "function",
+            (),
+            None,
+        )
+        logger.info(f"Worker {worker_id} [{p.pid}] started")
+        return p
+
     def start(self):
-        for _ in range(self.workers):
-            p = start_process(
-                self.start_fn,
-                "function",
-                (),
-                None,
-            )
-            logger.info(f"Worker [{p.pid}] started")
-            self.worker_processes.append(p)
+        for i in range(self.workers):
+            p = self.start_worker(i)
+            self.worker_processes[i] = p
 
     def stop(self):
-        for process in self.worker_processes:
+        for process in self.worker_processes.values():
             process.stop(sigint_timeout=5, sigkill_timeout=1)
-        self.worker_processes = []
+        self.worker_processes.clear()
 
     def restart(self):
         self.stop()
         self.start()
 
+    def monitor_workers(self):
+        while not self.stop_event.is_set():
+            try:
+                for worker_id, process in self.worker_processes.items():
+                    if not process.is_alive():
+                        logger.warning(f"Worker {worker_id} [{process.pid}] died unexpectedly")
+                        process.stop(sigint_timeout=1, sigkill_timeout=1)
+                        new_process = self.start_worker(worker_id)
+                        self.worker_processes[worker_id] = new_process
+                        logger.info(
+                            f"Restarted worker {worker_id} [{process.pid}] -> [{new_process.pid}]"
+                        )
+
+                time.sleep(MONITOR_THREAD_CHECK_DELAY)
+
+            except Exception as e:
+                logger.error(f"Error in worker monitoring: {e}")
+                time.sleep(MONITOR_THREAD_ERROR_DELAY)
+
     def __iter__(self):
         return self
 
@@ -79,12 +109,24 @@ class Master:
     def run(self):
         self.start()
         logger.info(f"Master process started: {os.getpid()}")
-        if self.reload:
-            for files_changed in self:
-                if files_changed:
-                    logger.warning(
-                        f"{', '.join(map(print_path, files_changed))} changed, reloading...",
-                    )
-                    self.restart()
-        else:
-            self.stop_event.wait()
+
+        # Start worker monitoring thread
+        self.monitor_thread = threading.Thread(target=self.monitor_workers, daemon=True)
+        self.monitor_thread.start()
+        logger.debug("Worker monitoring thread started")
+
+        try:
+            if self.reload:
+                for files_changed in self:
+                    if files_changed:
+                        logger.warning(
+                            f"{', '.join(map(print_path, files_changed))} changed, reloading...",
+                        )
+                        self.restart()
+            else:
+                self.stop_event.wait()
+        finally:
+            if self.monitor_thread and self.monitor_thread.is_alive():  # pragma: no cover
+                logger.debug("Wait for monitor worker to exit")
+                self.monitor_thread.join(timeout=MONITOR_THREAD_JOIN_TIMEOUT)
+            self.stop()

{mrok-0.2.1.dist-info → mrok-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mrok
-Version: 0.2.1
+Version: 0.2.2
 Summary: MPT Extensions OpenZiti Orchestrator
 Author: SoftwareOne AG
 License:                                  Apache License

{mrok-0.2.1.dist-info → mrok-0.2.2.dist-info}/RECORD

@@ -48,7 +48,7 @@ mrok/http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mrok/http/config.py,sha256=k8mjvD3ninJn-v1t-co-GSa3upm4b70bWyk3fwdcOh8,2161
 mrok/http/forwarder.py,sha256=mo-Z8B8Zg6kdDX-lWEiptRv-9kJU9cEdmg6gt6eF0cc,11374
 mrok/http/lifespan.py,sha256=9qevhD_5Y0f8fGTh2axdfWx7v1K4vnWtiUNyJLesOHE,262
-mrok/http/master.py,sha256=o_0Sxe2XuTgVAwvBbWkYcO3HkCcfvYP4rgxcuIDPwXo,2426
+mrok/http/master.py,sha256=TwU78yE_GQecogBs_PDpl3gY7_jWYNIRNaAXRzi0rvY,4152
 mrok/http/protocol.py,sha256=ap8jbLUvgbAH81ZJZCBkQiYR7mkV_eL3rpfwEkoE8sU,392
 mrok/http/server.py,sha256=Mj7C85fc-DXp-WTBWaOd7ag808oliLmFBH5bf-G2FHg,370
 mrok/ziti/__init__.py,sha256=20OWMiexRhOovZOX19zlX87-V78QyWnEnSZfyAftUdE,263
@@ -59,8 +59,8 @@ mrok/ziti/errors.py,sha256=yYCbVDwktnR0AYduqtynIjo73K3HOhIrwA_vQimvEd4,368
 mrok/ziti/identities.py,sha256=oE_3j6Y4xCr6uKNdprW55bxGsyKnmJt-MrxrylB2Ey4,5388
 mrok/ziti/pki.py,sha256=o2tySqHC8-7bvFuI2Tqxg9vX6H6ZSxWxfP_9x29e19M,1954
 mrok/ziti/services.py,sha256=JnznLTHNZjgbFwnBtv7y2XIp4NiQxLVawwP9EfWdVuM,3208
-mrok-0.2.1.dist-info/METADATA,sha256=6pEGhIuJ0KJNyG3QXfLouC71PaFpywYGg2AEWrrQFLg,15546
-mrok-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-mrok-0.2.1.dist-info/entry_points.txt,sha256=tloXwvU1uJicBJR2h-8HoVclPgwJWDwuREMHN8Zq-nU,38
-mrok-0.2.1.dist-info/licenses/LICENSE.txt,sha256=6PaICaoA3yNsZKLv5G6OKqSfLSoX7MakYqTDgJoTCBs,11346
-mrok-0.2.1.dist-info/RECORD,,
+mrok-0.2.2.dist-info/METADATA,sha256=aB7i_YKGCGzw0Eo2yOHE5u7uY4Y4wuDaj24245QH3fE,15546
+mrok-0.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+mrok-0.2.2.dist-info/entry_points.txt,sha256=tloXwvU1uJicBJR2h-8HoVclPgwJWDwuREMHN8Zq-nU,38
+mrok-0.2.2.dist-info/licenses/LICENSE.txt,sha256=6PaICaoA3yNsZKLv5G6OKqSfLSoX7MakYqTDgJoTCBs,11346
+mrok-0.2.2.dist-info/RECORD,,