mrok 0.2.1__tar.gz → 0.2.2__tar.gz

{mrok-0.2.1 → mrok-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mrok
-Version: 0.2.1
+Version: 0.2.2
 Summary: MPT Extensions OpenZiti Orchestrator
 Author: SoftwareOne AG
 License:                                  Apache License

mrok-0.2.2/mrok/http/master.py

@@ -0,0 +1,132 @@
+import logging
+import os
+import signal
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from watchfiles import watch
+from watchfiles.filters import PythonFilter
+from watchfiles.run import CombinedProcess, start_process
+
+logger = logging.getLogger("mrok.agent")
+
+MONITOR_THREAD_JOIN_TIMEOUT = 5
+MONITOR_THREAD_CHECK_DELAY = 1
+MONITOR_THREAD_ERROR_DELAY = 3
+
+
+def print_path(path):
+    try:
+        return f'"{path.relative_to(Path.cwd())}"'
+    except ValueError:
+        return f'"{path}"'
+
+
+class Master:
+    def __init__(
+        self,
+        start_fn: Callable,
+        workers: int,
+        reload: bool,
+    ):
+        self.start_fn = start_fn
+        self.workers = workers
+        self.reload = reload
+        self.worker_processes: dict[int, CombinedProcess] = {}
+        self.stop_event = threading.Event()
+        self.watch_filter = PythonFilter(ignore_paths=None)
+        self.watcher = watch(
+            Path.cwd(),
+            watch_filter=self.watch_filter,
+            stop_event=self.stop_event,
+            yield_on_timeout=True,
+        )
+        self.setup_signals_handler()
+        self.monitor_thread = None
+
+    def setup_signals_handler(self):
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            signal.signal(sig, self.handle_signal)
+
+    def handle_signal(self, *args, **kwargs):
+        self.stop_event.set()
+
+    def start_worker(self, worker_id: int):
+        """Start a single worker process"""
+        p = start_process(
+            self.start_fn,
+            "function",
+            (),
+            None,
+        )
+        logger.info(f"Worker {worker_id} [{p.pid}] started")
+        return p
+
+    def start(self):
+        for i in range(self.workers):
+            p = self.start_worker(i)
+            self.worker_processes[i] = p
+
+    def stop(self):
+        for process in self.worker_processes.values():
+            process.stop(sigint_timeout=5, sigkill_timeout=1)
+        self.worker_processes.clear()
+
+    def restart(self):
+        self.stop()
+        self.start()
+
+    def monitor_workers(self):
+        while not self.stop_event.is_set():
+            try:
+                for worker_id, process in self.worker_processes.items():
+                    if not process.is_alive():
+                        logger.warning(f"Worker {worker_id} [{process.pid}] died unexpectedly")
+                        process.stop(sigint_timeout=1, sigkill_timeout=1)
+                        new_process = self.start_worker(worker_id)
+                        self.worker_processes[worker_id] = new_process
+                        logger.info(
+                            f"Restarted worker {worker_id} [{process.pid}] -> [{new_process.pid}]"
+                        )
+
+                time.sleep(MONITOR_THREAD_CHECK_DELAY)
+
+            except Exception as e:
+                logger.error(f"Error in worker monitoring: {e}")
+                time.sleep(MONITOR_THREAD_ERROR_DELAY)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        changes = next(self.watcher)
+        if changes:
+            return list({Path(change[1]) for change in changes})
+        return None
+
+    def run(self):
+        self.start()
+        logger.info(f"Master process started: {os.getpid()}")
+
+        # Start worker monitoring thread
+        self.monitor_thread = threading.Thread(target=self.monitor_workers, daemon=True)
+        self.monitor_thread.start()
+        logger.debug("Worker monitoring thread started")
+
+        try:
+            if self.reload:
+                for files_changed in self:
+                    if files_changed:
+                        logger.warning(
+                            f"{', '.join(map(print_path, files_changed))} changed, reloading...",
+                        )
+                        self.restart()
+            else:
+                self.stop_event.wait()
+        finally:
+            if self.monitor_thread and self.monitor_thread.is_alive():  # pragma: no cover
+                logger.debug("Wait for monitor worker to exit")
+                self.monitor_thread.join(timeout=MONITOR_THREAD_JOIN_TIMEOUT)
+            self.stop()

{mrok-0.2.1 → mrok-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mrok"
-version = "0.2.1"
+version = "0.2.2"
 description = "MPT Extensions OpenZiti Orchestrator"
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [

{mrok-0.2.1 → mrok-0.2.2}/tests/http/test_master.py

@@ -1,4 +1,6 @@
 import signal
+import threading
+import time
 from collections.abc import Generator
 from pathlib import Path
 
@@ -42,7 +44,7 @@ def test_start(mocker: MockerFixture):
     start_fn = mocker.MagicMock()
     master = Master(start_fn, 3, False)
     master.start()
-    assert master.worker_processes == processes
+    assert master.worker_processes == {0: processes[0], 1: processes[1], 2: processes[2]}
     for i in range(3):
         assert mocked_start_process.mock_calls[i].args == (start_fn, "function", (), None)
 
@@ -51,7 +53,7 @@ def test_stop(mocker: MockerFixture):
     master = Master(mocker.MagicMock(), 3, False)
     p1 = mocker.MagicMock()
     p2 = mocker.MagicMock()
-    master.worker_processes = [p1, p2]
+    master.worker_processes = {0: p1, 2: p2}
     master.stop()
     p1.stop.assert_called_once_with(sigint_timeout=5, sigkill_timeout=1)
     p2.stop.assert_called_once_with(sigint_timeout=5, sigkill_timeout=1)
@@ -84,17 +86,20 @@ def test_next(mocker: MockerFixture):
 
 def test_run(mocker: MockerFixture):
     mocked_start = mocker.patch.object(Master, "start")
+    mocked_monitor_fn = mocker.patch.object(Master, "monitor_workers")
     master = Master(mocker.MagicMock(), 3, False)
     mocked_stop_event = mocker.MagicMock()
     master.stop_event = mocked_stop_event
     master.run()
     mocked_start.assert_called_once()
     mocked_stop_event.wait.assert_called_once()
+    mocked_monitor_fn.assert_called_once()
 
 
 def test_run_with_reload(mocker: MockerFixture):
     mocker.patch.object(Master, "start")
     mocked_restart = mocker.patch.object(Master, "restart")
+    mocked_monitor_fn = mocker.patch.object(Master, "monitor_workers")
 
     def watcher() -> Generator:
         yield {(Change.modified, "/file1.py")}
@@ -105,3 +110,57 @@ def test_run_with_reload(mocker: MockerFixture):
     master.run()
 
     mocked_restart.assert_called_once()
+    mocked_monitor_fn.assert_called_once()
+
+
+def test_monitor_workers_restarts_dead_process(mocker: MockerFixture):
+    mocker.patch("mrok.http.master.MONITOR_THREAD_CHECK_DELAY", 0.1)
+
+    mock_start_process = mocker.patch("mrok.http.master.start_process")
+    master = Master(mocker.MagicMock(), 3, False)
+
+    dead_process = mocker.Mock()
+    dead_process.is_alive.return_value = False
+    dead_process.pid = 12345
+
+    alive_process = mocker.Mock()
+    alive_process.is_alive.return_value = True
+    alive_process.pid = 12346
+
+    new_process = mocker.Mock()
+    new_process.pid = 12347
+
+    master.worker_processes = {0: dead_process, 1: alive_process}
+
+    mock_start_process.return_value = new_process
+
+    monitor_thread = threading.Thread(target=master.monitor_workers)
+    monitor_thread.start()
+    time.sleep(0.1)
+    master.stop_event.set()
+    monitor_thread.join()
+
+    dead_process.stop.assert_called_once_with(sigint_timeout=1, sigkill_timeout=1)
+    mock_start_process.assert_called_once_with(master.start_fn, "function", (), None)
+    assert master.worker_processes[0] == new_process
+    assert master.worker_processes[1] == alive_process
+
+
+def test_monitor_workers_handles_is_alive_exception(mocker: MockerFixture):
+    mocker.patch("mrok.http.master.MONITOR_THREAD_ERROR_DELAY", 0.1)
+    mock_logger = mocker.patch("mrok.http.master.logger.error")
+    master = Master(mocker.MagicMock(), 3, False)
+
+    problematic_process = mocker.Mock()
+    problematic_process.is_alive.side_effect = Exception("Test exception")
+
+    master.worker_processes = {0: problematic_process}
+
+    monitor_thread = threading.Thread(target=master.monitor_workers)
+    monitor_thread.start()
+
+    time.sleep(0.1)
+    master.stop_event.set()
+    monitor_thread.join()
+
+    assert mock_logger.mock_calls[0].args[0] == "Error in worker monitoring: Test exception"

mrok-0.2.1/mrok/http/master.py

@@ -1,90 +0,0 @@
-import logging
-import os
-import signal
-import threading
-from collections.abc import Callable
-from pathlib import Path
-
-from watchfiles import watch
-from watchfiles.filters import PythonFilter
-from watchfiles.run import CombinedProcess, start_process
-
-logger = logging.getLogger("mrok.agent")
-
-
-def print_path(path):
-    try:
-        return f'"{path.relative_to(Path.cwd())}"'
-    except ValueError:
-        return f'"{path}"'
-
-
-class Master:
-    def __init__(
-        self,
-        start_fn: Callable,
-        workers: int,
-        reload: bool,
-    ):
-        self.start_fn = start_fn
-        self.workers = workers
-        self.reload = reload
-        self.worker_processes: list[CombinedProcess] = []
-        self.stop_event = threading.Event()
-        self.watch_filter = PythonFilter(ignore_paths=None)
-        self.watcher = watch(
-            Path.cwd(),
-            watch_filter=self.watch_filter,
-            stop_event=self.stop_event,
-            yield_on_timeout=True,
-        )
-        self.setup_signals_handler()
-
-    def setup_signals_handler(self):
-        for sig in (signal.SIGINT, signal.SIGTERM):
-            signal.signal(sig, self.handle_signal)
-
-    def handle_signal(self, *args, **kwargs):
-        self.stop_event.set()
-
-    def start(self):
-        for _ in range(self.workers):
-            p = start_process(
-                self.start_fn,
-                "function",
-                (),
-                None,
-            )
-            logger.info(f"Worker [{p.pid}] started")
-            self.worker_processes.append(p)
-
-    def stop(self):
-        for process in self.worker_processes:
-            process.stop(sigint_timeout=5, sigkill_timeout=1)
-        self.worker_processes = []
-
-    def restart(self):
-        self.stop()
-        self.start()
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        changes = next(self.watcher)
-        if changes:
-            return list({Path(change[1]) for change in changes})
-        return None
-
-    def run(self):
-        self.start()
-        logger.info(f"Master process started: {os.getpid()}")
-        if self.reload:
-            for files_changed in self:
-                if files_changed:
-                    logger.warning(
-                        f"{', '.join(map(print_path, files_changed))} changed, reloading...",
-                    )
-                    self.restart()
-        else:
-            self.stop_event.wait()