quapp-common 0.0.15.dev1__tar.gz → 0.0.15.dev2__tar.gz

{quapp_common-0.0.15.dev1/quapp_common.egg-info → quapp_common-0.0.15.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-common
-Version: 0.0.15.dev1
+Version: 0.0.15.dev2
 Summary: Quapp common library supporting Quapp Platform for Quantum Computing
 Author-email: "CITYNOW Co. Ltd. " <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-common"
-version = "0.0.15.dev1"
+version = "0.0.15.dev2"
 description = "Quapp common library supporting Quapp Platform for Quantum Computing"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd. ", email = "corp@citynow.vn" }]

quapp_common-0.0.15.dev2/quapp_common/component/backend/job_heartbeat.py

@@ -0,0 +1,164 @@
+#  Quapp Platform Project
+#  job_heartbeat.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+import math
+import multiprocessing
+import os
+import sys
+import threading
+import time
+
+import requests
+
+from ...config.logging_config import logger
+
+REQUEST_TIMEOUT_SEC = 5.0
+
+DEFAULT_INTERVAL_SEC = 30.0
+MAX_INTERVAL_SEC = DEFAULT_INTERVAL_SEC
+ORPHAN_POLL_SEC = 5.0
+STOP_GRACE_SEC = 1.0
+TERMINAL_STATUSES = ('DONE', 'FAILED')
+
+_context = None
+_condition = threading.Condition()
+_process = None
+_stop_event = None
+_starting = False
+
+
+def heartbeat_interval_sec() -> float:
+    raw = os.getenv('HEARTBEAT_INTERVAL_SEC', '')
+    try:
+        value = float(raw)
+    except ValueError:
+        return DEFAULT_INTERVAL_SEC
+    if math.isfinite(value) and 0 < value <= MAX_INTERVAL_SEC:
+        return value
+    return DEFAULT_INTERVAL_SEC
+
+
+def _log_safely(level: str, message: str) -> None:
+    try:
+        getattr(logger, level)(message)
+    except Exception as exception:
+        try:
+            print(f'[job-heartbeat] {level}: {message} ({exception})',
+                  file=sys.stderr, flush=True)
+        except Exception:
+            pass
+
+
+def beat_once() -> int:
+    from .job_manager import JobManager, META, STATUS, SCHEDULER_CALLBACK_URL
+
+    sent = 0
+    for job_id, data in JobManager.list_jobs().items():
+        if data.get(STATUS) in TERMINAL_STATUSES:
+            continue
+        url = (data.get(META) or {}).get(SCHEDULER_CALLBACK_URL)
+        if not url:
+            continue
+        try:
+            response = requests.patch(
+                    url, json={'status': 'RUNNING', 'heartbeat': True},
+                    timeout=REQUEST_TIMEOUT_SEC)
+            sent += 1
+            _log_safely('debug',
+                        f'Heartbeat for job {job_id} returned {response.status_code}')
+        except Exception as exception:
+            _log_safely('debug',
+                        f'Heartbeat for job {job_id} failed: {exception}')
+    return sent
+
+
+def _wait_for_next_beat(parent_pid: int, stop_event, interval: float) -> bool:
+    deadline = time.monotonic() + interval
+    while True:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            return True
+        if stop_event.wait(min(remaining, ORPHAN_POLL_SEC)):
+            return False
+        if os.getppid() != parent_pid:
+            return False
+
+
+def _heartbeat_loop(parent_pid: int, stop_event) -> None:
+    while os.getppid() == parent_pid:
+        try:
+            beat_once()
+        except Exception as exception:
+            _log_safely('exception', f'Heartbeat loop error: {exception}')
+        if not _wait_for_next_beat(parent_pid, stop_event,
+                                   heartbeat_interval_sec()):
+            return
+
+
+def _is_running_or_starting(process) -> bool:
+    if process is None:
+        return False
+    return process.pid is None or process.is_alive()
+
+
+def _shutdown_process(process, stop_event) -> None:
+    if stop_event is not None:
+        stop_event.set()
+
+    if process is None or process.pid is None:
+        return
+
+    process.join(timeout=STOP_GRACE_SEC)
+    if process.is_alive():
+        process.terminate()
+        process.join(timeout=STOP_GRACE_SEC)
+    if process.is_alive():
+        process.kill()
+        process.join(timeout=STOP_GRACE_SEC)
+
+
+def ensure_started() -> None:
+    global _context, _process, _stop_event, _starting
+
+    with _condition:
+        while _starting:
+            _condition.wait()
+        if _is_running_or_starting(_process):
+            return
+        if _context is None:
+            _context = multiprocessing.get_context('fork')
+        stop_event = _context.Event()
+        process = _context.Process(
+                target=_heartbeat_loop, args=(os.getpid(), stop_event),
+                name='job-heartbeat', daemon=True)
+        _stop_event = stop_event
+        _process = process
+        _starting = True
+
+    try:
+        process.start()
+    finally:
+        with _condition:
+            _starting = False
+            if process.pid is None and _process is process:
+                _process = None
+                _stop_event = None
+            _condition.notify_all()
+
+    logger.info(
+            f'Job heartbeat started with interval {heartbeat_interval_sec()}s')
+
+
+def stop() -> None:
+    global _process, _stop_event
+
+    with _condition:
+        while _starting:
+            _condition.wait()
+        process = _process
+        stop_event = _stop_event
+        _process = None
+        _stop_event = None
+
+    _shutdown_process(process, stop_event)

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev2}/quapp_common/component/backend/job_manager.py

@@ -64,10 +64,14 @@ class JobManager:
     @classmethod
     def _shutdown(cls) -> None:
         """
-        Stop any listener threads in this process and shutdown Manager.
+        Stop the heartbeat process, then any listener threads in this process,
+        then shutdown Manager.
         """
         from .job_heartbeat import stop
-        stop()
+        try:
+            stop()
+        except Exception as exception:
+            logger.exception(f'Error stopping job heartbeat: {exception}')
 
         # stop listeners
         for key, (t, stop_ev) in list(cls._listener_threads.items()):
@@ -132,6 +136,10 @@ class JobManager:
                     UPDATED_AT: cls._get_current_utc_time()}
         cls._jobs[job_id] = new_data
         cls._publish(job_id, {"event": "updated", **new_data})
+
+        from .job_heartbeat import TERMINAL_STATUSES, ensure_started
+        if status not in TERMINAL_STATUSES:
+            ensure_started()
         return None
 
     @classmethod

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev2/quapp_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-common
-Version: 0.0.15.dev1
+Version: 0.0.15.dev2
 Summary: Quapp common library supporting Quapp Platform for Quantum Computing
 Author-email: "CITYNOW Co. Ltd. " <corp@citynow.vn>
 License: The MIT License (MIT)