quapp-common 0.0.15.dev1__tar.gz → 0.0.15.dev4__tar.gz

{quapp_common-0.0.15.dev1/quapp_common.egg-info → quapp_common-0.0.15.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-common
-Version: 0.0.15.dev1
+Version: 0.0.15.dev4
 Summary: Quapp common library supporting Quapp Platform for Quantum Computing
 Author-email: "CITYNOW Co. Ltd. " <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-common"
-version = "0.0.15.dev1"
+version = "0.0.15.dev4"
 description = "Quapp common library supporting Quapp Platform for Quantum Computing"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd. ", email = "corp@citynow.vn" }]

quapp_common-0.0.15.dev4/quapp_common/async_tasks/async_invocation_task.py

@@ -0,0 +1,209 @@
+#  Quapp Platform Project
+#  async_invocation_task.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+import asyncio
+import ctypes
+import multiprocessing
+import os
+import signal
+import threading
+import time
+
+from starlette.responses import JSONResponse
+
+from ..async_tasks.async_task import AsyncTask
+from ..component.backend.job_heartbeat import TERMINAL_STATUSES, ensure_started
+from ..component.backend.job_manager import JobManager
+from ..config.log_context import bind_job
+from ..config.logging_config import logger
+from ..factory.handler_factory import HandlerFactory
+from ..model.invocation import Event, Error, Success
+from ..util.invocation_failure import report_invocation_failure
+
+PR_SET_PDEATHSIG = 1
+ORPHAN_EXIT_CODE = 3
+PARENT_POLL_SEC = 5.0
+COMPUTE_PROCESS_PREFIX = 'job-compute'
+
+
+def fork_context():
+    return multiprocessing.get_context('fork')
+
+
+def set_parent_death_signal() -> bool:
+    try:
+        libc = ctypes.CDLL('libc.so.6', use_errno=True)
+        return libc.prctl(PR_SET_PDEATHSIG, int(signal.SIGKILL), 0, 0, 0) == 0
+    except Exception:
+        return False
+
+
+def describe_exit_code(exit_code: int) -> str:
+    if exit_code == -int(signal.SIGKILL):
+        return ('compute process was killed with SIGKILL, which usually means '
+                'the container ran out of memory')
+    if exit_code is not None and exit_code < 0:
+        try:
+            name = signal.Signals(-exit_code).name
+        except ValueError:
+            name = f'signal {-exit_code}'
+        return f'compute process was terminated by {name}'
+    if exit_code == ORPHAN_EXIT_CODE:
+        return 'compute process exited because its parent process was gone'
+    return f'compute process exited with code {exit_code}'
+
+
+def _watch_parent(parent_pid: int) -> None:
+    while True:
+        if os.getppid() != parent_pid:
+            os._exit(ORPHAN_EXIT_CODE)
+        time.sleep(PARENT_POLL_SEC)
+
+
+def run_compute(parent_pid: int, target) -> None:
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+    if not set_parent_death_signal():
+        logger.warning(
+                'Could not arm PR_SET_PDEATHSIG for the compute process; the '
+                'only remaining guard is a polling thread, which cannot run '
+                'while native code holds the GIL, so an orphaned compute '
+                'process may keep a GPU or a scheduler slot busy')
+
+    if os.getppid() != parent_pid:
+        os._exit(ORPHAN_EXIT_CODE)
+
+    watcher = threading.Thread(target=_watch_parent, args=(parent_pid,),
+                               name='compute-parent-watch', daemon=True)
+    watcher.start()
+
+    target()
+
+
+class AsyncInvocationTask(AsyncTask):
+    """
+    Handles asynchronous invocation tasks, designed to process events with
+    specific handlers and generate immediate responses.
+
+    The class is aimed at enabling tasks to be processed asynchronously, involving
+    a handler factory and functions for event processing and post-processing. It
+    executes the handler in a dedicated child process to ensure quick response to
+    clients, allowing them to poll for job status updates. Its primary purpose is
+    to facilitate efficient asynchronous processing of jobs without blocking the
+    main application flow.
+
+    Attributes:
+        handler_factory: Factory instance that produces job handlers.
+        event: Event to be processed by the created handler.
+        processing_fn: Function to be executed during the job processing stage.
+        post_processing_fn: Function to be executed post-job processing.
+    """
+
+    def __init__(self, handler_factory: HandlerFactory, event: Event,
+                 processing_fn, post_processing_fn):
+        super().__init__()
+        self.handler_factory = handler_factory
+        self.event = event
+        self.processing_fn = processing_fn
+        self.post_processing_fn = post_processing_fn
+
+    def do(self) -> JSONResponse:
+        """
+        Handles executing a job in a child process and provides an immediate response
+        with the job's ID and status. Logging is configured dynamically based on the
+        provided job ID.
+
+        Returns:
+            JSONResponse: A response object with the status and job ID if the job submission succeeds,
+            or an error message if there is an issue.
+
+        Raises:
+            Exception: If any error occurs during job processing, it is logged, and an appropriate error
+            response is returned.
+        """
+        try:
+            job_id = self.event.json().get('jobId')
+            if job_id is None:
+                raise Exception("Job ID is missing from event")
+
+            # This used to call logger.add(sink=sys.stderr, ...) for EVERY
+            # request: after N jobs there were N+1 sinks and every line printed
+            # N+1 times. The sinks are configured once in logging_config; all
+            # that is needed here is binding the context so every later line
+            # carries job_id/trace_id on its own.
+            job_log = bind_job(job_id)
+
+            def _run_handle():
+                return self.handler_factory.create_handler(self.event,
+                                                           self.processing_fn,
+                                                           self.post_processing_fn).handle()
+
+            JobManager.ensure_registry()
+            ensure_started()
+
+            loop = asyncio.get_running_loop()
+            process = fork_context().Process(
+                    target=run_compute, args=(os.getpid(), _run_handle),
+                    name=f'{COMPUTE_PROCESS_PREFIX}-{job_id}', daemon=False)
+
+            try:
+                process.start()
+            except Exception as start_exception:
+                job_log.bind(event='compute_process_start_failed').exception(
+                        f'Could not start the compute process for job '
+                        f'{job_id}: {start_exception}')
+                JobManager.set_failed(job_id)
+                report_invocation_failure(self.event, start_exception,
+                                          phase=None,
+                                          context={'jobId': job_id})
+                raise
+
+            self._watch_compute_process(loop, process, job_id, job_log)
+
+            # Respond immediately so clients can poll GET /jobs/{job_id}
+            return JSONResponse(status_code=200, content=Success(
+                    JobManager.get_job(job_id)).serialize())
+        except Exception as exception:
+            logger.exception(f"An error occurred: {exception}")
+            return JSONResponse(status_code=400,
+                                content=Error(exception).serialize())
+
+    def _watch_compute_process(self, loop, process, job_id, job_log) -> None:
+        sentinel = process.sentinel
+
+        def _on_process_exit():
+            loop.remove_reader(sentinel)
+            process.join()
+            exit_code = process.exitcode
+            if exit_code == 0:
+                return
+            self._report_compute_failure(loop, job_id, job_log, exit_code)
+
+        loop.add_reader(sentinel, _on_process_exit)
+
+    def _report_compute_failure(self, loop, job_id, job_log,
+                                exit_code) -> None:
+        reason = describe_exit_code(exit_code)
+
+        if JobManager.get_status(job_id) in TERMINAL_STATUSES:
+            job_log.bind(event='compute_process_exit_ignored').info(
+                    f'Job {job_id} already reached a terminal state, ignoring '
+                    f'that the {reason}')
+            return
+
+        job_log.bind(event='compute_process_failed').error(
+                f'Job {job_id} lost its compute process: the {reason}')
+        JobManager.set_failed(job_id)
+        loop.run_in_executor(None, self._send_failure_report, job_id, job_log,
+                             exit_code, reason)
+
+    def _send_failure_report(self, job_id, job_log, exit_code, reason) -> None:
+        try:
+            report_invocation_failure(self.event, RuntimeError(reason),
+                                      phase=None,
+                                      context={'jobId'   : job_id,
+                                               'exitCode': exit_code})
+        except Exception as exception:
+            job_log.bind(event='callback_failed').exception(
+                    f'Failed to report the lost compute process of job '
+                    f'{job_id}: {exception}')

quapp_common-0.0.15.dev4/quapp_common/component/backend/job_heartbeat.py

@@ -0,0 +1,164 @@
+#  Quapp Platform Project
+#  job_heartbeat.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+
+import math
+import multiprocessing
+import os
+import sys
+import threading
+import time
+
+import requests
+
+from ...config.logging_config import logger
+
+REQUEST_TIMEOUT_SEC = 5.0
+
+DEFAULT_INTERVAL_SEC = 30.0
+MAX_INTERVAL_SEC = DEFAULT_INTERVAL_SEC
+ORPHAN_POLL_SEC = 5.0
+STOP_GRACE_SEC = 1.0
+TERMINAL_STATUSES = ('DONE', 'FAILED')
+
+_context = None
+_condition = threading.Condition()
+_process = None
+_stop_event = None
+_starting = False
+
+
+def heartbeat_interval_sec() -> float:
+    raw = os.getenv('HEARTBEAT_INTERVAL_SEC', '')
+    try:
+        value = float(raw)
+    except ValueError:
+        return DEFAULT_INTERVAL_SEC
+    if math.isfinite(value) and 0 < value <= MAX_INTERVAL_SEC:
+        return value
+    return DEFAULT_INTERVAL_SEC
+
+
+def _log_safely(level: str, message: str) -> None:
+    try:
+        getattr(logger, level)(message)
+    except Exception as exception:
+        try:
+            print(f'[job-heartbeat] {level}: {message} ({exception})',
+                  file=sys.stderr, flush=True)
+        except Exception:
+            pass
+
+
+def beat_once() -> int:
+    from .job_manager import JobManager, META, STATUS, SCHEDULER_CALLBACK_URL
+
+    sent = 0
+    for job_id, data in JobManager.list_jobs().items():
+        if data.get(STATUS) in TERMINAL_STATUSES:
+            continue
+        url = (data.get(META) or {}).get(SCHEDULER_CALLBACK_URL)
+        if not url:
+            continue
+        try:
+            response = requests.patch(
+                    url, json={'status': 'RUNNING', 'heartbeat': True},
+                    timeout=REQUEST_TIMEOUT_SEC)
+            sent += 1
+            _log_safely('debug',
+                        f'Heartbeat for job {job_id} returned {response.status_code}')
+        except Exception as exception:
+            _log_safely('debug',
+                        f'Heartbeat for job {job_id} failed: {exception}')
+    return sent
+
+
+def _wait_for_next_beat(parent_pid: int, stop_event, interval: float) -> bool:
+    deadline = time.monotonic() + interval
+    while True:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            return True
+        if stop_event.wait(min(remaining, ORPHAN_POLL_SEC)):
+            return False
+        if os.getppid() != parent_pid:
+            return False
+
+
+def _heartbeat_loop(parent_pid: int, stop_event) -> None:
+    while os.getppid() == parent_pid:
+        try:
+            beat_once()
+        except Exception as exception:
+            _log_safely('exception', f'Heartbeat loop error: {exception}')
+        if not _wait_for_next_beat(parent_pid, stop_event,
+                                   heartbeat_interval_sec()):
+            return
+
+
+def _is_running_or_starting(process) -> bool:
+    if process is None:
+        return False
+    return process.pid is None or process.is_alive()
+
+
+def _shutdown_process(process, stop_event) -> None:
+    if stop_event is not None:
+        stop_event.set()
+
+    if process is None or process.pid is None:
+        return
+
+    process.join(timeout=STOP_GRACE_SEC)
+    if process.is_alive():
+        process.terminate()
+        process.join(timeout=STOP_GRACE_SEC)
+    if process.is_alive():
+        process.kill()
+        process.join(timeout=STOP_GRACE_SEC)
+
+
+def ensure_started() -> None:
+    global _context, _process, _stop_event, _starting
+
+    with _condition:
+        while _starting:
+            _condition.wait()
+        if _is_running_or_starting(_process):
+            return
+        if _context is None:
+            _context = multiprocessing.get_context('fork')
+        stop_event = _context.Event()
+        process = _context.Process(
+                target=_heartbeat_loop, args=(os.getpid(), stop_event),
+                name='job-heartbeat', daemon=True)
+        _stop_event = stop_event
+        _process = process
+        _starting = True
+
+    try:
+        process.start()
+    finally:
+        with _condition:
+            _starting = False
+            if process.pid is None and _process is process:
+                _process = None
+                _stop_event = None
+            _condition.notify_all()
+
+    logger.info(
+            f'Job heartbeat started with interval {heartbeat_interval_sec()}s')
+
+
+def stop() -> None:
+    global _process, _stop_event
+
+    with _condition:
+        while _starting:
+            _condition.wait()
+        process = _process
+        stop_event = _stop_event
+        _process = None
+        _stop_event = None
+
+    _shutdown_process(process, stop_event)

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev4}/quapp_common/component/backend/job_manager.py

@@ -8,7 +8,7 @@ import atexit
 import threading
 import time
 from datetime import timezone, datetime
-from multiprocessing import Manager, current_process
+from multiprocessing import Manager, current_process, parent_process
 from typing import Any, Callable, Dict, Iterable, Optional, Tuple
 
 from ...config.logging_config import logger
@@ -61,13 +61,25 @@ class JobManager:
             # Ensure a clean shutdown in this process
             atexit.register(cls._shutdown)
 
+    @classmethod
+    def ensure_registry(cls) -> None:
+        cls._ensure_initialized()
+
+    @classmethod
+    def is_root_process(cls) -> bool:
+        return parent_process() is None
+
     @classmethod
     def _shutdown(cls) -> None:
         """
-        Stop any listener threads in this process and shutdown Manager.
+        Stop the heartbeat process, then any listener threads in this process,
+        then shutdown Manager.
         """
         from .job_heartbeat import stop
-        stop()
+        try:
+            stop()
+        except Exception as exception:
+            logger.exception(f'Error stopping job heartbeat: {exception}')
 
         # stop listeners
         for key, (t, stop_ev) in list(cls._listener_threads.items()):
@@ -103,8 +115,9 @@ class JobManager:
         cls._jobs[job_id] = data  # proxy is process-safe
         cls._publish(job_id, {"event": "added", **data})
 
-        from .job_heartbeat import ensure_started
-        ensure_started()
+        if cls.is_root_process():
+            from .job_heartbeat import ensure_started
+            ensure_started()
 
     @classmethod
     def update_status(cls, job_id: str, status: str,
@@ -132,6 +145,10 @@ class JobManager:
                     UPDATED_AT: cls._get_current_utc_time()}
         cls._jobs[job_id] = new_data
         cls._publish(job_id, {"event": "updated", **new_data})
+
+        from .job_heartbeat import TERMINAL_STATUSES, ensure_started
+        if status not in TERMINAL_STATUSES and cls.is_root_process():
+            ensure_started()
         return None
 
     @classmethod

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev4}/quapp_common/config/thread_config.py

@@ -2,6 +2,9 @@
     QApp Platform Project thread_config.py Copyright © CITYNOW Co. Ltd. All rights reserved.
 """
 import contextvars
+import os
+import queue
+import threading
 from concurrent.futures import ThreadPoolExecutor
 
 
@@ -27,3 +30,17 @@ circuit_exporting_pool = ContextAwareThreadPoolExecutor(
 
 circuit_running_pool = ContextAwareThreadPoolExecutor(
         max_workers=20, thread_name_prefix='circuit-running-pool-')
+
+
+def reset_pools_after_fork() -> None:
+    for pool in (circuit_exporting_pool, circuit_running_pool):
+        pool._threads = set()
+        pool._idle_semaphore = threading.Semaphore(0)
+        pool._work_queue = queue.SimpleQueue()
+        pool._shutdown = False
+        pool._shutdown_lock = threading.Lock()
+        pool._broken = False
+
+
+if hasattr(os, 'register_at_fork'):
+    os.register_at_fork(after_in_child=reset_pools_after_fork)

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev4/quapp_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-common
-Version: 0.0.15.dev1
+Version: 0.0.15.dev4
 Summary: Quapp common library supporting Quapp Platform for Quantum Computing
 Author-email: "CITYNOW Co. Ltd. " <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_common-0.0.15.dev1 → quapp_common-0.0.15.dev4}/quapp_common.egg-info/SOURCES.txt

@@ -91,6 +91,7 @@ quapp_common/util/json_parser_utils.py
 quapp_common/util/response_utils.py
 tests/test_callback_payload.py
 tests/test_circuit_adapter.py
+tests/test_compute_process.py
 tests/test_dispatcher_runner_error.py
 tests/test_error_classifier.py
 tests/test_error_log_dedupe.py