quapp-common 0.0.15.dev2__tar.gz → 0.0.15.dev4__tar.gz

{quapp_common-0.0.15.dev2/quapp_common.egg-info → quapp_common-0.0.15.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-common
-Version: 0.0.15.dev2
+Version: 0.0.15.dev4
 Summary: Quapp common library supporting Quapp Platform for Quantum Computing
 Author-email: "CITYNOW Co. Ltd. " <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_common-0.0.15.dev2 → quapp_common-0.0.15.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-common"
-version = "0.0.15.dev2"
+version = "0.0.15.dev4"
 description = "Quapp common library supporting Quapp Platform for Quantum Computing"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd. ", email = "corp@citynow.vn" }]

quapp_common-0.0.15.dev4/quapp_common/async_tasks/async_invocation_task.py

@@ -0,0 +1,209 @@
+#  Quapp Platform Project
+#  async_invocation_task.py
+#  Copyright © CITYNOW Co. Ltd. All rights reserved.
+import asyncio
+import ctypes
+import multiprocessing
+import os
+import signal
+import threading
+import time
+
+from starlette.responses import JSONResponse
+
+from ..async_tasks.async_task import AsyncTask
+from ..component.backend.job_heartbeat import TERMINAL_STATUSES, ensure_started
+from ..component.backend.job_manager import JobManager
+from ..config.log_context import bind_job
+from ..config.logging_config import logger
+from ..factory.handler_factory import HandlerFactory
+from ..model.invocation import Event, Error, Success
+from ..util.invocation_failure import report_invocation_failure
+
+PR_SET_PDEATHSIG = 1
+ORPHAN_EXIT_CODE = 3
+PARENT_POLL_SEC = 5.0
+COMPUTE_PROCESS_PREFIX = 'job-compute'
+
+
+def fork_context():
+    return multiprocessing.get_context('fork')
+
+
+def set_parent_death_signal() -> bool:
+    try:
+        libc = ctypes.CDLL('libc.so.6', use_errno=True)
+        return libc.prctl(PR_SET_PDEATHSIG, int(signal.SIGKILL), 0, 0, 0) == 0
+    except Exception:
+        return False
+
+
+def describe_exit_code(exit_code: int) -> str:
+    if exit_code == -int(signal.SIGKILL):
+        return ('compute process was killed with SIGKILL, which usually means '
+                'the container ran out of memory')
+    if exit_code is not None and exit_code < 0:
+        try:
+            name = signal.Signals(-exit_code).name
+        except ValueError:
+            name = f'signal {-exit_code}'
+        return f'compute process was terminated by {name}'
+    if exit_code == ORPHAN_EXIT_CODE:
+        return 'compute process exited because its parent process was gone'
+    return f'compute process exited with code {exit_code}'
+
+
+def _watch_parent(parent_pid: int) -> None:
+    while True:
+        if os.getppid() != parent_pid:
+            os._exit(ORPHAN_EXIT_CODE)
+        time.sleep(PARENT_POLL_SEC)
+
+
+def run_compute(parent_pid: int, target) -> None:
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+    if not set_parent_death_signal():
+        logger.warning(
+                'Could not arm PR_SET_PDEATHSIG for the compute process; the '
+                'only remaining guard is a polling thread, which cannot run '
+                'while native code holds the GIL, so an orphaned compute '
+                'process may keep a GPU or a scheduler slot busy')
+
+    if os.getppid() != parent_pid:
+        os._exit(ORPHAN_EXIT_CODE)
+
+    watcher = threading.Thread(target=_watch_parent, args=(parent_pid,),
+                               name='compute-parent-watch', daemon=True)
+    watcher.start()
+
+    target()
+
+
+class AsyncInvocationTask(AsyncTask):
+    """
+    Handles asynchronous invocation tasks, designed to process events with
+    specific handlers and generate immediate responses.
+
+    The class is aimed at enabling tasks to be processed asynchronously, involving
+    a handler factory and functions for event processing and post-processing. It
+    executes the handler in a dedicated child process to ensure quick response to
+    clients, allowing them to poll for job status updates. Its primary purpose is
+    to facilitate efficient asynchronous processing of jobs without blocking the
+    main application flow.
+
+    Attributes:
+        handler_factory: Factory instance that produces job handlers.
+        event: Event to be processed by the created handler.
+        processing_fn: Function to be executed during the job processing stage.
+        post_processing_fn: Function to be executed post-job processing.
+    """
+
+    def __init__(self, handler_factory: HandlerFactory, event: Event,
+                 processing_fn, post_processing_fn):
+        super().__init__()
+        self.handler_factory = handler_factory
+        self.event = event
+        self.processing_fn = processing_fn
+        self.post_processing_fn = post_processing_fn
+
+    def do(self) -> JSONResponse:
+        """
+        Handles executing a job in a child process and provides an immediate response
+        with the job's ID and status. Logging is configured dynamically based on the
+        provided job ID.
+
+        Returns:
+            JSONResponse: A response object with the status and job ID if the job submission succeeds,
+            or an error message if there is an issue.
+
+        Raises:
+            Exception: If any error occurs during job processing, it is logged, and an appropriate error
+            response is returned.
+        """
+        try:
+            job_id = self.event.json().get('jobId')
+            if job_id is None:
+                raise Exception("Job ID is missing from event")
+
+            # This used to call logger.add(sink=sys.stderr, ...) for EVERY
+            # request: after N jobs there were N+1 sinks and every line printed
+            # N+1 times. The sinks are configured once in logging_config; all
+            # that is needed here is binding the context so every later line
+            # carries job_id/trace_id on its own.
+            job_log = bind_job(job_id)
+
+            def _run_handle():
+                return self.handler_factory.create_handler(self.event,
+                                                           self.processing_fn,
+                                                           self.post_processing_fn).handle()
+
+            JobManager.ensure_registry()
+            ensure_started()
+
+            loop = asyncio.get_running_loop()
+            process = fork_context().Process(
+                    target=run_compute, args=(os.getpid(), _run_handle),
+                    name=f'{COMPUTE_PROCESS_PREFIX}-{job_id}', daemon=False)
+
+            try:
+                process.start()
+            except Exception as start_exception:
+                job_log.bind(event='compute_process_start_failed').exception(
+                        f'Could not start the compute process for job '
+                        f'{job_id}: {start_exception}')
+                JobManager.set_failed(job_id)
+                report_invocation_failure(self.event, start_exception,
+                                          phase=None,
+                                          context={'jobId': job_id})
+                raise
+
+            self._watch_compute_process(loop, process, job_id, job_log)
+
+            # Respond immediately so clients can poll GET /jobs/{job_id}
+            return JSONResponse(status_code=200, content=Success(
+                    JobManager.get_job(job_id)).serialize())
+        except Exception as exception:
+            logger.exception(f"An error occurred: {exception}")
+            return JSONResponse(status_code=400,
+                                content=Error(exception).serialize())
+
+    def _watch_compute_process(self, loop, process, job_id, job_log) -> None:
+        sentinel = process.sentinel
+
+        def _on_process_exit():
+            loop.remove_reader(sentinel)
+            process.join()
+            exit_code = process.exitcode
+            if exit_code == 0:
+                return
+            self._report_compute_failure(loop, job_id, job_log, exit_code)
+
+        loop.add_reader(sentinel, _on_process_exit)
+
+    def _report_compute_failure(self, loop, job_id, job_log,
+                                exit_code) -> None:
+        reason = describe_exit_code(exit_code)
+
+        if JobManager.get_status(job_id) in TERMINAL_STATUSES:
+            job_log.bind(event='compute_process_exit_ignored').info(
+                    f'Job {job_id} already reached a terminal state, ignoring '
+                    f'that the {reason}')
+            return
+
+        job_log.bind(event='compute_process_failed').error(
+                f'Job {job_id} lost its compute process: the {reason}')
+        JobManager.set_failed(job_id)
+        loop.run_in_executor(None, self._send_failure_report, job_id, job_log,
+                             exit_code, reason)
+
+    def _send_failure_report(self, job_id, job_log, exit_code, reason) -> None:
+        try:
+            report_invocation_failure(self.event, RuntimeError(reason),
+                                      phase=None,
+                                      context={'jobId'   : job_id,
+                                               'exitCode': exit_code})
+        except Exception as exception:
+            job_log.bind(event='callback_failed').exception(
+                    f'Failed to report the lost compute process of job '
+                    f'{job_id}: {exception}')

{quapp_common-0.0.15.dev2 → quapp_common-0.0.15.dev4}/quapp_common/component/backend/job_manager.py

@@ -8,7 +8,7 @@ import atexit
 import threading
 import time
 from datetime import timezone, datetime
-from multiprocessing import Manager, current_process
+from multiprocessing import Manager, current_process, parent_process
 from typing import Any, Callable, Dict, Iterable, Optional, Tuple
 
 from ...config.logging_config import logger
@@ -61,6 +61,14 @@ class JobManager:
             # Ensure a clean shutdown in this process
             atexit.register(cls._shutdown)
 
+    @classmethod
+    def ensure_registry(cls) -> None:
+        cls._ensure_initialized()
+
+    @classmethod
+    def is_root_process(cls) -> bool:
+        return parent_process() is None
+
     @classmethod
     def _shutdown(cls) -> None:
         """
@@ -107,8 +115,9 @@ class JobManager:
         cls._jobs[job_id] = data  # proxy is process-safe
         cls._publish(job_id, {"event": "added", **data})
 
-        from .job_heartbeat import ensure_started
-        ensure_started()
+        if cls.is_root_process():
+            from .job_heartbeat import ensure_started
+            ensure_started()
 
     @classmethod
     def update_status(cls, job_id: str, status: str,
@@ -138,7 +147,7 @@ class JobManager:
         cls._publish(job_id, {"event": "updated", **new_data})
 
         from .job_heartbeat import TERMINAL_STATUSES, ensure_started
-        if status not in TERMINAL_STATUSES:
+        if status not in TERMINAL_STATUSES and cls.is_root_process():
             ensure_started()
         return None
 

{quapp_common-0.0.15.dev2 → quapp_common-0.0.15.dev4}/quapp_common/config/thread_config.py

@@ -2,6 +2,9 @@
     QApp Platform Project thread_config.py Copyright © CITYNOW Co. Ltd. All rights reserved.
 """
 import contextvars
+import os
+import queue
+import threading
 from concurrent.futures import ThreadPoolExecutor
 
 
@@ -27,3 +30,17 @@ circuit_exporting_pool = ContextAwareThreadPoolExecutor(
 
 circuit_running_pool = ContextAwareThreadPoolExecutor(
         max_workers=20, thread_name_prefix='circuit-running-pool-')
+
+
+def reset_pools_after_fork() -> None:
+    for pool in (circuit_exporting_pool, circuit_running_pool):
+        pool._threads = set()
+        pool._idle_semaphore = threading.Semaphore(0)
+        pool._work_queue = queue.SimpleQueue()
+        pool._shutdown = False
+        pool._shutdown_lock = threading.Lock()
+        pool._broken = False
+
+
+if hasattr(os, 'register_at_fork'):
+    os.register_at_fork(after_in_child=reset_pools_after_fork)

{quapp_common-0.0.15.dev2 → quapp_common-0.0.15.dev4/quapp_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-common
-Version: 0.0.15.dev2
+Version: 0.0.15.dev4
 Summary: Quapp common library supporting Quapp Platform for Quantum Computing
 Author-email: "CITYNOW Co. Ltd. " <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_common-0.0.15.dev2 → quapp_common-0.0.15.dev4}/quapp_common.egg-info/SOURCES.txt

@@ -91,6 +91,7 @@ quapp_common/util/json_parser_utils.py
 quapp_common/util/response_utils.py
 tests/test_callback_payload.py
 tests/test_circuit_adapter.py
+tests/test_compute_process.py
 tests/test_dispatcher_runner_error.py
 tests/test_error_classifier.py
 tests/test_error_log_dedupe.py