mint-sdk 1.2.4__tar.gz → 1.2.6__tar.gz

{mint_sdk-1.2.4 → mint_sdk-1.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: mint-sdk
-Version: 1.2.4
+Version: 1.2.6
 Summary: MINT Plugin SDK - Build analysis plugins for the MINT (Mass-spec INtegrated Toolkit) platform
 Project-URL: Homepage, https://github.com/MorscherLab/MINT
 Project-URL: Documentation, https://github.com/MorscherLab/MINT/tree/main/packages/sdk-python#readme

{mint_sdk-1.2.4 → mint_sdk-1.2.6}/src/mint_sdk/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.4'
-__version_tuple__ = version_tuple = (1, 2, 4)
+__version__ = version = '1.2.6'
+__version_tuple__ = version_tuple = (1, 2, 6)
 
 __commit_id__ = commit_id = None

{mint_sdk-1.2.4 → mint_sdk-1.2.6}/src/mint_sdk/docs/bundled/frontend.json

@@ -1,6 +1,6 @@
 {
   "sdk": "frontend",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "categories": {
     "components": [
       {

{mint_sdk-1.2.4 → mint_sdk-1.2.6}/src/mint_sdk/job_manager.py

@@ -21,6 +21,7 @@ import os
 import pickle
 import re
 import signal
+import traceback
 from collections import deque
 from collections.abc import Callable, Mapping
 from contextlib import suppress
@@ -83,8 +84,13 @@ from mint_sdk.temp_byte_quota import (
 )
 
 _CANCEL_GRACE_SECONDS = 3.0
+# SIGKILL cannot be caught, but the kernel still has to tear the worker down:
+# a 10 GB, 158-thread worker took ~1 s after SIGKILL to disappear.
+_WORKER_KILL_WAIT_SECONDS = 30.0
 _FINALIZER_RETRY_BASE_SECONDS = 0.1
 _MAX_PROGRESS_BYTES = 64 * 1024
+_MAX_ERROR_CHARS = 2000
+_MAX_TRACEBACK_CHARS = 64 * 1024
 _MEDIA_TYPE_PATTERN = re.compile(r"^[A-Za-z0-9!#$&^_.+-]+/[A-Za-z0-9!#$&^_.+-]+$")
 
 logger = logging.getLogger(__name__)
@@ -1568,10 +1574,12 @@ class JobManager:
                 item.job_id,
                 f"Job result exceeds the configured size limit (job {item.job_id})",
             )
-        except Exception:
+        except Exception as exc:
+            logger.exception("Service job failed (job %s)", item.job_id)
             state = runtime.jobs.get_snapshot(item.job_id)
             if state is not None and state.status not in TERMINAL_JOB_STATUSES:
-                runtime.jobs.fail(item.job_id, f"Service job failed (job {item.job_id})")
+                cause = exc.__cause__ or exc
+                runtime.jobs.fail(item.job_id, _job_error(_describe_exception(cause), item.job_id))
         finally:
             if running.draining:
                 runtime.jobs.finish_draining(item.job_id)
@@ -1651,7 +1659,17 @@ class JobManager:
                     if running.cancel_requested:
                         runtime.jobs.cancel(item.job_id)
                     else:
-                        runtime.jobs.fail(item.job_id, f"Job failed (job {item.job_id})")
+                        failure = payload if isinstance(payload, dict) else {}
+                        detail = failure.get("message")
+                        trace = failure.get("traceback")
+                        logger.error(
+                            "Job worker failed (job %s)\n%s",
+                            item.job_id,
+                            trace[:_MAX_TRACEBACK_CHARS] if isinstance(trace, str) else "",
+                        )
+                        if not isinstance(detail, str) or not detail:
+                            detail = "Job failed"
+                        runtime.jobs.fail(item.job_id, _job_error(detail, item.job_id))
                     break
                 if not running.process.is_alive():
                     if running.cancel_requested:
@@ -1707,7 +1725,7 @@ class JobManager:
             force=True,
         )
         if running.process.is_alive():
-            await asyncio.to_thread(running.process.join, 1.0)
+            await asyncio.to_thread(running.process.join, _WORKER_KILL_WAIT_SECONDS)
         running.reaped = not running.process.is_alive()
         return running.reaped
 
@@ -2085,10 +2103,13 @@ def _job_worker_entry(
             connection.send(("result_too_large", None))
             return
         connection.send(("result", payload))
-    except BaseException:
-        logger.exception("Job worker failed (job %s)", job_id)
-        with suppress(BrokenPipeError, EOFError, OSError):
-            connection.send(("error", None))
+    except BaseException as exc:
+        # The host logs the traceback; its handlers own the platform log file.
+        failure = {"message": _describe_exception(exc), "traceback": traceback.format_exc()}
+        try:
+            connection.send(("error", failure))
+        except (BrokenPipeError, EOFError, OSError):
+            logger.exception("Job worker failed (job %s)", job_id)
     finally:
         connection.close()
 
@@ -2117,6 +2138,16 @@ def _has_control_characters(value: str) -> bool:
     return any(ord(character) < 32 or ord(character) == 127 for character in value)
 
 
+def _describe_exception(exc: BaseException) -> str:
+    return f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
+
+
+def _job_error(detail: str, job_id: str) -> str:
+    """Bound a handler-supplied failure reason and tag it with the job id."""
+
+    return f"{detail[:_MAX_ERROR_CHARS]} (job {job_id})"
+
+
 def _send_worker_progress(connection: Connection, values: dict[str, Any]) -> None:
     if _json_size(values) > _MAX_PROGRESS_BYTES:
         raise ValueError("Job progress update exceeds the configured size limit")

{mint_sdk-1.2.4 → mint_sdk-1.2.6}/tests/test_job_manager.py

@@ -169,6 +169,10 @@ class _ServicePlugin:
         finally:
             self.active -= 1
 
+    @job(profile=ServiceJob(timeout=1))
+    async def raises(self, inputs: _Inputs) -> str:
+        raise ValueError(f"service broke on {inputs.label}")
+
     @job(profile=ServiceJob(timeout=0.01))
     async def times_out(self, inputs: _Inputs) -> str:
         await asyncio.sleep(inputs.delay)
@@ -280,6 +284,10 @@ class _Plugin:
         context.warn("Low signal")
         return inputs.label
 
+    @job(cpu=1)
+    def raises(self, inputs: _Inputs) -> str:
+        raise ValueError(f"worker broke on {inputs.label}")
+
     @job(cpu=1)
     def invalid_output(self, inputs: _Inputs) -> int:
         return inputs.label  # type: ignore[return-value]
@@ -718,6 +726,26 @@ async def test_service_jobs_timeout_and_cancel_without_worker_processes() -> Non
     assert cancelled is not None and cancelled.status.value == "cancelled"
 
 
+@pytest.mark.asyncio
+async def test_service_exception_message_is_reported_as_job_error() -> None:
+    manager = JobManager(JobManagerLimits(cpu_slots=1))
+    runtime = manager.register(_ServicePlugin())
+    session = runtime.create_session(owner_user_id="alice", tab_id="service")
+    try:
+        job_state = await runtime.submit(
+            "raises",
+            {"label": "sample.raw"},
+            actor=PluginActor(user_id="alice"),
+            session_id=session.id,
+            client_request_id="service-raises",
+        )
+        completed = await runtime.wait(job_state.job_id, timeout=5.0)
+    finally:
+        await manager.shutdown()
+
+    assert completed.error == f"ValueError: service broke on sample.raw (job {job_state.job_id})"
+
+
 @pytest.mark.asyncio
 async def test_service_cancel_is_bounded_when_handler_swallows_cancellation(
     monkeypatch: pytest.MonkeyPatch,
@@ -1600,6 +1628,51 @@ async def test_invalid_progress_fails_job_instead_of_leaving_it_running() -> Non
     assert completed.status.value == "failed"
 
 
+@pytest.mark.asyncio
+async def test_worker_exception_message_is_reported_as_job_error() -> None:
+    manager = JobManager(JobManagerLimits(cpu_slots=1))
+    runtime = manager.register(_Plugin())
+    session = runtime.create_session(owner_user_id="alice", tab_id="a")
+    try:
+        job_state = await runtime.submit(
+            "raises",
+            {"label": "sample.raw"},
+            actor=PluginActor(user_id="alice"),
+            session_id=session.id,
+            client_request_id="raises",
+        )
+        completed = await runtime.wait(job_state.job_id, timeout=5.0)
+    finally:
+        await manager.shutdown()
+
+    assert completed.error == f"ValueError: worker broke on sample.raw (job {job_state.job_id})"
+
+
+@pytest.mark.asyncio
+async def test_host_logs_worker_traceback(caplog: pytest.LogCaptureFixture) -> None:
+    manager = JobManager(JobManagerLimits(cpu_slots=1))
+    runtime = manager.register(_Plugin())
+    session = runtime.create_session(owner_user_id="alice", tab_id="a")
+    try:
+        job_state = await runtime.submit(
+            "raises",
+            {"label": "logged.raw"},
+            actor=PluginActor(user_id="alice"),
+            session_id=session.id,
+            client_request_id="raises-logged",
+        )
+        await runtime.wait(job_state.job_id, timeout=5.0)
+    finally:
+        await manager.shutdown()
+
+    assert any(
+        record.name == job_manager_module.__name__
+        and "Traceback" in record.getMessage()
+        and "worker broke on logged.raw" in record.getMessage()
+        for record in caplog.records
+    )
+
+
 @pytest.mark.asyncio
 async def test_structured_progress_and_warnings_cross_the_worker_boundary() -> None:
     manager = JobManager(JobManagerLimits(cpu_slots=1))
@@ -1845,9 +1918,8 @@ def test_worker_rejects_oversized_result_before_sending_payload(
     assert sent_messages == ["ready", "result_too_large"]
 
 
-def test_worker_handler_exception_logs_traceback(
+def test_worker_handler_exception_sends_traceback_to_host(
     monkeypatch: pytest.MonkeyPatch,
-    caplog: pytest.LogCaptureFixture,
 ) -> None:
     connection = Mock()
     definition = Mock(id="failing-job")
@@ -1870,9 +1942,11 @@ def test_worker_handler_exception_logs_traceback(
         connection,
     )
 
-    assert (
-        "Job worker failed (job job-id)" in caplog.text
-        and "RuntimeError: instrument offline" in caplog.text
+    message, payload = connection.send.call_args_list[-1].args[0]
+    assert (message, payload["message"], "Traceback" in payload["traceback"]) == (
+        "error",
+        "RuntimeError: instrument offline",
+        True,
     )
 
 
@@ -1908,6 +1982,32 @@ def test_force_cancel_terminates_the_worker_process_group(
     )
 
 
+class _SlowDyingProcess:
+    """A killed worker whose kernel teardown outlasts one second."""
+
+    pid = 4321
+
+    def __init__(self, seconds_to_die: float) -> None:
+        self.remaining = seconds_to_die
+
+    def join(self, timeout: float | None = None) -> None:
+        self.remaining -= timeout or 0
+
+    def is_alive(self) -> bool:
+        return self.remaining > 0
+
+
+def test_reap_waits_for_a_killed_worker_with_slow_teardown(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(job_manager_module, "_terminate_worker_process", Mock())
+    running = Mock(reaped=False, process=_SlowDyingProcess(seconds_to_die=5.0))
+
+    reaped = asyncio.run(JobManager._reap_worker(Mock(), running))
+
+    assert reaped is True
+
+
 def test_cancel_never_signals_the_host_process_group(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None: