datashare-python 0.10.1__tar.gz → 0.10.5__tar.gz

{datashare_python-0.10.1 → datashare_python-0.10.5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: datashare-python
-Version: 0.10.1
+Version: 0.10.5
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.10.1 → datashare_python-0.10.5}/datashare_python/interceptors.py

@@ -1,18 +1,20 @@
 import asyncio
 import dataclasses
 import datetime
+import logging
 import secrets
 from collections.abc import Callable, Generator, Mapping
 from contextlib import contextmanager
 from contextvars import ContextVar
 from copy import deepcopy
 from functools import wraps
-from inspect import signature
+from inspect import iscoroutinefunction, signature
 from types import UnionType
 from typing import (
     Annotated,
     Any,
     NoReturn,
+    ParamSpec,
     Self,
     TypeVar,
     get_args,
@@ -27,6 +29,7 @@ from temporalio.activity import _Definition
 from temporalio.api.common.v1 import Payload
 from temporalio.client import WorkflowHandle
 from temporalio.converter import DataConverter
+from temporalio.service import RPCError
 from temporalio.worker import (
     ActivityInboundInterceptor,
     ActivityOutboundInterceptor,
@@ -66,6 +69,8 @@ from .utils import (
     ProgressSignal,
 )
 
+logger = logging.getLogger(__name__)
+
 _TRACEPARENT = "traceparent"
 _DEFAULT_PAYLOAD_CONVERTER = DataConverter.default.payload_converter
 _PROGRESS_TYPES = {
@@ -190,6 +195,8 @@ class _TraceContextWorkflowOutboundInterceptor(WorkflowOutboundInterceptor):
 
 class _TraceContextActivityInboundInterceptor(ActivityInboundInterceptor):
     async def execute_activity(self, input: ExecuteActivityInput) -> Any:  # noqa: A002
+        info = activity.info()
+        logger.info("start executing activity: %s", info.activity_id)
         with _trace_context(input.headers):
             return await super().execute_activity(input)
 
@@ -327,7 +334,8 @@ def _get_progress_handler(
         weight=weight,
         min_progress_interval_s=min_progress_interval_s,
     )
-    return handler.progress
+    handler = _fail_safe(handler.progress, "emit progress", excs=(RPCError,))
+    return handler
 
 
 def _is_progress(t: type) -> bool:
@@ -431,9 +439,10 @@ class _HeartbeatInboundInterceptor(ActivityInboundInterceptor):
         heartbeat_task = None
         if heartbeat_timeout:
             period = heartbeat_timeout.total_seconds() / self._n_missed_before_timeout
-            heartbeat_task = asyncio.create_task(
-                _heartbeat_every(self._outbound.heartbeat, period)
-            )
+            # We don't want a failing hearbeat to fail the worker task so we just
+            # ignore any exception
+            heartbeat_fn = _fail_safe(self._outbound.heartbeat, "heartbeat")
+            heartbeat_task = asyncio.create_task(_heartbeat_every(heartbeat_fn, period))
         try:
             activity.heartbeat()
             return await super().execute_activity(input)
@@ -443,6 +452,40 @@ class _HeartbeatInboundInterceptor(ActivityInboundInterceptor):
                 await asyncio.wait([heartbeat_task])
 
 
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+def _fail_safe[P, T](
+    fn: Callable[P, T],
+    action: str,
+    excs: tuple[type[Exception]] | None = None,
+) -> Callable[P, T]:
+    if excs is None:
+        excs = (Exception,)
+    if iscoroutinefunction(fn):
+
+        @wraps(fn)
+        async def wrapper(*args, **kwargs) -> None:
+            try:
+                await fn(*args, **kwargs)
+            except excs as exc:
+                msg = f"failed to {action} due to {exc}"
+                logger.exception(msg)
+
+    else:
+
+        @wraps(fn)
+        def wrapper(*args, **kwargs) -> None:
+            try:
+                fn(*args, **kwargs)
+            except excs as exc:
+                msg = f"failed to {action} due to {exc}"
+                logger.exception(msg)
+
+    return wrapper
+
+
 def _sync_progress(
     progress_handler: AsyncProgressRateHandler,
 ) -> SyncProgressRateHandler:

{datashare_python-0.10.1 → datashare_python-0.10.5}/datashare_python/objects.py

@@ -14,7 +14,7 @@ from icij_common.registrable import Registrable
 from pydantic_core import PydanticCustomError, ValidationError, core_schema
 from pydantic_core.core_schema import PlainValidatorFunctionSchema
 from pydantic_extra_types.language_code import LanguageName
-from temporalio import workflow
+from temporalio import activity, workflow
 
 from .constants import TIKA_METADATA_RESOURCENAME
 
@@ -325,6 +325,8 @@ A = TypeVar("A", bound=TaskArgs)
 
 class ManifestEntry[A](DatashareModel, ABC):
     status: ManifestEntryStatus
+    # TODO: make this one non optional in the next major !
+    task_id: str | None
     label: str | None = None
     input: Annotated[
         dict[str, Any] | None,
@@ -336,7 +338,11 @@ class ManifestEntry[A](DatashareModel, ABC):
 
     @classmethod
     def complete(cls, args: A, label: str | None = None, **kwargs) -> Self:
+        task_id = None
+        if activity.in_activity():
+            task_id = activity.info().workflow_id
         return cls(
+            task_id=task_id,
             input=args.as_manifest_task_input(),
             label=label,
             status=ManifestEntryStatus.COMPLETE,
@@ -345,7 +351,11 @@ class ManifestEntry[A](DatashareModel, ABC):
 
     @classmethod
     def partial(cls, args: A, label: str | None = None, **kwargs) -> Self:
+        task_id = None
+        if activity.in_activity():
+            task_id = activity.info().workflow_id
         return cls(
+            task_id=task_id,
             input=args.as_manifest_task_input(),
             label=label,
             status=ManifestEntryStatus.PARTIAL,

{datashare_python-0.10.1 → datashare_python-0.10.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.10.1"
+version = "0.10.5"
 description = "Manage Python tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },