datashare-python 0.10.6__tar.gz → 0.10.8__tar.gz

{datashare_python-0.10.6 → datashare_python-0.10.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datashare-python
-Version: 0.10.6
+Version: 0.10.8
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.10.6 → datashare_python-0.10.8}/datashare_python/objects.py

@@ -1,3 +1,5 @@
+import hashlib
+import json
 import logging
 import os
 from abc import ABC
@@ -63,6 +65,12 @@ Predicate = Callable[[T], bool] | Callable[[T], Awaitable[bool]]
 class BaseModel(_BaseModel):
     model_config = merge_configs(icij_config(), no_enum_values_config())
 
+    def __hash__(self) -> int:
+        digest = hashlib.md5(
+            json.dumps(self.model_dump(mode="json"), sort_keys=True).encode()
+        ).digest()
+        return int.from_bytes(digest[:8])
+
 
 class DatashareModel(BaseModel):
     model_config = merge_configs(BaseModel.model_config, lowercamel_case_config())

{datashare_python-0.10.6 → datashare_python-0.10.8}/datashare_python/utils.py

@@ -32,6 +32,7 @@ import temporalio
 from aiofile import async_open
 from lru import LRU
 from pydantic import ValidationError
+from pydantic.alias_generators import to_snake
 from temporalio import activity, workflow
 from temporalio.api.common.v1 import Payload
 from temporalio.client import Client
@@ -62,6 +63,8 @@ from .objects import (
 )
 from .types_ import RawAsyncProgressHandler
 
+_CACHE_DIR = "__cache__"
+
 logger = logging.getLogger(__name__)
 
 DependencyLabel = str | None
@@ -225,7 +228,7 @@ def _unpack_positional_args(
 
 
 def with_retriables[**P, T](
-    retriables: set[type[Exception]] = None,
+    logger: logging.Logger, retriables: set[type[Exception]] = None
 ) -> Callable[[Callable[P, T]], Callable[P, T]]:
     if retriables is None:
 
@@ -237,6 +240,7 @@ def with_retriables[**P, T](
                     try:
                         return await activity_fn(*args, **kwargs)
                     except Exception as e:
+                        logger.exception("failing activity due to non retriable error")
                         raise fatal_error_from_exception(e) from e
             else:
 
@@ -245,6 +249,7 @@ def with_retriables[**P, T](
                     try:
                         return activity_fn(*args, **kwargs)
                     except Exception as e:
+                        logger.exception("failing activity due to non retriable error")
                         raise fatal_error_from_exception(e) from e
 
             return wrapper
@@ -259,8 +264,12 @@ def with_retriables[**P, T](
                 try:
                     return await activity_fn(*args, **kwargs)
                 except retriables:
+                    logger.exception(
+                        "retriable error occurred during activity, will retry !"
+                    )
                     raise
                 except Exception as e:
+                    logger.exception("failing activity due to non retriable error")
                     raise fatal_error_from_exception(e) from e
 
         else:
@@ -270,8 +279,12 @@ def with_retriables[**P, T](
                 try:
                     return activity_fn(*args, **kwargs)
                 except retriables:
+                    logger.exception(
+                        "retriable error occurred during activity, will retry !"
+                    )
                     raise
                 except Exception as e:
+                    logger.exception("failing activity due to non retriable error")
                     raise fatal_error_from_exception(e) from e
 
         return wrapper
@@ -280,12 +293,19 @@ def with_retriables[**P, T](
 
 
 def activity_defn[**P, T](
-    name: str, retriables: set[type[Exception]] = None
+    name: str,
+    retriables: set[type[Exception]] = None,
+    logger: logging.Logger | None = None,
 ) -> Callable[[Callable[P, T]], Callable[P, T]]:
 
+    if logger is None:
+        frame = inspect.stack()[1]
+        caller_module = inspect.getmodule(frame[0])
+        logger = logging.getLogger(caller_module.__name__)
+
     def decorator(activity_fn: Callable[P, T]) -> Callable[P, T]:
         activity_fn = positional_args_only(activity_fn)
-        activity_fn = with_retriables(retriables)(activity_fn)
+        activity_fn = with_retriables(logger, retriables)(activity_fn)
         activity_fn = activity.defn(activity_fn, name=name)
 
         is_async = inspect.iscoroutinefunction(activity_fn)
@@ -533,7 +553,11 @@ activity_contextual_id = contextual_id
 
 
 def _contextual_path(
-    *, wf_context: bool = True, act_context: bool = True, run_context: bool = False
+    *,
+    wf_context: bool = True,
+    act_context: bool = True,
+    run_context: bool = False,
+    caching_key: str | None = None,
 ) -> Path:
     act_info = activity.info()
     path = []
@@ -541,14 +565,18 @@ def _contextual_path(
         raise ValueError("at least one of wf_context and act_context must be True")
     if wf_context:
         path = [act_info.workflow_type]
-        path.append(act_info.workflow_id)
-        if run_context:
-            path.append(act_info.workflow_run_id)
+        if not caching_key:
+            path.append(act_info.workflow_id)
+            if run_context:
+                path.append(act_info.workflow_run_id)
     if act_context:
         path.append(act_info.activity_type)
-        path.append(act_info.activity_id)
-        if run_context:
-            path.append(act_info.activity_run_id)
+        if not caching_key:
+            path.append(act_info.activity_id)
+            if run_context:
+                path.append(act_info.activity_run_id)
+    if caching_key:
+        path += [_CACHE_DIR, caching_key]
     return Path(*path)
 
 
@@ -559,9 +587,13 @@ def activity_workdir(
     wf_context: bool = True,
     act_context: bool = True,
     run_context: bool = False,
+    caching_key: str | None = None,
 ) -> Path:
     ctx_path = _contextual_path(
-        wf_context=wf_context, act_context=act_context, run_context=run_context
+        wf_context=wf_context,
+        act_context=act_context,
+        run_context=run_context,
+        caching_key=caching_key,
     )
     return workdir.joinpath(project, ctx_path)
 
@@ -808,4 +840,10 @@ def async_enter_cm(
 
 
 def config_cache_key(config: BaseModel) -> str:
-    return str(hash(config))
+    # BaseModel support deterministic hash we can call hash on it
+    config_label = (
+        to_snake(config.__class__.__name__.replace("Config", ""))
+        .lower()
+        .replace("_", "-")
+    )
+    return f"{config_label}-{str(hash(config))}"

{datashare_python-0.10.6 → datashare_python-0.10.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.10.6"
+version = "0.10.8"
 description = "Manage Python tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },