datashare-python 0.12.6__tar.gz → 0.12.8__tar.gz

{datashare_python-0.12.6 → datashare_python-0.12.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datashare-python
-Version: 0.12.6
+Version: 0.12.8
 Summary: Manage Python tasks and local resources in Datashare
 Project-URL: Homepage, https://icij.github.io/datashare-python/
 Project-URL: Documentation, https://icij.github.io/datashare-python/

{datashare_python-0.12.6 → datashare_python-0.12.8}/datashare_python/config.py

@@ -114,6 +114,7 @@ class LoggingConfig(BaseModel):
 
 class ResourceCacheConfig(BaseModel):
     size: int = 1
+    refresh_interval: int | None = None
     exit_context_managers: bool = True
 
     def to_resource_cache(self) -> "SharedResources":
@@ -123,7 +124,9 @@ class ResourceCacheConfig(BaseModel):
         if self.exit_context_managers:
             eviction_callback = close_cm_callback
         return SharedResources(
-            cache_size=self.size, eviction_callback=eviction_callback
+            cache_size=self.size,
+            eviction_callback=eviction_callback,
+            refresh_interval=self.refresh_interval,
         )
 
 

datashare_python-0.12.8/datashare_python/constants.py

@@ -0,0 +1,212 @@
+from copy import deepcopy
+from pathlib import Path
+
+PACKAGE_DIR = Path(__file__).parent
+PACKAGE_ROOT = PACKAGE_DIR.parent
+
+
+DEFAULT_TEMPORAL_ADDRESS = "temporal:7233"
+
+DEFAULT_DS_ADDRESS = "http://localhost:8080"
+
+DEFAULT_NAMESPACE = "datashare-default"
+
+METADATA_JSON = "metadata.json"
+MANIFEST_JSON = "manifest.json"
+
+TIKA_METADATA_RESOURCENAME = "tika_metadata_resourcename"
+
+
+_LANGUAGES = {
+    "ABKHAZIAN": ("ab", "abk"),
+    "AFAR": ("aa", "aar"),
+    "AFRIKAANS": ("af", "afr"),
+    "AKAN": ("ak", "aka"),
+    "ALBANIAN": ("sq", "sqi"),
+    "AMHARIC": ("am", "amh"),
+    "ARABIC": ("ar", "ara"),
+    "ARAGONESE": ("an", "arg"),
+    "ARMENIAN": ("hy", "hye"),
+    "ASSAMESE": ("as", "asm"),
+    "AVARIC": ("av", "ava"),
+    "AVESTAN": ("ae", "ave"),
+    "AYMARA": ("ay", "aym"),
+    "AZERBAIJANI": ("az", "aze"),
+    "BAMBARA": ("bm", "bam"),
+    "BASHKIR": ("ba", "bak"),
+    "BASQUE": ("eu", "eus"),
+    "BELARUSIAN": ("be", "bel"),
+    "BENGALI": ("bn", "ben"),
+    "BIHARI": ("bh", "bih"),
+    "BISLAMA": ("bi", "bis"),
+    "BOSNIAN": ("bs", "bos"),
+    "BRETON": ("br", "bre"),
+    "BULGARIAN": ("bg", "bul"),
+    "BURMESE": ("my", "mya"),
+    "CATALAN": ("ca", "cat"),
+    "CHAMORRO": ("ch", "cha"),
+    "CHECHEN": ("ce", "che"),
+    "CHICHEWA": ("ny", "nya"),
+    "CHINESE": ("zh", "zho"),
+    "CHUVASH": ("cv", "chv"),
+    "CORNISH": ("kw", "cor"),
+    "CORSICAN": ("co", "cos"),
+    "CREE": ("cr", "cre"),
+    "CROATIAN": ("hr", "hrv"),
+    "CZECH": ("cs", "ces"),
+    "DANISH": ("da", "dan"),
+    "MALDIVIAN": ("dv", "div"),
+    "DUTCH": ("nl", "nld"),
+    "DZONGKHA": ("dz", "dzo"),
+    "ENGLISH": ("en", "eng"),
+    "ESPERANTO": ("eo", "epo"),
+    "ESTONIAN": ("et", "est"),
+    "EWE": ("ee", "ewe"),
+    "FAROESE": ("fo", "fao"),
+    "FIJIAN": ("fj", "fij"),
+    "FINNISH": ("fi", "fin"),
+    "FRENCH": ("fr", "fra"),
+    "FULAH": ("ff", "ful"),
+    "GALICIAN": ("gl", "glg"),
+    "GEORGIAN": ("ka", "kat"),
+    "GERMAN": ("de", "deu"),
+    "GREEK": ("el", "ell"),
+    "GUARANI": ("gn", "grn"),
+    "GUJARATI": ("gu", "guj"),
+    "HAITIAN": ("ht", "hat"),
+    "HAUSA": ("ha", "hau"),
+    "HEBREW": ("he", "heb"),
+    "HERERO": ("hz", "her"),
+    "HINDI": ("hi", "hin"),
+    "HIRIMOTU": ("ho", "hmo"),
+    "HUNGARIAN": ("hu", "hun"),
+    "INTERLINGUA": ("ia", "ina"),
+    "INDONESIAN": ("id", "ind"),
+    "INTERLIGUE": ("ie", "ile"),
+    "IRISH": ("ga", "gle"),
+    "IGBO": ("ig", "ibo"),
+    "INUPIAQ": ("ik", "ipk"),
+    "IDO": ("io", "ido"),
+    "ICELANDIC": ("is", "isl"),
+    "ITALIAN": ("it", "ita"),
+    "INUKTITUT": ("iu", "iku"),
+    "JAPANESE": ("ja", "jpn"),
+    "JAVANESE": ("jv", "jav"),
+    "GREENLANDIC": ("kl", "kal"),
+    "KANNADA": ("kn", "kan"),
+    "KANURI": ("kr", "kau"),
+    "KASHMIRI": ("ks", "kas"),
+    "KAZAKH": ("kk", "kaz"),
+    "CENTRALKHMER": ("km", "khm"),
+    "KIKUYU": ("ki", "kik"),
+    "KINYARWANDA": ("rw", "kin"),
+    "KIRGHIZ": ("ky", "kir"),
+    "KOMI": ("kv", "kom"),
+    "KONGO": ("kg", "kon"),
+    "KOREAN": ("ko", "kor"),
+    "KURDISH": ("ku", "kur"),
+    "KUANYAMA": ("kj", "kua"),
+    "LATIN": ("la", "lat"),
+    "LUXEMBOURGISH": ("lb", "ltz"),
+    "GANDA": ("lg", "lug"),
+    "LIMBURGAN": ("li", "lim"),
+    "LINGALA": ("ln", "lin"),
+    "LAO": ("lo", "lao"),
+    "LITHUANIAN": ("lt", "lit"),
+    "LUBAKATANGA": ("lu", "lub"),
+    "LATVIAN": ("lv", "lav"),
+    "MANX": ("gv", "glv"),
+    "MACEDONIAN": ("mk", "mkd"),
+    "MALAGASY": ("mg", "mlg"),
+    "MALAY": ("ms", "msa"),
+    "MALAYALAM": ("ml", "mal"),
+    "MALTESE": ("mt", "mlt"),
+    "MAORI": ("mi", "mri"),
+    "MARATHI": ("mr", "mar"),
+    "MARSHALLESE": ("mh", "mah"),
+    "MONGOLIAN": ("mn", "mon"),
+    "NAURU": ("na", "nau"),
+    "NAVAJO": ("nv", "nav"),
+    "NORTHNDEBELE": ("nd", "nde"),
+    "NEPALI": ("ne", "nep"),
+    "NDONGA": ("ng", "ndo"),
+    "NORWEGIANBOKMAL": ("nb", "nob"),
+    "NORWEGIANNYNORSK": ("nn", "nno"),
+    "NORWEGIAN": ("no", "nor"),
+    "SICHUANYI": ("ii", "iii"),
+    "SOUTHNDEBELE": ("nr", "nbl"),
+    "OCCITAN": ("oc", "oci"),
+    "OJIBWA": ("oj", "oji"),
+    "OLDSLAVONIC": ("cu", "chu"),
+    "OROMO": ("om", "orm"),
+    "ORIYA": ("or", "ori"),
+    "OSSETIAN": ("os", "oss"),
+    "PANJABI": ("pa", "pan"),
+    "PALI": ("pi", "pli"),
+    "PERSIAN": ("fa", "fas"),
+    "POLISH": ("pl", "pol"),
+    "PASHTO": ("ps", "pus"),
+    "PORTUGUESE": ("pt", "por"),
+    "QUECHUA": ("qu", "que"),
+    "ROMANSH": ("rm", "roh"),
+    "ROMANIAN": ("ro", "ron"),
+    "RUNDI": ("rn", "run"),
+    "MOLDAVIAN": ("ro", "ron"),
+    "MOLDOVAN": ("ro", "ron"),
+    "RUSSIAN": ("ru", "rus"),
+    "SANSKRIT": ("sa", "san"),
+    "SARDINIAN": ("sc", "srd"),
+    "SINDHI": ("sd", "snd"),
+    "NORTHERNSAMI": ("se", "sme"),
+    "SAMOAN": ("sm", "smo"),
+    "SANGO": ("sg", "sag"),
+    "SERBIAN": ("sr", "srp"),
+    "GAELIC": ("gd", "gla"),
+    "SHONA": ("sn", "sna"),
+    "SINHALESE": ("si", "sin"),
+    "SLOVAK": ("sk", "slk"),
+    "SLOVENIAN": ("sl", "slv"),
+    "SOMALI": ("so", "som"),
+    "SOUTHERNSOTHO": ("st", "sot"),
+    "SPANISH": ("es", "spa"),
+    "SUNDANESE": ("su", "sun"),
+    "SWAHILI": ("sw", "swa"),
+    "SWATI": ("ss", "ssw"),
+    "SWEDISH": ("sv", "swe"),
+    "TAMIL": ("ta", "tam"),
+    "TELUGU": ("te", "tel"),
+    "TAJIK": ("tg", "tgk"),
+    "THAI": ("th", "tha"),
+    "TIGRINYA": ("ti", "tir"),
+    "TIBETAN": ("bo", "bod"),
+    "TURKMEN": ("tk", "tuk"),
+    "TAGALOG": ("tl", "tgl"),
+    "TSWANA": ("tn", "tsn"),
+    "TONGA": ("to", "ton"),
+    "TURKISH": ("tr", "tur"),
+    "TSONGA": ("ts", "tso"),
+    "TATAR": ("tt", "tat"),
+    "TWI": ("tw", "twi"),
+    "TAHITIAN": ("ty", "tah"),
+    "UIGHUR": ("ug", "uig"),
+    "UKRAINIAN": ("uk", "ukr"),
+    "URDU": ("ur", "urd"),
+    "UZBEK": ("uz", "uzb"),
+    "VENDA": ("ve", "ven"),
+    "VIETNAMESE": ("vi", "vie"),
+    "VOLAPUK": ("vo", "vol"),
+    "WALLOON": ("wa", "wln"),
+    "WELSH": ("cy", "cym"),
+    "WOLOF": ("wo", "wol"),
+    "WESTERNFRISIAN": ("fy", "fry"),
+    "XHOSA": ("xh", "xho"),
+    "YIDDISH": ("yi", "yid"),
+    "YORUBA": ("yo", "yor"),
+    "ZHUANG": ("za", "zha"),
+    "ZULU": ("zu", "zul"),
+    "UNKNOWN": ("un", "ukn"),
+}
+
+
+def get_datashare_languages() -> dict[str, tuple[str, str]]:
+    return deepcopy(_LANGUAGES)

{datashare_python-0.12.6 → datashare_python-0.12.8}/datashare_python/objects.py

@@ -14,12 +14,11 @@ from typing import Annotated, Any, ClassVar, Literal, Self, TypeVar, cast, final
 
 import langcodes
 from icij_common.registrable import Registrable
-from pydantic_core import PydanticCustomError, ValidationError, core_schema
+from pydantic_core import PydanticCustomError, core_schema
 from pydantic_core.core_schema import PlainValidatorFunctionSchema
-from pydantic_extra_types.language_code import LanguageName
 from temporalio import activity, workflow
 
-from .constants import TIKA_METADATA_RESOURCENAME
+from .constants import TIKA_METADATA_RESOURCENAME, get_datashare_languages
 
 with workflow.unsafe.imports_passed_through():
     from icij_common.es import (
@@ -81,7 +80,7 @@ class DatashareModel(BaseModel):
 
 
 class DatashareLanguage(str):
-    _language_type_adapter: ClassVar[TypeAdapter] = TypeAdapter(LanguageName)
+    _languages: ClassVar[dict[str, tuple[str, str]]] = get_datashare_languages()
 
     @classmethod
     def _validate(cls, __input_value: str, _: core_schema.ValidationInfo) -> Self:
@@ -91,8 +90,8 @@ class DatashareLanguage(str):
             )
         try:
             # Use pydantic provided validation
-            cls._language_type_adapter.validate_python(__input_value.title())
-        except ValidationError as e:
+            cls._languages[__input_value]
+        except KeyError as e:
             raise PydanticCustomError(
                 "datashare_language", "Unknown Datashare language"
             ) from e
@@ -109,16 +108,16 @@ class DatashareLanguage(str):
         )
 
     @property
-    def as_language_name(self) -> LanguageName:
-        return LanguageName(self.title())
+    def as_language_name(self) -> str:
+        return self
 
     @property
     def alpha2(self) -> str | None:
-        return self.as_language_name.alpha2
+        return self._languages[self][0]
 
     @property
     def alpha3(self) -> str:
-        return self.as_language_name.alpha3
+        return self._languages[self][1]
 
 
 @unique

{datashare_python-0.12.6 → datashare_python-0.12.8}/datashare_python/utils.py

@@ -822,9 +822,11 @@ class SharedResources:
         self,
         cache_size: int = 1,
         eviction_callback: Callable[[Any, Any], None] | None = None,
+        refresh_interval: int | None = None,
     ) -> None:
         self._eviction_callback = eviction_callback
         self._cache = LRU(cache_size, self._on_evict)
+        self._refresh_interval = refresh_interval
         self._sentinel = object()
         # Leased resources are tracked by id, evicting a leased resource defers the
         # eviction callback until its last lease is released
@@ -854,6 +856,14 @@ class SharedResources:
         """
         value = self._get_and_acquire(key)
         if value is self._sentinel:
+            # The factory can be long to complete (that's one of the reason we want to
+            # cache its results). Because we don't lock the factory call, together with
+            # the cache update, in case of concurrent access, quicker factories will
+            # complete first and will stay longer in the cache. This is fine, the
+            # alternative is to lock factory + cache update to avoid concurrent access
+            # but this will mean waiting for the first factory call to complete, this
+            # can potentially imply longer waits than no getting the value from the
+            # cache
             value = default_factory()
             self._cache_and_acquire(key, value)
         try:
@@ -873,6 +883,14 @@ class SharedResources:
         """
         value = self._get_and_acquire(key)
         if value is self._sentinel:
+            # The factory can be long to complete (that's one of the reason we want to
+            # cache its results). Because we don't lock the factory call, together with
+            # the cache update, in case of concurrent access, quicker factories will
+            # complete first and will stay longer in the cache. This is fine, the
+            # alternative is to lock factory + cache update to avoid concurrent access
+            # but this will mean waiting for the first factory call to complete, this
+            # can potentially imply longer waits than no getting the value from the
+            # cache
             value = await default_factory()
             self._cache_and_acquire(key, value)
         try:
@@ -884,28 +902,31 @@ class SharedResources:
         with self._lock:
             value = self._cache.get(key, self._sentinel)
             if value is not self._sentinel:
-                self._acquire(value)
+                # If refresh needs to happen, we set the value to the sentinel.
+                # The old value will be release with the last user call to _release.
+                # Returning the sentinel will trigger the call to the factory
+                if (
+                    self._refresh_interval is not None
+                    and self._n_leases[self._vkey(value)] >= self._refresh_interval
+                ):
+                    self._pending_evictions[self._vkey(value)] = (key, value)
+                    value = self._sentinel
+                else:
+                    self._acquire(value)
         return value
 
     def _cache_and_acquire(self, key: str, value: Any) -> None:
-        # The factory can be long to complete (that's one of the reason we want to
-        # cache its results). Because we don't lock the factory call, together with
-        # the cache update, in case of concurrent access, quicker factories will
-        # complete first and will stay longer in the cache. This is fine, the
-        # alternative is to lock factory + cache update to avoid concurrent access but
-        # this will mean waiting for the first factory call to complete, this can
-        # potentially imply longer waits than no getting the value from the cache
         with self._lock:
             self._acquire(value)
         # Can evict other resources, must be called without holding the lock
         self._cache[key] = value
 
     def _acquire(self, value: Any) -> None:
-        value_id = id(value)
+        value_id = self._vkey(value)
         self._n_leases[value_id] = self._n_leases.get(value_id, 0) + 1
 
     def _release(self, value: Any) -> None:
-        value_id = id(value)
+        value_id = self._vkey(value)
         with self._lock:
             self._n_leases[value_id] -= 1
             if self._n_leases[value_id]:
@@ -917,8 +938,8 @@ class SharedResources:
 
     def _on_evict(self, key: Any, value: Any) -> None:
         with self._lock:
-            if id(value) in self._n_leases:
-                self._pending_evictions[id(value)] = (key, value)
+            if self._vkey(value) in self._n_leases:
+                self._pending_evictions[self._vkey(value)] = (key, value)
                 return
         self._evict(key, value)
 
@@ -926,6 +947,9 @@ class SharedResources:
         if self._eviction_callback is not None:
             self._eviction_callback(key, value)
 
+    def _vkey(self, value: Any) -> int:
+        return id(value)
+
 
 def close_cm_callback(key: str, value: Any) -> None:  # noqa: ARG001
     if hasattr(value, "__exit__"):

{datashare_python-0.12.6 → datashare_python-0.12.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datashare-python"
-version = "0.12.6"
+version = "0.12.8"
 description = "Manage Python tasks and local resources in Datashare"
 authors = [
   { name = "Clément Doumouro", email = "cdoumouro@icij.org" },

datashare_python-0.12.6/datashare_python/constants.py

@@ -1,16 +0,0 @@
-from pathlib import Path
-
-PACKAGE_DIR = Path(__file__).parent
-PACKAGE_ROOT = PACKAGE_DIR.parent
-
-
-DEFAULT_TEMPORAL_ADDRESS = "temporal:7233"
-
-DEFAULT_DS_ADDRESS = "http://localhost:8080"
-
-DEFAULT_NAMESPACE = "datashare-default"
-
-METADATA_JSON = "metadata.json"
-MANIFEST_JSON = "manifest.json"
-
-TIKA_METADATA_RESOURCENAME = "tika_metadata_resourcename"