arkindex-base-worker 0.3.6rc5__py3-none-any.whl → 0.3.7.post1__py3-none-any.whl

arkindex_worker/worker/base.py

@@ -1,6 +1,7 @@
 """
 The base class for all Arkindex workers.
 """
+
 import argparse
 import json
 import logging
@@ -12,15 +13,8 @@ from tempfile import mkdtemp
 import gnupg
 import yaml
 from apistar.exceptions import ErrorResponse
-from tenacity import (
-    before_sleep_log,
-    retry,
-    retry_if_exception,
-    stop_after_attempt,
-    wait_exponential,
-)
 
-from arkindex import ArkindexClient, options_from_env
+from arkindex import options_from_env
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     check_version,
@@ -30,18 +24,7 @@ from arkindex_worker.cache import (
     merge_parents_cache,
 )
 from arkindex_worker.utils import close_delete_file, extract_tar_zst_archive
-
-
-def _is_500_error(exc: Exception) -> bool:
-    """
-    Check if an Arkindex API error has a HTTP 5xx error code.
-    Used to retry most API calls in [BaseWorker][arkindex_worker.worker.base.BaseWorker].
-    :param exc: Exception to check
-    """
-    if not isinstance(exc, ErrorResponse):
-        return False
-
-    return 500 <= exc.status_code < 600
+from teklia_toolbox.requests import get_arkindex_client
 
 
 class ExtrasDirNotFoundError(Exception):
@@ -72,7 +55,7 @@ class BaseWorker:
         self.parser.add_argument(
             "-c",
             "--config",
-            help="Alternative configuration file when running without a Worker Version ID",
+            help="Alternative configuration file when running without a Worker Run ID",
             type=open,
         )
         self.parser.add_argument(
@@ -94,7 +77,7 @@ class BaseWorker:
             "--dev",
             help=(
                 "Run worker in developer mode. "
-                "Worker will be in read-only state even if a worker_version is supplied. "
+                "Worker will be in read-only state even if a worker run is supplied. "
             ),
             action="store_true",
             default=False,
@@ -148,6 +131,13 @@ class BaseWorker:
         # there is at least one available sqlite database either given or in the parent tasks
         self.use_cache = False
 
+        # model_version_id will be updated in configure() using the worker_run's model version
+        # or in configure_for_developers() from the environment
+        self.model_version_id = None
+        # model_details will be updated in configure() using the worker_run's model version
+        # or in configure_for_developers() from the environment
+        self.model_details = {}
+
         # task_parents will be updated in configure_cache() if the cache is supported,
         # if the task ID is set and if no database is passed as argument
         self.task_parents = []
@@ -176,12 +166,20 @@ class BaseWorker:
         """
         return self.args.dev or self.worker_run_id is None
 
+    @property
+    def worker_version_id(self):
+        """Deprecated property previously used to retrieve the current WorkerVersion ID.
+
+        :raises DeprecationWarning: Whenever `worker_version_id` is used.
+        """
+        raise DeprecationWarning("`worker_version_id` usage is deprecated")
+
     def setup_api_client(self):
         """
         Create an ArkindexClient to make API requests towards Arkindex instances.
         """
         # Build Arkindex API client from environment variables
-        self.api_client = ArkindexClient(**options_from_env())
+        self.api_client = get_arkindex_client(**options_from_env())
         logger.debug(f"Setup Arkindex API client on {self.api_client.document.url}")
 
     def configure_for_developers(self):
@@ -243,25 +241,21 @@ class BaseWorker:
 
         # Load worker version information
         worker_version = worker_run["worker_version"]
-
-        # Store worker version id
-        self.worker_version_id = worker_version["id"]
-
         self.worker_details = worker_version["worker"]
 
         logger.info(f"Loaded {worker_run['summary']} from API")
 
         # Load model version configuration when available
         model_version = worker_run.get("model_version")
-        if model_version and model_version.get("configuration"):
+        if model_version:
             logger.info("Loaded model version configuration from WorkerRun")
-            self.model_configuration.update(model_version.get("configuration"))
+            self.model_configuration.update(model_version["configuration"])
 
             # Set model_version ID as worker attribute
-            self.model_version_id = model_version.get("id")
+            self.model_version_id = model_version["id"]
 
             # Set model details as worker attribute
-            self.model_details = model_version.get("model")
+            self.model_details = model_version["model"]
 
         # Retrieve initial configuration from API
         self.config = worker_version["configuration"].get("configuration", {})
@@ -477,23 +471,9 @@ class BaseWorker:
             # Clean up
             shutil.rmtree(base_extracted_path)
 
-    @retry(
-        retry=retry_if_exception(_is_500_error),
-        wait=wait_exponential(multiplier=2, min=3),
-        reraise=True,
-        stop=stop_after_attempt(5),
-        before_sleep=before_sleep_log(logger, logging.INFO),
-    )
     def request(self, *args, **kwargs):
         """
         Wrapper around the ``ArkindexClient.request`` method.
-
-        The API call will be retried up to 5 times in case of HTTP 5xx errors,
-        with an exponential sleep time of 3, 4, 8 and 16 seconds between calls.
-        If the 5th call still causes an HTTP 5xx error, the exception is re-raised
-        and the caller should catch it.
-
-        Log messages are displayed when an HTTP 5xx error occurs, before waiting for the next call.
         """
         return self.api_client.request(*args, **kwargs)
 

arkindex_worker/worker/classification.py

@@ -2,8 +2,6 @@
 ElementsWorker methods for classifications and ML classes.
 """
 
-from uuid import UUID
-
 from apistar.exceptions import ErrorResponse
 from peewee import IntegrityError
 
@@ -154,13 +152,6 @@ class ClassificationMixin:
             # Detect already existing classification
             if e.status_code == 400 and "non_field_errors" in e.content:
                 if (
-                    "The fields element, worker_version, ml_class must make a unique set."
-                    in e.content["non_field_errors"]
-                ):
-                    logger.warning(
-                        f"This worker version has already set {ml_class} on element {element.id}"
-                    )
-                elif (
                     "The fields element, worker_run, ml_class must make a unique set."
                     in e.content["non_field_errors"]
                 ):
@@ -185,10 +176,14 @@ class ClassificationMixin:
         Create multiple classifications at once on the given element through the API.
 
         :param element: The element to create classifications on.
-        :param classifications: The classifications to create, a list of dicts. Each of them contains
-            a **ml_class_id** (str), the ID of the MLClass for this classification;
-            a **confidence** (float), the confidence score, between 0 and 1;
-            a **high_confidence** (bool), the high confidence state of the classification.
+        :param classifications: A list of dicts representing a classification each, with the following keys:
+
+            ml_class (str)
+                Required. Name of the MLClass to use.
+            confidence (float)
+                Required. Confidence score for the classification. Must be between 0 and 1.
+            high_confidence (bool)
+                Optional. Whether or not the classification is of high confidence.
 
         :returns: List of created classifications, as returned in the ``classifications`` field by
            the ``CreateClassifications`` API endpoint.
@@ -201,18 +196,10 @@ class ClassificationMixin:
         ), "classifications shouldn't be null and should be of type list"
 
         for index, classification in enumerate(classifications):
-            ml_class_id = classification.get("ml_class_id")
+            ml_class = classification.get("ml_class")
             assert (
-                ml_class_id and isinstance(ml_class_id, str)
-            ), f"Classification at index {index} in classifications: ml_class_id shouldn't be null and should be of type str"
-
-            # Make sure it's a valid UUID
-            try:
-                UUID(ml_class_id)
-            except ValueError as e:
-                raise ValueError(
-                    f"Classification at index {index} in classifications: ml_class_id is not a valid uuid."
-                ) from e
+                ml_class and isinstance(ml_class, str)
+            ), f"Classification at index {index} in classifications: ml_class shouldn't be null and should be of type str"
 
             confidence = classification.get("confidence")
             assert (
@@ -238,7 +225,13 @@ class ClassificationMixin:
             body={
                 "parent": str(element.id),
                 "worker_run_id": self.worker_run_id,
-                "classifications": classifications,
+                "classifications": [
+                    {
+                        **classification,
+                        "ml_class": self.get_ml_class_id(classification["ml_class"]),
+                    }
+                    for classification in classifications
+                ],
             },
         )["classifications"]
 

arkindex_worker/worker/dataset.py

@@ -6,7 +6,8 @@ from collections.abc import Iterator
 from enum import Enum
 
 from arkindex_worker import logger
-from arkindex_worker.models import Dataset, Element
+from arkindex_worker.cache import unsupported_cache
+from arkindex_worker.models import Dataset, Element, Set
 
 
 class DatasetState(Enum):
@@ -36,38 +37,43 @@ class DatasetState(Enum):
 
 
 class DatasetMixin:
-    def list_process_datasets(self) -> Iterator[Dataset]:
+    def list_process_sets(self) -> Iterator[Set]:
         """
-        List datasets associated to the worker's process. This helper is not available in developer mode.
+        List dataset sets associated to the worker's process. This helper is not available in developer mode.
 
-        :returns: An iterator of ``Dataset`` objects built from the ``ListProcessDatasets`` API endpoint.
+        :returns: An iterator of ``Set`` objects built from the ``ListProcessSets`` API endpoint.
         """
         assert not self.is_read_only, "This helper is not available in read-only mode."
 
         results = self.api_client.paginate(
-            "ListProcessDatasets", id=self.process_information["id"]
+            "ListProcessSets", id=self.process_information["id"]
         )
 
-        return map(Dataset, list(results))
+        return map(
+            lambda result: Set(
+                name=result["set_name"], dataset=Dataset(**result["dataset"])
+            ),
+            results,
+        )
 
-    def list_dataset_elements(self, dataset: Dataset) -> Iterator[tuple[str, Element]]:
+    def list_set_elements(self, dataset_set: Set) -> Iterator[Element]:
         """
-        List elements in a dataset.
+        List elements in a dataset set.
 
-        :param dataset: Dataset to find elements in.
-        :returns: An iterator of tuples built from the ``ListDatasetElements`` API endpoint.
+        :param dataset_set: Set to find elements in.
+        :returns: An iterator of Element built from the ``ListDatasetElements`` API endpoint.
         """
-        assert dataset and isinstance(
-            dataset, Dataset
-        ), "dataset shouldn't be null and should be a Dataset"
+        assert dataset_set and isinstance(
+            dataset_set, Set
+        ), "dataset_set shouldn't be null and should be a Set"
 
-        results = self.api_client.paginate("ListDatasetElements", id=dataset.id)
-
-        def format_result(result):
-            return (result["set"], Element(**result["element"]))
+        results = self.api_client.paginate(
+            "ListDatasetElements", id=dataset_set.dataset.id, set=dataset_set.name
+        )
 
-        return map(format_result, list(results))
+        return map(lambda result: Element(**result["element"]), results)
 
+    @unsupported_cache
     def update_dataset_state(self, dataset: Dataset, state: DatasetState) -> Dataset:
         """
         Partially updates a dataset state through the API.

arkindex_worker/worker/element.py

@@ -1,14 +1,16 @@
 """
 ElementsWorker methods for elements and element types.
 """
+
 from collections.abc import Iterable
 from typing import NamedTuple
 from uuid import UUID
+from warnings import warn
 
 from peewee import IntegrityError
 
 from arkindex_worker import logger
-from arkindex_worker.cache import CachedElement, CachedImage
+from arkindex_worker.cache import CachedElement, CachedImage, unsupported_cache
 from arkindex_worker.models import Element
 
 
@@ -29,6 +31,7 @@ class MissingTypeError(Exception):
 
 
 class ElementMixin:
+    @unsupported_cache
     def create_required_types(self, element_types: list[ElementType]):
         """Creates given element types in the corpus.
 
@@ -81,6 +84,7 @@ class ElementMixin:
 
         return True
 
+    @unsupported_cache
     def create_sub_element(
         self,
         element: Element,
@@ -283,6 +287,7 @@ class ElementMixin:
 
         return created_ids
 
+    @unsupported_cache
     def create_element_parent(
         self,
         parent: Element,
@@ -422,6 +427,13 @@ class ElementMixin:
         """
         List children of an element.
 
+        Warns:
+        ----
+        The following parameters are **deprecated**:
+
+        - `transcription_worker_version` in favor of `transcription_worker_run`
+        - `worker_version` in favor of `worker_run`
+
         :param element: Parent element to find children of.
         :param folder: Restrict to or exclude elements with folder types.
            This parameter is not supported when caching is enabled.
@@ -429,9 +441,9 @@ class ElementMixin:
            This parameter is not supported when caching is enabled.
         :param recursive: Look for elements recursively (grand-children, etc.)
            This parameter is not supported when caching is enabled.
-        :param transcription_worker_version: Restrict to elements that have a transcription created by a worker version with this UUID.
+        :param transcription_worker_version: **Deprecated** Restrict to elements that have a transcription created by a worker version with this UUID. Set to False to look for elements that have a manual transcription.
            This parameter is not supported when caching is enabled.
-        :param transcription_worker_run: Restrict to elements that have a transcription created by a worker run with this UUID.
+        :param transcription_worker_run: Restrict to elements that have a transcription created by a worker run with this UUID. Set to False to look for elements that have a manual transcription.
            This parameter is not supported when caching is enabled.
         :param type: Restrict to elements with a specific type slug
            This parameter is not supported when caching is enabled.
@@ -447,7 +459,7 @@ class ElementMixin:
         :param with_zone: Include the ``zone`` attribute in the response,
            holding the element's image and polygon.
            This parameter is not supported when caching is enabled.
-        :param worker_version: Restrict to elements created by a worker version with this UUID.
+        :param worker_version: **Deprecated** Restrict to elements created by a worker version with this UUID.
         :param worker_run: Restrict to elements created by a worker run with this UUID.
         :return: An iterable of dicts from the ``ListElementChildren`` API endpoint,
            or an iterable of [CachedElement][arkindex_worker.cache.CachedElement] when caching is enabled.
@@ -466,6 +478,11 @@ class ElementMixin:
             assert isinstance(recursive, bool), "recursive should be of type bool"
             query_params["recursive"] = recursive
         if transcription_worker_version is not None:
+            warn(
+                "`transcription_worker_version` usage is deprecated. Consider using `transcription_worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
             assert isinstance(
                 transcription_worker_version, str | bool
             ), "transcription_worker_version should be of type str or bool"
@@ -506,6 +523,11 @@ class ElementMixin:
             assert isinstance(with_zone, bool), "with_zone should be of type bool"
             query_params["with_zone"] = with_zone
         if worker_version is not None:
+            warn(
+                "`worker_version` usage is deprecated. Consider using `worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
             assert isinstance(
                 worker_version, str | bool
             ), "worker_version should be of type str or bool"
@@ -584,6 +606,13 @@ class ElementMixin:
         """
         List parents of an element.
 
+        Warns:
+        ----
+        The following parameters are **deprecated**:
+
+        - `transcription_worker_version` in favor of `transcription_worker_run`
+        - `worker_version` in favor of `worker_run`
+
         :param element: Child element to find parents of.
         :param folder: Restrict to or exclude elements with folder types.
            This parameter is not supported when caching is enabled.
@@ -591,7 +620,7 @@ class ElementMixin:
            This parameter is not supported when caching is enabled.
         :param recursive: Look for elements recursively (grand-children, etc.)
            This parameter is not supported when caching is enabled.
-        :param transcription_worker_version: Restrict to elements that have a transcription created by a worker version with this UUID.
+        :param transcription_worker_version: **Deprecated** Restrict to elements that have a transcription created by a worker version with this UUID.
            This parameter is not supported when caching is enabled.
         :param transcription_worker_run: Restrict to elements that have a transcription created by a worker run with this UUID.
            This parameter is not supported when caching is enabled.
@@ -609,7 +638,7 @@ class ElementMixin:
         :param with_zone: Include the ``zone`` attribute in the response,
            holding the element's image and polygon.
            This parameter is not supported when caching is enabled.
-        :param worker_version: Restrict to elements created by a worker version with this UUID.
+        :param worker_version: **Deprecated** Restrict to elements created by a worker version with this UUID.
         :param worker_run: Restrict to elements created by a worker run with this UUID.
         :return: An iterable of dicts from the ``ListElementParents`` API endpoint,
            or an iterable of [CachedElement][arkindex_worker.cache.CachedElement] when caching is enabled.
@@ -628,6 +657,11 @@ class ElementMixin:
             assert isinstance(recursive, bool), "recursive should be of type bool"
             query_params["recursive"] = recursive
         if transcription_worker_version is not None:
+            warn(
+                "`transcription_worker_version` usage is deprecated. Consider using `transcription_worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
             assert isinstance(
                 transcription_worker_version, str | bool
             ), "transcription_worker_version should be of type str or bool"
@@ -668,6 +702,11 @@ class ElementMixin:
             assert isinstance(with_zone, bool), "with_zone should be of type bool"
             query_params["with_zone"] = with_zone
         if worker_version is not None:
+            warn(
+                "`worker_version` usage is deprecated. Consider using `worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
             assert isinstance(
                 worker_version, str | bool
             ), "worker_version should be of type str or bool"

arkindex_worker/worker/entity.py

@@ -4,11 +4,16 @@ ElementsWorker methods for entities.
 
 from operator import itemgetter
 from typing import TypedDict
+from warnings import warn
 
 from peewee import IntegrityError
 
 from arkindex_worker import logger
-from arkindex_worker.cache import CachedEntity, CachedTranscriptionEntity
+from arkindex_worker.cache import (
+    CachedEntity,
+    CachedTranscriptionEntity,
+    unsupported_cache,
+)
 from arkindex_worker.models import Element, Transcription
 
 
@@ -28,6 +33,7 @@ class MissingEntityType(Exception):
 
 
 class EntityMixin:
+    @unsupported_cache
     def check_required_entity_types(
         self, entity_types: list[str], create_missing: bool = True
     ):
@@ -205,6 +211,7 @@ class EntityMixin:
                 )
         return transcription_ent
 
+    @unsupported_cache
     def create_transcription_entities(
         self,
         transcription: Transcription,
@@ -297,13 +304,21 @@ class EntityMixin:
         self,
         transcription: Transcription,
         worker_version: str | bool | None = None,
+        worker_run: str | bool | None = None,
     ):
         """
         List existing entities on a transcription
         This method does not support cache
 
+        Warns:
+        ----
+        The following parameters are **deprecated**:
+
+        - `worker_version` in favor of `worker_run`
+
         :param transcription: The transcription to list entities on.
-        :param worker_version: Restrict to entities created by a worker version with this UUID. Set to False to look for manually created transcriptions.
+        :param worker_version: **Deprecated** Restrict to entities created by a worker version with this UUID. Set to False to look for manually created entities.
+        :param worker_run: Restrict to entities created by a worker run with this UUID. Set to False to look for manually created entities.
         """
         query_params = {}
         assert transcription and isinstance(
@@ -311,6 +326,11 @@ class EntityMixin:
         ), "transcription shouldn't be null and should be a Transcription"
 
         if worker_version is not None:
+            warn(
+                "`worker_version` usage is deprecated. Consider using `worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
             assert isinstance(
                 worker_version, str | bool
             ), "worker_version should be of type str or bool"
@@ -320,6 +340,15 @@ class EntityMixin:
                     worker_version is False
                 ), "if of type bool, worker_version can only be set to False"
             query_params["worker_version"] = worker_version
+        if worker_run is not None:
+            assert isinstance(
+                worker_run, str | bool
+            ), "worker_run should be of type str or bool"
+            if isinstance(worker_run, bool):
+                assert (
+                    worker_run is False
+                ), "if of type bool, worker_run can only be set to False"
+            query_params["worker_run"] = worker_run
 
         return self.api_client.paginate(
             "ListTranscriptionEntities", id=transcription.id, **query_params
@@ -351,8 +380,9 @@ class EntityMixin:
                 "ListCorpusEntities", id=self.corpus_id, **query_params
             )
         }
+        count = len(self.entities)
         logger.info(
-            f"Loaded {len(self.entities)} entities in corpus ({self.corpus_id})"
+            f'Loaded {count} entit{"ies" if count > 1 else "y"} in corpus ({self.corpus_id})'
         )
 
     def list_corpus_entity_types(
@@ -367,6 +397,7 @@ class EntityMixin:
                 "ListCorpusEntityTypes", id=self.corpus_id
             )
         }
+        count = len(self.entity_types)
         logger.info(
-            f"Loaded {len(self.entity_types)} entity types in corpus ({self.corpus_id})."
+            f'Loaded {count} entity type{"s"[:count>1]} in corpus ({self.corpus_id}).'
         )

arkindex_worker/worker/metadata.py

@@ -5,7 +5,7 @@ ElementsWorker methods for metadata.
 from enum import Enum
 
 from arkindex_worker import logger
-from arkindex_worker.cache import CachedElement
+from arkindex_worker.cache import CachedElement, unsupported_cache
 from arkindex_worker.models import Element
 
 
@@ -50,12 +50,13 @@ class MetaType(Enum):
 
     URL = "url"
     """
-    A metadata with a string value that should be interpreted as an URL.
+    A metadata with a string value that should be interpreted as a URL.
     Only the ``http`` and ``https`` schemes are allowed.
     """
 
 
 class MetaDataMixin:
+    @unsupported_cache
     def create_metadata(
         self,
         element: Element | CachedElement,
@@ -106,17 +107,18 @@ class MetaDataMixin:
 
         return metadata["id"]
 
-    def create_metadatas(
+    @unsupported_cache
+    def create_metadata_bulk(
         self,
         element: Element | CachedElement,
-        metadatas: list[dict[str, MetaType | str | int | float | None]],
+        metadata_list: list[dict[str, MetaType | str | int | float | None]],
     ) -> list[dict[str, str]]:
         """
         Create multiple metadata on an existing element.
         This method does not support cache.
 
         :param element: The element to create multiple metadata on.
-        :param metadatas: The list of dict whose keys are the following:
+        :param metadata_list: The list of dict whose keys are the following:
             - type: MetaType
             - name: str
             - value: str | int | float
@@ -126,13 +128,13 @@ class MetaDataMixin:
             element, Element | CachedElement
         ), "element shouldn't be null and should be of type Element or CachedElement"
 
-        assert metadatas and isinstance(
-            metadatas, list
-        ), "type shouldn't be null and should be of type list of Dict"
+        assert metadata_list and isinstance(
+            metadata_list, list
+        ), "metadata_list shouldn't be null and should be of type list of dict"
 
         # Make a copy to avoid modifying the metadata_list argument
         metas = []
-        for index, metadata in enumerate(metadatas):
+        for index, metadata in enumerate(metadata_list):
             assert isinstance(
                 metadata, dict
             ), f"Element at index {index} in metadata_list: Should be of type dict"
@@ -178,16 +180,24 @@ class MetaDataMixin:
         return created_metadata_list
 
     def list_element_metadata(
-        self, element: Element | CachedElement
+        self, element: Element | CachedElement, load_parents: bool | None = None
     ) -> list[dict[str, str]]:
         """
         List all metadata linked to an element.
         This method does not support cache.
 
         :param element: The element to list metadata on.
+        :param load_parents: Also include all metadata from the element's parents in the response.
         """
         assert element and isinstance(
             element, Element | CachedElement
         ), "element shouldn't be null and should be of type Element or CachedElement"
 
-        return self.api_client.paginate("ListElementMetaData", id=element.id)
+        query_params = {}
+        if load_parents is not None:
+            assert isinstance(load_parents, bool), "load_parents should be of type bool"
+            query_params["load_parents"] = load_parents
+
+        return self.api_client.paginate(
+            "ListElementMetaData", id=element.id, **query_params
+        )

arkindex_worker/worker/training.py

@@ -81,6 +81,13 @@ class TrainingMixin:
 
     model_version = None
 
+    @property
+    def is_finetuning(self) -> bool:
+        """
+        Whether or not this worker is fine-tuning an existing model version.
+        """
+        return bool(self.model_version_id)
+
     @skip_if_read_only
     def publish_model_version(
         self,
@@ -276,8 +283,17 @@ class TrainingMixin:
                 },
             )
         except ErrorResponse as e:
+            # Temporary fix while waiting for `ValidateModelVersion` refactoring as it can
+            # return errors even when the model version is properly validated
+            if e.status_code in [403, 500]:
+                logger.warning(
+                    f'An error occurred while validating model version {self.model_version["id"]}, please check its status.'
+                )
+                return
+
             if e.status_code != 409:
                 raise e
+
             logger.warning(
                 f"An available model version exists with hash {hash}, using it instead of the pending version."
             )