arkindex-base-worker 0.4.0rc5__py3-none-any.whl → 0.5.0__py3-none-any.whl

arkindex_worker/worker/entity.py

@@ -8,20 +8,17 @@ from warnings import warn
 
 from peewee import IntegrityError
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
-    CachedEntity,
     CachedTranscriptionEntity,
     unsupported_cache,
 )
-from arkindex_worker.models import Element, Transcription
-from arkindex_worker.utils import (
-    pluralize,
-)
+from arkindex_worker.models import Transcription
+from arkindex_worker.utils import pluralize
 
 
 class Entity(TypedDict):
-    name: str
     type_id: str
     length: int
     offset: int
@@ -36,24 +33,85 @@ class MissingEntityType(Exception):
 
 
 class EntityMixin:
+    def list_corpus_entity_types(self):
+        """
+        Loads available entity types in corpus.
+        """
+        self.entity_types = {
+            entity_type["name"]: entity_type["id"]
+            for entity_type in self.api_client.paginate(
+                "ListCorpusEntityTypes", id=self.corpus_id
+            )
+        }
+        count = len(self.entity_types)
+        logger.info(
+            f"Loaded {count} entity {pluralize('type', count)} in corpus ({self.corpus_id})."
+        )
+
     @unsupported_cache
+    def create_entity_type(self, name: str) -> None:
+        """
+        Create an entity type on the given corpus.
+
+        :param name: Name of the entity type.
+        """
+        assert name and isinstance(name, str), (
+            "name shouldn't be null and should be of type str"
+        )
+
+        try:
+            entity_type = self.api_client.request(
+                "CreateEntityType",
+                body={
+                    "name": name,
+                    "corpus": self.corpus_id,
+                },
+            )
+            self.entity_types[name] = entity_type["id"]
+            logger.info(f"Created a new entity type with name `{name}`.")
+        except ErrorResponse as e:
+            # Only reload for 400 errors
+            if e.status_code != 400:
+                raise
+
+            # Reload and make sure we have the element type now
+            logger.warning(
+                f"Unable to create the entity type `{name}`. Refreshing corpus entity types cache."
+            )
+            self.list_corpus_entity_types()
+            assert name in self.entity_types, (
+                f"Missing entity type `{name}` even after refreshing."
+            )
+
     def check_required_entity_types(
         self, entity_types: list[str], create_missing: bool = True
-    ):
-        """Checks that every entity type needed is available in the corpus.
+    ) -> None:
+        """
+        Check that every entity type needed is available in the corpus.
         Missing ones may be created automatically if needed.
 
         :param entity_types: Entity type names to search.
         :param create_missing: Whether the missing types should be created. Defaults to True.
-        :raises MissingEntityType: When an entity type is missing and cannot create.
+        :raises MissingEntityType: When an entity type is missing and cannot be created.
         """
-        # Retrieve entity_type ID
+        assert entity_types and isinstance(entity_types, list), (
+            "entity_types shouldn't be null and should be of type list"
+        )
+
+        for index, entity_type in enumerate(entity_types):
+            assert isinstance(entity_type, str), (
+                f"Entity type at index {index} in entity_types: Should be of type str"
+            )
+
+        assert create_missing is not None and isinstance(create_missing, bool), (
+            "create_missing shouldn't be null and should be of type bool"
+        )
+
         if not self.entity_types:
-            # Load entity_types of corpus
             self.list_corpus_entity_types()
 
         for entity_type in entity_types:
-            # Do nothing if type already exists
+            # Do nothing if the type already exists
             if entity_type in self.entity_types:
                 continue
 
@@ -63,98 +121,23 @@ class EntityMixin:
                     f"Entity type `{entity_type}` was not in the corpus."
                 )
 
-            # Create type if non-existent
-            self.entity_types[entity_type] = self.api_client.request(
-                "CreateEntityType",
-                body={
-                    "name": entity_type,
-                    "corpus": self.corpus_id,
-                },
-            )["id"]
-            logger.info(f"Created a new entity type with name `{entity_type}`.")
-
-    def create_entity(
-        self,
-        name: str,
-        type: str,
-        metas=None,
-        validated=None,
-    ):
-        """
-        Create an entity on the given corpus.
-        If cache support is enabled, a [CachedEntity][arkindex_worker.cache.CachedEntity] will also be created.
-
-        :param name: Name of the entity.
-        :param type: Type of the entity.
-        """
-        assert name and isinstance(
-            name, str
-        ), "name shouldn't be null and should be of type str"
-        assert type and isinstance(
-            type, str
-        ), "type shouldn't be null and should be of type str"
-        metas = metas or {}
-        if metas:
-            assert isinstance(metas, dict), "metas should be of type dict"
-        if validated is not None:
-            assert isinstance(validated, bool), "validated should be of type bool"
-        if self.is_read_only:
-            logger.warning("Cannot create entity as this worker is in read-only mode")
-            return
-
-        # Retrieve entity_type ID
-        if not self.entity_types:
-            # Load entity_types of corpus
-            self.list_corpus_entity_types()
-
-        entity_type_id = self.entity_types.get(type)
-        assert entity_type_id, f"Entity type `{type}` not found in the corpus."
-
-        entity = self.api_client.request(
-            "CreateEntity",
-            body={
-                "name": name,
-                "type_id": entity_type_id,
-                "metas": metas,
-                "validated": validated,
-                "corpus": self.corpus_id,
-                "worker_run_id": self.worker_run_id,
-            },
-        )
-
-        if self.use_cache:
-            # Store entity in local cache
-            try:
-                to_insert = [
-                    {
-                        "id": entity["id"],
-                        "type": type,
-                        "name": name,
-                        "validated": validated if validated is not None else False,
-                        "metas": metas,
-                        "worker_run_id": self.worker_run_id,
-                    }
-                ]
-                CachedEntity.insert_many(to_insert).execute()
-            except IntegrityError as e:
-                logger.warning(f"Couldn't save created entity in local cache: {e}")
-
-        return entity["id"]
+            # Create the type if non-existent
+            self.create_entity_type(entity_type)
 
     def create_transcription_entity(
         self,
         transcription: Transcription,
-        entity: str,
+        type_id: str,
         offset: int,
         length: int,
         confidence: float | None = None,
     ) -> dict[str, str | int] | None:
         """
-        Create a link between an existing entity and an existing transcription.
+        Create an entity on an existing transcription.
         If cache support is enabled, a `CachedTranscriptionEntity` will also be created.
 
         :param transcription: Transcription to create the entity on.
-        :param entity: UUID of the existing entity.
+        :param type_id: UUID of the entity type.
         :param offset: Starting position of the entity in the transcription's text,
            as a 0-based index.
         :param length: Length of the entity in the transcription's text.
@@ -162,18 +145,18 @@ class EntityMixin:
         :returns: A dict as returned by the ``CreateTranscriptionEntity`` API endpoint,
            or None if the worker is in read-only mode.
         """
-        assert transcription and isinstance(
-            transcription, Transcription
-        ), "transcription shouldn't be null and should be a Transcription"
-        assert entity and isinstance(
-            entity, str
-        ), "entity shouldn't be null and should be of type str"
-        assert (
-            offset is not None and isinstance(offset, int) and offset >= 0
-        ), "offset shouldn't be null and should be a positive integer"
-        assert (
-            length is not None and isinstance(length, int) and length > 0
-        ), "length shouldn't be null and should be a strictly positive integer"
+        assert transcription and isinstance(transcription, Transcription), (
+            "transcription shouldn't be null and should be a Transcription"
+        )
+        assert type_id and isinstance(type_id, str), (
+            "type_id shouldn't be null and should be of type str"
+        )
+        assert offset is not None and isinstance(offset, int) and offset >= 0, (
+            "offset shouldn't be null and should be a positive integer"
+        )
+        assert length is not None and isinstance(length, int) and length > 0, (
+            "length shouldn't be null and should be a strictly positive integer"
+        )
         assert (
             confidence is None or isinstance(confidence, float) and 0 <= confidence <= 1
         ), "confidence should be null or a float in [0..1] range"
@@ -184,7 +167,7 @@ class EntityMixin:
             return
 
         body = {
-            "entity": entity,
+            "type_id": type_id,
             "length": length,
             "offset": offset,
             "worker_run_id": self.worker_run_id,
@@ -192,7 +175,7 @@ class EntityMixin:
         if confidence is not None:
             body["confidence"] = confidence
 
-        transcription_ent = self.api_client.request(
+        tr_entity = self.api_client.request(
             "CreateTranscriptionEntity",
             id=transcription.id,
             body=body,
@@ -203,7 +186,7 @@ class EntityMixin:
             try:
                 CachedTranscriptionEntity.create(
                     transcription=transcription.id,
-                    entity=entity,
+                    type=tr_entity["type"]["name"],
                     offset=offset,
                     length=length,
                     worker_run_id=self.worker_run_id,
@@ -213,7 +196,8 @@ class EntityMixin:
                 logger.warning(
                     f"Couldn't save created transcription entity in local cache: {e}"
                 )
-        return transcription_ent
+
+        return tr_entity
 
     @unsupported_cache
     def create_transcription_entities(
@@ -222,14 +206,11 @@ class EntityMixin:
         entities: list[Entity],
     ) -> list[dict[str, str]]:
         """
-        Create multiple entities attached to a transcription in a single API request.
+        Create multiple entities on a transcription in a single API request.
 
         :param transcription: Transcription to create the entity on.
         :param entities: List of dicts, one per element. Each dict can have the following keys:
 
-            name (str)
-               Required. Name of the entity.
-
             type_id (str)
                Required. ID of the EntityType of the entity.
 
@@ -242,49 +223,45 @@ class EntityMixin:
             confidence (float or None)
                 Optional confidence score, between 0.0 and 1.0.
 
-        :return: List of dicts, with each dict having a two keys, `transcription_entity_id` and `entity_id`, holding the UUID of each created object.
+        :return: List of strings, holding the UUID of each created object.
         """
-        assert transcription and isinstance(
-            transcription, Transcription
-        ), "transcription shouldn't be null and should be of type Transcription"
+        assert transcription and isinstance(transcription, Transcription), (
+            "transcription shouldn't be null and should be of type Transcription"
+        )
 
-        assert entities and isinstance(
-            entities, list
-        ), "entities shouldn't be null and should be of type list"
+        assert entities and isinstance(entities, list), (
+            "entities shouldn't be null and should be of type list"
+        )
 
         for index, entity in enumerate(entities):
-            assert isinstance(
-                entity, dict
-            ), f"Entity at index {index} in entities: Should be of type dict"
-
-            name = entity.get("name")
-            assert (
-                name and isinstance(name, str)
-            ), f"Entity at index {index} in entities: name shouldn't be null and should be of type str"
+            assert isinstance(entity, dict), (
+                f"Entity at index {index} in entities: Should be of type dict"
+            )
 
             type_id = entity.get("type_id")
-            assert (
-                type_id and isinstance(type_id, str)
-            ), f"Entity at index {index} in entities: type_id shouldn't be null and should be of type str"
+            assert type_id and isinstance(type_id, str), (
+                f"Entity at index {index} in entities: type_id shouldn't be null and should be of type str"
+            )
 
             offset = entity.get("offset")
-            assert (
-                offset is not None and isinstance(offset, int) and offset >= 0
-            ), f"Entity at index {index} in entities: offset shouldn't be null and should be a positive integer"
+            assert offset is not None and isinstance(offset, int) and offset >= 0, (
+                f"Entity at index {index} in entities: offset shouldn't be null and should be a positive integer"
+            )
 
             length = entity.get("length")
-            assert (
-                length is not None and isinstance(length, int) and length > 0
-            ), f"Entity at index {index} in entities: length shouldn't be null and should be a strictly positive integer"
+            assert length is not None and isinstance(length, int) and length > 0, (
+                f"Entity at index {index} in entities: length shouldn't be null and should be a strictly positive integer"
+            )
 
             confidence = entity.get("confidence")
-            assert (
-                confidence is None
-                or (isinstance(confidence, float) and 0 <= confidence <= 1)
-            ), f"Entity at index {index} in entities: confidence should be None or a float in [0..1] range"
+            assert confidence is None or (
+                isinstance(confidence, float) and 0 <= confidence <= 1
+            ), (
+                f"Entity at index {index} in entities: confidence should be None or a float in [0..1] range"
+            )
 
         assert len(entities) == len(
-            set(map(itemgetter("offset", "length", "name", "type_id"), entities))
+            set(map(itemgetter("offset", "length", "type_id"), entities))
         ), "entities should be unique"
 
         if self.is_read_only:
@@ -293,16 +270,16 @@ class EntityMixin:
             )
             return
 
-        created_entities = self.api_client.request(
+        created_tr_entities = self.api_client.request(
             "CreateTranscriptionEntities",
             id=transcription.id,
             body={
                 "worker_run_id": self.worker_run_id,
-                "entities": entities,
+                "transcription_entities": entities,
             },
-        )["entities"]
+        )["transcription_entities"]
 
-        return created_entities
+        return created_tr_entities
 
     def list_transcription_entities(
         self,
@@ -325,9 +302,9 @@ class EntityMixin:
         :param worker_run: Restrict to entities created by a worker run with this UUID. Set to False to look for manually created entities.
         """
         query_params = {}
-        assert transcription and isinstance(
-            transcription, Transcription
-        ), "transcription shouldn't be null and should be a Transcription"
+        assert transcription and isinstance(transcription, Transcription), (
+            "transcription shouldn't be null and should be a Transcription"
+        )
 
         if worker_version is not None:
             warn(
@@ -335,71 +312,25 @@ class EntityMixin:
                 DeprecationWarning,
                 stacklevel=1,
             )
-            assert isinstance(
-                worker_version, str | bool
-            ), "worker_version should be of type str or bool"
+            assert isinstance(worker_version, str | bool), (
+                "worker_version should be of type str or bool"
+            )
 
             if isinstance(worker_version, bool):
-                assert (
-                    worker_version is False
-                ), "if of type bool, worker_version can only be set to False"
+                assert worker_version is False, (
+                    "if of type bool, worker_version can only be set to False"
+                )
             query_params["worker_version"] = worker_version
         if worker_run is not None:
-            assert isinstance(
-                worker_run, str | bool
-            ), "worker_run should be of type str or bool"
+            assert isinstance(worker_run, str | bool), (
+                "worker_run should be of type str or bool"
+            )
             if isinstance(worker_run, bool):
-                assert (
-                    worker_run is False
-                ), "if of type bool, worker_run can only be set to False"
+                assert worker_run is False, (
+                    "if of type bool, worker_run can only be set to False"
+                )
             query_params["worker_run"] = worker_run
 
         return self.api_client.paginate(
             "ListTranscriptionEntities", id=transcription.id, **query_params
         )
-
-    def list_corpus_entities(
-        self,
-        name: str | None = None,
-        parent: Element | None = None,
-    ):
-        """
-        List all entities in the worker's corpus and store them in the ``self.entities`` cache.
-        :param name: Filter entities by part of their name (case-insensitive)
-        :param parent: Restrict entities to those linked to all transcriptions of an element and all its descendants. Note that links to metadata are ignored.
-        """
-        query_params = {}
-
-        if name is not None:
-            assert name and isinstance(name, str), "name should be of type str"
-            query_params["name"] = name
-
-        if parent is not None:
-            assert isinstance(parent, Element), "parent should be of type Element"
-            query_params["parent"] = parent.id
-
-        self.entities = {
-            entity["id"]: entity
-            for entity in self.api_client.paginate(
-                "ListCorpusEntities", id=self.corpus_id, **query_params
-            )
-        }
-        count = len(self.entities)
-        logger.info(
-            f'Loaded {count} {pluralize("entity", count)} in corpus ({self.corpus_id})'
-        )
-
-    def list_corpus_entity_types(self):
-        """
-        Loads available entity types in corpus.
-        """
-        self.entity_types = {
-            entity_type["name"]: entity_type["id"]
-            for entity_type in self.api_client.paginate(
-                "ListCorpusEntityTypes", id=self.corpus_id
-            )
-        }
-        count = len(self.entity_types)
-        logger.info(
-            f'Loaded {count} entity {pluralize("type", count)} in corpus ({self.corpus_id}).'
-        )

arkindex_worker/worker/image.py

@@ -14,8 +14,8 @@ class ImageMixin:
         :param url: URL of the image.
         :returns: The created image.
         """
-        assert url and isinstance(
-            url, str
-        ), "url shouldn't be null and should be of type str"
+        assert url and isinstance(url, str), (
+            "url shouldn't be null and should be of type str"
+        )
 
         return Image(self.api_client.request("CreateIIIFURL", body={"url": url}))

arkindex_worker/worker/metadata.py

@@ -64,7 +64,6 @@ class MetaDataMixin:
         type: MetaType,
         name: str,
         value: str,
-        entity: str | None = None,
     ) -> str:
         """
         Create a metadata on the given element through API.
@@ -73,23 +72,20 @@ class MetaDataMixin:
         :param type: Type of the metadata.
         :param name: Name of the metadata.
         :param value: Value of the metadata.
-        :param entity: UUID of an entity this metadata is related to.
         :returns: UUID of the created metadata.
         """
-        assert element and isinstance(
-            element, Element | CachedElement
-        ), "element shouldn't be null and should be of type Element or CachedElement"
-        assert type and isinstance(
-            type, MetaType
-        ), "type shouldn't be null and should be of type MetaType"
-        assert name and isinstance(
-            name, str
-        ), "name shouldn't be null and should be of type str"
-        assert value and isinstance(
-            value, str
-        ), "value shouldn't be null and should be of type str"
-        if entity:
-            assert isinstance(entity, str), "entity should be of type str"
+        assert element and isinstance(element, Element | CachedElement), (
+            "element shouldn't be null and should be of type Element or CachedElement"
+        )
+        assert type and isinstance(type, MetaType), (
+            "type shouldn't be null and should be of type MetaType"
+        )
+        assert name and isinstance(name, str), (
+            "name shouldn't be null and should be of type str"
+        )
+        assert value and isinstance(value, str), (
+            "value shouldn't be null and should be of type str"
+        )
         if self.is_read_only:
             logger.warning("Cannot create metadata as this worker is in read-only mode")
             return
@@ -101,7 +97,6 @@ class MetaDataMixin:
                 "type": type.value,
                 "name": name,
                 "value": value,
-                "entity_id": entity,
                 "worker_run_id": self.worker_run_id,
             },
         )
@@ -125,48 +120,42 @@ class MetaDataMixin:
             - type: MetaType
             - name: str
             - value: str | int | float
-            - entity_id: str | None
         :param batch_size: The size of each batch, which will be used to split the publication to avoid API errors.
 
         :returns: A list of dicts as returned in the ``metadata_list`` field by the ``CreateMetaDataBulk`` API endpoint.
         """
-        assert element and isinstance(
-            element, Element | CachedElement
-        ), "element shouldn't be null and should be of type Element or CachedElement"
+        assert element and isinstance(element, Element | CachedElement), (
+            "element shouldn't be null and should be of type Element or CachedElement"
+        )
 
-        assert metadata_list and isinstance(
-            metadata_list, list
-        ), "metadata_list shouldn't be null and should be of type list of dict"
+        assert metadata_list and isinstance(metadata_list, list), (
+            "metadata_list shouldn't be null and should be of type list of dict"
+        )
 
         # Make a copy to avoid modifying the metadata_list argument
         metas = []
         for index, metadata in enumerate(metadata_list):
-            assert isinstance(
-                metadata, dict
-            ), f"Element at index {index} in metadata_list: Should be of type dict"
+            assert isinstance(metadata, dict), (
+                f"Element at index {index} in metadata_list: Should be of type dict"
+            )
 
             assert metadata.get("type") and isinstance(
                 metadata.get("type"), MetaType
             ), "type shouldn't be null and should be of type MetaType"
 
-            assert metadata.get("name") and isinstance(
-                metadata.get("name"), str
-            ), "name shouldn't be null and should be of type str"
+            assert metadata.get("name") and isinstance(metadata.get("name"), str), (
+                "name shouldn't be null and should be of type str"
+            )
 
             assert metadata.get("value") is not None and isinstance(
                 metadata.get("value"), str | float | int
             ), "value shouldn't be null and should be of type (str or float or int)"
 
-            assert metadata.get("entity_id") is None or isinstance(
-                metadata.get("entity_id"), str
-            ), "entity_id should be None or a str"
-
             metas.append(
                 {
                     "type": metadata.get("type").value,
                     "name": metadata.get("name"),
                     "value": metadata.get("value"),
-                    "entity_id": metadata.get("entity_id"),
                 }
             )
 
@@ -199,9 +188,9 @@ class MetaDataMixin:
         :param element: The element to list metadata on.
         :param load_parents: Also include all metadata from the element's parents in the response.
         """
-        assert element and isinstance(
-            element, Element | CachedElement
-        ), "element shouldn't be null and should be of type Element or CachedElement"
+        assert element and isinstance(element, Element | CachedElement), (
+            "element shouldn't be null and should be of type Element or CachedElement"
+        )
 
         query_params = {}
         if load_parents is not None:

arkindex_worker/worker/process.py

@@ -1,5 +1,11 @@
+from collections.abc import Iterator
 from enum import Enum
 
+from arkindex_worker.cache import unsupported_cache
+
+# Increases the number of elements returned per page by the API
+PROCESS_ELEMENTS_PAGE_SIZE = 500
+
 
 class ActivityState(Enum):
     """
@@ -66,3 +72,21 @@ class ProcessMode(Enum):
     """
     Export processes.
     """
+
+
+class ProcessMixin:
+    @unsupported_cache
+    def list_process_elements(self, with_image: bool = False) -> Iterator[dict]:
+        """
+        List the elements of a process.
+
+        :param with_image: whether or not to include zone and image information in the elements response.
+        :returns: the process' elements.
+        """
+        return self.api_client.paginate(
+            "ListProcessElements",
+            id=self.process_information["id"],
+            with_image=with_image,
+            allow_missing_data=True,
+            page_size=PROCESS_ELEMENTS_PAGE_SIZE,
+        )