arkindex-base-worker 0.5.0b3__tar.gz → 0.5.0rc1__tar.gz

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.0b3
+Version: 0.5.0rc1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.0b3
+Version: 0.5.0rc1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -46,8 +46,7 @@ tests/test_elements_worker/test_element_create_multiple.py
 tests/test_elements_worker/test_element_create_single.py
 tests/test_elements_worker/test_element_list_children.py
 tests/test_elements_worker/test_element_list_parents.py
-tests/test_elements_worker/test_entity_create.py
-tests/test_elements_worker/test_entity_list_and_check.py
+tests/test_elements_worker/test_entity.py
 tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
 tests/test_elements_worker/test_process.py

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/arkindex_worker/cache.py

@@ -206,23 +206,6 @@ class CachedClassification(Model):
         table_name = "classifications"
 
 
-class CachedEntity(Model):
-    """
-    Cache entity table
-    """
-
-    id = UUIDField(primary_key=True)
-    type = CharField(max_length=50)
-    name = TextField()
-    validated = BooleanField(default=False)
-    metas = JSONField(null=True)
-    worker_run_id = UUIDField(null=True)
-
-    class Meta:
-        database = db
-        table_name = "entities"
-
-
 class CachedTranscriptionEntity(Model):
     """
     Cache transcription entity table
@@ -231,14 +214,14 @@ class CachedTranscriptionEntity(Model):
     transcription = ForeignKeyField(
         CachedTranscription, backref="transcription_entities"
     )
-    entity = ForeignKeyField(CachedEntity, backref="transcription_entities")
+    type = CharField(max_length=50)
     offset = IntegerField(constraints=[Check("offset >= 0")])
     length = IntegerField(constraints=[Check("length > 0")])
     worker_run_id = UUIDField(null=True)
     confidence = FloatField(null=True)
 
     class Meta:
-        primary_key = CompositeKey("transcription", "entity")
+        primary_key = CompositeKey("transcription", "type")
         database = db
         table_name = "transcription_entities"
 
@@ -272,12 +255,11 @@ MODELS = [
     CachedElement,
     CachedTranscription,
     CachedClassification,
-    CachedEntity,
     CachedTranscriptionEntity,
     CachedDataset,
     CachedDatasetElement,
 ]
-SQL_VERSION = 3
+SQL_VERSION = 4
 
 
 def init_cache_db(path: Path):
@@ -365,7 +347,6 @@ def merge_parents_cache(paths: list, current_database: Path):
             f"REPLACE INTO elements SELECT * FROM source_{idx}.elements;",
             f"REPLACE INTO transcriptions SELECT * FROM source_{idx}.transcriptions;",
             f"REPLACE INTO classifications SELECT * FROM source_{idx}.classifications;",
-            f"REPLACE INTO entities SELECT * FROM source_{idx}.entities;",
             f"REPLACE INTO transcription_entities SELECT * FROM source_{idx}.transcription_entities;",
             f"REPLACE INTO datasets SELECT * FROM source_{idx}.datasets;",
             f"REPLACE INTO dataset_elements SELECT * FROM source_{idx}.dataset_elements;",

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/arkindex_worker/worker/entity.py

@@ -11,16 +11,14 @@ from peewee import IntegrityError
 from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
-    CachedEntity,
     CachedTranscriptionEntity,
     unsupported_cache,
 )
-from arkindex_worker.models import Element, Transcription
+from arkindex_worker.models import Transcription
 from arkindex_worker.utils import pluralize
 
 
 class Entity(TypedDict):
-    name: str
     type_id: str
     length: int
     offset: int
@@ -126,88 +124,20 @@ class EntityMixin:
             # Create the type if non-existent
             self.create_entity_type(entity_type)
 
-    def create_entity(
-        self,
-        name: str,
-        type: str,
-        metas=None,
-        validated=None,
-    ):
-        """
-        Create an entity on the given corpus.
-        If cache support is enabled, a [CachedEntity][arkindex_worker.cache.CachedEntity] will also be created.
-
-        :param name: Name of the entity.
-        :param type: Type of the entity.
-        """
-        assert name and isinstance(name, str), (
-            "name shouldn't be null and should be of type str"
-        )
-        assert type and isinstance(type, str), (
-            "type shouldn't be null and should be of type str"
-        )
-        metas = metas or {}
-        if metas:
-            assert isinstance(metas, dict), "metas should be of type dict"
-        if validated is not None:
-            assert isinstance(validated, bool), "validated should be of type bool"
-        if self.is_read_only:
-            logger.warning("Cannot create entity as this worker is in read-only mode")
-            return
-
-        # Retrieve entity_type ID
-        if not self.entity_types:
-            # Load entity_types of corpus
-            self.list_corpus_entity_types()
-
-        entity_type_id = self.entity_types.get(type)
-        assert entity_type_id, f"Entity type `{type}` not found in the corpus."
-
-        entity = self.api_client.request(
-            "CreateEntity",
-            body={
-                "name": name,
-                "type_id": entity_type_id,
-                "metas": metas,
-                "validated": validated,
-                "corpus": self.corpus_id,
-                "worker_run_id": self.worker_run_id,
-            },
-        )
-
-        if self.use_cache:
-            # Store entity in local cache
-            try:
-                to_insert = [
-                    {
-                        "id": entity["id"],
-                        "type": type,
-                        "name": name,
-                        "validated": validated if validated is not None else False,
-                        "metas": metas,
-                        "worker_run_id": self.worker_run_id,
-                    }
-                ]
-                CachedEntity.insert_many(to_insert).execute()
-            except IntegrityError as e:
-                logger.warning(f"Couldn't save created entity in local cache: {e}")
-
-        return entity["id"]
-
     def create_transcription_entity(
         self,
         transcription: Transcription,
-        entity: str,
+        type_id: str,
         offset: int,
         length: int,
         confidence: float | None = None,
     ) -> dict[str, str | int] | None:
         """
-        Create a link between an existing entity and an existing transcription.
+        Create an entity on an existing transcription.
         If cache support is enabled, a `CachedTranscriptionEntity` will also be created.
 
         :param transcription: Transcription to create the entity on.
-        :param entity: UUID of the existing entity.
+        :param type_id: UUID of the entity type.
         :param offset: Starting position of the entity in the transcription's text,
            as a 0-based index.
         :param length: Length of the entity in the transcription's text.
@@ -218,8 +148,8 @@ class EntityMixin:
         assert transcription and isinstance(transcription, Transcription), (
             "transcription shouldn't be null and should be a Transcription"
         )
-        assert entity and isinstance(entity, str), (
-            "entity shouldn't be null and should be of type str"
+        assert type_id and isinstance(type_id, str), (
+            "type_id shouldn't be null and should be of type str"
         )
         assert offset is not None and isinstance(offset, int) and offset >= 0, (
             "offset shouldn't be null and should be a positive integer"
@@ -237,7 +167,7 @@ class EntityMixin:
             return
 
         body = {
-            "entity": entity,
+            "type_id": type_id,
             "length": length,
             "offset": offset,
             "worker_run_id": self.worker_run_id,
@@ -245,7 +175,7 @@ class EntityMixin:
         if confidence is not None:
             body["confidence"] = confidence
 
-        transcription_ent = self.api_client.request(
+        tr_entity = self.api_client.request(
             "CreateTranscriptionEntity",
             id=transcription.id,
             body=body,
@@ -256,7 +186,7 @@ class EntityMixin:
             try:
                 CachedTranscriptionEntity.create(
                     transcription=transcription.id,
-                    entity=entity,
+                    type=tr_entity["type"]["name"],
                     offset=offset,
                     length=length,
                     worker_run_id=self.worker_run_id,
@@ -267,7 +197,7 @@ class EntityMixin:
                     f"Couldn't save created transcription entity in local cache: {e}"
                 )
 
-        return transcription_ent
+        return tr_entity
 
     @unsupported_cache
     def create_transcription_entities(
@@ -276,14 +206,11 @@ class EntityMixin:
         entities: list[Entity],
     ) -> list[dict[str, str]]:
         """
-        Create multiple entities attached to a transcription in a single API request.
+        Create multiple entities on a transcription in a single API request.
 
         :param transcription: Transcription to create the entity on.
         :param entities: List of dicts, one per element. Each dict can have the following keys:
 
-            name (str)
-               Required. Name of the entity.
-
             type_id (str)
                Required. ID of the EntityType of the entity.
 
@@ -296,7 +223,7 @@ class EntityMixin:
             confidence (float or None)
                 Optional confidence score, between 0.0 and 1.0.
 
-        :return: List of dicts, with each dict having a two keys, `transcription_entity_id` and `entity_id`, holding the UUID of each created object.
+        :return: List of strings, holding the UUID of each created object.
         """
         assert transcription and isinstance(transcription, Transcription), (
             "transcription shouldn't be null and should be of type Transcription"
@@ -311,11 +238,6 @@ class EntityMixin:
                 f"Entity at index {index} in entities: Should be of type dict"
             )
 
-            name = entity.get("name")
-            assert name and isinstance(name, str), (
-                f"Entity at index {index} in entities: name shouldn't be null and should be of type str"
-            )
-
             type_id = entity.get("type_id")
             assert type_id and isinstance(type_id, str), (
                 f"Entity at index {index} in entities: type_id shouldn't be null and should be of type str"
@@ -339,7 +261,7 @@ class EntityMixin:
             )
 
         assert len(entities) == len(
-            set(map(itemgetter("offset", "length", "name", "type_id"), entities))
+            set(map(itemgetter("offset", "length", "type_id"), entities))
         ), "entities should be unique"
 
         if self.is_read_only:
@@ -348,16 +270,16 @@ class EntityMixin:
             )
             return
 
-        created_entities = self.api_client.request(
+        created_tr_entities = self.api_client.request(
             "CreateTranscriptionEntities",
             id=transcription.id,
             body={
                 "worker_run_id": self.worker_run_id,
-                "entities": entities,
+                "transcription_entities": entities,
             },
-        )["entities"]
+        )["transcription_entities"]
 
-        return created_entities
+        return created_tr_entities
 
     def list_transcription_entities(
         self,
@@ -412,34 +334,3 @@ class EntityMixin:
         return self.api_client.paginate(
             "ListTranscriptionEntities", id=transcription.id, **query_params
         )
-
-    def list_corpus_entities(
-        self,
-        name: str | None = None,
-        parent: Element | None = None,
-    ):
-        """
-        List all entities in the worker's corpus and store them in the ``self.entities`` cache.
-        :param name: Filter entities by part of their name (case-insensitive)
-        :param parent: Restrict entities to those linked to all transcriptions of an element and all its descendants. Note that links to metadata are ignored.
-        """
-        query_params = {}
-
-        if name is not None:
-            assert name and isinstance(name, str), "name should be of type str"
-            query_params["name"] = name
-
-        if parent is not None:
-            assert isinstance(parent, Element), "parent should be of type Element"
-            query_params["parent"] = parent.id
-
-        self.entities = {
-            entity["id"]: entity
-            for entity in self.api_client.paginate(
-                "ListCorpusEntities", id=self.corpus_id, **query_params
-            )
-        }
-        count = len(self.entities)
-        logger.info(
-            f"Loaded {count} {pluralize('entity', count)} in corpus ({self.corpus_id})"
-        )

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/arkindex_worker/worker/metadata.py

@@ -64,7 +64,6 @@ class MetaDataMixin:
         type: MetaType,
         name: str,
         value: str,
-        entity: str | None = None,
     ) -> str:
         """
         Create a metadata on the given element through API.
@@ -73,7 +72,6 @@ class MetaDataMixin:
         :param type: Type of the metadata.
         :param name: Name of the metadata.
         :param value: Value of the metadata.
-        :param entity: UUID of an entity this metadata is related to.
         :returns: UUID of the created metadata.
         """
         assert element and isinstance(element, Element | CachedElement), (
@@ -88,8 +86,6 @@ class MetaDataMixin:
         assert value and isinstance(value, str), (
             "value shouldn't be null and should be of type str"
         )
-        if entity:
-            assert isinstance(entity, str), "entity should be of type str"
         if self.is_read_only:
             logger.warning("Cannot create metadata as this worker is in read-only mode")
             return
@@ -101,7 +97,6 @@ class MetaDataMixin:
                 "type": type.value,
                 "name": name,
                 "value": value,
-                "entity_id": entity,
                 "worker_run_id": self.worker_run_id,
             },
         )
@@ -125,7 +120,6 @@ class MetaDataMixin:
             - type: MetaType
             - name: str
             - value: str | int | float
-            - entity_id: str | None
         :param batch_size: The size of each batch, which will be used to split the publication to avoid API errors.
 
         :returns: A list of dicts as returned in the ``metadata_list`` field by the ``CreateMetaDataBulk`` API endpoint.
@@ -157,16 +151,11 @@ class MetaDataMixin:
                 metadata.get("value"), str | float | int
             ), "value shouldn't be null and should be of type (str or float or int)"
 
-            assert metadata.get("entity_id") is None or isinstance(
-                metadata.get("entity_id"), str
-            ), "entity_id should be None or a str"
-
             metas.append(
                 {
                     "type": metadata.get("type").value,
                     "name": metadata.get("name"),
                     "value": metadata.get("value"),
-                    "entity_id": metadata.get("entity_id"),
                 }
             )
 

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "arkindex-base-worker"
-version = "0.5.0b3"
+version = "0.5.0rc1"
 description = "Base Worker to easily build Arkindex ML workflows"
 license = { file = "LICENSE" }
 dependencies = [

{arkindex_base_worker-0.5.0b3 → arkindex_base_worker-0.5.0rc1}/tests/test_cache.py

@@ -60,9 +60,8 @@ def test_create_tables(tmp_path):
 CREATE TABLE "dataset_elements" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "dataset_id" TEXT NOT NULL, "set_name" VARCHAR(255) NOT NULL, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"), FOREIGN KEY ("dataset_id") REFERENCES "datasets" ("id"))
 CREATE TABLE "datasets" ("id" TEXT NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, "state" VARCHAR(255) NOT NULL DEFAULT 'open', "sets" TEXT NOT NULL)
 CREATE TABLE "elements" ("id" TEXT NOT NULL PRIMARY KEY, "parent_id" TEXT, "type" VARCHAR(50) NOT NULL, "image_id" TEXT, "polygon" text, "rotation_angle" INTEGER NOT NULL, "mirrored" INTEGER NOT NULL, "initial" INTEGER NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, FOREIGN KEY ("image_id") REFERENCES "images" ("id"))
-CREATE TABLE "entities" ("id" TEXT NOT NULL PRIMARY KEY, "type" VARCHAR(50) NOT NULL, "name" TEXT NOT NULL, "validated" INTEGER NOT NULL, "metas" text, "worker_run_id" TEXT)
 CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL)
-CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "entity_id" TEXT NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "entity_id"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"), FOREIGN KEY ("entity_id") REFERENCES "entities" ("id"))
+CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "type" VARCHAR(50) NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "type"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"))
 CREATE TABLE "transcriptions" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "text" TEXT NOT NULL, "confidence" REAL, "orientation" VARCHAR(50) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))"""
 
     actual_schema = "\n".join(