arkindex-base-worker 0.3.6rc2__py3-none-any.whl → 0.3.6rc4__py3-none-any.whl

{arkindex_base_worker-0.3.6rc2.dist-info → arkindex_base_worker-0.3.6rc4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.6rc2
+Version: 0.3.6rc4
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -18,6 +18,14 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: arkindex-client ==1.0.14
+Requires-Dist: peewee ==3.17.0
+Requires-Dist: Pillow ==10.1.0
+Requires-Dist: pymdown-extensions ==10.3.1
+Requires-Dist: python-gnupg ==0.5.1
+Requires-Dist: shapely ==2.0.2
+Requires-Dist: tenacity ==8.2.3
+Requires-Dist: zstandard ==0.22.0
 Provides-Extra: docs
 Requires-Dist: black ==23.11.0 ; extra == 'docs'
 Requires-Dist: doc8 ==1.1.1 ; extra == 'docs'

{arkindex_base_worker-0.3.6rc2.dist-info → arkindex_base_worker-0.3.6rc4.dist-info}/RECORD

@@ -3,12 +3,12 @@ arkindex_worker/cache.py,sha256=ZbXJo-O24W8x6nbS0IJm32Tas9CKLaHBBeyQyvF-Nyo,1090
 arkindex_worker/image.py,sha256=uwfUE9hy0Iw-e3vU7OHmLSqouxbznWq08SykXmPD1Cs,14107
 arkindex_worker/models.py,sha256=DVrZPIurSiOoHvj3t_Szwd0j1t6pnwBx_dqwhNakzN0,9528
 arkindex_worker/utils.py,sha256=_lC1-RYvNWXEkK-AuF4_FraoggP1tYPdalNFSj4jDb4,6885
-arkindex_worker/worker/__init__.py,sha256=4_kDCgjFp7oUCdhnNUEoLbrpUyr2MYK5JStrUYmgUDc,19062
+arkindex_worker/worker/__init__.py,sha256=Iun6jhuakKdCGKjQtgqDWEpWO1HrxK34RoxdzE5gcRs,19322
 arkindex_worker/worker/base.py,sha256=4eG4v4vejvFv9UtTRhxEZkXEBVzlFd3rILHK8lt-mbc,19397
 arkindex_worker/worker/classification.py,sha256=CoMIj7SFFlt90W1r5FQmsB80qK9Zfltcm3k-37FSHA0,10693
 arkindex_worker/worker/dataset.py,sha256=LRZU_KkOuCRkxlkdqw1PHYnu1zmoQfm_OiY8Sqt6mi0,2754
 arkindex_worker/worker/element.py,sha256=5z6yAO2jKCF4U7Tz2cGdFX34Zxdo0gBWkU-ciIZ69bQ,30705
-arkindex_worker/worker/entity.py,sha256=WJANdUUvFdR_6c3wYwfckWdZqeq-W8TciDPm-YcmkCE,13397
+arkindex_worker/worker/entity.py,sha256=l0gCoeaoUBFU7pv2iC4pHkSQVjiIur4M15P7Mg_WlaA,13601
 arkindex_worker/worker/metadata.py,sha256=PnzyHkPyb-mtgItzRi4s-_f0dsEOM3ak8F_bFoqp3O0,6225
 arkindex_worker/worker/task.py,sha256=cz3wJNPgogZv1lm_3lm7WScitQtYQtL6H6I7Xokq208,1475
 arkindex_worker/worker/training.py,sha256=rhg4TPFo0ignnCkgbekUYmNXX8u2iZGyeM1VCs1R5kI,10140
@@ -17,7 +17,7 @@ arkindex_worker/worker/version.py,sha256=uL-OrwuFZB8TNU6ePmdKIL3g3e-GE2tqHEWBRpX
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/conftest.py,sha256=BEeyspbvxwSpFvQ1taj8ysnu7wAE3d7pWUDMmmKC5Ho,21837
 tests/test_base_worker.py,sha256=AF1pjvNckN80LVyLJ4ILXJ122fEnWtztK7ZncVDI_Ms,24976
-tests/test_cache.py,sha256=6pdgdsU20pn-CMLbYA1Af0A2QZ_0eiQ0UqtlIMoujfU,10425
+tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
 tests/test_dataset_worker.py,sha256=XxBCLRroCIq97P37_qWc9I7QiyE3zUL7fLAw1J_BI7E,27703
 tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
 tests/test_image.py,sha256=nIT0NhmuHtD9I1rcMSvqSfjQqvTE5Ko8tAQGLIkm_zo,15232
@@ -28,13 +28,13 @@ tests/test_elements_worker/test_classifications.py,sha256=0_6vryoQc2-s3CQWANsEva
 tests/test_elements_worker/test_cli.py,sha256=DdCRKobesehL61c5QwaZOZCde7bsTlmeSN2iosQ5_2s,2873
 tests/test_elements_worker/test_dataset.py,sha256=ElDPrYTTt8KzaZ_Xf5uslUD6_kiGZybntO10uqCquLo,12063
 tests/test_elements_worker/test_elements.py,sha256=F7azXQBNec4QLH62uXaGgAgg82GEz0q98pYXH3Sh640,75761
-tests/test_elements_worker/test_entities.py,sha256=epfiLH-cs0g8nKr4e8som7N_10lbogJIXyLyMMiz460,33792
+tests/test_elements_worker/test_entities.py,sha256=ZOFB3ckKJvNG2kIPUX_kz_378k3uQrJmvYHpR_xiVuo,33789
 tests/test_elements_worker/test_metadata.py,sha256=c3kXPYRXVPDnGim28Ncg5YO4I0ejh3qyi7dBvbSYxMU,17739
 tests/test_elements_worker/test_task.py,sha256=FCpxE9UpouKXgjGvWgNHEai_Hiy2d1YmqRG-_v2s27s,6312
 tests/test_elements_worker/test_training.py,sha256=WeG-cDuJ-YhPgfKH47TtXBxyargtLuk7c8tsik2WnL8,8414
 tests/test_elements_worker/test_transcriptions.py,sha256=6UWGriQVwEORunJYW11mGcD16voZGFY41i_NIdXuqnI,68750
 tests/test_elements_worker/test_worker.py,sha256=zD8sY5yZFhuUr1txVX8z7bSgW4I2jNuzH5i1TM3qkZI,16491
-arkindex_base_worker-0.3.6rc2.dist-info/METADATA,sha256=UWtS8SfoCWWXb5N7BQHa8SrJqpxDed-Rb754uNVqenE,1536
-arkindex_base_worker-0.3.6rc2.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-arkindex_base_worker-0.3.6rc2.dist-info/top_level.txt,sha256=TtagLI8LSv7GE7nG8MQqDFAJ5bNDPJn7Z5vizOgrWkA,22
-arkindex_base_worker-0.3.6rc2.dist-info/RECORD,,
+arkindex_base_worker-0.3.6rc4.dist-info/METADATA,sha256=5ug8gbQxLwb87ArFDtHn32bekEexXmiTEysRPA3UKio,1814
+arkindex_base_worker-0.3.6rc4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+arkindex_base_worker-0.3.6rc4.dist-info/top_level.txt,sha256=TtagLI8LSv7GE7nG8MQqDFAJ5bNDPJn7Z5vizOgrWkA,22
+arkindex_base_worker-0.3.6rc4.dist-info/RECORD,,

{arkindex_base_worker-0.3.6rc2.dist-info → arkindex_base_worker-0.3.6rc4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

arkindex_worker/worker/__init__.py

@@ -387,7 +387,7 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
         """
 
         def format_split(
-            split: tuple[str, Iterator[tuple[str, Element]]]
+            split: tuple[str, Iterator[tuple[str, Element]]],
         ) -> tuple[str, list[Element]]:
             return (split[0], list(map(itemgetter(1), list(split[1]))))
 
@@ -440,6 +440,8 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
         failed = 0
         for i, item in enumerate(datasets, start=1):
             dataset = None
+            dataset_artifact = None
+
             try:
                 if not self.is_read_only:
                     # Just use the result of list_datasets as the dataset
@@ -465,7 +467,7 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
                     self.update_dataset_state(dataset, DatasetState.Building)
                 else:
                     logger.info(f"Downloading data for {dataset} ({i}/{count})")
-                    self.download_dataset_artifact(dataset)
+                    dataset_artifact = self.download_dataset_artifact(dataset)
 
                 # Process the dataset
                 self.process_dataset(dataset)
@@ -496,6 +498,10 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
                     # Try to update the state to Error regardless of the response
                     with contextlib.suppress(Exception):
                         self.update_dataset_state(dataset, DatasetState.Error)
+            finally:
+                # Cleanup the dataset artifact if it was downloaded, no matter what
+                if dataset_artifact:
+                    dataset_artifact.unlink(missing_ok=True)
 
         if failed:
             logger.error(

arkindex_worker/worker/entity.py

@@ -331,8 +331,7 @@ class EntityMixin:
         parent: Element | None = None,
     ):
         """
-        List all entities in the worker's corpus
-        This method does not support cache
+        List all entities in the worker's corpus and store them in the ``self.entities`` cache.
         :param name: Filter entities by part of their name (case-insensitive)
         :param parent: Restrict entities to those linked to all transcriptions of an element and all its descendants. Note that links to metadata are ignored.
         """
@@ -346,8 +345,14 @@ class EntityMixin:
             assert isinstance(parent, Element), "parent should be of type Element"
             query_params["parent"] = parent.id
 
-        return self.api_client.paginate(
-            "ListCorpusEntities", id=self.corpus_id, **query_params
+        self.entities = {
+            entity["id"]: entity
+            for entity in self.api_client.paginate(
+                "ListCorpusEntities", id=self.corpus_id, **query_params
+            )
+        }
+        logger.info(
+            f"Loaded {len(self.entities)} entities in corpus ({self.corpus_id})"
         )
 
     def list_corpus_entity_types(

tests/test_cache.py

@@ -53,6 +53,9 @@ def test_create_tables(tmp_path):
     init_cache_db(db_path)
     create_tables()
 
+    # WARNING: If you are updating this schema following a development you have made
+    # in base-worker, make sure to upgrade the arkindex_worker.cache.SQL_VERSION in
+    # the same merge request as your changes.
     expected_schema = """CREATE TABLE "classifications" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "class_name" TEXT NOT NULL, "confidence" REAL NOT NULL, "state" VARCHAR(10) NOT NULL, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))
 CREATE TABLE "dataset_elements" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "dataset_id" TEXT NOT NULL, "set_name" VARCHAR(255) NOT NULL, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"), FOREIGN KEY ("dataset_id") REFERENCES "datasets" ("id"))
 CREATE TABLE "datasets" ("id" TEXT NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, "state" VARCHAR(255) NOT NULL DEFAULT 'open', "sets" TEXT NOT NULL)

tests/test_elements_worker/test_entities.py

@@ -741,12 +741,13 @@ def test_list_corpus_entities(responses, mock_elements_worker):
         },
     )
 
-    # list is required to actually do the request
-    assert list(mock_elements_worker.list_corpus_entities()) == [
-        {
+    mock_elements_worker.list_corpus_entities()
+
+    assert mock_elements_worker.entities == {
+        "fake_entity_id": {
             "id": "fake_entity_id",
         }
-    ]
+    }
 
     assert len(responses.calls) == len(BASE_API_CALLS) + 1
     assert [