arkindex-base-worker 0.4.0rc3__tar.gz → 0.4.0rc5__tar.gz

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc3
+Version: 0.4.0rc5
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,6 +37,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -44,7 +45,7 @@ Requires-Dist: humanize==4.10.0
 Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==10.4.0
 Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.5
+Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.7b1
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc3
+Version: 0.4.0rc5
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,6 +37,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -44,7 +45,7 @@ Requires-Dist: humanize==4.10.0
 Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==10.4.0
 Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.5
+Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.7b1
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -36,17 +36,25 @@ tests/test_image.py
 tests/test_merge.py
 tests/test_utils.py
 tests/test_elements_worker/__init__.py
-tests/test_elements_worker/test_classifications.py
+tests/test_elements_worker/test_classification.py
 tests/test_elements_worker/test_cli.py
 tests/test_elements_worker/test_corpus.py
 tests/test_elements_worker/test_dataset.py
-tests/test_elements_worker/test_elements.py
-tests/test_elements_worker/test_entities.py
+tests/test_elements_worker/test_element.py
+tests/test_elements_worker/test_element_create_multiple.py
+tests/test_elements_worker/test_element_create_single.py
+tests/test_elements_worker/test_element_list_children.py
+tests/test_elements_worker/test_element_list_parents.py
+tests/test_elements_worker/test_entity_create.py
+tests/test_elements_worker/test_entity_list_and_check.py
 tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
-tests/test_elements_worker/test_transcriptions.py
+tests/test_elements_worker/test_transcription_create.py
+tests/test_elements_worker/test_transcription_create_with_elements.py
+tests/test_elements_worker/test_transcription_list.py
+tests/test_elements_worker/test_version.py
 tests/test_elements_worker/test_worker.py
 worker-demo/tests/__init__.py
 worker-demo/tests/conftest.py

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_base_worker.egg-info/requires.txt

@@ -2,7 +2,7 @@ humanize==4.10.0
 peewee~=3.17
 Pillow==10.4.0
 python-gnupg==0.5.2
-shapely==2.0.5
+shapely==2.0.6
 teklia-toolbox==0.1.7b1
 zstandard==0.22.0
 

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_worker/worker/__init__.py

@@ -133,8 +133,12 @@ class ElementsWorker(
         Whether or not WorkerActivity support has been enabled on the DataImport
         used to run this worker.
         """
-        if self.is_read_only or self.process_mode == ProcessMode.Dataset:
-            # Worker activities are also disabled when running an ElementsWorker in a Dataset process.
+        if self.is_read_only or self.process_mode in [
+            ProcessMode.Dataset,
+            ProcessMode.Export,
+        ]:
+            # Worker activities are also disabled when running an ElementsWorker in a Dataset process
+            # and when running export processes.
             return False
         assert (
             self.process_information

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_worker/worker/entity.py

@@ -16,9 +16,6 @@ from arkindex_worker.cache import (
 )
 from arkindex_worker.models import Element, Transcription
 from arkindex_worker.utils import (
-    DEFAULT_BATCH_SIZE,
-    batch_publication,
-    make_batches,
     pluralize,
 )
 
@@ -219,12 +216,10 @@ class EntityMixin:
         return transcription_ent
 
     @unsupported_cache
-    @batch_publication
     def create_transcription_entities(
         self,
         transcription: Transcription,
         entities: list[Entity],
-        batch_size: int = DEFAULT_BATCH_SIZE,
     ) -> list[dict[str, str]]:
         """
         Create multiple entities attached to a transcription in a single API request.
@@ -247,8 +242,6 @@ class EntityMixin:
             confidence (float or None)
                 Optional confidence score, between 0.0 and 1.0.
 
-        :param batch_size: The size of each batch, which will be used to split the publication to avoid API errors.
-
         :return: List of dicts, with each dict having a two keys, `transcription_entity_id` and `entity_id`, holding the UUID of each created object.
         """
         assert transcription and isinstance(
@@ -300,18 +293,14 @@ class EntityMixin:
             )
             return
 
-        created_entities = [
-            created_entity
-            for batch in make_batches(entities, "entity", batch_size)
-            for created_entity in self.api_client.request(
-                "CreateTranscriptionEntities",
-                id=transcription.id,
-                body={
-                    "worker_run_id": self.worker_run_id,
-                    "entities": batch,
-                },
-            )["entities"]
-        ]
+        created_entities = self.api_client.request(
+            "CreateTranscriptionEntities",
+            id=transcription.id,
+            body={
+                "worker_run_id": self.worker_run_id,
+                "entities": entities,
+            },
+        )["entities"]
 
         return created_entities
 

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/arkindex_worker/worker/process.py

@@ -61,3 +61,8 @@ class ProcessMode(Enum):
     """
     Dataset processes.
     """
+
+    Export = "export"
+    """
+    Export processes.
+    """

{arkindex_base_worker-0.4.0rc3 → arkindex_base_worker-0.4.0rc5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "arkindex-base-worker"
-version = "0.4.0rc3"
+version = "0.4.0rc5"
 description = "Base Worker to easily build Arkindex ML workflows"
 license = { file = "LICENSE" }
 dependencies = [
@@ -12,7 +12,7 @@ dependencies = [
     "peewee~=3.17",
     "Pillow==10.4.0",
     "python-gnupg==0.5.2",
-    "shapely==2.0.5",
+    "shapely==2.0.6",
     "teklia-toolbox==0.1.7b1",
     "zstandard==0.22.0",
 ]
@@ -32,6 +32,7 @@ classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 [project.urls]

arkindex_base_worker-0.4.0rc3/tests/test_elements_worker/test_classifications.py → arkindex_base_worker-0.4.0rc5/tests/test_elements_worker/test_classification.py

@@ -17,6 +17,92 @@ from . import BASE_API_CALLS
 DELETE_PARAMETER = "DELETE_PARAMETER"
 
 
+def test_load_corpus_classes_api_error(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=418,
+    )
+
+    assert not mock_elements_worker.classes
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        # We do 5 retries
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert not mock_elements_worker.classes
+
+
+def test_load_corpus_classes(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=200,
+        json={
+            "count": 3,
+            "next": None,
+            "results": [
+                {
+                    "id": "0000",
+                    "name": "good",
+                },
+                {
+                    "id": "1111",
+                    "name": "average",
+                },
+                {
+                    "id": "2222",
+                    "name": "bad",
+                },
+            ],
+        },
+    )
+
+    assert not mock_elements_worker.classes
+    mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert mock_elements_worker.classes == {
+        "good": "0000",
+        "average": "1111",
+        "bad": "2222",
+    }
+
+
 def test_get_ml_class_id_load_classes(responses, mock_elements_worker):
     responses.add(
         responses.GET,