arkindex-base-worker 0.4.0rc3__py3-none-any.whl → 0.4.0rc5__py3-none-any.whl

{arkindex_base_worker-0.4.0rc3.dist-info → arkindex_base_worker-0.4.0rc5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc3
+Version: 0.4.0rc5
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,24 +37,25 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize ==4.10.0
-Requires-Dist: peewee ~=3.17
-Requires-Dist: Pillow ==10.4.0
-Requires-Dist: python-gnupg ==0.5.2
-Requires-Dist: shapely ==2.0.5
-Requires-Dist: teklia-toolbox ==0.1.7b1
-Requires-Dist: zstandard ==0.22.0
+Requires-Dist: humanize==4.10.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==10.4.0
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7b1
+Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
-Requires-Dist: black ==24.4.2 ; extra == 'docs'
-Requires-Dist: mkdocs-material ==9.5.33 ; extra == 'docs'
-Requires-Dist: mkdocstrings-python ==1.11.1 ; extra == 'docs'
+Requires-Dist: black==24.4.2; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest ==8.3.2 ; extra == 'tests'
-Requires-Dist: pytest-mock ==3.14.0 ; extra == 'tests'
-Requires-Dist: pytest-responses ==0.5.1 ; extra == 'tests'
+Requires-Dist: pytest==8.3.2; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 

{arkindex_base_worker-0.4.0rc3.dist-info → arkindex_base_worker-0.4.0rc5.dist-info}/RECORD

@@ -1,18 +1,18 @@
 arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,162
-arkindex_worker/cache.py,sha256=FTlB0coXofn5zTNRTcVIvh709mcw4a1bPGqkwWjKs3w,11248
+arkindex_worker/cache.py,sha256=qTblc_zKdYC47Wip6_O9Jf5qBkQW2ozQQrg-nsx1WuY,11221
 arkindex_worker/image.py,sha256=oEgVCrSHiGh3D5-UXfM6PvT17TttSxC0115irpvB3Dw,18581
 arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
 arkindex_worker/utils.py,sha256=q1EeLdC6ebYIH-C0LOAqw2cNpjCjVoP-Vbr-39mF4w0,9884
-arkindex_worker/worker/__init__.py,sha256=tcqxrox9EpOjaN2EQgXumiABKpWHLsJiynPC2_sZuOQ,15880
+arkindex_worker/worker/__init__.py,sha256=m255Cle3nE_FtAXbbJj_v1aS9ClT6hpDlrUFXTPiqkI,15985
 arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
 arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
 arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
 arkindex_worker/worker/dataset.py,sha256=LwzKwNFX4FqfLxh29LSvJydPwRw3VHaB1wjuFhUshsE,5267
 arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6e9Y4,45309
-arkindex_worker/worker/entity.py,sha256=ThhP22xOYR5Z4P1VH_pOl_y_uDKZFeQVDqxO6aRkIhg,15227
+arkindex_worker/worker/entity.py,sha256=s5wjX6_JfTyk4qfMoV0OWfOXUx6T-9WpOiEpaoaCEFM,14808
 arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
 arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
-arkindex_worker/worker/process.py,sha256=I1rBt3Y8bV4zcPr8N1E2NRZ0UClSTqhExsO9CPcP41E,1012
+arkindex_worker/worker/process.py,sha256=IAJaiiCizK4vpPmMQD0yYSB6IIoyy7yU-5JKaiuPb7o,1073
 arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
 arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
 arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
@@ -24,29 +24,37 @@ tests/test_base_worker.py,sha256=2EIYcd_3f9O0zB5WiGIQV0Cn9wndLvnEnSfcAE1qWWU,306
 tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
 tests/test_dataset_worker.py,sha256=gApYz0LArHr1cNn079_fa_BQABF6RVQYuM1Tc4m3NsQ,22089
 tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
-tests/test_image.py,sha256=J3jqB5OhcdCpB6n0UnwivxrMlne8YjFLXhq1gBMANrs,26711
+tests/test_image.py,sha256=03E24JVa7TZJfuwQyfVEBe3RAq3R993IMl1AHXRr7zY,25497
 tests/test_merge.py,sha256=TuOeUS0UCz66DPOQFFhc4NQBxIjZL9f5czi4XnvGrr4,8270
 tests/test_utils.py,sha256=_WJUPnt-pM_TQ0er4yjPZy-u_LePrHq1lxwk_teky7M,2544
 tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
-tests/test_elements_worker/test_classifications.py,sha256=GtVyi9bg4YTd7nyw8u6IjQZYBwFMwoVZdrfSBc5UybU,27780
+tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
 tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
-tests/test_elements_worker/test_corpus.py,sha256=OAbwgaQtHmcmPkcAl9Kuceun_BvMasnZvYj4_EdfugY,5483
+tests/test_elements_worker/test_corpus.py,sha256=kscJyM8k1njYJJFGuvliVzn89lWh41mEyDCCawnp3W8,5483
 tests/test_elements_worker/test_dataset.py,sha256=00IlOZv9YFlZ23rGXyR-HLbKLQxGelZ1Bf9lEZYA0IY,11412
-tests/test_elements_worker/test_elements.py,sha256=l5YTfm0CzBTQyZvdOplhhza-gpPSz-8RVix1YUzAwhM,115497
-tests/test_elements_worker/test_entities.py,sha256=nrCvkdJdjsyOrbD6R-H8NvxREZxciiR6CGIObXzeg50,36182
+tests/test_elements_worker/test_element.py,sha256=lb5tLjl0jsixX0OWVhBAaKLE9GKkBw79kFHDNGommaQ,12535
+tests/test_elements_worker/test_element_create_multiple.py,sha256=arYFGmxc0517ZUii6k__G_UQQatuNIASTC8MXvUrSwk,21887
+tests/test_elements_worker/test_element_create_single.py,sha256=Fa9zm12J2rQ3VrUe3yIlHAc7Vty_eQYb_YGnNPQB3IE,16697
+tests/test_elements_worker/test_element_list_children.py,sha256=2zH4h663w3EduqpzQr-7bf9zIDzO1x2WxdUYYHsIHkI,31358
+tests/test_elements_worker/test_element_list_parents.py,sha256=TXeGW-a3W-7GmB2QrhJH9mMnvxuybeAwQ4tL3iIxwXo,16734
+tests/test_elements_worker/test_entity_create.py,sha256=9Tjr9KA2yo44VFV283q_cs6XbbVguUMDNfCj-DILSJg,29353
+tests/test_elements_worker/test_entity_list_and_check.py,sha256=ENBLaqbXlRUDbHRvQla3080a0HJltrWAPYWNohUA9NU,4992
 tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
 tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
 tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
 tests/test_elements_worker/test_training.py,sha256=Qxi9EzGr_uKcn2Fh5aE6jNrq1K8QKLiOiSew4upASPs,8721
-tests/test_elements_worker/test_transcriptions.py,sha256=iq-nR_st7Q9E_nD7knrKGY57g36J6nYSEzbPk9y-cxY,77061
-tests/test_elements_worker/test_worker.py,sha256=VdprIWezB3dJdE8vNOrS71RQugqUysHlveOWTQate-8,10804
+tests/test_elements_worker/test_transcription_create.py,sha256=yznO9B_BVsOR0Z_VY5ZL8gJp0ZPCz_4sPUs5dXtixAg,29281
+tests/test_elements_worker/test_transcription_create_with_elements.py,sha256=tmcyglgssEqMnt1Mdy_u6X1m2wgLWTo_HdWst3GrK2k,33056
+tests/test_elements_worker/test_transcription_list.py,sha256=ikz7HYPCoQWTdTRCd382SB-y-T2BbigPLlIcx5Eow-I,15324
+tests/test_elements_worker/test_version.py,sha256=xqCgcgukTFJzkMgYfQG-8mTbu0o2fdYjWC07FktThfw,2125
+tests/test_elements_worker/test_worker.py,sha256=fnFwkcDb6tx5i7lbelifeHD-BrGz6o5UBEojPCdtuAo,23474
 worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.4.0rc3.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
-arkindex_base_worker-0.4.0rc3.dist-info/METADATA,sha256=eDT7HxTvEz2yg4U_lbzkuigNWFu4JTqaLTnY0fqSCiM,3306
-arkindex_base_worker-0.4.0rc3.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-arkindex_base_worker-0.4.0rc3.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
-arkindex_base_worker-0.4.0rc3.dist-info/RECORD,,
+arkindex_base_worker-0.4.0rc5.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.4.0rc5.dist-info/METADATA,sha256=EFoMgnh4SUvYV0yIdBSfA7aoaZiTA7kYrJTrH2la3mY,3338
+arkindex_base_worker-0.4.0rc5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+arkindex_base_worker-0.4.0rc5.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
+arkindex_base_worker-0.4.0rc5.dist-info/RECORD,,

{arkindex_base_worker-0.4.0rc3.dist-info → arkindex_base_worker-0.4.0rc5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

arkindex_worker/worker/__init__.py

@@ -133,8 +133,12 @@ class ElementsWorker(
         Whether or not WorkerActivity support has been enabled on the DataImport
         used to run this worker.
         """
-        if self.is_read_only or self.process_mode == ProcessMode.Dataset:
-            # Worker activities are also disabled when running an ElementsWorker in a Dataset process.
+        if self.is_read_only or self.process_mode in [
+            ProcessMode.Dataset,
+            ProcessMode.Export,
+        ]:
+            # Worker activities are also disabled when running an ElementsWorker in a Dataset process
+            # and when running export processes.
             return False
         assert (
             self.process_information

arkindex_worker/worker/entity.py

@@ -16,9 +16,6 @@ from arkindex_worker.cache import (
 )
 from arkindex_worker.models import Element, Transcription
 from arkindex_worker.utils import (
-    DEFAULT_BATCH_SIZE,
-    batch_publication,
-    make_batches,
     pluralize,
 )
 
@@ -219,12 +216,10 @@ class EntityMixin:
         return transcription_ent
 
     @unsupported_cache
-    @batch_publication
     def create_transcription_entities(
         self,
         transcription: Transcription,
         entities: list[Entity],
-        batch_size: int = DEFAULT_BATCH_SIZE,
     ) -> list[dict[str, str]]:
         """
         Create multiple entities attached to a transcription in a single API request.
@@ -247,8 +242,6 @@ class EntityMixin:
             confidence (float or None)
                 Optional confidence score, between 0.0 and 1.0.
 
-        :param batch_size: The size of each batch, which will be used to split the publication to avoid API errors.
-
         :return: List of dicts, with each dict having a two keys, `transcription_entity_id` and `entity_id`, holding the UUID of each created object.
         """
         assert transcription and isinstance(
@@ -300,18 +293,14 @@ class EntityMixin:
             )
             return
 
-        created_entities = [
-            created_entity
-            for batch in make_batches(entities, "entity", batch_size)
-            for created_entity in self.api_client.request(
-                "CreateTranscriptionEntities",
-                id=transcription.id,
-                body={
-                    "worker_run_id": self.worker_run_id,
-                    "entities": batch,
-                },
-            )["entities"]
-        ]
+        created_entities = self.api_client.request(
+            "CreateTranscriptionEntities",
+            id=transcription.id,
+            body={
+                "worker_run_id": self.worker_run_id,
+                "entities": entities,
+            },
+        )["entities"]
 
         return created_entities
 

arkindex_worker/worker/process.py

@@ -61,3 +61,8 @@ class ProcessMode(Enum):
     """
     Dataset processes.
     """
+
+    Export = "export"
+    """
+    Export processes.
+    """

tests/test_elements_worker/{test_classifications.py → test_classification.py}

@@ -17,6 +17,92 @@ from . import BASE_API_CALLS
 DELETE_PARAMETER = "DELETE_PARAMETER"
 
 
+def test_load_corpus_classes_api_error(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=418,
+    )
+
+    assert not mock_elements_worker.classes
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        # We do 5 retries
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert not mock_elements_worker.classes
+
+
+def test_load_corpus_classes(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=200,
+        json={
+            "count": 3,
+            "next": None,
+            "results": [
+                {
+                    "id": "0000",
+                    "name": "good",
+                },
+                {
+                    "id": "1111",
+                    "name": "average",
+                },
+                {
+                    "id": "2222",
+                    "name": "bad",
+                },
+            ],
+        },
+    )
+
+    assert not mock_elements_worker.classes
+    mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert mock_elements_worker.classes == {
+        "good": "0000",
+        "average": "1111",
+        "bad": "2222",
+    }
+
+
 def test_get_ml_class_id_load_classes(responses, mock_elements_worker):
     responses.add(
         responses.GET,

tests/test_elements_worker/test_corpus.py

@@ -9,6 +9,37 @@ from tests import CORPUS_ID
 from tests.test_elements_worker import BASE_API_CALLS
 
 
+def test_download_export_not_a_uuid(responses, mock_elements_worker):
+    with pytest.raises(ValueError, match="export_id is not a valid uuid."):
+        mock_elements_worker.download_export("mon export")
+
+
+def test_download_export(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        status=302,
+        body=b"some SQLite export",
+        content_type="application/x-sqlite3",
+        stream=True,
+    )
+
+    export = mock_elements_worker.download_export(
+        "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+    )
+    assert export.name == "/tmp/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        ),
+    ]
+
+
 def mock_list_exports_call(responses, export_id):
     responses.add(
         responses.GET,
@@ -135,34 +166,3 @@ def test_download_latest_export(responses, mock_elements_worker):
         ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
         ("GET", f"http://testserver/api/v1/export/{export_id}/"),
     ]
-
-
-def test_download_export_not_a_uuid(responses, mock_elements_worker):
-    with pytest.raises(ValueError, match="export_id is not a valid uuid."):
-        mock_elements_worker.download_export("mon export")
-
-
-def test_download_export(responses, mock_elements_worker):
-    responses.add(
-        responses.GET,
-        "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
-        status=302,
-        body=b"some SQLite export",
-        content_type="application/x-sqlite3",
-        stream=True,
-    )
-
-    export = mock_elements_worker.download_export(
-        "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
-    )
-    assert export.name == "/tmp/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
-
-    assert len(responses.calls) == len(BASE_API_CALLS) + 1
-    assert [
-        (call.request.method, call.request.url) for call in responses.calls
-    ] == BASE_API_CALLS + [
-        (
-            "GET",
-            "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
-        ),
-    ]