arkindex-base-worker 0.4.0rc2__py3-none-any.whl → 0.4.0rc4__py3-none-any.whl

{arkindex_base_worker-0.4.0rc2.dist-info → arkindex_base_worker-0.4.0rc4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc2
+Version: 0.4.0rc4
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,24 +37,25 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize ==4.10.0
-Requires-Dist: peewee ~=3.17
-Requires-Dist: Pillow ==10.4.0
-Requires-Dist: python-gnupg ==0.5.2
-Requires-Dist: shapely ==2.0.5
-Requires-Dist: teklia-toolbox ==0.1.5
-Requires-Dist: zstandard ==0.22.0
+Requires-Dist: humanize==4.10.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==10.4.0
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7b1
+Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
-Requires-Dist: black ==24.4.2 ; extra == 'docs'
-Requires-Dist: mkdocs-material ==9.5.33 ; extra == 'docs'
-Requires-Dist: mkdocstrings-python ==1.10.8 ; extra == 'docs'
+Requires-Dist: black==24.4.2; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest ==8.3.2 ; extra == 'tests'
-Requires-Dist: pytest-mock ==3.14.0 ; extra == 'tests'
-Requires-Dist: pytest-responses ==0.5.1 ; extra == 'tests'
+Requires-Dist: pytest==8.3.2; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 

arkindex_base_worker-0.4.0rc4.dist-info/RECORD

@@ -0,0 +1,60 @@
+arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,162
+arkindex_worker/cache.py,sha256=qTblc_zKdYC47Wip6_O9Jf5qBkQW2ozQQrg-nsx1WuY,11221
+arkindex_worker/image.py,sha256=oEgVCrSHiGh3D5-UXfM6PvT17TttSxC0115irpvB3Dw,18581
+arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
+arkindex_worker/utils.py,sha256=q1EeLdC6ebYIH-C0LOAqw2cNpjCjVoP-Vbr-39mF4w0,9884
+arkindex_worker/worker/__init__.py,sha256=tcqxrox9EpOjaN2EQgXumiABKpWHLsJiynPC2_sZuOQ,15880
+arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
+arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
+arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
+arkindex_worker/worker/dataset.py,sha256=LwzKwNFX4FqfLxh29LSvJydPwRw3VHaB1wjuFhUshsE,5267
+arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6e9Y4,45309
+arkindex_worker/worker/entity.py,sha256=ThhP22xOYR5Z4P1VH_pOl_y_uDKZFeQVDqxO6aRkIhg,15227
+arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
+arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
+arkindex_worker/worker/process.py,sha256=I1rBt3Y8bV4zcPr8N1E2NRZ0UClSTqhExsO9CPcP41E,1012
+arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
+arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
+arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
+arkindex_worker/worker/version.py,sha256=JIT7OI3Mo7RPkNrjOB9hfqrsG-FYygz_zi4l8PbkuAo,1960
+hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
+tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
+tests/conftest.py,sha256=2ocZ2x-mZQrNe9zvWwhWk2_4ExdaBHIB74SvtDlExRE,21580
+tests/test_base_worker.py,sha256=2EIYcd_3f9O0zB5WiGIQV0Cn9wndLvnEnSfcAE1qWWU,30607
+tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
+tests/test_dataset_worker.py,sha256=gApYz0LArHr1cNn079_fa_BQABF6RVQYuM1Tc4m3NsQ,22089
+tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
+tests/test_image.py,sha256=03E24JVa7TZJfuwQyfVEBe3RAq3R993IMl1AHXRr7zY,25497
+tests/test_merge.py,sha256=TuOeUS0UCz66DPOQFFhc4NQBxIjZL9f5czi4XnvGrr4,8270
+tests/test_utils.py,sha256=_WJUPnt-pM_TQ0er4yjPZy-u_LePrHq1lxwk_teky7M,2544
+tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
+tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
+tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
+tests/test_elements_worker/test_corpus.py,sha256=kscJyM8k1njYJJFGuvliVzn89lWh41mEyDCCawnp3W8,5483
+tests/test_elements_worker/test_dataset.py,sha256=00IlOZv9YFlZ23rGXyR-HLbKLQxGelZ1Bf9lEZYA0IY,11412
+tests/test_elements_worker/test_element.py,sha256=lb5tLjl0jsixX0OWVhBAaKLE9GKkBw79kFHDNGommaQ,12535
+tests/test_elements_worker/test_element_create_multiple.py,sha256=arYFGmxc0517ZUii6k__G_UQQatuNIASTC8MXvUrSwk,21887
+tests/test_elements_worker/test_element_create_single.py,sha256=Fa9zm12J2rQ3VrUe3yIlHAc7Vty_eQYb_YGnNPQB3IE,16697
+tests/test_elements_worker/test_element_list_children.py,sha256=2zH4h663w3EduqpzQr-7bf9zIDzO1x2WxdUYYHsIHkI,31358
+tests/test_elements_worker/test_element_list_parents.py,sha256=TXeGW-a3W-7GmB2QrhJH9mMnvxuybeAwQ4tL3iIxwXo,16734
+tests/test_elements_worker/test_entity_create.py,sha256=TkEoQDmwuP1AlSYf42WXJVMMa0lIEPMn6PGy5ExA_5c,31343
+tests/test_elements_worker/test_entity_list_and_check.py,sha256=ENBLaqbXlRUDbHRvQla3080a0HJltrWAPYWNohUA9NU,4992
+tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
+tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
+tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
+tests/test_elements_worker/test_training.py,sha256=Qxi9EzGr_uKcn2Fh5aE6jNrq1K8QKLiOiSew4upASPs,8721
+tests/test_elements_worker/test_transcription_create.py,sha256=yznO9B_BVsOR0Z_VY5ZL8gJp0ZPCz_4sPUs5dXtixAg,29281
+tests/test_elements_worker/test_transcription_create_with_elements.py,sha256=tmcyglgssEqMnt1Mdy_u6X1m2wgLWTo_HdWst3GrK2k,33056
+tests/test_elements_worker/test_transcription_list.py,sha256=ikz7HYPCoQWTdTRCd382SB-y-T2BbigPLlIcx5Eow-I,15324
+tests/test_elements_worker/test_version.py,sha256=xqCgcgukTFJzkMgYfQG-8mTbu0o2fdYjWC07FktThfw,2125
+tests/test_elements_worker/test_worker.py,sha256=fnFwkcDb6tx5i7lbelifeHD-BrGz6o5UBEojPCdtuAo,23474
+worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
+worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
+worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
+worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
+arkindex_base_worker-0.4.0rc4.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.4.0rc4.dist-info/METADATA,sha256=Du-kBAZ-QL6SjvASaOK1Nxsw8yqN8Efx4L1JR-dA720,3338
+arkindex_base_worker-0.4.0rc4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+arkindex_base_worker-0.4.0rc4.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
+arkindex_base_worker-0.4.0rc4.dist-info/RECORD,,

{arkindex_base_worker-0.4.0rc2.dist-info → arkindex_base_worker-0.4.0rc4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

arkindex_worker/worker/__init__.py

@@ -10,8 +10,7 @@ from collections.abc import Iterable
 from itertools import chain
 from pathlib import Path
 
-from apistar.exceptions import ErrorResponse
-
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Dataset, Element, Set

arkindex_worker/worker/base.py

@@ -12,9 +12,9 @@ from tempfile import mkdtemp
 
 import gnupg
 import yaml
-from apistar.exceptions import ErrorResponse
 
 from arkindex import options_from_env
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     check_version,

arkindex_worker/worker/classification.py

@@ -2,9 +2,9 @@
 ElementsWorker methods for classifications and ML classes.
 """
 
-from apistar.exceptions import ErrorResponse
 from peewee import IntegrityError
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedClassification, CachedElement
 from arkindex_worker.models import Element

arkindex_worker/worker/corpus.py

@@ -5,6 +5,7 @@ BaseWorker methods for corpora.
 from enum import Enum
 from operator import itemgetter
 from tempfile import _TemporaryFileWrapper
+from uuid import UUID
 
 from arkindex_worker import logger
 
@@ -36,6 +37,25 @@ class CorpusExportState(Enum):
 
 
 class CorpusMixin:
+    def download_export(self, export_id: str) -> _TemporaryFileWrapper:
+        """
+        Download an export.
+
+        :param export_id: UUID of the export to download
+        :returns: The downloaded export stored in a temporary file.
+        """
+        try:
+            UUID(export_id)
+        except ValueError as e:
+            raise ValueError("export_id is not a valid uuid.") from e
+
+        logger.info(f"Downloading export ({export_id})...")
+        export: _TemporaryFileWrapper = self.api_client.request(
+            "DownloadExport", id=export_id
+        )
+        logger.info(f"Downloaded export ({export_id}) @ `{export.name}`")
+        return export
+
     def download_latest_export(self) -> _TemporaryFileWrapper:
         """
         Download the latest export in `done` state of the current corpus.
@@ -62,10 +82,5 @@ class CorpusMixin:
 
         # Download latest export
         export_id: str = exports[0]["id"]
-        logger.info(f"Downloading export ({export_id})...")
-        export: _TemporaryFileWrapper = self.api_client.request(
-            "DownloadExport", id=export_id
-        )
-        logger.info(f"Downloaded export ({export_id}) @ `{export.name}`")
 
-        return export
+        return self.download_export(export_id)

arkindex_worker/worker/entity.py

@@ -302,7 +302,7 @@ class EntityMixin:
 
         created_entities = [
             created_entity
-            for batch in make_batches(entities, "entities", batch_size)
+            for batch in make_batches(entities, "entity", batch_size)
             for created_entity in self.api_client.request(
                 "CreateTranscriptionEntities",
                 id=transcription.id,

arkindex_worker/worker/task.py

@@ -5,8 +5,7 @@ BaseWorker methods for tasks.
 import uuid
 from collections.abc import Iterator
 
-from apistar.compat import DownloadedFile
-
+from arkindex.compat import DownloadedFile
 from arkindex_worker.models import Artifact
 
 

arkindex_worker/worker/training.py

@@ -9,8 +9,8 @@ from typing import NewType
 from uuid import UUID
 
 import requests
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.utils import close_delete_file, create_tar_zst_archive
 

tests/test_dataset_worker.py

@@ -3,8 +3,8 @@ import uuid
 from argparse import ArgumentTypeError
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Set
 from arkindex_worker.worker.dataset import (
     DatasetState,

tests/test_elements_worker/{test_classifications.py → test_classification.py}

@@ -3,8 +3,8 @@ import re
 from uuid import UUID
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import CachedClassification, CachedElement
 from arkindex_worker.models import Element
 from arkindex_worker.utils import DEFAULT_BATCH_SIZE
@@ -17,6 +17,92 @@ from . import BASE_API_CALLS
 DELETE_PARAMETER = "DELETE_PARAMETER"
 
 
+def test_load_corpus_classes_api_error(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=418,
+    )
+
+    assert not mock_elements_worker.classes
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        # We do 5 retries
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert not mock_elements_worker.classes
+
+
+def test_load_corpus_classes(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=200,
+        json={
+            "count": 3,
+            "next": None,
+            "results": [
+                {
+                    "id": "0000",
+                    "name": "good",
+                },
+                {
+                    "id": "1111",
+                    "name": "average",
+                },
+                {
+                    "id": "2222",
+                    "name": "bad",
+                },
+            ],
+        },
+    )
+
+    assert not mock_elements_worker.classes
+    mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert mock_elements_worker.classes == {
+        "good": "0000",
+        "average": "1111",
+        "bad": "2222",
+    }
+
+
 def test_get_ml_class_id_load_classes(responses, mock_elements_worker):
     responses.add(
         responses.GET,

tests/test_elements_worker/test_corpus.py

@@ -2,13 +2,44 @@ import re
 import uuid
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.worker.corpus import CorpusExportState
 from tests import CORPUS_ID
 from tests.test_elements_worker import BASE_API_CALLS
 
 
+def test_download_export_not_a_uuid(responses, mock_elements_worker):
+    with pytest.raises(ValueError, match="export_id is not a valid uuid."):
+        mock_elements_worker.download_export("mon export")
+
+
+def test_download_export(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        status=302,
+        body=b"some SQLite export",
+        content_type="application/x-sqlite3",
+        stream=True,
+    )
+
+    export = mock_elements_worker.download_export(
+        "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+    )
+    assert export.name == "/tmp/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        ),
+    ]
+
+
 def mock_list_exports_call(responses, export_id):
     responses.add(
         responses.GET,

tests/test_elements_worker/test_dataset.py

@@ -2,8 +2,8 @@ import json
 import logging
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Element, Set
 from arkindex_worker.worker.dataset import DatasetState
 from tests import PROCESS_ID