arkindex-base-worker 0.4.0rc2__py3-none-any.whl → 0.4.0rc3__py3-none-any.whl

{arkindex_base_worker-0.4.0rc2.dist-info → arkindex_base_worker-0.4.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc2
+Version: 0.4.0rc3
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -45,12 +45,12 @@ Requires-Dist: peewee ~=3.17
 Requires-Dist: Pillow ==10.4.0
 Requires-Dist: python-gnupg ==0.5.2
 Requires-Dist: shapely ==2.0.5
-Requires-Dist: teklia-toolbox ==0.1.5
+Requires-Dist: teklia-toolbox ==0.1.7b1
 Requires-Dist: zstandard ==0.22.0
 Provides-Extra: docs
 Requires-Dist: black ==24.4.2 ; extra == 'docs'
 Requires-Dist: mkdocs-material ==9.5.33 ; extra == 'docs'
-Requires-Dist: mkdocstrings-python ==1.10.8 ; extra == 'docs'
+Requires-Dist: mkdocstrings-python ==1.11.1 ; extra == 'docs'
 Provides-Extra: tests
 Requires-Dist: pytest ==8.3.2 ; extra == 'tests'
 Requires-Dist: pytest-mock ==3.14.0 ; extra == 'tests'

{arkindex_base_worker-0.4.0rc2.dist-info → arkindex_base_worker-0.4.0rc3.dist-info}/RECORD

@@ -3,18 +3,18 @@ arkindex_worker/cache.py,sha256=FTlB0coXofn5zTNRTcVIvh709mcw4a1bPGqkwWjKs3w,1124
 arkindex_worker/image.py,sha256=oEgVCrSHiGh3D5-UXfM6PvT17TttSxC0115irpvB3Dw,18581
 arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
 arkindex_worker/utils.py,sha256=q1EeLdC6ebYIH-C0LOAqw2cNpjCjVoP-Vbr-39mF4w0,9884
-arkindex_worker/worker/__init__.py,sha256=4kj94AbdACO-q7w0krmv2QkCWbP7vLLgZk9sowZUBNw,15880
-arkindex_worker/worker/base.py,sha256=hIorYsLy88xR4-dhIsaQHbWCz5FMoja-gfRPCJhT3eY,19843
-arkindex_worker/worker/classification.py,sha256=ECm1cnQPOj_9m-CoO0e182ElSySAUOoyddHrORbShhc,10951
-arkindex_worker/worker/corpus.py,sha256=s9bCxOszJMwRq1WWAmKjWq888mjDfbaJ18Wo7h-rNOw,1827
+arkindex_worker/worker/__init__.py,sha256=tcqxrox9EpOjaN2EQgXumiABKpWHLsJiynPC2_sZuOQ,15880
+arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
+arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
+arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
 arkindex_worker/worker/dataset.py,sha256=LwzKwNFX4FqfLxh29LSvJydPwRw3VHaB1wjuFhUshsE,5267
 arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6e9Y4,45309
-arkindex_worker/worker/entity.py,sha256=qGjQvOVXfP84rER0Dkui6q-rb9nTWerHVG0Z5voB8pU,15229
+arkindex_worker/worker/entity.py,sha256=ThhP22xOYR5Z4P1VH_pOl_y_uDKZFeQVDqxO6aRkIhg,15227
 arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
 arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
 arkindex_worker/worker/process.py,sha256=I1rBt3Y8bV4zcPr8N1E2NRZ0UClSTqhExsO9CPcP41E,1012
-arkindex_worker/worker/task.py,sha256=1O9zrWXxe3na3TOcoHX5Pxn1875v7EU08BSsCPnb62g,1519
-arkindex_worker/worker/training.py,sha256=qnBFEk11JOWWPLTbjF-lZ9iFBdTPpQzZAzQ9a03J1j4,10874
+arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
+arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
 arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
 arkindex_worker/worker/version.py,sha256=JIT7OI3Mo7RPkNrjOB9hfqrsG-FYygz_zi4l8PbkuAo,1960
 hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
@@ -22,31 +22,31 @@ tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
 tests/conftest.py,sha256=2ocZ2x-mZQrNe9zvWwhWk2_4ExdaBHIB74SvtDlExRE,21580
 tests/test_base_worker.py,sha256=2EIYcd_3f9O0zB5WiGIQV0Cn9wndLvnEnSfcAE1qWWU,30607
 tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
-tests/test_dataset_worker.py,sha256=r5SWdXSWKXGBFAgH6ouF2Nk-k6lLJ6SgXjKYD2Vu_Es,22088
+tests/test_dataset_worker.py,sha256=gApYz0LArHr1cNn079_fa_BQABF6RVQYuM1Tc4m3NsQ,22089
 tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
 tests/test_image.py,sha256=J3jqB5OhcdCpB6n0UnwivxrMlne8YjFLXhq1gBMANrs,26711
 tests/test_merge.py,sha256=TuOeUS0UCz66DPOQFFhc4NQBxIjZL9f5czi4XnvGrr4,8270
 tests/test_utils.py,sha256=_WJUPnt-pM_TQ0er4yjPZy-u_LePrHq1lxwk_teky7M,2544
 tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
-tests/test_elements_worker/test_classifications.py,sha256=fXZ8cSzIWwZ6LHsY7tKsy9-Pp9fKyKUStIXS4ViBcek,27779
+tests/test_elements_worker/test_classifications.py,sha256=GtVyi9bg4YTd7nyw8u6IjQZYBwFMwoVZdrfSBc5UybU,27780
 tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
-tests/test_elements_worker/test_corpus.py,sha256=c_LUHvkJIYgk_wXF06VQPNOoWfiZ06XpjOXrJ7MRiBc,4479
-tests/test_elements_worker/test_dataset.py,sha256=lSXqubhg1EEq2Y2goE8Y2RYaqIpM9Iejq6fGNW2BczU,11411
-tests/test_elements_worker/test_elements.py,sha256=JZmqddzxuEFhGNnYj8iE4kA7mODYbMGNf7STnA8CVL4,115496
-tests/test_elements_worker/test_entities.py,sha256=oav2dtvWWavQe1l3Drbxw1Ta2ocUJEVxJfDQ_r6-rYQ,36181
-tests/test_elements_worker/test_image.py,sha256=_E3UGdDOwTo1MW5KMS81PrdeSPBPWinWYoQPNy2F9Ro,2077
-tests/test_elements_worker/test_metadata.py,sha256=cm2NNaXxBYmYMkPexSPVTAqb2skDTB4mliwQCLz8Y98,22293
-tests/test_elements_worker/test_task.py,sha256=7Sr3fbjdgWUXJUhJEiC9CwnbhQIQX3rCInmHMIrmA38,5573
+tests/test_elements_worker/test_corpus.py,sha256=OAbwgaQtHmcmPkcAl9Kuceun_BvMasnZvYj4_EdfugY,5483
+tests/test_elements_worker/test_dataset.py,sha256=00IlOZv9YFlZ23rGXyR-HLbKLQxGelZ1Bf9lEZYA0IY,11412
+tests/test_elements_worker/test_elements.py,sha256=l5YTfm0CzBTQyZvdOplhhza-gpPSz-8RVix1YUzAwhM,115497
+tests/test_elements_worker/test_entities.py,sha256=nrCvkdJdjsyOrbD6R-H8NvxREZxciiR6CGIObXzeg50,36182
+tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
+tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
+tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
 tests/test_elements_worker/test_training.py,sha256=Qxi9EzGr_uKcn2Fh5aE6jNrq1K8QKLiOiSew4upASPs,8721
-tests/test_elements_worker/test_transcriptions.py,sha256=FNY6E26iTKqe7LP9LO72By4oV4g9hBIZYTU9BAc_w7I,77060
-tests/test_elements_worker/test_worker.py,sha256=AuFDyqncIusT-rMMY4sEay9MqGvoNuSuZQq-5rHN02U,10803
+tests/test_elements_worker/test_transcriptions.py,sha256=iq-nR_st7Q9E_nD7knrKGY57g36J6nYSEzbPk9y-cxY,77061
+tests/test_elements_worker/test_worker.py,sha256=VdprIWezB3dJdE8vNOrS71RQugqUysHlveOWTQate-8,10804
 worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.4.0rc2.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
-arkindex_base_worker-0.4.0rc2.dist-info/METADATA,sha256=Akdq43GSmQL2Sr7VpaBGMjKlQtb1z8lIEUUBqoXoWg8,3304
-arkindex_base_worker-0.4.0rc2.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-arkindex_base_worker-0.4.0rc2.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
-arkindex_base_worker-0.4.0rc2.dist-info/RECORD,,
+arkindex_base_worker-0.4.0rc3.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.4.0rc3.dist-info/METADATA,sha256=eDT7HxTvEz2yg4U_lbzkuigNWFu4JTqaLTnY0fqSCiM,3306
+arkindex_base_worker-0.4.0rc3.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+arkindex_base_worker-0.4.0rc3.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
+arkindex_base_worker-0.4.0rc3.dist-info/RECORD,,

{arkindex_base_worker-0.4.0rc2.dist-info → arkindex_base_worker-0.4.0rc3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

arkindex_worker/worker/__init__.py

@@ -10,8 +10,7 @@ from collections.abc import Iterable
 from itertools import chain
 from pathlib import Path
 
-from apistar.exceptions import ErrorResponse
-
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Dataset, Element, Set

arkindex_worker/worker/base.py

@@ -12,9 +12,9 @@ from tempfile import mkdtemp
 
 import gnupg
 import yaml
-from apistar.exceptions import ErrorResponse
 
 from arkindex import options_from_env
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     check_version,

arkindex_worker/worker/classification.py

@@ -2,9 +2,9 @@
 ElementsWorker methods for classifications and ML classes.
 """
 
-from apistar.exceptions import ErrorResponse
 from peewee import IntegrityError
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedClassification, CachedElement
 from arkindex_worker.models import Element

arkindex_worker/worker/corpus.py

@@ -5,6 +5,7 @@ BaseWorker methods for corpora.
 from enum import Enum
 from operator import itemgetter
 from tempfile import _TemporaryFileWrapper
+from uuid import UUID
 
 from arkindex_worker import logger
 
@@ -36,6 +37,25 @@ class CorpusExportState(Enum):
 
 
 class CorpusMixin:
+    def download_export(self, export_id: str) -> _TemporaryFileWrapper:
+        """
+        Download an export.
+
+        :param export_id: UUID of the export to download
+        :returns: The downloaded export stored in a temporary file.
+        """
+        try:
+            UUID(export_id)
+        except ValueError as e:
+            raise ValueError("export_id is not a valid uuid.") from e
+
+        logger.info(f"Downloading export ({export_id})...")
+        export: _TemporaryFileWrapper = self.api_client.request(
+            "DownloadExport", id=export_id
+        )
+        logger.info(f"Downloaded export ({export_id}) @ `{export.name}`")
+        return export
+
     def download_latest_export(self) -> _TemporaryFileWrapper:
         """
         Download the latest export in `done` state of the current corpus.
@@ -62,10 +82,5 @@ class CorpusMixin:
 
         # Download latest export
         export_id: str = exports[0]["id"]
-        logger.info(f"Downloading export ({export_id})...")
-        export: _TemporaryFileWrapper = self.api_client.request(
-            "DownloadExport", id=export_id
-        )
-        logger.info(f"Downloaded export ({export_id}) @ `{export.name}`")
 
-        return export
+        return self.download_export(export_id)

arkindex_worker/worker/entity.py

@@ -302,7 +302,7 @@ class EntityMixin:
 
         created_entities = [
             created_entity
-            for batch in make_batches(entities, "entities", batch_size)
+            for batch in make_batches(entities, "entity", batch_size)
             for created_entity in self.api_client.request(
                 "CreateTranscriptionEntities",
                 id=transcription.id,

arkindex_worker/worker/task.py

@@ -5,8 +5,7 @@ BaseWorker methods for tasks.
 import uuid
 from collections.abc import Iterator
 
-from apistar.compat import DownloadedFile
-
+from arkindex.compat import DownloadedFile
 from arkindex_worker.models import Artifact
 
 

arkindex_worker/worker/training.py

@@ -9,8 +9,8 @@ from typing import NewType
 from uuid import UUID
 
 import requests
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.utils import close_delete_file, create_tar_zst_archive
 

tests/test_dataset_worker.py

@@ -3,8 +3,8 @@ import uuid
 from argparse import ArgumentTypeError
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Set
 from arkindex_worker.worker.dataset import (
     DatasetState,

tests/test_elements_worker/test_classifications.py

@@ -3,8 +3,8 @@ import re
 from uuid import UUID
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import CachedClassification, CachedElement
 from arkindex_worker.models import Element
 from arkindex_worker.utils import DEFAULT_BATCH_SIZE

tests/test_elements_worker/test_corpus.py

@@ -2,8 +2,8 @@ import re
 import uuid
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.worker.corpus import CorpusExportState
 from tests import CORPUS_ID
 from tests.test_elements_worker import BASE_API_CALLS
@@ -135,3 +135,34 @@ def test_download_latest_export(responses, mock_elements_worker):
         ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
         ("GET", f"http://testserver/api/v1/export/{export_id}/"),
     ]
+
+
+def test_download_export_not_a_uuid(responses, mock_elements_worker):
+    with pytest.raises(ValueError, match="export_id is not a valid uuid."):
+        mock_elements_worker.download_export("mon export")
+
+
+def test_download_export(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        status=302,
+        body=b"some SQLite export",
+        content_type="application/x-sqlite3",
+        stream=True,
+    )
+
+    export = mock_elements_worker.download_export(
+        "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+    )
+    assert export.name == "/tmp/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        ),
+    ]

tests/test_elements_worker/test_dataset.py

@@ -2,8 +2,8 @@ import json
 import logging
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Element, Set
 from arkindex_worker.worker.dataset import DatasetState
 from tests import PROCESS_ID

tests/test_elements_worker/test_elements.py

@@ -4,9 +4,9 @@ from argparse import Namespace
 from uuid import UUID
 
 import pytest
-from apistar.exceptions import ErrorResponse
 from responses import matchers
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import (
     SQL_VERSION,
     CachedElement,

tests/test_elements_worker/test_entities.py

@@ -3,9 +3,9 @@ import re
 from uuid import UUID
 
 import pytest
-from apistar.exceptions import ErrorResponse
 from responses import matchers
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import (
     CachedElement,
     CachedEntity,

tests/test_elements_worker/test_image.py

@@ -1,7 +1,8 @@
 import json
 
 import pytest
-from apistar.exceptions import ErrorResponse
+
+from arkindex.exceptions import ErrorResponse
 
 from . import BASE_API_CALLS
 

tests/test_elements_worker/test_metadata.py

@@ -2,8 +2,8 @@ import json
 import re
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex.mock import MockApiClient
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Element

tests/test_elements_worker/test_task.py

@@ -1,8 +1,8 @@
 import uuid
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Artifact
 from tests import FIXTURES_DIR
 from tests.test_elements_worker import BASE_API_CALLS

tests/test_elements_worker/test_transcriptions.py

@@ -3,9 +3,9 @@ import re
 from uuid import UUID
 
 import pytest
-from apistar.exceptions import ErrorResponse
 from playhouse.shortcuts import model_to_dict
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import CachedElement, CachedTranscription
 from arkindex_worker.models import Element
 from arkindex_worker.utils import DEFAULT_BATCH_SIZE

tests/test_elements_worker/test_worker.py

@@ -2,8 +2,8 @@ import json
 import sys
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.worker import ActivityState, ElementsWorker