arkindex-base-worker 0.4.0rc2__tar.gz → 0.4.0rc4__tar.gz

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc2
+Version: 0.4.0rc4
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,6 +37,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -44,13 +45,13 @@ Requires-Dist: humanize==4.10.0
 Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==10.4.0
 Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.5
-Requires-Dist: teklia-toolbox==0.1.5
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7b1
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
 Requires-Dist: black==24.4.2; extra == "docs"
 Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.10.8; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
 Provides-Extra: tests
 Requires-Dist: pytest==8.3.2; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc2
+Version: 0.4.0rc4
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,6 +37,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -44,13 +45,13 @@ Requires-Dist: humanize==4.10.0
 Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==10.4.0
 Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.5
-Requires-Dist: teklia-toolbox==0.1.5
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7b1
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
 Requires-Dist: black==24.4.2; extra == "docs"
 Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.10.8; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
 Provides-Extra: tests
 Requires-Dist: pytest==8.3.2; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -36,17 +36,25 @@ tests/test_image.py
 tests/test_merge.py
 tests/test_utils.py
 tests/test_elements_worker/__init__.py
-tests/test_elements_worker/test_classifications.py
+tests/test_elements_worker/test_classification.py
 tests/test_elements_worker/test_cli.py
 tests/test_elements_worker/test_corpus.py
 tests/test_elements_worker/test_dataset.py
-tests/test_elements_worker/test_elements.py
-tests/test_elements_worker/test_entities.py
+tests/test_elements_worker/test_element.py
+tests/test_elements_worker/test_element_create_multiple.py
+tests/test_elements_worker/test_element_create_single.py
+tests/test_elements_worker/test_element_list_children.py
+tests/test_elements_worker/test_element_list_parents.py
+tests/test_elements_worker/test_entity_create.py
+tests/test_elements_worker/test_entity_list_and_check.py
 tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
-tests/test_elements_worker/test_transcriptions.py
+tests/test_elements_worker/test_transcription_create.py
+tests/test_elements_worker/test_transcription_create_with_elements.py
+tests/test_elements_worker/test_transcription_list.py
+tests/test_elements_worker/test_version.py
 tests/test_elements_worker/test_worker.py
 worker-demo/tests/__init__.py
 worker-demo/tests/conftest.py

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_base_worker.egg-info/requires.txt

@@ -2,14 +2,14 @@ humanize==4.10.0
 peewee~=3.17
 Pillow==10.4.0
 python-gnupg==0.5.2
-shapely==2.0.5
-teklia-toolbox==0.1.5
+shapely==2.0.6
+teklia-toolbox==0.1.7b1
 zstandard==0.22.0
 
 [docs]
 black==24.4.2
 mkdocs-material==9.5.33
-mkdocstrings-python==1.10.8
+mkdocstrings-python==1.11.1
 
 [tests]
 pytest==8.3.2

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/__init__.py

@@ -10,8 +10,7 @@ from collections.abc import Iterable
 from itertools import chain
 from pathlib import Path
 
-from apistar.exceptions import ErrorResponse
-
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Dataset, Element, Set

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/base.py

@@ -12,9 +12,9 @@ from tempfile import mkdtemp
 
 import gnupg
 import yaml
-from apistar.exceptions import ErrorResponse
 
 from arkindex import options_from_env
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     check_version,

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/classification.py

@@ -2,9 +2,9 @@
 ElementsWorker methods for classifications and ML classes.
 """
 
-from apistar.exceptions import ErrorResponse
 from peewee import IntegrityError
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedClassification, CachedElement
 from arkindex_worker.models import Element

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/corpus.py

@@ -5,6 +5,7 @@ BaseWorker methods for corpora.
 from enum import Enum
 from operator import itemgetter
 from tempfile import _TemporaryFileWrapper
+from uuid import UUID
 
 from arkindex_worker import logger
 
@@ -36,6 +37,25 @@ class CorpusExportState(Enum):
 
 
 class CorpusMixin:
+    def download_export(self, export_id: str) -> _TemporaryFileWrapper:
+        """
+        Download an export.
+
+        :param export_id: UUID of the export to download
+        :returns: The downloaded export stored in a temporary file.
+        """
+        try:
+            UUID(export_id)
+        except ValueError as e:
+            raise ValueError("export_id is not a valid uuid.") from e
+
+        logger.info(f"Downloading export ({export_id})...")
+        export: _TemporaryFileWrapper = self.api_client.request(
+            "DownloadExport", id=export_id
+        )
+        logger.info(f"Downloaded export ({export_id}) @ `{export.name}`")
+        return export
+
     def download_latest_export(self) -> _TemporaryFileWrapper:
         """
         Download the latest export in `done` state of the current corpus.
@@ -62,10 +82,5 @@ class CorpusMixin:
 
         # Download latest export
         export_id: str = exports[0]["id"]
-        logger.info(f"Downloading export ({export_id})...")
-        export: _TemporaryFileWrapper = self.api_client.request(
-            "DownloadExport", id=export_id
-        )
-        logger.info(f"Downloaded export ({export_id}) @ `{export.name}`")
 
-        return export
+        return self.download_export(export_id)

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/entity.py

@@ -302,7 +302,7 @@ class EntityMixin:
 
         created_entities = [
             created_entity
-            for batch in make_batches(entities, "entities", batch_size)
+            for batch in make_batches(entities, "entity", batch_size)
             for created_entity in self.api_client.request(
                 "CreateTranscriptionEntities",
                 id=transcription.id,

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/task.py

@@ -5,8 +5,7 @@ BaseWorker methods for tasks.
 import uuid
 from collections.abc import Iterator
 
-from apistar.compat import DownloadedFile
-
+from arkindex.compat import DownloadedFile
 from arkindex_worker.models import Artifact
 
 

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/arkindex_worker/worker/training.py

@@ -9,8 +9,8 @@ from typing import NewType
 from uuid import UUID
 
 import requests
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.utils import close_delete_file, create_tar_zst_archive
 

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "arkindex-base-worker"
-version = "0.4.0rc2"
+version = "0.4.0rc4"
 description = "Base Worker to easily build Arkindex ML workflows"
 license = { file = "LICENSE" }
 dependencies = [
@@ -12,8 +12,8 @@ dependencies = [
     "peewee~=3.17",
     "Pillow==10.4.0",
     "python-gnupg==0.5.2",
-    "shapely==2.0.5",
-    "teklia-toolbox==0.1.5",
+    "shapely==2.0.6",
+    "teklia-toolbox==0.1.7b1",
     "zstandard==0.22.0",
 ]
 authors = [
@@ -32,6 +32,7 @@ classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 [project.urls]
@@ -45,7 +46,7 @@ Authors = "https://teklia.com"
 docs = [
     "black==24.4.2",
     "mkdocs-material==9.5.33",
-    "mkdocstrings-python==1.10.8",
+    "mkdocstrings-python==1.11.1",
 ]
 tests = [
     "pytest==8.3.2",
@@ -95,7 +96,6 @@ select = [
 known-first-party = ["arkindex", "arkindex_common", "arkindex_worker", "teklia_toolbox"]
 known-third-party = [
     "PIL",
-    "apistar",
     "gnupg",
     "peewee",
     "playhouse",

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/tests/test_dataset_worker.py

@@ -3,8 +3,8 @@ import uuid
 from argparse import ArgumentTypeError
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Set
 from arkindex_worker.worker.dataset import (
     DatasetState,

arkindex_base_worker-0.4.0rc2/tests/test_elements_worker/test_classifications.py → arkindex_base_worker-0.4.0rc4/tests/test_elements_worker/test_classification.py

@@ -3,8 +3,8 @@ import re
 from uuid import UUID
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.cache import CachedClassification, CachedElement
 from arkindex_worker.models import Element
 from arkindex_worker.utils import DEFAULT_BATCH_SIZE
@@ -17,6 +17,92 @@ from . import BASE_API_CALLS
 DELETE_PARAMETER = "DELETE_PARAMETER"
 
 
+def test_load_corpus_classes_api_error(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=418,
+    )
+
+    assert not mock_elements_worker.classes
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        # We do 5 retries
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert not mock_elements_worker.classes
+
+
+def test_load_corpus_classes(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        status=200,
+        json={
+            "count": 3,
+            "next": None,
+            "results": [
+                {
+                    "id": "0000",
+                    "name": "good",
+                },
+                {
+                    "id": "1111",
+                    "name": "average",
+                },
+                {
+                    "id": "2222",
+                    "name": "bad",
+                },
+            ],
+        },
+    )
+
+    assert not mock_elements_worker.classes
+    mock_elements_worker.load_corpus_classes()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{CORPUS_ID}/classes/",
+        ),
+    ]
+    assert mock_elements_worker.classes == {
+        "good": "0000",
+        "average": "1111",
+        "bad": "2222",
+    }
+
+
 def test_get_ml_class_id_load_classes(responses, mock_elements_worker):
     responses.add(
         responses.GET,

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/tests/test_elements_worker/test_corpus.py

@@ -2,13 +2,44 @@ import re
 import uuid
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.worker.corpus import CorpusExportState
 from tests import CORPUS_ID
 from tests.test_elements_worker import BASE_API_CALLS
 
 
+def test_download_export_not_a_uuid(responses, mock_elements_worker):
+    with pytest.raises(ValueError, match="export_id is not a valid uuid."):
+        mock_elements_worker.download_export("mon export")
+
+
+def test_download_export(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        status=302,
+        body=b"some SQLite export",
+        content_type="application/x-sqlite3",
+        stream=True,
+    )
+
+    export = mock_elements_worker.download_export(
+        "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+    )
+    assert export.name == "/tmp/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        ),
+    ]
+
+
 def mock_list_exports_call(responses, export_id):
     responses.add(
         responses.GET,

{arkindex_base_worker-0.4.0rc2 → arkindex_base_worker-0.4.0rc4}/tests/test_elements_worker/test_dataset.py

@@ -2,8 +2,8 @@ import json
 import logging
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Element, Set
 from arkindex_worker.worker.dataset import DatasetState
 from tests import PROCESS_ID