arkindex-base-worker 0.4.0rc5__py3-none-any.whl → 0.4.0rc6__py3-none-any.whl

{arkindex_base_worker-0.4.0rc5.dist-info → arkindex_base_worker-0.4.0rc6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0rc5
+Version: 0.4.0rc6
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,19 +41,19 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.10.0
+Requires-Dist: humanize==4.11.0
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==10.4.0
-Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: Pillow==11.0.0
+Requires-Dist: python-gnupg==0.5.3
 Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.7b1
-Requires-Dist: zstandard==0.22.0
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
-Requires-Dist: black==24.4.2; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
+Requires-Dist: black==24.10.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.2; extra == "tests"
+Requires-Dist: pytest==8.3.4; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 

{arkindex_base_worker-0.4.0rc5.dist-info → arkindex_base_worker-0.4.0rc6.dist-info}/RECORD

@@ -3,7 +3,7 @@ arkindex_worker/cache.py,sha256=qTblc_zKdYC47Wip6_O9Jf5qBkQW2ozQQrg-nsx1WuY,1122
 arkindex_worker/image.py,sha256=oEgVCrSHiGh3D5-UXfM6PvT17TttSxC0115irpvB3Dw,18581
 arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
 arkindex_worker/utils.py,sha256=q1EeLdC6ebYIH-C0LOAqw2cNpjCjVoP-Vbr-39mF4w0,9884
-arkindex_worker/worker/__init__.py,sha256=m255Cle3nE_FtAXbbJj_v1aS9ClT6hpDlrUFXTPiqkI,15985
+arkindex_worker/worker/__init__.py,sha256=0_YHeOe31KR_8ynbnYMIMwnSQTVbKkkeLGmnlTMhFx0,16234
 arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
 arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
 arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
@@ -12,7 +12,7 @@ arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6
 arkindex_worker/worker/entity.py,sha256=s5wjX6_JfTyk4qfMoV0OWfOXUx6T-9WpOiEpaoaCEFM,14808
 arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
 arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
-arkindex_worker/worker/process.py,sha256=IAJaiiCizK4vpPmMQD0yYSB6IIoyy7yU-5JKaiuPb7o,1073
+arkindex_worker/worker/process.py,sha256=9TEHpMcBax1wc6PrWMMrdXe2uNfqyVj7n_dAYZRBGnY,1854
 arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
 arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
 arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
@@ -41,20 +41,21 @@ tests/test_elements_worker/test_entity_create.py,sha256=9Tjr9KA2yo44VFV283q_cs6X
 tests/test_elements_worker/test_entity_list_and_check.py,sha256=ENBLaqbXlRUDbHRvQla3080a0HJltrWAPYWNohUA9NU,4992
 tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
 tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
+tests/test_elements_worker/test_process.py,sha256=y4RoVhPfyHzR795fw7-_FXElBcKo3fy4Ew_HI-kxJic,3088
 tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
 tests/test_elements_worker/test_training.py,sha256=Qxi9EzGr_uKcn2Fh5aE6jNrq1K8QKLiOiSew4upASPs,8721
 tests/test_elements_worker/test_transcription_create.py,sha256=yznO9B_BVsOR0Z_VY5ZL8gJp0ZPCz_4sPUs5dXtixAg,29281
 tests/test_elements_worker/test_transcription_create_with_elements.py,sha256=tmcyglgssEqMnt1Mdy_u6X1m2wgLWTo_HdWst3GrK2k,33056
 tests/test_elements_worker/test_transcription_list.py,sha256=ikz7HYPCoQWTdTRCd382SB-y-T2BbigPLlIcx5Eow-I,15324
 tests/test_elements_worker/test_version.py,sha256=xqCgcgukTFJzkMgYfQG-8mTbu0o2fdYjWC07FktThfw,2125
-tests/test_elements_worker/test_worker.py,sha256=fnFwkcDb6tx5i7lbelifeHD-BrGz6o5UBEojPCdtuAo,23474
+tests/test_elements_worker/test_worker.py,sha256=pLUgjyrrXrzVD6T-kdH1ppk5Yn_iDuI8JdFGweTEMXE,25156
 worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.4.0rc5.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
-arkindex_base_worker-0.4.0rc5.dist-info/METADATA,sha256=EFoMgnh4SUvYV0yIdBSfA7aoaZiTA7kYrJTrH2la3mY,3338
-arkindex_base_worker-0.4.0rc5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-arkindex_base_worker-0.4.0rc5.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
-arkindex_base_worker-0.4.0rc5.dist-info/RECORD,,
+arkindex_base_worker-0.4.0rc6.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.4.0rc6.dist-info/METADATA,sha256=gJd_0X7A26nuBe2EsIPHwap1XV2KnJBq2QwjBBB3Wi0,3339
+arkindex_base_worker-0.4.0rc6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+arkindex_base_worker-0.4.0rc6.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
+arkindex_base_worker-0.4.0rc6.dist-info/RECORD,,

arkindex_worker/worker/__init__.py

@@ -27,7 +27,7 @@ from arkindex_worker.worker.element import ElementMixin
 from arkindex_worker.worker.entity import EntityMixin
 from arkindex_worker.worker.image import ImageMixin
 from arkindex_worker.worker.metadata import MetaDataMixin, MetaType  # noqa: F401
-from arkindex_worker.worker.process import ActivityState, ProcessMode
+from arkindex_worker.worker.process import ActivityState, ProcessMixin, ProcessMode
 from arkindex_worker.worker.task import TaskMixin
 from arkindex_worker.worker.transcription import TranscriptionMixin
 from arkindex_worker.worker.version import WorkerVersionMixin
@@ -44,6 +44,7 @@ class ElementsWorker(
     EntityMixin,
     MetaDataMixin,
     ImageMixin,
+    ProcessMixin,
 ):
     """
     Base class for ML workers that operate on Arkindex elements.
@@ -119,6 +120,9 @@ class ElementsWorker(
             return list(
                 chain.from_iterable(map(self.list_set_elements, self.list_sets()))
             )
+        elif self.process_mode == ProcessMode.Export:
+            # For export mode processes, use list_process_elements and return element IDs
+            return {item["id"] for item in self.list_process_elements()}
 
         invalid_element_ids = list(filter(invalid_element_id, out))
         assert (

arkindex_worker/worker/process.py

@@ -1,5 +1,11 @@
+from collections.abc import Iterator
 from enum import Enum
 
+from arkindex_worker.cache import unsupported_cache
+
+# Increases the number of elements returned per page by the API
+PROCESS_ELEMENTS_PAGE_SIZE = 500
+
 
 class ActivityState(Enum):
     """
@@ -66,3 +72,21 @@ class ProcessMode(Enum):
     """
     Export processes.
     """
+
+
+class ProcessMixin:
+    @unsupported_cache
+    def list_process_elements(self, with_image: bool = False) -> Iterator[dict]:
+        """
+        List the elements of a process.
+
+        :param with_image: whether or not to include zone and image information in the elements response.
+        :returns: the process' elements.
+        """
+        return self.api_client.paginate(
+            "ListProcessElements",
+            id=self.process_information["id"],
+            with_image=with_image,
+            allow_missing_data=True,
+            page_size=PROCESS_ELEMENTS_PAGE_SIZE,
+        )

tests/test_elements_worker/test_process.py

@@ -0,0 +1,89 @@
+import pytest
+
+from tests import PROCESS_ID
+
+
+@pytest.mark.parametrize(
+    ("with_image", "elements"),
+    [
+        (
+            False,
+            [
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 1",
+                    "confidence": 1,
+                    "image_id": None,
+                    "image_width": None,
+                    "image_height": None,
+                    "image_url": None,
+                    "polygon": None,
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa0",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 2",
+                    "confidence": 1,
+                    "image_id": None,
+                    "image_width": None,
+                    "image_height": None,
+                    "image_url": None,
+                    "polygon": None,
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+            ],
+        ),
+        (
+            True,
+            [
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 1",
+                    "confidence": 1,
+                    "image_id": "aaa2aaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "image_width": 76,
+                    "image_height": 138,
+                    "image_url": "http://somewhere.com/iiif/image.jpeg",
+                    "polygon": [[0, 0], [0, 40], [20, 40], [20, 0]],
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa0",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 2",
+                    "confidence": 1,
+                    "image_id": "aaa2aaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "image_width": 138,
+                    "image_height": 76,
+                    "image_url": "http://somewhere.com/iiif/image.jpeg",
+                    "polygon": [[0, 0], [0, 40], [20, 40], [20, 0]],
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+            ],
+        ),
+    ],
+)
+def test_list_process_elements_with_image(
+    responses, mock_elements_worker, with_image, elements
+):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/process/{PROCESS_ID}/elements/?page_size=500&with_count=true&with_image={with_image}",
+        status=200,
+        json={
+            "count": 2,
+            "next": None,
+            "results": elements,
+        },
+    )
+    assert (
+        list(mock_elements_worker.list_process_elements(with_image=with_image))
+        == elements
+    )

tests/test_elements_worker/test_worker.py

@@ -16,6 +16,7 @@ from arkindex_worker.models import Element
 from arkindex_worker.worker import ActivityState, ElementsWorker
 from arkindex_worker.worker.dataset import DatasetState
 from arkindex_worker.worker.process import ProcessMode
+from tests import PROCESS_ID
 
 from . import BASE_API_CALLS
 
@@ -523,6 +524,51 @@ def test_get_elements_both_args_error(mocker, mock_elements_worker, tmp_path):
         worker.get_elements()
 
 
+def test_get_elements_export_process(mock_elements_worker, responses):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/process/{PROCESS_ID}/elements/?page_size=500&with_count=true&with_image=False",
+        status=200,
+        json={
+            "count": 2,
+            "next": None,
+            "results": [
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 1",
+                    "confidence": 1,
+                    "image_id": None,
+                    "image_width": None,
+                    "image_height": None,
+                    "image_url": None,
+                    "polygon": None,
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa0",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 2",
+                    "confidence": 1,
+                    "image_id": None,
+                    "image_width": None,
+                    "image_height": None,
+                    "image_url": None,
+                    "polygon": None,
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+            ],
+        },
+    )
+    mock_elements_worker.process_information["mode"] = "export"
+    assert set(mock_elements_worker.get_elements()) == {
+        "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+        "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa0",
+    }
+
+
 @pytest.mark.usefixtures("_mock_worker_run_api")
 def test_activities_disabled(responses, monkeypatch):
     """Test worker process elements without updating activities when they are disabled for the process"""