arkindex-base-worker 0.5.1b4__py3-none-any.whl → 0.5.1.post1__py3-none-any.whl

{arkindex_base_worker-0.5.1b4.dist-info → arkindex_base_worker-0.5.1.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.1b4
+Version: 0.5.1.post1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,16 +41,15 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.12.3
+Requires-Dist: humanize==4.14.0
 Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==11.3.0
-Requires-Dist: python-gnupg==0.5.4
+Requires-Dist: python-gnupg==0.5.5
 Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.11
-Requires-Dist: zstandard==0.23.0
+Requires-Dist: zstandard==0.25.0
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.5; extra == "tests"
-Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-mock==3.15.1; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 Dynamic: license-file
 

{arkindex_base_worker-0.5.1b4.dist-info → arkindex_base_worker-0.5.1.post1.dist-info}/RECORD

@@ -1,18 +1,18 @@
-arkindex_base_worker-0.5.1b4.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.5.1.post1.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
 arkindex_worker/__init__.py,sha256=Sdt5KXn8EgURb2MurYVrUWaHbH3iFA1XLRo0Lc5AJ44,250
-arkindex_worker/cache.py,sha256=x1d1oVF297ItLoZnPkZQoEefa39ZigrwRoHC_6az94k,10731
-arkindex_worker/image.py,sha256=GvIpW7LNSalVw3Obt9nySDWnW7-NbC0__SWREEQqVCk,20696
-arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
-arkindex_worker/utils.py,sha256=MbbJT8oh8DMHHR-vidFeXdUH0TSXGWm7ZDGWzrRXoEY,9933
+arkindex_worker/cache.py,sha256=XpEXMSnbhYCvrJquwA9XXqZo-ajMLpaCxKG5wH3Gp6Y,10959
+arkindex_worker/image.py,sha256=sGE8to5iykXv25bpkftOEWzlh5NzBZSKy4lSRoHYHPU,20929
+arkindex_worker/models.py,sha256=7GnKqpWPOSxyR_eKlDNVBe_r3TcE4ofK-1GzaonJEdM,10132
+arkindex_worker/utils.py,sha256=Eqg5pGAuOmuwMT3EhKTQDMek7wHC1KzZL7XXqYVVfHY,10977
 arkindex_worker/worker/__init__.py,sha256=SzD0s1_m6gMV02EUF-NeciqZdVPA4dpXI84tSj-g494,17869
-arkindex_worker/worker/base.py,sha256=fbRJ5vDON3DfSQfwxFqto85HY8Dw2_YgOmnm5cxbQ2g,21725
+arkindex_worker/worker/base.py,sha256=-R_aLMJHbR6X1uM-U0zExsF_KLy5Wl3WJ_YMGO9We0I,22153
 arkindex_worker/worker/classification.py,sha256=qvykymkgd4nGywHCxL8obo4egstoGsmWNS4Ztc1qNWQ,11024
 arkindex_worker/worker/corpus.py,sha256=MeIMod7jkWyX0frtD0a37rhumnMV3p9ZOC1xwAoXrAA,2291
 arkindex_worker/worker/dataset.py,sha256=tVaPx43vaH-KTtx4w5V06e26ha8XPfiJTRzBXlu928Y,5273
 arkindex_worker/worker/element.py,sha256=sLfnf09AfJ5tSCKQ7cAkl7WsGhjsfq14swsT30MDnYk,47385
 arkindex_worker/worker/entity.py,sha256=Aj6EOfzHEm7qQV-Egm0YKLZgCrLS_3ggOKTY81M2JbI,12323
 arkindex_worker/worker/image.py,sha256=L6Ikuf0Z0RxJk7JarY5PggJGrYSHLaPK0vn0dy0CIaQ,623
-arkindex_worker/worker/metadata.py,sha256=rBjU057xngwrf32vAo-2cpgYfmrdEj3lfDg_kv4-zr0,6810
+arkindex_worker/worker/metadata.py,sha256=keZdOdUthSH2hAw9iet5pN7rzWihTUYjZHRGTEjaltw,6843
 arkindex_worker/worker/process.py,sha256=9TEHpMcBax1wc6PrWMMrdXe2uNfqyVj7n_dAYZRBGnY,1854
 arkindex_worker/worker/task.py,sha256=nYfMSFm_d-4t8y4PO4HjFBnLsZf7IsDjkS7-A2Pgnac,1525
 arkindex_worker/worker/training.py,sha256=tyQOHcwv--_wdYz6CgLEe1YM7kwwwKN30LvGTsnWd78,10923
@@ -21,15 +21,15 @@ examples/standalone/python/worker.py,sha256=Zr4s4pHvgexEjlkixLFYZp1UuwMLeoTxjyNG
 examples/tooled/python/worker.py,sha256=kIYlHLsO5UpwX4XtERRq4tf2qTsvqKK30C-w8t0yyhA,1821
 hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
 tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
-tests/conftest.py,sha256=OaajGB0FS0Jh3rFhL8a3HdaN84XzIxNbRORvzI5W2ns,24928
-tests/test_base_worker.py,sha256=3YjhjxSWVjEWFYS8m8pYYoaVAhHFkJLNTs0QPQIkBDM,32651
-tests/test_cache.py,sha256=nnEFfAAqtYHk2ymOwN0spXJd8nrRiwp3voj0tOmIbQ8,10407
+tests/conftest.py,sha256=Tp7YFK17NATwF2yAcBwi0QFNyKSXtLS0VhZ-zZngsQI,24343
+tests/test_base_worker.py,sha256=lwS4X3atS2ktEKd1XdogmN3mbzq-tO206-k_0EDITlw,29302
+tests/test_cache.py,sha256=_wztzh94EwVrb8UvpFqgl2aa2_FLaCcJKaqunCYR5Dw,10435
 tests/test_dataset_worker.py,sha256=iDJM2C4PfQNH0r4_QqSWoPt8BcM0geUUdODtWY0Z9PA,22412
-tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
-tests/test_image.py,sha256=NEIp5evr6QoTWgJ-_fze19IEFm_hG6YEcuW1kxnxS_I,28013
+tests/test_element.py,sha256=hlj5VSF4plwC7uz9R4LGOOXZJQcHZiYCIDZT5V6EIB8,14334
+tests/test_image.py,sha256=yAM5mMfpQcIurT1KLHmu0AhSX2Qm3YvCu7afyZ3XUdU,28314
 tests/test_merge.py,sha256=REpZ13jkq_qm_4L5URQgFy5lxvPZtXxQEiWfYLMdmF0,7956
 tests/test_modern_config.py,sha256=Bm-a4LYQXgLZWQX7AmVyfJW0LNoLy1wj2d2GjzDkcBk,2683
-tests/test_utils.py,sha256=nYL1s2ViZoLoMiNpLGDaWwxf8dJ1D8aT522AO-PVaEQ,3607
+tests/test_utils.py,sha256=tgzNqyJMpddpeFWEjgsew_yDzmqnCA9HDaA5IpevAcM,5353
 tests/test_elements_worker/__init__.py,sha256=2t3NciCIOun_N-Wv63FWGsTm5W9N3mbwAWVuFORlMg8,308
 tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
 tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
@@ -55,7 +55,7 @@ worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.5.1b4.dist-info/METADATA,sha256=hLR30IODhVUqKVq_4qXv_AcNswb0afCA-2Nce13UPno,3137
-arkindex_base_worker-0.5.1b4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-arkindex_base_worker-0.5.1b4.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
-arkindex_base_worker-0.5.1b4.dist-info/RECORD,,
+arkindex_base_worker-0.5.1.post1.dist-info/METADATA,sha256=guiF6br4V1kfGBTRwX1ezI4b1oRG7RYdMGPnXq2jvIU,3094
+arkindex_base_worker-0.5.1.post1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+arkindex_base_worker-0.5.1.post1.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
+arkindex_base_worker-0.5.1.post1.dist-info/RECORD,,

arkindex_worker/cache.py

@@ -73,6 +73,7 @@ class CachedImage(Model):
     width = IntegerField()
     height = IntegerField()
     url = TextField()
+    version = IntegerField(default=2)
 
     class Meta:
         database = db
@@ -157,6 +158,10 @@ class CachedElement(Model):
             else:
                 resize = f"{max_width or ''},{max_height or ''}"
 
+        # Use `max` instead of `full` for IIIF 3, since `full` was deprecated in 2.1 then removed in 3.0
+        if self.image.version == 3 and resize == "full":
+            resize = "max"
+
         url = self.image.url
         if not url.endswith("/"):
             url += "/"
@@ -259,7 +264,7 @@ MODELS = [
     CachedDataset,
     CachedDatasetElement,
 ]
-SQL_VERSION = 4
+SQL_VERSION = 5
 
 
 def init_cache_db(path: Path):

arkindex_worker/image.py

@@ -366,6 +366,10 @@ def download_tiles(url: str) -> Image:
     logger.debug("Downloading image information")
     info = _retried_request(url + "info.json").json()
 
+    # Use `max` instead of `full` for IIIF 3, since `full` was deprecated in 2.1 then removed in 3.0
+    # With IIIF 3, the image's ID will be at `id`, while IIIF 2 will use `@id``
+    resize = "max" if "id" in info else "full"
+
     image_width, image_height = info.get("width"), info.get("height")
     assert image_width and image_height, "Missing image dimensions in info.json"
     assert info.get("tiles"), (
@@ -391,7 +395,7 @@ def download_tiles(url: str) -> Image:
 
             logger.debug(f"Downloading tile {tile_x},{tile_y}")
             resp = _retried_request(
-                f"{url}{region_x},{region_y},{region_width},{region_height}/full/0/default.jpg"
+                f"{url}{region_x},{region_y},{region_width},{region_height}/{resize}/0/default.jpg"
             )
 
             tile_img = Image.open(BytesIO(resp.content))

arkindex_worker/models.py

@@ -87,6 +87,11 @@ class Element(MagicDict):
         url = self.zone.image.get("s3_url")
         if url:
             return url
+
+        # Use `max` instead of `full` for IIIF 3, since `full` was deprecated in 2.1 then removed in 3.0
+        if self.zone.image.server.get("version", 2) == 3 and size == "full":
+            size = "max"
+
         url = self.zone.image.url
         if not url.endswith("/"):
             url += "/"

arkindex_worker/utils.py

@@ -4,6 +4,7 @@ import logging
 import os
 import tarfile
 import tempfile
+import zipfile
 from collections.abc import Callable, Generator
 from itertools import islice
 from pathlib import Path
@@ -225,6 +226,32 @@ def create_tar_zst_archive(
     return zst_fd, zst_archive, zst_hash, tar_hash
 
 
+def create_zip_archive(source: Path, destination: Path | None = None) -> Path:
+    """Helper to create a ZIP archive from a source folder.
+
+    :param source: Path to the folder whose content should be archived.
+    :param destination: Path to the created archive, defaults to None. If unspecified, a temporary file will be created.
+    :return: The file descriptor of the created tempfile (if one was created), path to the archive.
+    """
+    # Parse destination and create a tmpfile if none was specified
+    file_d, destination = (
+        tempfile.mkstemp(prefix="teklia-", suffix=".zip")
+        if destination is None
+        else (None, destination)
+    )
+    destination = Path(destination)
+    logger.debug(f"Compressing file to {destination}")
+
+    with zipfile.ZipFile(
+        destination, mode="w", compression=zipfile.ZIP_DEFLATED
+    ) as archive:
+        for p in source.rglob("*"):
+            relpath = p.relative_to(source)
+            archive.write(p, arcname=relpath)
+
+        return archive, destination
+
+
 DEFAULT_BATCH_SIZE = 50
 """Batch size used for bulk publication to Arkindex"""
 

arkindex_worker/worker/base.py

@@ -265,6 +265,11 @@ class BaseWorker:
             if not item["secret"]:
                 return (item["key"], item["value"])
 
+            # The secret may not be picked by the user
+            if item["value"] is None:
+                logger.info(f"Optional secret `{item['key']}` is not set")
+                return (item["key"], None)
+
             # Load secret, only available in Arkindex EE
             try:
                 secret = self.load_secret(Path(item["value"]))
@@ -276,6 +281,19 @@ class BaseWorker:
 
             return (item["key"], secret)
 
+        # Load model version configuration when available
+        # Workers will use model version ID and details to download the model
+        model_version = worker_run.get("model_version")
+        if model_version:
+            logger.info("Loaded model version configuration from WorkerRun")
+            self.model_configuration.update(model_version["configuration"])
+
+            # Set model_version ID as worker attribute
+            self.model_version_id = model_version["id"]
+
+            # Set model details as worker attribute
+            self.model_details = model_version["model"]
+
         # Load worker run information
         try:
             config = self.api_client.request(
@@ -295,6 +313,9 @@ class BaseWorker:
             }
             logger.info("Using modern configuration")
 
+            # Reset the model configuration to make sure workers rely on the single new source
+            self.model_configuration = {}
+
             return  # Stop here once we have modern configuration
 
         except ErrorResponse as e:
@@ -303,18 +324,6 @@ class BaseWorker:
             logger.info("Modern configuration is not available")
 
         # Use old-style configuration with local merge
-        # Load model version configuration when available
-        model_version = worker_run.get("model_version")
-        if model_version:
-            logger.info("Loaded model version configuration from WorkerRun")
-            self.model_configuration.update(model_version["configuration"])
-
-            # Set model_version ID as worker attribute
-            self.model_version_id = model_version["id"]
-
-            # Set model details as worker attribute
-            self.model_details = model_version["model"]
-
         # Retrieve initial configuration from API
         self.config = worker_version["configuration"].get("configuration", {})
         if "user_configuration" in worker_version["configuration"]:

arkindex_worker/worker/metadata.py

@@ -20,10 +20,10 @@ class MetaType(Enum):
     A regular string with no special interpretation.
     """
 
-    HTML = "html"
+    Markdown = "markdown"
     """
-    A metadata with a string value that should be interpreted as HTML content.
-    The allowed HTML tags are restricted for security reasons.
+    A metadata with a string value that should be interpreted as Markdown content.
+    HTML is allowed, but the allowed HTML tags are restricted for security reasons.
     """
 
     Date = "date"

tests/conftest.py

@@ -103,12 +103,6 @@ def _mock_worker_run_api(responses):
     payload = {
         "id": "56785678-5678-5678-5678-567856785678",
         "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "configuration": {
@@ -180,12 +174,6 @@ def _mock_worker_run_no_revision_api(responses):
     payload = {
         "id": "56785678-5678-5678-5678-567856785678",
         "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "configuration": {
@@ -355,12 +343,6 @@ def mock_elements_worker_consume_wa(monkeypatch, responses, mock_elements_worker
         json={
             "id": "56785678-5678-5678-5678-567856785678",
             "parents": [],
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
             "worker_version": {
                 "id": "12341234-1234-1234-1234-123412341234",
                 "configuration": {

tests/test_base_worker.py

@@ -13,6 +13,29 @@ from arkindex_worker.worker import BaseWorker, ElementsWorker
 from arkindex_worker.worker.base import ExtrasDirNotFoundError
 from tests import CORPUS_ID, FIXTURES_DIR
 
+SIMPLE_PAYLOAD = {
+    "id": "56785678-5678-5678-5678-567856785678",
+    "parents": [],
+    "worker_version": {
+        "id": "12341234-1234-1234-1234-123412341234",
+        "worker": {
+            "id": "deadbeef-1234-5678-1234-worker",
+            "name": "Fake worker",
+            "slug": "fake_worker",
+            "type": "classifier",
+        },
+        "revision": {"hash": "deadbeef1234"},
+        "configuration": {"configuration": {}},
+    },
+    "configuration": None,
+    "model_version": None,
+    "process": {
+        "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+        "corpus": CORPUS_ID,
+    },
+    "summary": "Worker Fake worker @ 123412",
+}
+
 
 def test_init_default_local_share():
     worker = BaseWorker()
@@ -149,38 +172,13 @@ def test_configure_worker_run(mocker, responses, caplog):
 
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
-    user_configuration = {
-        "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
-        "name": "BBB",
-        "configuration": {"a": "b"},
-    }
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": user_configuration,
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
+        **SIMPLE_PAYLOAD,
+        "configuration": {
+            "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+            "name": "BBB",
+            "configuration": {"a": "b"},
         },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(
@@ -262,8 +260,7 @@ def test_configure_user_configuration_defaults(mocker, responses):
     worker.args = worker.parser.parse_args()
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
+        **SIMPLE_PAYLOAD,
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "worker": {
@@ -293,12 +290,6 @@ def test_configure_user_configuration_defaults(mocker, responses):
                 "param_5": True,
             },
         },
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -340,30 +331,12 @@ def test_configure_user_config_debug(mocker, responses, debug):
     mocker.patch.object(sys, "argv", ["worker"])
     assert logger.level == logging.NOTSET
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {
             "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
             "name": "BBB",
             "configuration": {"debug": debug},
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -393,32 +366,8 @@ def test_configure_worker_run_missing_conf(mocker, responses):
     mocker.patch.object(sys, "argv", ["worker"])
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {"id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "BBB"},
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -446,28 +395,7 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
 
-    payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {},
-        },
-        "model_version": None,
-        "configuration": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
-    }
+    payload = SIMPLE_PAYLOAD
     responses.add(
         responses.GET,
         "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
@@ -491,20 +419,7 @@ def test_configure_load_model_configuration(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": None,
+        **SIMPLE_PAYLOAD,
         "model_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "model": {
@@ -517,11 +432,6 @@ def test_configure_load_model_configuration(mocker, responses):
                 "param3": None,
             },
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(

tests/test_cache.py

@@ -60,7 +60,7 @@ def test_create_tables(tmp_path):
 CREATE TABLE "dataset_elements" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "dataset_id" TEXT NOT NULL, "set_name" VARCHAR(255) NOT NULL, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"), FOREIGN KEY ("dataset_id") REFERENCES "datasets" ("id"))
 CREATE TABLE "datasets" ("id" TEXT NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, "state" VARCHAR(255) NOT NULL DEFAULT 'open', "sets" TEXT NOT NULL)
 CREATE TABLE "elements" ("id" TEXT NOT NULL PRIMARY KEY, "parent_id" TEXT, "type" VARCHAR(50) NOT NULL, "image_id" TEXT, "polygon" text, "rotation_angle" INTEGER NOT NULL, "mirrored" INTEGER NOT NULL, "initial" INTEGER NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, FOREIGN KEY ("image_id") REFERENCES "images" ("id"))
-CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL)
+CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL, "version" INTEGER NOT NULL)
 CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "type" VARCHAR(50) NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "type"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"))
 CREATE TABLE "transcriptions" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "text" TEXT NOT NULL, "confidence" REAL, "orientation" VARCHAR(50) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))"""
 

tests/test_element.py

@@ -5,26 +5,36 @@ from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Element
 
 
-def test_no_image_url():
+@pytest.mark.parametrize(
+    ("zone", "expected_url"),
+    [
+        (None, None),
+        (
+            {"image": {"url": "http://something/", "server": {"version": 2}}},
+            "http://something/full/full/0/default.jpg",
+        ),
+        (
+            {"image": {"url": "http://something", "server": {"version": 2}}},
+            "http://something/full/full/0/default.jpg",
+        ),
+        (
+            {"image": {"url": "http://something/", "server": {"version": 3}}},
+            "http://something/full/max/0/default.jpg",
+        ),
+    ],
+)
+def test_image_url(zone, expected_url):
     url = Element({"zone": None}).image_url()
     assert not url
 
 
-def test_image_url_iiif():
-    url = Element({"zone": {"image": {"url": "http://something/"}}}).image_url()
-    assert url == "http://something/full/full/0/default.jpg"
-
-
 def test_image_url_iiif_resize():
-    url = Element({"zone": {"image": {"url": "http://something/"}}}).image_url(500)
+    url = Element(
+        {"zone": {"image": {"url": "http://something/", "server": {"version": 2}}}}
+    ).image_url(500)
     assert url == "http://something/full/500/0/default.jpg"
 
 
-def test_image_url_iiif_append_slash():
-    url = Element({"zone": {"image": {"url": "http://something"}}}).image_url()
-    assert url == "http://something/full/full/0/default.jpg"
-
-
 def test_image_url_s3():
     url = Element(
         {
@@ -418,6 +428,36 @@ def test_open_image_rotation_mirror(mocker):
     )
 
 
+def test_open_image_iiif_3(mocker):
+    open_mock = mocker.patch(
+        "arkindex_worker.image.open_image", return_value="an image!"
+    )
+    elt = Element(
+        {
+            "zone": {
+                "image": {
+                    "url": "http://something",
+                    "server": {
+                        "max_width": None,
+                        "max_height": None,
+                        "version": 3,
+                    },
+                },
+                "polygon": [[0, 0], [181, 0], [181, 240], [0, 240], [0, 0]],
+            },
+            "rotation_angle": 0,
+            "mirrored": False,
+        },
+    )
+    assert elt.open_image(use_full_image=True) == "an image!"
+    assert open_mock.call_count == 1
+    assert open_mock.call_args == mocker.call(
+        "http://something/full/max/0/default.jpg",
+        rotation_angle=0,
+        mirrored=False,
+    )
+
+
 def test_setattr_setitem():
     element = Element({"name": "something"})
     element.type = "page"

tests/test_image.py

@@ -113,21 +113,37 @@ def test_update_pillow_image_size_limit(max_image_pixels, expected_image_pixels)
     assert Image.MAX_IMAGE_PIXELS == MAX_IMAGE_PIXELS
 
 
-def test_download_tiles(responses):
+@pytest.mark.parametrize(
+    ("id_key", "resize"),
+    [
+        # IIIF version 2
+        ("@id", "full"),
+        # IIIF version 3
+        ("id", "max"),
+    ],
+)
+def test_download_tiles(responses, id_key, resize):
     expected = Image.open(FULL_IMAGE).convert("RGB")
     tile_bytes = TILE.read_bytes()
 
     responses.add(
         responses.GET,
         "http://nowhere/info.json",
-        json={"width": 543, "height": 720, "tiles": [{"width": 181, "height": 240}]},
+        json={
+            id_key: "http://nowhere",
+            "width": 543,
+            "height": 720,
+            "tiles": [
+                {"width": 181, "height": 240},
+            ],
+        },
     )
 
     for x in (0, 181, 362):
         for y in (0, 240, 480):
             responses.add(
                 responses.GET,
-                f"http://nowhere/{x},{y},181,240/full/0/default.jpg",
+                f"http://nowhere/{x},{y},181,240/{resize}/0/default.jpg",
                 body=tile_bytes,
             )
 

tests/test_utils.py

@@ -1,4 +1,5 @@
 import logging
+import zipfile
 
 import pytest
 
@@ -7,6 +8,7 @@ from arkindex_worker.utils import (
     DEFAULT_BATCH_SIZE,
     batch_publication,
     close_delete_file,
+    create_zip_archive,
     extract_tar_zst_archive,
     parse_source_id,
 )
@@ -118,3 +120,43 @@ def test_batch_publication_decorator_alongside_unsupported_cache(caplog):
             "This API helper `custom_publication_in_batches_without_cache` did not update the cache database",
         ),
     ]
+
+
+def test_zip_archive():
+    # Create zip archive from fixtures
+    _, archive = create_zip_archive(FIXTURES_DIR / "extract_parent_archives/expected")
+
+    # Check the files in the archive
+    with zipfile.ZipFile(archive, mode="r") as f:
+        assert sorted(f.namelist()) == [
+            "test/",
+            "test/images/",
+            "test/images/f2649ce7-333e-44d2-ae73-387f18aad1f6.png",
+            "test/labels/",
+            "test/labels/f2649ce7-333e-44d2-ae73-387f18aad1f6.png",
+            "test/labels_json/",
+            "test/labels_json/f2649ce7-333e-44d2-ae73-387f18aad1f6.json",
+            "train/",
+            "train/images/",
+            "train/images/98115546-df07-448c-a2f0-34aa24789b77.png",
+            "train/images/ebeaa451-9287-4df7-9c40-07eb25cadb78.png",
+            "train/labels/",
+            "train/labels/98115546-df07-448c-a2f0-34aa24789b77.png",
+            "train/labels/ebeaa451-9287-4df7-9c40-07eb25cadb78.png",
+            "train/labels_json/",
+            "train/labels_json/98115546-df07-448c-a2f0-34aa24789b77.json",
+            "train/labels_json/ebeaa451-9287-4df7-9c40-07eb25cadb78.json",
+            "val/",
+            "val/images/",
+            "val/images/2987176d-4338-40f2-90d9-6d2cb4fd4a00.png",
+            "val/images/e3f91312-9201-45b7-9c32-e04a97ff1334.png",
+            "val/labels/",
+            "val/labels/2987176d-4338-40f2-90d9-6d2cb4fd4a00.png",
+            "val/labels/e3f91312-9201-45b7-9c32-e04a97ff1334.png",
+            "val/labels_json/",
+            "val/labels_json/2987176d-4338-40f2-90d9-6d2cb4fd4a00.json",
+            "val/labels_json/e3f91312-9201-45b7-9c32-e04a97ff1334.json",
+        ]
+
+    # Cleanup
+    archive.unlink()