arkindex-base-worker 0.5.1b5__py3-none-any.whl → 0.5.1.post1__py3-none-any.whl

{arkindex_base_worker-0.5.1b5.dist-info → arkindex_base_worker-0.5.1.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.1b5
+Version: 0.5.1.post1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,16 +41,15 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.12.3
+Requires-Dist: humanize==4.14.0
 Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==11.3.0
-Requires-Dist: python-gnupg==0.5.4
+Requires-Dist: python-gnupg==0.5.5
 Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.11
-Requires-Dist: zstandard==0.23.0
+Requires-Dist: zstandard==0.25.0
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.5; extra == "tests"
-Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-mock==3.15.1; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 Dynamic: license-file
 

{arkindex_base_worker-0.5.1b5.dist-info → arkindex_base_worker-0.5.1.post1.dist-info}/RECORD

@@ -1,18 +1,18 @@
-arkindex_base_worker-0.5.1b5.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.5.1.post1.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
 arkindex_worker/__init__.py,sha256=Sdt5KXn8EgURb2MurYVrUWaHbH3iFA1XLRo0Lc5AJ44,250
-arkindex_worker/cache.py,sha256=x1d1oVF297ItLoZnPkZQoEefa39ZigrwRoHC_6az94k,10731
-arkindex_worker/image.py,sha256=GvIpW7LNSalVw3Obt9nySDWnW7-NbC0__SWREEQqVCk,20696
-arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
-arkindex_worker/utils.py,sha256=MbbJT8oh8DMHHR-vidFeXdUH0TSXGWm7ZDGWzrRXoEY,9933
+arkindex_worker/cache.py,sha256=XpEXMSnbhYCvrJquwA9XXqZo-ajMLpaCxKG5wH3Gp6Y,10959
+arkindex_worker/image.py,sha256=sGE8to5iykXv25bpkftOEWzlh5NzBZSKy4lSRoHYHPU,20929
+arkindex_worker/models.py,sha256=7GnKqpWPOSxyR_eKlDNVBe_r3TcE4ofK-1GzaonJEdM,10132
+arkindex_worker/utils.py,sha256=Eqg5pGAuOmuwMT3EhKTQDMek7wHC1KzZL7XXqYVVfHY,10977
 arkindex_worker/worker/__init__.py,sha256=SzD0s1_m6gMV02EUF-NeciqZdVPA4dpXI84tSj-g494,17869
-arkindex_worker/worker/base.py,sha256=-ASOgLzemHOAz7EnPZ9hT6wON_9GJ7uadxx7oZLgUD4,22149
+arkindex_worker/worker/base.py,sha256=-R_aLMJHbR6X1uM-U0zExsF_KLy5Wl3WJ_YMGO9We0I,22153
 arkindex_worker/worker/classification.py,sha256=qvykymkgd4nGywHCxL8obo4egstoGsmWNS4Ztc1qNWQ,11024
 arkindex_worker/worker/corpus.py,sha256=MeIMod7jkWyX0frtD0a37rhumnMV3p9ZOC1xwAoXrAA,2291
 arkindex_worker/worker/dataset.py,sha256=tVaPx43vaH-KTtx4w5V06e26ha8XPfiJTRzBXlu928Y,5273
 arkindex_worker/worker/element.py,sha256=sLfnf09AfJ5tSCKQ7cAkl7WsGhjsfq14swsT30MDnYk,47385
 arkindex_worker/worker/entity.py,sha256=Aj6EOfzHEm7qQV-Egm0YKLZgCrLS_3ggOKTY81M2JbI,12323
 arkindex_worker/worker/image.py,sha256=L6Ikuf0Z0RxJk7JarY5PggJGrYSHLaPK0vn0dy0CIaQ,623
-arkindex_worker/worker/metadata.py,sha256=rBjU057xngwrf32vAo-2cpgYfmrdEj3lfDg_kv4-zr0,6810
+arkindex_worker/worker/metadata.py,sha256=keZdOdUthSH2hAw9iet5pN7rzWihTUYjZHRGTEjaltw,6843
 arkindex_worker/worker/process.py,sha256=9TEHpMcBax1wc6PrWMMrdXe2uNfqyVj7n_dAYZRBGnY,1854
 arkindex_worker/worker/task.py,sha256=nYfMSFm_d-4t8y4PO4HjFBnLsZf7IsDjkS7-A2Pgnac,1525
 arkindex_worker/worker/training.py,sha256=tyQOHcwv--_wdYz6CgLEe1YM7kwwwKN30LvGTsnWd78,10923
@@ -23,13 +23,13 @@ hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
 tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
 tests/conftest.py,sha256=Tp7YFK17NATwF2yAcBwi0QFNyKSXtLS0VhZ-zZngsQI,24343
 tests/test_base_worker.py,sha256=lwS4X3atS2ktEKd1XdogmN3mbzq-tO206-k_0EDITlw,29302
-tests/test_cache.py,sha256=nnEFfAAqtYHk2ymOwN0spXJd8nrRiwp3voj0tOmIbQ8,10407
+tests/test_cache.py,sha256=_wztzh94EwVrb8UvpFqgl2aa2_FLaCcJKaqunCYR5Dw,10435
 tests/test_dataset_worker.py,sha256=iDJM2C4PfQNH0r4_QqSWoPt8BcM0geUUdODtWY0Z9PA,22412
-tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
-tests/test_image.py,sha256=NEIp5evr6QoTWgJ-_fze19IEFm_hG6YEcuW1kxnxS_I,28013
+tests/test_element.py,sha256=hlj5VSF4plwC7uz9R4LGOOXZJQcHZiYCIDZT5V6EIB8,14334
+tests/test_image.py,sha256=yAM5mMfpQcIurT1KLHmu0AhSX2Qm3YvCu7afyZ3XUdU,28314
 tests/test_merge.py,sha256=REpZ13jkq_qm_4L5URQgFy5lxvPZtXxQEiWfYLMdmF0,7956
 tests/test_modern_config.py,sha256=Bm-a4LYQXgLZWQX7AmVyfJW0LNoLy1wj2d2GjzDkcBk,2683
-tests/test_utils.py,sha256=nYL1s2ViZoLoMiNpLGDaWwxf8dJ1D8aT522AO-PVaEQ,3607
+tests/test_utils.py,sha256=tgzNqyJMpddpeFWEjgsew_yDzmqnCA9HDaA5IpevAcM,5353
 tests/test_elements_worker/__init__.py,sha256=2t3NciCIOun_N-Wv63FWGsTm5W9N3mbwAWVuFORlMg8,308
 tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
 tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
@@ -55,7 +55,7 @@ worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.5.1b5.dist-info/METADATA,sha256=pVX35tc38qCuTCXwhXS2veoEJpTMJ3KWh3a1CUZQ3yg,3137
-arkindex_base_worker-0.5.1b5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-arkindex_base_worker-0.5.1b5.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
-arkindex_base_worker-0.5.1b5.dist-info/RECORD,,
+arkindex_base_worker-0.5.1.post1.dist-info/METADATA,sha256=guiF6br4V1kfGBTRwX1ezI4b1oRG7RYdMGPnXq2jvIU,3094
+arkindex_base_worker-0.5.1.post1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+arkindex_base_worker-0.5.1.post1.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
+arkindex_base_worker-0.5.1.post1.dist-info/RECORD,,

arkindex_worker/cache.py

@@ -73,6 +73,7 @@ class CachedImage(Model):
     width = IntegerField()
     height = IntegerField()
     url = TextField()
+    version = IntegerField(default=2)
 
     class Meta:
         database = db
@@ -157,6 +158,10 @@ class CachedElement(Model):
             else:
                 resize = f"{max_width or ''},{max_height or ''}"
 
+        # Use `max` instead of `full` for IIIF 3, since `full` was deprecated in 2.1 then removed in 3.0
+        if self.image.version == 3 and resize == "full":
+            resize = "max"
+
         url = self.image.url
         if not url.endswith("/"):
             url += "/"
@@ -259,7 +264,7 @@ MODELS = [
     CachedDataset,
     CachedDatasetElement,
 ]
-SQL_VERSION = 4
+SQL_VERSION = 5
 
 
 def init_cache_db(path: Path):

arkindex_worker/image.py

@@ -366,6 +366,10 @@ def download_tiles(url: str) -> Image:
     logger.debug("Downloading image information")
     info = _retried_request(url + "info.json").json()
 
+    # Use `max` instead of `full` for IIIF 3, since `full` was deprecated in 2.1 then removed in 3.0
+    # With IIIF 3, the image's ID will be at `id`, while IIIF 2 will use `@id``
+    resize = "max" if "id" in info else "full"
+
     image_width, image_height = info.get("width"), info.get("height")
     assert image_width and image_height, "Missing image dimensions in info.json"
     assert info.get("tiles"), (
@@ -391,7 +395,7 @@ def download_tiles(url: str) -> Image:
 
             logger.debug(f"Downloading tile {tile_x},{tile_y}")
             resp = _retried_request(
-                f"{url}{region_x},{region_y},{region_width},{region_height}/full/0/default.jpg"
+                f"{url}{region_x},{region_y},{region_width},{region_height}/{resize}/0/default.jpg"
             )
 
             tile_img = Image.open(BytesIO(resp.content))

arkindex_worker/models.py

@@ -87,6 +87,11 @@ class Element(MagicDict):
         url = self.zone.image.get("s3_url")
         if url:
             return url
+
+        # Use `max` instead of `full` for IIIF 3, since `full` was deprecated in 2.1 then removed in 3.0
+        if self.zone.image.server.get("version", 2) == 3 and size == "full":
+            size = "max"
+
         url = self.zone.image.url
         if not url.endswith("/"):
             url += "/"

arkindex_worker/utils.py

@@ -4,6 +4,7 @@ import logging
 import os
 import tarfile
 import tempfile
+import zipfile
 from collections.abc import Callable, Generator
 from itertools import islice
 from pathlib import Path
@@ -225,6 +226,32 @@ def create_tar_zst_archive(
     return zst_fd, zst_archive, zst_hash, tar_hash
 
 
+def create_zip_archive(source: Path, destination: Path | None = None) -> Path:
+    """Helper to create a ZIP archive from a source folder.
+
+    :param source: Path to the folder whose content should be archived.
+    :param destination: Path to the created archive, defaults to None. If unspecified, a temporary file will be created.
+    :return: The file descriptor of the created tempfile (if one was created), path to the archive.
+    """
+    # Parse destination and create a tmpfile if none was specified
+    file_d, destination = (
+        tempfile.mkstemp(prefix="teklia-", suffix=".zip")
+        if destination is None
+        else (None, destination)
+    )
+    destination = Path(destination)
+    logger.debug(f"Compressing file to {destination}")
+
+    with zipfile.ZipFile(
+        destination, mode="w", compression=zipfile.ZIP_DEFLATED
+    ) as archive:
+        for p in source.rglob("*"):
+            relpath = p.relative_to(source)
+            archive.write(p, arcname=relpath)
+
+        return archive, destination
+
+
 DEFAULT_BATCH_SIZE = 50
 """Batch size used for bulk publication to Arkindex"""
 

arkindex_worker/worker/base.py

@@ -267,7 +267,7 @@ class BaseWorker:
 
             # The secret may not be picked by the user
             if item["value"] is None:
-                logger.info(f"Skipping optional secret {item['key']}")
+                logger.info(f"Optional secret `{item['key']}` is not set")
                 return (item["key"], None)
 
             # Load secret, only available in Arkindex EE

arkindex_worker/worker/metadata.py

@@ -20,10 +20,10 @@ class MetaType(Enum):
     A regular string with no special interpretation.
     """
 
-    HTML = "html"
+    Markdown = "markdown"
     """
-    A metadata with a string value that should be interpreted as HTML content.
-    The allowed HTML tags are restricted for security reasons.
+    A metadata with a string value that should be interpreted as Markdown content.
+    HTML is allowed, but the allowed HTML tags are restricted for security reasons.
     """
 
     Date = "date"

tests/test_cache.py

@@ -60,7 +60,7 @@ def test_create_tables(tmp_path):
 CREATE TABLE "dataset_elements" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "dataset_id" TEXT NOT NULL, "set_name" VARCHAR(255) NOT NULL, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"), FOREIGN KEY ("dataset_id") REFERENCES "datasets" ("id"))
 CREATE TABLE "datasets" ("id" TEXT NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, "state" VARCHAR(255) NOT NULL DEFAULT 'open', "sets" TEXT NOT NULL)
 CREATE TABLE "elements" ("id" TEXT NOT NULL PRIMARY KEY, "parent_id" TEXT, "type" VARCHAR(50) NOT NULL, "image_id" TEXT, "polygon" text, "rotation_angle" INTEGER NOT NULL, "mirrored" INTEGER NOT NULL, "initial" INTEGER NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, FOREIGN KEY ("image_id") REFERENCES "images" ("id"))
-CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL)
+CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL, "version" INTEGER NOT NULL)
 CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "type" VARCHAR(50) NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "type"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"))
 CREATE TABLE "transcriptions" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "text" TEXT NOT NULL, "confidence" REAL, "orientation" VARCHAR(50) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))"""
 

tests/test_element.py

@@ -5,26 +5,36 @@ from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Element
 
 
-def test_no_image_url():
+@pytest.mark.parametrize(
+    ("zone", "expected_url"),
+    [
+        (None, None),
+        (
+            {"image": {"url": "http://something/", "server": {"version": 2}}},
+            "http://something/full/full/0/default.jpg",
+        ),
+        (
+            {"image": {"url": "http://something", "server": {"version": 2}}},
+            "http://something/full/full/0/default.jpg",
+        ),
+        (
+            {"image": {"url": "http://something/", "server": {"version": 3}}},
+            "http://something/full/max/0/default.jpg",
+        ),
+    ],
+)
+def test_image_url(zone, expected_url):
     url = Element({"zone": None}).image_url()
     assert not url
 
 
-def test_image_url_iiif():
-    url = Element({"zone": {"image": {"url": "http://something/"}}}).image_url()
-    assert url == "http://something/full/full/0/default.jpg"
-
-
 def test_image_url_iiif_resize():
-    url = Element({"zone": {"image": {"url": "http://something/"}}}).image_url(500)
+    url = Element(
+        {"zone": {"image": {"url": "http://something/", "server": {"version": 2}}}}
+    ).image_url(500)
     assert url == "http://something/full/500/0/default.jpg"
 
 
-def test_image_url_iiif_append_slash():
-    url = Element({"zone": {"image": {"url": "http://something"}}}).image_url()
-    assert url == "http://something/full/full/0/default.jpg"
-
-
 def test_image_url_s3():
     url = Element(
         {
@@ -418,6 +428,36 @@ def test_open_image_rotation_mirror(mocker):
     )
 
 
+def test_open_image_iiif_3(mocker):
+    open_mock = mocker.patch(
+        "arkindex_worker.image.open_image", return_value="an image!"
+    )
+    elt = Element(
+        {
+            "zone": {
+                "image": {
+                    "url": "http://something",
+                    "server": {
+                        "max_width": None,
+                        "max_height": None,
+                        "version": 3,
+                    },
+                },
+                "polygon": [[0, 0], [181, 0], [181, 240], [0, 240], [0, 0]],
+            },
+            "rotation_angle": 0,
+            "mirrored": False,
+        },
+    )
+    assert elt.open_image(use_full_image=True) == "an image!"
+    assert open_mock.call_count == 1
+    assert open_mock.call_args == mocker.call(
+        "http://something/full/max/0/default.jpg",
+        rotation_angle=0,
+        mirrored=False,
+    )
+
+
 def test_setattr_setitem():
     element = Element({"name": "something"})
     element.type = "page"

tests/test_image.py

@@ -113,21 +113,37 @@ def test_update_pillow_image_size_limit(max_image_pixels, expected_image_pixels)
     assert Image.MAX_IMAGE_PIXELS == MAX_IMAGE_PIXELS
 
 
-def test_download_tiles(responses):
+@pytest.mark.parametrize(
+    ("id_key", "resize"),
+    [
+        # IIIF version 2
+        ("@id", "full"),
+        # IIIF version 3
+        ("id", "max"),
+    ],
+)
+def test_download_tiles(responses, id_key, resize):
     expected = Image.open(FULL_IMAGE).convert("RGB")
     tile_bytes = TILE.read_bytes()
 
     responses.add(
         responses.GET,
         "http://nowhere/info.json",
-        json={"width": 543, "height": 720, "tiles": [{"width": 181, "height": 240}]},
+        json={
+            id_key: "http://nowhere",
+            "width": 543,
+            "height": 720,
+            "tiles": [
+                {"width": 181, "height": 240},
+            ],
+        },
     )
 
     for x in (0, 181, 362):
         for y in (0, 240, 480):
             responses.add(
                 responses.GET,
-                f"http://nowhere/{x},{y},181,240/full/0/default.jpg",
+                f"http://nowhere/{x},{y},181,240/{resize}/0/default.jpg",
                 body=tile_bytes,
             )
 

tests/test_utils.py

@@ -1,4 +1,5 @@
 import logging
+import zipfile
 
 import pytest
 
@@ -7,6 +8,7 @@ from arkindex_worker.utils import (
     DEFAULT_BATCH_SIZE,
     batch_publication,
     close_delete_file,
+    create_zip_archive,
     extract_tar_zst_archive,
     parse_source_id,
 )
@@ -118,3 +120,43 @@ def test_batch_publication_decorator_alongside_unsupported_cache(caplog):
             "This API helper `custom_publication_in_batches_without_cache` did not update the cache database",
         ),
     ]
+
+
+def test_zip_archive():
+    # Create zip archive from fixtures
+    _, archive = create_zip_archive(FIXTURES_DIR / "extract_parent_archives/expected")
+
+    # Check the files in the archive
+    with zipfile.ZipFile(archive, mode="r") as f:
+        assert sorted(f.namelist()) == [
+            "test/",
+            "test/images/",
+            "test/images/f2649ce7-333e-44d2-ae73-387f18aad1f6.png",
+            "test/labels/",
+            "test/labels/f2649ce7-333e-44d2-ae73-387f18aad1f6.png",
+            "test/labels_json/",
+            "test/labels_json/f2649ce7-333e-44d2-ae73-387f18aad1f6.json",
+            "train/",
+            "train/images/",
+            "train/images/98115546-df07-448c-a2f0-34aa24789b77.png",
+            "train/images/ebeaa451-9287-4df7-9c40-07eb25cadb78.png",
+            "train/labels/",
+            "train/labels/98115546-df07-448c-a2f0-34aa24789b77.png",
+            "train/labels/ebeaa451-9287-4df7-9c40-07eb25cadb78.png",
+            "train/labels_json/",
+            "train/labels_json/98115546-df07-448c-a2f0-34aa24789b77.json",
+            "train/labels_json/ebeaa451-9287-4df7-9c40-07eb25cadb78.json",
+            "val/",
+            "val/images/",
+            "val/images/2987176d-4338-40f2-90d9-6d2cb4fd4a00.png",
+            "val/images/e3f91312-9201-45b7-9c32-e04a97ff1334.png",
+            "val/labels/",
+            "val/labels/2987176d-4338-40f2-90d9-6d2cb4fd4a00.png",
+            "val/labels/e3f91312-9201-45b7-9c32-e04a97ff1334.png",
+            "val/labels_json/",
+            "val/labels_json/2987176d-4338-40f2-90d9-6d2cb4fd4a00.json",
+            "val/labels_json/e3f91312-9201-45b7-9c32-e04a97ff1334.json",
+        ]
+
+    # Cleanup
+    archive.unlink()