arkindex-base-worker 0.3.7rc9__tar.gz → 0.4.0__tar.gz

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.7rc9
+Version: 0.4.0
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,28 +37,24 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: peewee==3.17.0
-Requires-Dist: Pillow==10.2.0
-Requires-Dist: pymdown-extensions==10.7
-Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.3
-Requires-Dist: teklia-toolbox==0.1.4rc3
-Requires-Dist: zstandard==0.22.0
+Requires-Dist: humanize==4.11.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==11.0.0
+Requires-Dist: python-gnupg==0.5.3
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
-Requires-Dist: black==24.2.0; extra == "docs"
-Requires-Dist: doc8==1.1.1; extra == "docs"
-Requires-Dist: mkdocs==1.5.3; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.10; extra == "docs"
-Requires-Dist: mkdocstrings==0.24.0; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.8.0; extra == "docs"
-Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Requires-Dist: black==24.10.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest==8.0.1; extra == "tests"
-Requires-Dist: pytest-mock==3.12.0; extra == "tests"
+Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
@@ -73,7 +69,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/README.md

@@ -10,7 +10,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.7rc9
+Version: 0.4.0
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,28 +37,24 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: peewee==3.17.0
-Requires-Dist: Pillow==10.2.0
-Requires-Dist: pymdown-extensions==10.7
-Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.3
-Requires-Dist: teklia-toolbox==0.1.4rc3
-Requires-Dist: zstandard==0.22.0
+Requires-Dist: humanize==4.11.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==11.0.0
+Requires-Dist: python-gnupg==0.5.3
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
-Requires-Dist: black==24.2.0; extra == "docs"
-Requires-Dist: doc8==1.1.1; extra == "docs"
-Requires-Dist: mkdocs==1.5.3; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.10; extra == "docs"
-Requires-Dist: mkdocstrings==0.24.0; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.8.0; extra == "docs"
-Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Requires-Dist: black==24.10.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest==8.0.1; extra == "tests"
-Requires-Dist: pytest-mock==3.12.0; extra == "tests"
+Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
@@ -73,7 +69,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -14,10 +14,13 @@ arkindex_worker/utils.py
 arkindex_worker/worker/__init__.py
 arkindex_worker/worker/base.py
 arkindex_worker/worker/classification.py
+arkindex_worker/worker/corpus.py
 arkindex_worker/worker/dataset.py
 arkindex_worker/worker/element.py
 arkindex_worker/worker/entity.py
+arkindex_worker/worker/image.py
 arkindex_worker/worker/metadata.py
+arkindex_worker/worker/process.py
 arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
@@ -33,15 +36,26 @@ tests/test_image.py
 tests/test_merge.py
 tests/test_utils.py
 tests/test_elements_worker/__init__.py
-tests/test_elements_worker/test_classifications.py
+tests/test_elements_worker/test_classification.py
 tests/test_elements_worker/test_cli.py
+tests/test_elements_worker/test_corpus.py
 tests/test_elements_worker/test_dataset.py
-tests/test_elements_worker/test_elements.py
-tests/test_elements_worker/test_entities.py
+tests/test_elements_worker/test_element.py
+tests/test_elements_worker/test_element_create_multiple.py
+tests/test_elements_worker/test_element_create_single.py
+tests/test_elements_worker/test_element_list_children.py
+tests/test_elements_worker/test_element_list_parents.py
+tests/test_elements_worker/test_entity_create.py
+tests/test_elements_worker/test_entity_list_and_check.py
+tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
+tests/test_elements_worker/test_process.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
-tests/test_elements_worker/test_transcriptions.py
+tests/test_elements_worker/test_transcription_create.py
+tests/test_elements_worker/test_transcription_create_with_elements.py
+tests/test_elements_worker/test_transcription_list.py
+tests/test_elements_worker/test_version.py
 tests/test_elements_worker/test_worker.py
 worker-demo/tests/__init__.py
 worker-demo/tests/conftest.py

arkindex_base_worker-0.4.0/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,17 @@
+humanize==4.11.0
+peewee~=3.17
+Pillow==11.0.0
+python-gnupg==0.5.3
+shapely==2.0.6
+teklia-toolbox==0.1.7
+zstandard==0.23.0
+
+[docs]
+black==24.10.0
+mkdocs-material==9.5.48
+mkdocstrings-python==1.12.2
+
+[tests]
+pytest==8.3.4
+pytest-mock==3.14.0
+pytest-responses==0.5.1

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/arkindex_worker/image.py

@@ -2,13 +2,18 @@
 Helper methods to download and open IIIF images, and manage polygons.
 """
 
+import functools
+import os
 import re
+import tempfile
 from collections import namedtuple
+from collections.abc import Generator, Iterator
 from io import BytesIO
 from math import ceil
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import humanize
 import requests
 from PIL import Image
 from shapely.affinity import rotate, scale, translate
@@ -21,6 +26,7 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
+from arkindex_worker.utils import pluralize
 from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
@@ -39,8 +45,57 @@ IIIF_URL = re.compile(r"\w+:\/{2}.+\/.+\/.+\/.+\/(?P<size>.+)\/!?\d+\/\w+\.\w+")
 IIIF_FULL = "full"
 # Maximum size available
 IIIF_MAX = "max"
+# Ratio to resize image
+IMAGE_RATIO = [1, 0.9, 0.85, 0.80, 0.75, 0.70, 0.60, 0.50, 0.40, 0.30]
 
 
+def update_pillow_image_size_limit(func):
+    """
+    Update Pillow Image size limit
+    """
+
+    @functools.wraps(func)
+    def wrapper(
+        *args,
+        max_image_pixels: str | int | None = os.getenv("ARKINDEX_MAX_IMAGE_PIXELS"),
+        **kwargs,
+    ):
+        """
+        Wrapper to update Pillow Image size limit and restore it at the end of the function.
+
+        :param *args: Positional arguments passed to the function.
+        :param max_image_pixels: Pillow Image size limit to use.
+        :param **kwargs: Keyword arguments passed to the function.
+        """
+        MAX_IMAGE_PIXELS = Image.MAX_IMAGE_PIXELS
+
+        # Override Pillow Image size limit
+        if max_image_pixels is not None:
+            max_image_pixels = int(max_image_pixels)
+            # Override Pillow limit for detecting decompression bombs, disabled if set to 0
+            if max_image_pixels == 0:
+                logger.warning(
+                    "Pillow Image size limit is completely disabled, make sure you trust the image source."
+                )
+                Image.MAX_IMAGE_PIXELS = None
+            else:
+                Image.MAX_IMAGE_PIXELS = max_image_pixels
+
+        try:
+            results = func(*args, **kwargs)
+        except:
+            # Restore initial Pillow Image size limit
+            Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
+            raise
+
+        # Restore initial Pillow Image size limit
+        Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
+        return results
+
+    return wrapper
+
+
+@update_pillow_image_size_limit
 def open_image(
     path: str,
     mode: str | None = "RGB",
@@ -148,6 +203,70 @@ def upload_image(image: Image, url: str) -> requests.Response:
     return resp
 
 
+def resized_images(
+    *args,
+    element: "Element",
+    max_pixels: int | None = None,
+    max_bytes: int | None = None,
+    **kwargs,
+) -> Iterator[Generator[tempfile.NamedTemporaryFile, None, None]]:
+    """
+    Build resized images according to the pixel and byte limits.
+
+    :param *args: Positional arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
+    :param element: Element whose image needs to be resized.
+    :param max_pixels: Maximum pixel size of the resized images.
+    :param max_bytes: Maximum byte size of the resized images.
+    :param **kwargs: Keyword arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
+    :returns: An iterator of the temporary file of the resized image.
+    """
+    _, _, element_width, element_height = polygon_bounding_box(element.polygon)
+
+    logger.info(f"This element's image sizes are ({element_width} x {element_height}).")
+    if max_pixels and max(element_width, element_height) > max_pixels:
+        logger.warning(
+            f"Maximum image input size supported is ({max_pixels} x {max_pixels})."
+        )
+        logger.warning("The image will be resized.")
+
+    element_pixel, param = (
+        (element_width, "max_width")
+        if element_width > element_height
+        else (element_height, "max_height")
+    )
+
+    for resized_pixel in sorted(
+        set(
+            min(round(ratio * element_pixel), max_pixels or element_pixel)
+            for ratio in IMAGE_RATIO
+        ),
+        reverse=True,
+    ):
+        with element.open_image_tempfile(
+            *args, **{**kwargs, param: resized_pixel}
+        ) as image:
+            pillow_image = Image.open(image)
+            if (
+                pillow_image.width != element_width
+                or pillow_image.height != element_height
+            ):
+                logger.warning(
+                    f"The image was resized to ({pillow_image.width} x {pillow_image.height})."
+                )
+
+            # The image is still too large
+            image_size = Path(image.name).stat().st_size
+            if max_bytes and image_size > max_bytes:
+                logger.warning(f"The image size is {humanize.naturalsize(image_size)}.")
+                logger.warning(
+                    f"Maximum image input size supported is {humanize.naturalsize(max_bytes)}."
+                )
+                logger.warning("The image will be resized.")
+                continue
+
+            yield image
+
+
 def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
@@ -164,7 +283,7 @@ def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
 def _retry_log(retry_state, *args, **kwargs):
     logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
-        f"retrying in {retry_state.idle_for} seconds"
+        f'retrying in {retry_state.idle_for} {pluralize("second", retry_state.idle_for)}'
     )
 
 

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/arkindex_worker/models.py

@@ -261,6 +261,12 @@ class Transcription(ArkindexModel):
     """
 
 
+class Image(ArkindexModel):
+    """
+    Describes an Arkindex image.
+    """
+
+
 class Dataset(ArkindexModel):
     """
     Describes an Arkindex dataset.

{arkindex-base-worker-0.3.7rc9 → arkindex_base_worker-0.4.0}/arkindex_worker/utils.py

@@ -1,15 +1,41 @@
 import hashlib
+import inspect
 import logging
 import os
 import tarfile
 import tempfile
+from collections.abc import Callable, Generator
+from itertools import islice
 from pathlib import Path
+from typing import Any
 
-import zstandard
 import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
+
+def pluralize(singular: str, count: int) -> str:
+    """Pluralize a noun, if necessary, using simplified rules of English pluralization and a list of exceptions.
+
+    :param str singular: A singular noun describing an object
+    :param int count: The object count, to determine whether to pluralize or not
+    :return str: The noun in its singular or plural form
+    """
+    if count == 1:
+        return singular
+
+    some_exceptions = {
+        "child": "children",
+        "class": "classes",
+        "entity": "entities",
+        "metadata": "metadata",
+    }
+    if singular in some_exceptions:
+        return some_exceptions[singular]
+
+    return singular + "s"
+
+
 MANUAL_SOURCE = "manual"
 
 
@@ -38,7 +64,7 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
     :param compressed_archive: Path to the target ZST-compressed archive
     :return: File descriptor and path to the uncompressed tar archive
     """
-    dctx = zstandard.ZstdDecompressor()
+    dctx = zstd.ZstdDecompressor()
     archive_fd, archive_path = tempfile.mkstemp(suffix=".tar")
     archive_path = Path(archive_path)
 
@@ -50,7 +76,7 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
         ):
             dctx.copy_stream(compressed, decompressed)
         logger.debug(f"Successfully uncompressed archive {compressed_archive}")
-    except zstandard.ZstdError as e:
+    except zstd.ZstdError as e:
         raise Exception(f"Couldn't uncompressed archive: {e}") from e
 
     return archive_fd, archive_path
@@ -129,7 +155,7 @@ def zstd_compress(
                 archive_hasher.update(compressed_chunk)
                 archive_file.write(compressed_chunk)
         logger.debug(f"Successfully compressed {source}")
-    except zstandard.ZstdError as e:
+    except zstd.ZstdError as e:
         raise Exception(f"Couldn't compress archive: {e}") from e
     return file_d, destination, archive_hasher.hexdigest()
 
@@ -197,3 +223,58 @@ def create_tar_zst_archive(
     close_delete_file(tar_fd, tar_archive)
 
     return zst_fd, zst_archive, zst_hash, tar_hash
+
+
+DEFAULT_BATCH_SIZE = 50
+"""Batch size used for bulk publication to Arkindex"""
+
+
+def batch_publication(func: Callable) -> Callable:
+    """
+    Decorator for functions that should raise an error when the value passed through the ``batch_size`` parameter is **not** a strictly positive integer.
+
+    :param func: The function to wrap with the ``batch_size`` check
+    :return: The function passing the ``batch_size`` check
+    """
+    signature = inspect.signature(func)
+
+    def wrapper(self, *args, **kwargs):
+        bound_func = signature.bind(self, *args, **kwargs)
+        bound_func.apply_defaults()
+        batch_size = bound_func.arguments.get("batch_size")
+        assert (
+            batch_size is not None and isinstance(batch_size, int) and batch_size > 0
+        ), "batch_size shouldn't be null and should be a strictly positive integer"
+
+        return func(self, *args, **kwargs)
+
+    wrapper.__name__ = func.__name__
+    return wrapper
+
+
+def make_batches(
+    objects: list, singular_name: str, batch_size: int
+) -> Generator[list[Any]]:
+    """Split an object list in successive batches of maximum size ``batch_size``.
+
+    :param objects: The object list to divide in batches of ``batch_size`` size
+    :param singular_name: The singular form of the noun associated with the object list
+    :param batch_size: The maximum size of each batch to split the object list
+    :return: A generator of successive batches containing ``batch_size`` items from ``objects``
+    """
+    count = len(objects)
+    logger.info(
+        f"Creating batches of size {batch_size} to process {count} {pluralize(singular_name, count)}"
+    )
+
+    index = 1
+    iterator = iter(objects)
+    while batch := list(islice(iterator, batch_size)):
+        count = len(batch)
+        logger.info(
+            f"Processing batch {index} containing {count} {pluralize(singular_name, count)}..."
+        )
+
+        yield batch
+
+        index += 1