arkindex-base-worker 0.3.7rc4__py3-none-any.whl → 0.5.0a1__py3-none-any.whl

{arkindex_base_worker-0.3.7rc4.dist-info → arkindex_base_worker-0.5.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: arkindex-base-worker
-Version: 0.3.7rc4
+Version: 0.5.0a1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,26 +37,25 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client ==1.0.14
-Requires-Dist: peewee ==3.17.0
-Requires-Dist: Pillow ==10.2.0
-Requires-Dist: pymdown-extensions ==10.7
-Requires-Dist: python-gnupg ==0.5.2
-Requires-Dist: shapely ==2.0.3
-Requires-Dist: tenacity ==8.2.3
-Requires-Dist: zstandard ==0.22.0
+Requires-Dist: humanize==4.11.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==11.0.0
+Requires-Dist: python-gnupg==0.5.3
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
-Requires-Dist: black ==24.2.0 ; extra == 'docs'
-Requires-Dist: doc8 ==1.1.1 ; extra == 'docs'
-Requires-Dist: mkdocs ==1.5.3 ; extra == 'docs'
-Requires-Dist: mkdocs-material ==9.5.10 ; extra == 'docs'
-Requires-Dist: mkdocstrings ==0.24.0 ; extra == 'docs'
-Requires-Dist: mkdocstrings-python ==1.8.0 ; extra == 'docs'
-Requires-Dist: recommonmark ==0.7.1 ; extra == 'docs'
+Requires-Dist: black==24.10.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 
@@ -70,7 +69,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

arkindex_base_worker-0.5.0a1.dist-info/RECORD

@@ -0,0 +1,61 @@
+arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,162
+arkindex_worker/cache.py,sha256=qTblc_zKdYC47Wip6_O9Jf5qBkQW2ozQQrg-nsx1WuY,11221
+arkindex_worker/image.py,sha256=D4CdTZKbzFULdRNy-flsilAdfNPP2WSV01dkxQnfGeA,20770
+arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
+arkindex_worker/utils.py,sha256=MbbJT8oh8DMHHR-vidFeXdUH0TSXGWm7ZDGWzrRXoEY,9933
+arkindex_worker/worker/__init__.py,sha256=0_YHeOe31KR_8ynbnYMIMwnSQTVbKkkeLGmnlTMhFx0,16234
+arkindex_worker/worker/base.py,sha256=7Pmw-UQSxV-xkW8NO5cXsxJ8W8szzyppMaNjq_az81A,19844
+arkindex_worker/worker/classification.py,sha256=zECSNzGCZFzoPoDVZN4kuGYRNLzMQLBaRt3q1jnBSaA,10952
+arkindex_worker/worker/corpus.py,sha256=0TQFOwZ6Te-CZi6lgkZY1wzyJ5wO9LAmcVQtqHvZpPk,2291
+arkindex_worker/worker/dataset.py,sha256=LwzKwNFX4FqfLxh29LSvJydPwRw3VHaB1wjuFhUshsE,5267
+arkindex_worker/worker/element.py,sha256=Qvvq9kJnAHNATHW7zi96eIY1x-0MsR-T5rrSJg6e9Y4,45309
+arkindex_worker/worker/entity.py,sha256=DG8oVAdy-r18fliTjnzGI1j6l7SOFmyIBmE6JlE6A8g,14799
+arkindex_worker/worker/image.py,sha256=t_Az6IGnj0EZyvcA4XxfPikOUjn_pztgsyxTkFZhaXU,621
+arkindex_worker/worker/metadata.py,sha256=VRajtd2kaBvar9GercX4knvR6l1WFYjoCdJWU9ccKgk,7291
+arkindex_worker/worker/process.py,sha256=9TEHpMcBax1wc6PrWMMrdXe2uNfqyVj7n_dAYZRBGnY,1854
+arkindex_worker/worker/task.py,sha256=r1j7_qbdNu2Z8H8HbGzO3P3qdx-2N1pBbUPFDca0rqg,1519
+arkindex_worker/worker/training.py,sha256=H8FmCdzGcDW-WMMwcgvmZPlN5tPHwGo0BXn12qmzj8g,10875
+arkindex_worker/worker/transcription.py,sha256=52RY9kYsiR1sz9FxOigyo12Ker3VDbQ4U42gK9DpR3g,21146
+arkindex_worker/worker/version.py,sha256=JIT7OI3Mo7RPkNrjOB9hfqrsG-FYygz_zi4l8PbkuAo,1960
+hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
+tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
+tests/conftest.py,sha256=Z9amrKmVtFltzTUUm07fGDrT4m540biaTpjedmplyzc,21536
+tests/test_base_worker.py,sha256=2EIYcd_3f9O0zB5WiGIQV0Cn9wndLvnEnSfcAE1qWWU,30607
+tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
+tests/test_dataset_worker.py,sha256=z8ydliUlwW2j-irgLAotJMacgJXkVvF5TgsWLyCn1Jo,22087
+tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
+tests/test_image.py,sha256=mYyRfDXGLLzcQQtmaM7GR3jt7ScxsLLog16pUVHrH3M,27824
+tests/test_merge.py,sha256=TuOeUS0UCz66DPOQFFhc4NQBxIjZL9f5czi4XnvGrr4,8270
+tests/test_utils.py,sha256=nYL1s2ViZoLoMiNpLGDaWwxf8dJ1D8aT522AO-PVaEQ,3607
+tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
+tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
+tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
+tests/test_elements_worker/test_corpus.py,sha256=kscJyM8k1njYJJFGuvliVzn89lWh41mEyDCCawnp3W8,5483
+tests/test_elements_worker/test_dataset.py,sha256=00IlOZv9YFlZ23rGXyR-HLbKLQxGelZ1Bf9lEZYA0IY,11412
+tests/test_elements_worker/test_element.py,sha256=lb5tLjl0jsixX0OWVhBAaKLE9GKkBw79kFHDNGommaQ,12535
+tests/test_elements_worker/test_element_create_multiple.py,sha256=arYFGmxc0517ZUii6k__G_UQQatuNIASTC8MXvUrSwk,21887
+tests/test_elements_worker/test_element_create_single.py,sha256=Fa9zm12J2rQ3VrUe3yIlHAc7Vty_eQYb_YGnNPQB3IE,16697
+tests/test_elements_worker/test_element_list_children.py,sha256=2zH4h663w3EduqpzQr-7bf9zIDzO1x2WxdUYYHsIHkI,31358
+tests/test_elements_worker/test_element_list_parents.py,sha256=TXeGW-a3W-7GmB2QrhJH9mMnvxuybeAwQ4tL3iIxwXo,16734
+tests/test_elements_worker/test_entity_create.py,sha256=9Tjr9KA2yo44VFV283q_cs6XbbVguUMDNfCj-DILSJg,29353
+tests/test_elements_worker/test_entity_list_and_check.py,sha256=ENBLaqbXlRUDbHRvQla3080a0HJltrWAPYWNohUA9NU,4992
+tests/test_elements_worker/test_image.py,sha256=BljMNKgec_9a5bzNzFpYZIvSbuvwsWDfdqLHVJaTa7M,2079
+tests/test_elements_worker/test_metadata.py,sha256=Xfggy-vxw5DZ3hFKx3sB7OYb2d1tu1RiNK8fvKJIaBs,22294
+tests/test_elements_worker/test_process.py,sha256=y4RoVhPfyHzR795fw7-_FXElBcKo3fy4Ew_HI-kxJic,3088
+tests/test_elements_worker/test_task.py,sha256=wTUWqN9UhfKmJn3IcFY75EW4I1ulRhisflmY1kmP47s,5574
+tests/test_elements_worker/test_training.py,sha256=3W2LzpqxekvRiX42m_PvWcVel7ynQJmzO8gKcLmCMQI,8717
+tests/test_elements_worker/test_transcription_create.py,sha256=yznO9B_BVsOR0Z_VY5ZL8gJp0ZPCz_4sPUs5dXtixAg,29281
+tests/test_elements_worker/test_transcription_create_with_elements.py,sha256=tmcyglgssEqMnt1Mdy_u6X1m2wgLWTo_HdWst3GrK2k,33056
+tests/test_elements_worker/test_transcription_list.py,sha256=ikz7HYPCoQWTdTRCd382SB-y-T2BbigPLlIcx5Eow-I,15324
+tests/test_elements_worker/test_version.py,sha256=xqCgcgukTFJzkMgYfQG-8mTbu0o2fdYjWC07FktThfw,2125
+tests/test_elements_worker/test_worker.py,sha256=HDw_UQdiMUzlBd4-jRvC-B3pNrZmmpps4sfZ9a87JVY,25378
+worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
+worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
+worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
+worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
+arkindex_base_worker-0.5.0a1.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.5.0a1.dist-info/METADATA,sha256=eP4wgAkBFUHBWvNVcASdUXsxxz_0AMtQTgjJPuBlCCQ,3336
+arkindex_base_worker-0.5.0a1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+arkindex_base_worker-0.5.0a1.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
+arkindex_base_worker-0.5.0a1.dist-info/RECORD,,

{arkindex_base_worker-0.3.7rc4.dist-info → arkindex_base_worker-0.5.0a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{arkindex_base_worker-0.3.7rc4.dist-info → arkindex_base_worker-0.5.0a1.dist-info}/top_level.txt

@@ -1,2 +1,4 @@
 arkindex_worker
+hooks
 tests
+worker-demo

arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

arkindex_worker/image.py

@@ -2,13 +2,20 @@
 Helper methods to download and open IIIF images, and manage polygons.
 """
 
+import base64
+import functools
+import os
 import re
+import tempfile
 from collections import namedtuple
+from collections.abc import Generator, Iterator
 from io import BytesIO
 from math import ceil
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import humanize
+import numpy as np
 import requests
 from PIL import Image
 from shapely.affinity import rotate, scale, translate
@@ -21,6 +28,8 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
+from arkindex_worker.utils import pluralize
+from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
 if TYPE_CHECKING:
@@ -38,8 +47,57 @@ IIIF_URL = re.compile(r"\w+:\/{2}.+\/.+\/.+\/.+\/(?P<size>.+)\/!?\d+\/\w+\.\w+")
 IIIF_FULL = "full"
 # Maximum size available
 IIIF_MAX = "max"
+# Ratios to resize images: 1.0, 0.95, [...], 0.1, 0.05
+IMAGE_RATIOS = np.arange(1, 0, -0.05).round(2).tolist()
 
 
+def update_pillow_image_size_limit(func):
+    """
+    Update Pillow Image size limit
+    """
+
+    @functools.wraps(func)
+    def wrapper(
+        *args,
+        max_image_pixels: str | int | None = os.getenv("ARKINDEX_MAX_IMAGE_PIXELS"),
+        **kwargs,
+    ):
+        """
+        Wrapper to update Pillow Image size limit and restore it at the end of the function.
+
+        :param *args: Positional arguments passed to the function.
+        :param max_image_pixels: Pillow Image size limit to use.
+        :param **kwargs: Keyword arguments passed to the function.
+        """
+        MAX_IMAGE_PIXELS = Image.MAX_IMAGE_PIXELS
+
+        # Override Pillow Image size limit
+        if max_image_pixels is not None:
+            max_image_pixels = int(max_image_pixels)
+            # Override Pillow limit for detecting decompression bombs, disabled if set to 0
+            if max_image_pixels == 0:
+                logger.warning(
+                    "Pillow Image size limit is completely disabled, make sure you trust the image source."
+                )
+                Image.MAX_IMAGE_PIXELS = None
+            else:
+                Image.MAX_IMAGE_PIXELS = max_image_pixels
+
+        try:
+            results = func(*args, **kwargs)
+        except:
+            # Restore initial Pillow Image size limit
+            Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
+            raise
+
+        # Restore initial Pillow Image size limit
+        Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
+        return results
+
+    return wrapper
+
+
+@update_pillow_image_size_limit
 def open_image(
     path: str,
     mode: str | None = "RGB",
@@ -147,6 +205,111 @@ def upload_image(image: Image, url: str) -> requests.Response:
     return resp
 
 
+def resized_images(
+    *args,
+    element: "Element",
+    max_pixels_short: int | None = None,
+    max_pixels_long: int | None = None,
+    max_bytes: int | None = None,
+    use_base64: bool = False,
+    **kwargs,
+) -> Iterator[Generator[tempfile._TemporaryFileWrapper | str]]:
+    """
+    Build resized images according to pixel and byte limits.
+
+    :param *args: Positional arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
+    :param element: Element whose image needs to be resized.
+    :param max_pixels_short: Maximum pixel size of the resized images' short side.
+    :param max_pixels_long: Maximum pixel size of the resized images' long side.
+    :param max_bytes: Maximum byte size of the resized images.
+    :param use_base64: Whether or not to encode resized images in base64 before calculating their size.
+    :param **kwargs: Keyword arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
+    :returns: An iterator of temporary files for resized images OR an iterator of base64-encoded strings if `use_base64` is set.
+    """
+    _, _, element_width, element_height = polygon_bounding_box(element.polygon)
+    logger.info(
+        f"This element's image dimensions are ({element_width} x {element_height})."
+    )
+
+    portrait_format = element_width <= element_height
+    max_pixels_width, max_pixels_height = (
+        (max_pixels_short, max_pixels_long)
+        if portrait_format
+        else (max_pixels_long, max_pixels_short)
+    )
+
+    # The image dimension is already within the pixel limitation, no need to resize the image
+    if max_pixels_width and max_pixels_width >= element_width:
+        max_pixels_width = None
+    if max_pixels_height and max_pixels_height >= element_height:
+        max_pixels_height = None
+
+    if (max_pixels_width and element_width > max_pixels_width) or (
+        max_pixels_height and element_height > max_pixels_height
+    ):
+        logger.warning(
+            f"Maximum image dimensions supported are ({max_pixels_width or element_width} x {max_pixels_height or element_height})."
+        )
+        logger.warning("The image will be resized.")
+
+    # No limitations provided, we keep the image initial dimensions
+    if max_pixels_width is None and max_pixels_height is None:
+        open_image_param, max_value = (
+            ("max_height", element_height)
+            if portrait_format
+            else ("max_width", element_width)
+        )
+    # A limitation is only given for the height, we resize it
+    elif max_pixels_width is None:
+        open_image_param, max_value = ("max_height", max_pixels_height)
+    # A limitation is only given for the width, we resize it
+    elif max_pixels_height is None:
+        open_image_param, max_value = ("max_width", max_pixels_width)
+    # Limitations are provided for both sides:
+    # - we resize only the one with the biggest scale factor
+    # - the remaining one will automatically fall within the other limitation
+    else:
+        width_rescaling_factor = element_width / max_pixels_width
+        height_rescaling_factor = element_height / max_pixels_height
+        open_image_param, max_value = (
+            ("max_height", max_pixels_height)
+            if height_rescaling_factor > width_rescaling_factor
+            else ("max_width", max_pixels_width)
+        )
+
+    resized_pixels = set(
+        min(round(ratio * max_value), max_value) for ratio in IMAGE_RATIOS
+    )
+    for resized_pixel in sorted(resized_pixels, reverse=True):
+        with element.open_image_tempfile(
+            *args, **{**kwargs, open_image_param: resized_pixel}
+        ) as image:
+            pillow_image = Image.open(image)
+            if (
+                pillow_image.width != element_width
+                or pillow_image.height != element_height
+            ):
+                logger.warning(
+                    f"The image was resized to ({pillow_image.width} x {pillow_image.height})."
+                )
+
+            image_size = Path(image.name).stat().st_size
+            if use_base64:
+                image = base64.b64encode(Path(image.name).read_bytes()).decode("utf-8")
+                image_size = len(image)
+
+            # The image is still too heavy
+            if max_bytes and image_size > max_bytes:
+                logger.warning(f"The image size is {humanize.naturalsize(image_size)}.")
+                logger.warning(
+                    f"Maximum image input size supported is {humanize.naturalsize(max_bytes)}."
+                )
+                logger.warning("The image will be resized.")
+                continue
+
+            yield image
+
+
 def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
@@ -163,7 +326,7 @@ def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
 def _retry_log(retry_state, *args, **kwargs):
     logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
-        f"retrying in {retry_state.idle_for} seconds"
+        f'retrying in {retry_state.idle_for} {pluralize("second", retry_state.idle_for)}'
     )
 
 
@@ -175,7 +338,9 @@ def _retry_log(retry_state, *args, **kwargs):
     reraise=True,
 )
 def _retried_request(url, *args, method=requests.get, **kwargs):
-    resp = method(url, *args, timeout=DOWNLOAD_TIMEOUT, **kwargs)
+    resp = method(
+        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+    )
     resp.raise_for_status()
     return resp
 

arkindex_worker/models.py

@@ -20,6 +20,8 @@ class MagicDict(dict):
         Automagically convert lists and dicts to MagicDicts and lists of MagicDicts
         Allows for nested access: foo.bar.baz
         """
+        if isinstance(item, Dataset):
+            return item
         if isinstance(item, list):
             return list(map(self._magify, item))
         if isinstance(item, dict):
@@ -259,6 +261,12 @@ class Transcription(ArkindexModel):
     """
 
 
+class Image(ArkindexModel):
+    """
+    Describes an Arkindex image.
+    """
+
+
 class Dataset(ArkindexModel):
     """
     Describes an Arkindex dataset.
@@ -272,6 +280,16 @@ class Dataset(ArkindexModel):
         return f"{self.id}.tar.zst"
 
 
+class Set(MagicDict):
+    """
+    Describes an Arkindex dataset set.
+    """
+
+    def __str__(self):
+        # Not using ArkindexModel.__str__ as we do not retrieve the Set ID
+        return f"{self.__class__.__name__} ({self.name}) from {self.dataset}"
+
+
 class Artifact(ArkindexModel):
     """
     Describes an Arkindex artifact.

arkindex_worker/utils.py

@@ -1,15 +1,54 @@
 import hashlib
+import inspect
 import logging
 import os
 import tarfile
 import tempfile
+from collections.abc import Callable, Generator
+from itertools import islice
 from pathlib import Path
+from typing import Any
 
-import zstandard
 import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
+
+def pluralize(singular: str, count: int) -> str:
+    """Pluralize a noun, if necessary, using simplified rules of English pluralization and a list of exceptions.
+
+    :param str singular: A singular noun describing an object
+    :param int count: The object count, to determine whether to pluralize or not
+    :return str: The noun in its singular or plural form
+    """
+    if count == 1:
+        return singular
+
+    some_exceptions = {
+        "child": "children",
+        "class": "classes",
+        "entity": "entities",
+        "metadata": "metadata",
+    }
+    if singular in some_exceptions:
+        return some_exceptions[singular]
+
+    return singular + "s"
+
+
+MANUAL_SOURCE = "manual"
+
+
+def parse_source_id(value: str) -> bool | str | None:
+    """
+    Parse a UUID argument (Worker Version, Worker Run, ...) to use it directly in the API.
+    Arkindex API filters generally expect `False` to filter manual sources.
+    """
+    if value == MANUAL_SOURCE:
+        return False
+    return value or None
+
+
 CHUNK_SIZE = 1024
 """Chunk Size used for ZSTD compression"""
 
@@ -25,7 +64,7 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
     :param compressed_archive: Path to the target ZST-compressed archive
     :return: File descriptor and path to the uncompressed tar archive
     """
-    dctx = zstandard.ZstdDecompressor()
+    dctx = zstd.ZstdDecompressor()
     archive_fd, archive_path = tempfile.mkstemp(suffix=".tar")
     archive_path = Path(archive_path)
 
@@ -37,7 +76,7 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
         ):
             dctx.copy_stream(compressed, decompressed)
         logger.debug(f"Successfully uncompressed archive {compressed_archive}")
-    except zstandard.ZstdError as e:
+    except zstd.ZstdError as e:
         raise Exception(f"Couldn't uncompressed archive: {e}") from e
 
     return archive_fd, archive_path
@@ -116,7 +155,7 @@ def zstd_compress(
                 archive_hasher.update(compressed_chunk)
                 archive_file.write(compressed_chunk)
         logger.debug(f"Successfully compressed {source}")
-    except zstandard.ZstdError as e:
+    except zstd.ZstdError as e:
         raise Exception(f"Couldn't compress archive: {e}") from e
     return file_d, destination, archive_hasher.hexdigest()
 
@@ -184,3 +223,58 @@ def create_tar_zst_archive(
     close_delete_file(tar_fd, tar_archive)
 
     return zst_fd, zst_archive, zst_hash, tar_hash
+
+
+DEFAULT_BATCH_SIZE = 50
+"""Batch size used for bulk publication to Arkindex"""
+
+
+def batch_publication(func: Callable) -> Callable:
+    """
+    Decorator for functions that should raise an error when the value passed through the ``batch_size`` parameter is **not** a strictly positive integer.
+
+    :param func: The function to wrap with the ``batch_size`` check
+    :return: The function passing the ``batch_size`` check
+    """
+    signature = inspect.signature(func)
+
+    def wrapper(self, *args, **kwargs):
+        bound_func = signature.bind(self, *args, **kwargs)
+        bound_func.apply_defaults()
+        batch_size = bound_func.arguments.get("batch_size")
+        assert (
+            batch_size is not None and isinstance(batch_size, int) and batch_size > 0
+        ), "batch_size shouldn't be null and should be a strictly positive integer"
+
+        return func(self, *args, **kwargs)
+
+    wrapper.__name__ = func.__name__
+    return wrapper
+
+
+def make_batches(
+    objects: list, singular_name: str, batch_size: int
+) -> Generator[list[Any]]:
+    """Split an object list in successive batches of maximum size ``batch_size``.
+
+    :param objects: The object list to divide in batches of ``batch_size`` size
+    :param singular_name: The singular form of the noun associated with the object list
+    :param batch_size: The maximum size of each batch to split the object list
+    :return: A generator of successive batches containing ``batch_size`` items from ``objects``
+    """
+    count = len(objects)
+    logger.info(
+        f"Creating batches of size {batch_size} to process {count} {pluralize(singular_name, count)}"
+    )
+
+    index = 1
+    iterator = iter(objects)
+    while batch := list(islice(iterator, batch_size)):
+        count = len(batch)
+        logger.info(
+            f"Processing batch {index} containing {count} {pluralize(singular_name, count)}..."
+        )
+
+        yield batch
+
+        index += 1