arkindex-base-worker 0.3.7rc5__tar.gz → 0.5.0a1__tar.gz

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: arkindex-base-worker
-Version: 0.3.7rc5
+Version: 0.5.0a1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,26 +37,25 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client==1.0.14
-Requires-Dist: peewee==3.17.0
-Requires-Dist: Pillow==10.2.0
-Requires-Dist: pymdown-extensions==10.7
-Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.3
-Requires-Dist: tenacity==8.2.3
-Requires-Dist: zstandard==0.22.0
+Requires-Dist: humanize==4.11.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==11.0.0
+Requires-Dist: python-gnupg==0.5.3
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
-Requires-Dist: black==24.2.0; extra == "docs"
-Requires-Dist: doc8==1.1.1; extra == "docs"
-Requires-Dist: mkdocs==1.5.3; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.10; extra == "docs"
-Requires-Dist: mkdocstrings==0.24.0; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.8.0; extra == "docs"
-Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Requires-Dist: black==24.10.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 
@@ -70,7 +69,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/README.md

@@ -10,7 +10,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: arkindex-base-worker
-Version: 0.3.7rc5
+Version: 0.5.0a1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,26 +37,25 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client==1.0.14
-Requires-Dist: peewee==3.17.0
-Requires-Dist: Pillow==10.2.0
-Requires-Dist: pymdown-extensions==10.7
-Requires-Dist: python-gnupg==0.5.2
-Requires-Dist: shapely==2.0.3
-Requires-Dist: tenacity==8.2.3
-Requires-Dist: zstandard==0.22.0
+Requires-Dist: humanize==4.11.0
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==11.0.0
+Requires-Dist: python-gnupg==0.5.3
+Requires-Dist: shapely==2.0.6
+Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
-Requires-Dist: black==24.2.0; extra == "docs"
-Requires-Dist: doc8==1.1.1; extra == "docs"
-Requires-Dist: mkdocs==1.5.3; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.10; extra == "docs"
-Requires-Dist: mkdocstrings==0.24.0; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.8.0; extra == "docs"
-Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Requires-Dist: black==24.10.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 
@@ -70,7 +69,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -1,9 +1,6 @@
 LICENSE
 README.md
-docs-requirements.txt
 pyproject.toml
-requirements.txt
-setup.py
 arkindex_base_worker.egg-info/PKG-INFO
 arkindex_base_worker.egg-info/SOURCES.txt
 arkindex_base_worker.egg-info/dependency_links.txt
@@ -17,14 +14,18 @@ arkindex_worker/utils.py
 arkindex_worker/worker/__init__.py
 arkindex_worker/worker/base.py
 arkindex_worker/worker/classification.py
+arkindex_worker/worker/corpus.py
 arkindex_worker/worker/dataset.py
 arkindex_worker/worker/element.py
 arkindex_worker/worker/entity.py
+arkindex_worker/worker/image.py
 arkindex_worker/worker/metadata.py
+arkindex_worker/worker/process.py
 arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
 arkindex_worker/worker/version.py
+hooks/pre_gen_project.py
 tests/__init__.py
 tests/conftest.py
 tests/test_base_worker.py
@@ -35,13 +36,29 @@ tests/test_image.py
 tests/test_merge.py
 tests/test_utils.py
 tests/test_elements_worker/__init__.py
-tests/test_elements_worker/test_classifications.py
+tests/test_elements_worker/test_classification.py
 tests/test_elements_worker/test_cli.py
+tests/test_elements_worker/test_corpus.py
 tests/test_elements_worker/test_dataset.py
-tests/test_elements_worker/test_elements.py
-tests/test_elements_worker/test_entities.py
+tests/test_elements_worker/test_element.py
+tests/test_elements_worker/test_element_create_multiple.py
+tests/test_elements_worker/test_element_create_single.py
+tests/test_elements_worker/test_element_list_children.py
+tests/test_elements_worker/test_element_list_parents.py
+tests/test_elements_worker/test_entity_create.py
+tests/test_elements_worker/test_entity_list_and_check.py
+tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
+tests/test_elements_worker/test_process.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
-tests/test_elements_worker/test_transcriptions.py
-tests/test_elements_worker/test_worker.py
+tests/test_elements_worker/test_transcription_create.py
+tests/test_elements_worker/test_transcription_create_with_elements.py
+tests/test_elements_worker/test_transcription_list.py
+tests/test_elements_worker/test_version.py
+tests/test_elements_worker/test_worker.py
+worker-demo/tests/__init__.py
+worker-demo/tests/conftest.py
+worker-demo/tests/test_worker.py
+worker-demo/worker_demo/__init__.py
+worker-demo/worker_demo/worker.py

arkindex_base_worker-0.5.0a1/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,17 @@
+humanize==4.11.0
+peewee~=3.17
+Pillow==11.0.0
+python-gnupg==0.5.3
+shapely==2.0.6
+teklia-toolbox==0.1.7
+zstandard==0.23.0
+
+[docs]
+black==24.10.0
+mkdocs-material==9.5.48
+mkdocstrings-python==1.12.2
+
+[tests]
+pytest==8.3.4
+pytest-mock==3.14.0
+pytest-responses==0.5.1

arkindex_base_worker-0.5.0a1/arkindex_base_worker.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+arkindex_worker
+dist
+docs
+hooks
+tests
+worker-demo

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if not (self.is_read_only or self.use_cache):
+        if self.use_cache:
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/arkindex_worker/image.py

@@ -2,13 +2,20 @@
 Helper methods to download and open IIIF images, and manage polygons.
 """
 
+import base64
+import functools
+import os
 import re
+import tempfile
 from collections import namedtuple
+from collections.abc import Generator, Iterator
 from io import BytesIO
 from math import ceil
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import humanize
+import numpy as np
 import requests
 from PIL import Image
 from shapely.affinity import rotate, scale, translate
@@ -21,6 +28,8 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
+from arkindex_worker.utils import pluralize
+from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
 if TYPE_CHECKING:
@@ -38,8 +47,57 @@ IIIF_URL = re.compile(r"\w+:\/{2}.+\/.+\/.+\/.+\/(?P<size>.+)\/!?\d+\/\w+\.\w+")
 IIIF_FULL = "full"
 # Maximum size available
 IIIF_MAX = "max"
+# Ratios to resize images: 1.0, 0.95, [...], 0.1, 0.05
+IMAGE_RATIOS = np.arange(1, 0, -0.05).round(2).tolist()
 
 
+def update_pillow_image_size_limit(func):
+    """
+    Update Pillow Image size limit
+    """
+
+    @functools.wraps(func)
+    def wrapper(
+        *args,
+        max_image_pixels: str | int | None = os.getenv("ARKINDEX_MAX_IMAGE_PIXELS"),
+        **kwargs,
+    ):
+        """
+        Wrapper to update Pillow Image size limit and restore it at the end of the function.
+
+        :param *args: Positional arguments passed to the function.
+        :param max_image_pixels: Pillow Image size limit to use.
+        :param **kwargs: Keyword arguments passed to the function.
+        """
+        MAX_IMAGE_PIXELS = Image.MAX_IMAGE_PIXELS
+
+        # Override Pillow Image size limit
+        if max_image_pixels is not None:
+            max_image_pixels = int(max_image_pixels)
+            # Override Pillow limit for detecting decompression bombs, disabled if set to 0
+            if max_image_pixels == 0:
+                logger.warning(
+                    "Pillow Image size limit is completely disabled, make sure you trust the image source."
+                )
+                Image.MAX_IMAGE_PIXELS = None
+            else:
+                Image.MAX_IMAGE_PIXELS = max_image_pixels
+
+        try:
+            results = func(*args, **kwargs)
+        except:
+            # Restore initial Pillow Image size limit
+            Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
+            raise
+
+        # Restore initial Pillow Image size limit
+        Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
+        return results
+
+    return wrapper
+
+
+@update_pillow_image_size_limit
 def open_image(
     path: str,
     mode: str | None = "RGB",
@@ -147,6 +205,111 @@ def upload_image(image: Image, url: str) -> requests.Response:
     return resp
 
 
+def resized_images(
+    *args,
+    element: "Element",
+    max_pixels_short: int | None = None,
+    max_pixels_long: int | None = None,
+    max_bytes: int | None = None,
+    use_base64: bool = False,
+    **kwargs,
+) -> Iterator[Generator[tempfile._TemporaryFileWrapper | str]]:
+    """
+    Build resized images according to pixel and byte limits.
+
+    :param *args: Positional arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
+    :param element: Element whose image needs to be resized.
+    :param max_pixels_short: Maximum pixel size of the resized images' short side.
+    :param max_pixels_long: Maximum pixel size of the resized images' long side.
+    :param max_bytes: Maximum byte size of the resized images.
+    :param use_base64: Whether or not to encode resized images in base64 before calculating their size.
+    :param **kwargs: Keyword arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
+    :returns: An iterator of temporary files for resized images OR an iterator of base64-encoded strings if `use_base64` is set.
+    """
+    _, _, element_width, element_height = polygon_bounding_box(element.polygon)
+    logger.info(
+        f"This element's image dimensions are ({element_width} x {element_height})."
+    )
+
+    portrait_format = element_width <= element_height
+    max_pixels_width, max_pixels_height = (
+        (max_pixels_short, max_pixels_long)
+        if portrait_format
+        else (max_pixels_long, max_pixels_short)
+    )
+
+    # The image dimension is already within the pixel limitation, no need to resize the image
+    if max_pixels_width and max_pixels_width >= element_width:
+        max_pixels_width = None
+    if max_pixels_height and max_pixels_height >= element_height:
+        max_pixels_height = None
+
+    if (max_pixels_width and element_width > max_pixels_width) or (
+        max_pixels_height and element_height > max_pixels_height
+    ):
+        logger.warning(
+            f"Maximum image dimensions supported are ({max_pixels_width or element_width} x {max_pixels_height or element_height})."
+        )
+        logger.warning("The image will be resized.")
+
+    # No limitations provided, we keep the image initial dimensions
+    if max_pixels_width is None and max_pixels_height is None:
+        open_image_param, max_value = (
+            ("max_height", element_height)
+            if portrait_format
+            else ("max_width", element_width)
+        )
+    # A limitation is only given for the height, we resize it
+    elif max_pixels_width is None:
+        open_image_param, max_value = ("max_height", max_pixels_height)
+    # A limitation is only given for the width, we resize it
+    elif max_pixels_height is None:
+        open_image_param, max_value = ("max_width", max_pixels_width)
+    # Limitations are provided for both sides:
+    # - we resize only the one with the biggest scale factor
+    # - the remaining one will automatically fall within the other limitation
+    else:
+        width_rescaling_factor = element_width / max_pixels_width
+        height_rescaling_factor = element_height / max_pixels_height
+        open_image_param, max_value = (
+            ("max_height", max_pixels_height)
+            if height_rescaling_factor > width_rescaling_factor
+            else ("max_width", max_pixels_width)
+        )
+
+    resized_pixels = set(
+        min(round(ratio * max_value), max_value) for ratio in IMAGE_RATIOS
+    )
+    for resized_pixel in sorted(resized_pixels, reverse=True):
+        with element.open_image_tempfile(
+            *args, **{**kwargs, open_image_param: resized_pixel}
+        ) as image:
+            pillow_image = Image.open(image)
+            if (
+                pillow_image.width != element_width
+                or pillow_image.height != element_height
+            ):
+                logger.warning(
+                    f"The image was resized to ({pillow_image.width} x {pillow_image.height})."
+                )
+
+            image_size = Path(image.name).stat().st_size
+            if use_base64:
+                image = base64.b64encode(Path(image.name).read_bytes()).decode("utf-8")
+                image_size = len(image)
+
+            # The image is still too heavy
+            if max_bytes and image_size > max_bytes:
+                logger.warning(f"The image size is {humanize.naturalsize(image_size)}.")
+                logger.warning(
+                    f"Maximum image input size supported is {humanize.naturalsize(max_bytes)}."
+                )
+                logger.warning("The image will be resized.")
+                continue
+
+            yield image
+
+
 def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
@@ -163,7 +326,7 @@ def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
 def _retry_log(retry_state, *args, **kwargs):
     logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
-        f"retrying in {retry_state.idle_for} seconds"
+        f'retrying in {retry_state.idle_for} {pluralize("second", retry_state.idle_for)}'
     )
 
 
@@ -175,7 +338,9 @@ def _retry_log(retry_state, *args, **kwargs):
     reraise=True,
 )
 def _retried_request(url, *args, method=requests.get, **kwargs):
-    resp = method(url, *args, timeout=DOWNLOAD_TIMEOUT, **kwargs)
+    resp = method(
+        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+    )
     resp.raise_for_status()
     return resp
 

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/arkindex_worker/models.py

@@ -20,6 +20,8 @@ class MagicDict(dict):
         Automagically convert lists and dicts to MagicDicts and lists of MagicDicts
         Allows for nested access: foo.bar.baz
         """
+        if isinstance(item, Dataset):
+            return item
         if isinstance(item, list):
             return list(map(self._magify, item))
         if isinstance(item, dict):
@@ -259,6 +261,12 @@ class Transcription(ArkindexModel):
     """
 
 
+class Image(ArkindexModel):
+    """
+    Describes an Arkindex image.
+    """
+
+
 class Dataset(ArkindexModel):
     """
     Describes an Arkindex dataset.
@@ -272,6 +280,16 @@ class Dataset(ArkindexModel):
         return f"{self.id}.tar.zst"
 
 
+class Set(MagicDict):
+    """
+    Describes an Arkindex dataset set.
+    """
+
+    def __str__(self):
+        # Not using ArkindexModel.__str__ as we do not retrieve the Set ID
+        return f"{self.__class__.__name__} ({self.name}) from {self.dataset}"
+
+
 class Artifact(ArkindexModel):
     """
     Describes an Arkindex artifact.

{arkindex-base-worker-0.3.7rc5 → arkindex_base_worker-0.5.0a1}/arkindex_worker/utils.py

@@ -1,15 +1,54 @@
 import hashlib
+import inspect
 import logging
 import os
 import tarfile
 import tempfile
+from collections.abc import Callable, Generator
+from itertools import islice
 from pathlib import Path
+from typing import Any
 
-import zstandard
 import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
+
+def pluralize(singular: str, count: int) -> str:
+    """Pluralize a noun, if necessary, using simplified rules of English pluralization and a list of exceptions.
+
+    :param str singular: A singular noun describing an object
+    :param int count: The object count, to determine whether to pluralize or not
+    :return str: The noun in its singular or plural form
+    """
+    if count == 1:
+        return singular
+
+    some_exceptions = {
+        "child": "children",
+        "class": "classes",
+        "entity": "entities",
+        "metadata": "metadata",
+    }
+    if singular in some_exceptions:
+        return some_exceptions[singular]
+
+    return singular + "s"
+
+
+MANUAL_SOURCE = "manual"
+
+
+def parse_source_id(value: str) -> bool | str | None:
+    """
+    Parse a UUID argument (Worker Version, Worker Run, ...) to use it directly in the API.
+    Arkindex API filters generally expect `False` to filter manual sources.
+    """
+    if value == MANUAL_SOURCE:
+        return False
+    return value or None
+
+
 CHUNK_SIZE = 1024
 """Chunk Size used for ZSTD compression"""
 
@@ -25,7 +64,7 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
     :param compressed_archive: Path to the target ZST-compressed archive
     :return: File descriptor and path to the uncompressed tar archive
     """
-    dctx = zstandard.ZstdDecompressor()
+    dctx = zstd.ZstdDecompressor()
     archive_fd, archive_path = tempfile.mkstemp(suffix=".tar")
     archive_path = Path(archive_path)
 
@@ -37,7 +76,7 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
         ):
             dctx.copy_stream(compressed, decompressed)
         logger.debug(f"Successfully uncompressed archive {compressed_archive}")
-    except zstandard.ZstdError as e:
+    except zstd.ZstdError as e:
         raise Exception(f"Couldn't uncompressed archive: {e}") from e
 
     return archive_fd, archive_path
@@ -116,7 +155,7 @@ def zstd_compress(
                 archive_hasher.update(compressed_chunk)
                 archive_file.write(compressed_chunk)
         logger.debug(f"Successfully compressed {source}")
-    except zstandard.ZstdError as e:
+    except zstd.ZstdError as e:
         raise Exception(f"Couldn't compress archive: {e}") from e
     return file_d, destination, archive_hasher.hexdigest()
 
@@ -184,3 +223,58 @@ def create_tar_zst_archive(
     close_delete_file(tar_fd, tar_archive)
 
     return zst_fd, zst_archive, zst_hash, tar_hash
+
+
+DEFAULT_BATCH_SIZE = 50
+"""Batch size used for bulk publication to Arkindex"""
+
+
+def batch_publication(func: Callable) -> Callable:
+    """
+    Decorator for functions that should raise an error when the value passed through the ``batch_size`` parameter is **not** a strictly positive integer.
+
+    :param func: The function to wrap with the ``batch_size`` check
+    :return: The function passing the ``batch_size`` check
+    """
+    signature = inspect.signature(func)
+
+    def wrapper(self, *args, **kwargs):
+        bound_func = signature.bind(self, *args, **kwargs)
+        bound_func.apply_defaults()
+        batch_size = bound_func.arguments.get("batch_size")
+        assert (
+            batch_size is not None and isinstance(batch_size, int) and batch_size > 0
+        ), "batch_size shouldn't be null and should be a strictly positive integer"
+
+        return func(self, *args, **kwargs)
+
+    wrapper.__name__ = func.__name__
+    return wrapper
+
+
+def make_batches(
+    objects: list, singular_name: str, batch_size: int
+) -> Generator[list[Any]]:
+    """Split an object list in successive batches of maximum size ``batch_size``.
+
+    :param objects: The object list to divide in batches of ``batch_size`` size
+    :param singular_name: The singular form of the noun associated with the object list
+    :param batch_size: The maximum size of each batch to split the object list
+    :return: A generator of successive batches containing ``batch_size`` items from ``objects``
+    """
+    count = len(objects)
+    logger.info(
+        f"Creating batches of size {batch_size} to process {count} {pluralize(singular_name, count)}"
+    )
+
+    index = 1
+    iterator = iter(objects)
+    while batch := list(islice(iterator, batch_size)):
+        count = len(batch)
+        logger.info(
+            f"Processing batch {index} containing {count} {pluralize(singular_name, count)}..."
+        )
+
+        yield batch
+
+        index += 1