arkindex-base-worker 0.4.0__tar.gz → 0.4.0a1__tar.gz

{arkindex_base_worker-0.4.0 → arkindex_base_worker-0.4.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0
+Version: 0.4.0a1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,23 +37,21 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.11.0
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==11.0.0
-Requires-Dist: python-gnupg==0.5.3
-Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.7
-Requires-Dist: zstandard==0.23.0
+Requires-Dist: Pillow==10.3.0
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.3
+Requires-Dist: teklia-toolbox==0.1.5
+Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
-Requires-Dist: black==24.10.0; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
+Requires-Dist: black==24.4.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.17; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.9.2; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest==8.1.1; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 

{arkindex_base_worker-0.4.0 → arkindex_base_worker-0.4.0a1}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.4.0
+Version: 0.4.0a1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -37,23 +37,21 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.11.0
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==11.0.0
-Requires-Dist: python-gnupg==0.5.3
-Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.7
-Requires-Dist: zstandard==0.23.0
+Requires-Dist: Pillow==10.3.0
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.3
+Requires-Dist: teklia-toolbox==0.1.5
+Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
-Requires-Dist: black==24.10.0; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
+Requires-Dist: black==24.4.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.17; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.9.2; extra == "docs"
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.4; extra == "tests"
+Requires-Dist: pytest==8.1.1; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 

{arkindex_base_worker-0.4.0 → arkindex_base_worker-0.4.0a1}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -20,7 +20,6 @@ arkindex_worker/worker/element.py
 arkindex_worker/worker/entity.py
 arkindex_worker/worker/image.py
 arkindex_worker/worker/metadata.py
-arkindex_worker/worker/process.py
 arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
@@ -36,26 +35,17 @@ tests/test_image.py
 tests/test_merge.py
 tests/test_utils.py
 tests/test_elements_worker/__init__.py
-tests/test_elements_worker/test_classification.py
+tests/test_elements_worker/test_classifications.py
 tests/test_elements_worker/test_cli.py
 tests/test_elements_worker/test_corpus.py
 tests/test_elements_worker/test_dataset.py
-tests/test_elements_worker/test_element.py
-tests/test_elements_worker/test_element_create_multiple.py
-tests/test_elements_worker/test_element_create_single.py
-tests/test_elements_worker/test_element_list_children.py
-tests/test_elements_worker/test_element_list_parents.py
-tests/test_elements_worker/test_entity_create.py
-tests/test_elements_worker/test_entity_list_and_check.py
+tests/test_elements_worker/test_elements.py
+tests/test_elements_worker/test_entities.py
 tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
-tests/test_elements_worker/test_process.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
-tests/test_elements_worker/test_transcription_create.py
-tests/test_elements_worker/test_transcription_create_with_elements.py
-tests/test_elements_worker/test_transcription_list.py
-tests/test_elements_worker/test_version.py
+tests/test_elements_worker/test_transcriptions.py
 tests/test_elements_worker/test_worker.py
 worker-demo/tests/__init__.py
 worker-demo/tests/conftest.py

arkindex_base_worker-0.4.0a1/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,16 @@
+peewee~=3.17
+Pillow==10.3.0
+python-gnupg==0.5.2
+shapely==2.0.3
+teklia-toolbox==0.1.5
+zstandard==0.22.0
+
+[docs]
+black==24.4.0
+mkdocs-material==9.5.17
+mkdocstrings-python==1.9.2
+
+[tests]
+pytest==8.1.1
+pytest-mock==3.14.0
+pytest-responses==0.5.1

{arkindex_base_worker-0.4.0 → arkindex_base_worker-0.4.0a1}/arkindex_worker/cache.py

@@ -380,7 +380,7 @@ def unsupported_cache(func):
     def wrapper(self, *args, **kwargs):
         results = func(self, *args, **kwargs)
 
-        if self.use_cache:
+        if not (self.is_read_only or self.use_cache):
             logger.warning(
                 f"This API helper `{func.__name__}` did not update the cache database"
             )

{arkindex_base_worker-0.4.0 → arkindex_base_worker-0.4.0a1}/arkindex_worker/image.py

@@ -2,18 +2,13 @@
 Helper methods to download and open IIIF images, and manage polygons.
 """
 
-import functools
-import os
 import re
-import tempfile
 from collections import namedtuple
-from collections.abc import Generator, Iterator
 from io import BytesIO
 from math import ceil
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-import humanize
 import requests
 from PIL import Image
 from shapely.affinity import rotate, scale, translate
@@ -26,7 +21,6 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
-from arkindex_worker.utils import pluralize
 from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
@@ -45,57 +39,8 @@ IIIF_URL = re.compile(r"\w+:\/{2}.+\/.+\/.+\/.+\/(?P<size>.+)\/!?\d+\/\w+\.\w+")
 IIIF_FULL = "full"
 # Maximum size available
 IIIF_MAX = "max"
-# Ratio to resize image
-IMAGE_RATIO = [1, 0.9, 0.85, 0.80, 0.75, 0.70, 0.60, 0.50, 0.40, 0.30]
 
 
-def update_pillow_image_size_limit(func):
-    """
-    Update Pillow Image size limit
-    """
-
-    @functools.wraps(func)
-    def wrapper(
-        *args,
-        max_image_pixels: str | int | None = os.getenv("ARKINDEX_MAX_IMAGE_PIXELS"),
-        **kwargs,
-    ):
-        """
-        Wrapper to update Pillow Image size limit and restore it at the end of the function.
-
-        :param *args: Positional arguments passed to the function.
-        :param max_image_pixels: Pillow Image size limit to use.
-        :param **kwargs: Keyword arguments passed to the function.
-        """
-        MAX_IMAGE_PIXELS = Image.MAX_IMAGE_PIXELS
-
-        # Override Pillow Image size limit
-        if max_image_pixels is not None:
-            max_image_pixels = int(max_image_pixels)
-            # Override Pillow limit for detecting decompression bombs, disabled if set to 0
-            if max_image_pixels == 0:
-                logger.warning(
-                    "Pillow Image size limit is completely disabled, make sure you trust the image source."
-                )
-                Image.MAX_IMAGE_PIXELS = None
-            else:
-                Image.MAX_IMAGE_PIXELS = max_image_pixels
-
-        try:
-            results = func(*args, **kwargs)
-        except:
-            # Restore initial Pillow Image size limit
-            Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
-            raise
-
-        # Restore initial Pillow Image size limit
-        Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
-        return results
-
-    return wrapper
-
-
-@update_pillow_image_size_limit
 def open_image(
     path: str,
     mode: str | None = "RGB",
@@ -203,70 +148,6 @@ def upload_image(image: Image, url: str) -> requests.Response:
     return resp
 
 
-def resized_images(
-    *args,
-    element: "Element",
-    max_pixels: int | None = None,
-    max_bytes: int | None = None,
-    **kwargs,
-) -> Iterator[Generator[tempfile.NamedTemporaryFile, None, None]]:
-    """
-    Build resized images according to the pixel and byte limits.
-
-    :param *args: Positional arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
-    :param element: Element whose image needs to be resized.
-    :param max_pixels: Maximum pixel size of the resized images.
-    :param max_bytes: Maximum byte size of the resized images.
-    :param **kwargs: Keyword arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
-    :returns: An iterator of the temporary file of the resized image.
-    """
-    _, _, element_width, element_height = polygon_bounding_box(element.polygon)
-
-    logger.info(f"This element's image sizes are ({element_width} x {element_height}).")
-    if max_pixels and max(element_width, element_height) > max_pixels:
-        logger.warning(
-            f"Maximum image input size supported is ({max_pixels} x {max_pixels})."
-        )
-        logger.warning("The image will be resized.")
-
-    element_pixel, param = (
-        (element_width, "max_width")
-        if element_width > element_height
-        else (element_height, "max_height")
-    )
-
-    for resized_pixel in sorted(
-        set(
-            min(round(ratio * element_pixel), max_pixels or element_pixel)
-            for ratio in IMAGE_RATIO
-        ),
-        reverse=True,
-    ):
-        with element.open_image_tempfile(
-            *args, **{**kwargs, param: resized_pixel}
-        ) as image:
-            pillow_image = Image.open(image)
-            if (
-                pillow_image.width != element_width
-                or pillow_image.height != element_height
-            ):
-                logger.warning(
-                    f"The image was resized to ({pillow_image.width} x {pillow_image.height})."
-                )
-
-            # The image is still too large
-            image_size = Path(image.name).stat().st_size
-            if max_bytes and image_size > max_bytes:
-                logger.warning(f"The image size is {humanize.naturalsize(image_size)}.")
-                logger.warning(
-                    f"Maximum image input size supported is {humanize.naturalsize(max_bytes)}."
-                )
-                logger.warning("The image will be resized.")
-                continue
-
-            yield image
-
-
 def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
@@ -283,7 +164,7 @@ def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
 def _retry_log(retry_state, *args, **kwargs):
     logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
-        f'retrying in {retry_state.idle_for} {pluralize("second", retry_state.idle_for)}'
+        f"retrying in {retry_state.idle_for} seconds"
     )
 
 

{arkindex_base_worker-0.4.0 → arkindex_base_worker-0.4.0a1}/arkindex_worker/utils.py

@@ -1,41 +1,14 @@
 import hashlib
-import inspect
 import logging
 import os
 import tarfile
 import tempfile
-from collections.abc import Callable, Generator
-from itertools import islice
 from pathlib import Path
-from typing import Any
 
 import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
-
-def pluralize(singular: str, count: int) -> str:
-    """Pluralize a noun, if necessary, using simplified rules of English pluralization and a list of exceptions.
-
-    :param str singular: A singular noun describing an object
-    :param int count: The object count, to determine whether to pluralize or not
-    :return str: The noun in its singular or plural form
-    """
-    if count == 1:
-        return singular
-
-    some_exceptions = {
-        "child": "children",
-        "class": "classes",
-        "entity": "entities",
-        "metadata": "metadata",
-    }
-    if singular in some_exceptions:
-        return some_exceptions[singular]
-
-    return singular + "s"
-
-
 MANUAL_SOURCE = "manual"
 
 
@@ -223,58 +196,3 @@ def create_tar_zst_archive(
     close_delete_file(tar_fd, tar_archive)
 
     return zst_fd, zst_archive, zst_hash, tar_hash
-
-
-DEFAULT_BATCH_SIZE = 50
-"""Batch size used for bulk publication to Arkindex"""
-
-
-def batch_publication(func: Callable) -> Callable:
-    """
-    Decorator for functions that should raise an error when the value passed through the ``batch_size`` parameter is **not** a strictly positive integer.
-
-    :param func: The function to wrap with the ``batch_size`` check
-    :return: The function passing the ``batch_size`` check
-    """
-    signature = inspect.signature(func)
-
-    def wrapper(self, *args, **kwargs):
-        bound_func = signature.bind(self, *args, **kwargs)
-        bound_func.apply_defaults()
-        batch_size = bound_func.arguments.get("batch_size")
-        assert (
-            batch_size is not None and isinstance(batch_size, int) and batch_size > 0
-        ), "batch_size shouldn't be null and should be a strictly positive integer"
-
-        return func(self, *args, **kwargs)
-
-    wrapper.__name__ = func.__name__
-    return wrapper
-
-
-def make_batches(
-    objects: list, singular_name: str, batch_size: int
-) -> Generator[list[Any]]:
-    """Split an object list in successive batches of maximum size ``batch_size``.
-
-    :param objects: The object list to divide in batches of ``batch_size`` size
-    :param singular_name: The singular form of the noun associated with the object list
-    :param batch_size: The maximum size of each batch to split the object list
-    :return: A generator of successive batches containing ``batch_size`` items from ``objects``
-    """
-    count = len(objects)
-    logger.info(
-        f"Creating batches of size {batch_size} to process {count} {pluralize(singular_name, count)}"
-    )
-
-    index = 1
-    iterator = iter(objects)
-    while batch := list(islice(iterator, batch_size)):
-        count = len(batch)
-        logger.info(
-            f"Processing batch {index} containing {count} {pluralize(singular_name, count)}..."
-        )
-
-        yield batch
-
-        index += 1