arkindex-base-worker 0.4.0rc5__tar.gz → 0.5.0__tar.gz

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.4.0rc5
+Version: 0.5.0
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,21 +41,18 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.10.0
+Requires-Dist: humanize==4.12.3
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==10.4.0
-Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: Pillow==11.3.0
+Requires-Dist: python-gnupg==0.5.4
 Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.7b1
-Requires-Dist: zstandard==0.22.0
-Provides-Extra: docs
-Requires-Dist: black==24.4.2; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
+Requires-Dist: teklia-toolbox==0.1.11
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.2; extra == "tests"
+Requires-Dist: pytest==8.3.5; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
+Dynamic: license-file
 
 # Arkindex base Worker
 
@@ -65,7 +62,7 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 ## Documentation
 
-The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+The [documentation](https://workers.arkindex.org/) is made with [Antora](https://antora.org/) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
 
 ## Create a new worker using our template
 

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/README.md

@@ -6,7 +6,7 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 ## Documentation
 
-The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+The [documentation](https://workers.arkindex.org/) is made with [Antora](https://antora.org/) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
 
 ## Create a new worker using our template
 

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.4.0rc5
+Version: 0.5.0
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,21 +41,18 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: humanize==4.10.0
+Requires-Dist: humanize==4.12.3
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==10.4.0
-Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: Pillow==11.3.0
+Requires-Dist: python-gnupg==0.5.4
 Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.7b1
-Requires-Dist: zstandard==0.22.0
-Provides-Extra: docs
-Requires-Dist: black==24.4.2; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.33; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.11.1; extra == "docs"
+Requires-Dist: teklia-toolbox==0.1.11
+Requires-Dist: zstandard==0.23.0
 Provides-Extra: tests
-Requires-Dist: pytest==8.3.2; extra == "tests"
+Requires-Dist: pytest==8.3.5; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
+Dynamic: license-file
 
 # Arkindex base Worker
 
@@ -65,7 +62,7 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 ## Documentation
 
-The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+The [documentation](https://workers.arkindex.org/) is made with [Antora](https://antora.org/) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
 
 ## Create a new worker using our template
 

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -24,7 +24,8 @@ arkindex_worker/worker/process.py
 arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
-arkindex_worker/worker/version.py
+examples/standalone/python/worker.py
+examples/tooled/python/worker.py
 hooks/pre_gen_project.py
 tests/__init__.py
 tests/conftest.py
@@ -45,16 +46,15 @@ tests/test_elements_worker/test_element_create_multiple.py
 tests/test_elements_worker/test_element_create_single.py
 tests/test_elements_worker/test_element_list_children.py
 tests/test_elements_worker/test_element_list_parents.py
-tests/test_elements_worker/test_entity_create.py
-tests/test_elements_worker/test_entity_list_and_check.py
+tests/test_elements_worker/test_entity.py
 tests/test_elements_worker/test_image.py
 tests/test_elements_worker/test_metadata.py
+tests/test_elements_worker/test_process.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
 tests/test_elements_worker/test_transcription_create.py
 tests/test_elements_worker/test_transcription_create_with_elements.py
 tests/test_elements_worker/test_transcription_list.py
-tests/test_elements_worker/test_version.py
 tests/test_elements_worker/test_worker.py
 worker-demo/tests/__init__.py
 worker-demo/tests/conftest.py

arkindex_base_worker-0.5.0/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,12 @@
+humanize==4.12.3
+peewee~=3.17
+Pillow==11.3.0
+python-gnupg==0.5.4
+shapely==2.0.6
+teklia-toolbox==0.1.11
+zstandard==0.23.0
+
+[tests]
+pytest==8.3.5
+pytest-mock==3.14.0
+pytest-responses==0.5.1

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_base_worker.egg-info/top_level.txt

@@ -1,6 +1,7 @@
 arkindex_worker
 dist
 docs
+examples
 hooks
 tests
 worker-demo

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_worker/__init__.py

@@ -1,3 +1,4 @@
+import importlib.metadata
 import logging
 
 logging.basicConfig(
@@ -5,3 +6,5 @@ logging.basicConfig(
     format="%(asctime)s %(levelname)s/%(name)s: %(message)s",
 )
 logger = logging.getLogger(__name__)
+
+VERSION = importlib.metadata.version("arkindex-base-worker")

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_worker/cache.py

@@ -206,23 +206,6 @@ class CachedClassification(Model):
         table_name = "classifications"
 
 
-class CachedEntity(Model):
-    """
-    Cache entity table
-    """
-
-    id = UUIDField(primary_key=True)
-    type = CharField(max_length=50)
-    name = TextField()
-    validated = BooleanField(default=False)
-    metas = JSONField(null=True)
-    worker_run_id = UUIDField(null=True)
-
-    class Meta:
-        database = db
-        table_name = "entities"
-
-
 class CachedTranscriptionEntity(Model):
     """
     Cache transcription entity table
@@ -231,14 +214,14 @@ class CachedTranscriptionEntity(Model):
     transcription = ForeignKeyField(
         CachedTranscription, backref="transcription_entities"
     )
-    entity = ForeignKeyField(CachedEntity, backref="transcription_entities")
+    type = CharField(max_length=50)
     offset = IntegerField(constraints=[Check("offset >= 0")])
     length = IntegerField(constraints=[Check("length > 0")])
     worker_run_id = UUIDField(null=True)
     confidence = FloatField(null=True)
 
     class Meta:
-        primary_key = CompositeKey("transcription", "entity")
+        primary_key = CompositeKey("transcription", "type")
         database = db
         table_name = "transcription_entities"
 
@@ -272,12 +255,11 @@ MODELS = [
     CachedElement,
     CachedTranscription,
     CachedClassification,
-    CachedEntity,
     CachedTranscriptionEntity,
     CachedDataset,
     CachedDatasetElement,
 ]
-SQL_VERSION = 3
+SQL_VERSION = 4
 
 
 def init_cache_db(path: Path):
@@ -327,9 +309,9 @@ def check_version(cache_path: str | Path):
         except OperationalError:
             version = None
 
-        assert (
-            version == SQL_VERSION
-        ), f"The SQLite database {cache_path} does not have the correct cache version, it should be {SQL_VERSION}"
+        assert version == SQL_VERSION, (
+            f"The SQLite database {cache_path} does not have the correct cache version, it should be {SQL_VERSION}"
+        )
 
 
 def merge_parents_cache(paths: list, current_database: Path):
@@ -365,7 +347,6 @@ def merge_parents_cache(paths: list, current_database: Path):
             f"REPLACE INTO elements SELECT * FROM source_{idx}.elements;",
             f"REPLACE INTO transcriptions SELECT * FROM source_{idx}.transcriptions;",
             f"REPLACE INTO classifications SELECT * FROM source_{idx}.classifications;",
-            f"REPLACE INTO entities SELECT * FROM source_{idx}.entities;",
             f"REPLACE INTO transcription_entities SELECT * FROM source_{idx}.transcription_entities;",
             f"REPLACE INTO datasets SELECT * FROM source_{idx}.datasets;",
             f"REPLACE INTO dataset_elements SELECT * FROM source_{idx}.dataset_elements;",

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_worker/image.py

@@ -2,6 +2,7 @@
 Helper methods to download and open IIIF images, and manage polygons.
 """
 
+import base64
 import functools
 import os
 import re
@@ -14,6 +15,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import humanize
+import numpy as np
 import requests
 from PIL import Image
 from shapely.affinity import rotate, scale, translate
@@ -22,10 +24,11 @@ from tenacity import (
     retry,
     retry_if_exception_type,
     stop_after_attempt,
-    wait_exponential,
+    wait_chain,
+    wait_fixed,
 )
 
-from arkindex_worker import logger
+from arkindex_worker import VERSION, logger
 from arkindex_worker.utils import pluralize
 from teklia_toolbox.requests import should_verify_cert
 
@@ -39,14 +42,16 @@ DOWNLOAD_TIMEOUT = (30, 60)
 
 BoundingBox = namedtuple("BoundingBox", ["x", "y", "width", "height"])
 
+# Specific User-Agent to bypass potential server limitations
+IIIF_USER_AGENT = f"Teklia/Workers {VERSION}"
 # To parse IIIF Urls
 IIIF_URL = re.compile(r"\w+:\/{2}.+\/.+\/.+\/.+\/(?P<size>.+)\/!?\d+\/\w+\.\w+")
 # Full size of the region
 IIIF_FULL = "full"
 # Maximum size available
 IIIF_MAX = "max"
-# Ratio to resize image
-IMAGE_RATIO = [1, 0.9, 0.85, 0.80, 0.75, 0.70, 0.60, 0.50, 0.40, 0.30]
+# Ratios to resize images: 1.0, 0.95, [...], 0.1, 0.05
+IMAGE_RATIOS = np.arange(1, 0, -0.05).round(2).tolist()
 
 
 def update_pillow_image_size_limit(func):
@@ -206,44 +211,81 @@ def upload_image(image: Image, url: str) -> requests.Response:
 def resized_images(
     *args,
     element: "Element",
-    max_pixels: int | None = None,
+    max_pixels_short: int | None = None,
+    max_pixels_long: int | None = None,
     max_bytes: int | None = None,
+    use_base64: bool = False,
     **kwargs,
-) -> Iterator[Generator[tempfile.NamedTemporaryFile, None, None]]:
+) -> Iterator[Generator[tempfile._TemporaryFileWrapper | str]]:
     """
-    Build resized images according to the pixel and byte limits.
+    Build resized images according to pixel and byte limits.
 
     :param *args: Positional arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
     :param element: Element whose image needs to be resized.
-    :param max_pixels: Maximum pixel size of the resized images.
+    :param max_pixels_short: Maximum pixel size of the resized images' short side.
+    :param max_pixels_long: Maximum pixel size of the resized images' long side.
     :param max_bytes: Maximum byte size of the resized images.
+    :param use_base64: Whether or not to encode resized images in base64 before calculating their size.
     :param **kwargs: Keyword arguments passed to [arkindex_worker.models.Element.open_image_tempfile][].
-    :returns: An iterator of the temporary file of the resized image.
+    :returns: An iterator of temporary files for resized images OR an iterator of base64-encoded strings if `use_base64` is set.
     """
     _, _, element_width, element_height = polygon_bounding_box(element.polygon)
+    logger.info(
+        f"This element's image dimensions are ({element_width} x {element_height})."
+    )
+
+    portrait_format = element_width <= element_height
+    max_pixels_width, max_pixels_height = (
+        (max_pixels_short, max_pixels_long)
+        if portrait_format
+        else (max_pixels_long, max_pixels_short)
+    )
 
-    logger.info(f"This element's image sizes are ({element_width} x {element_height}).")
-    if max_pixels and max(element_width, element_height) > max_pixels:
+    # The image dimension is already within the pixel limitation, no need to resize the image
+    if max_pixels_width and max_pixels_width >= element_width:
+        max_pixels_width = None
+    if max_pixels_height and max_pixels_height >= element_height:
+        max_pixels_height = None
+
+    if (max_pixels_width and element_width > max_pixels_width) or (
+        max_pixels_height and element_height > max_pixels_height
+    ):
         logger.warning(
-            f"Maximum image input size supported is ({max_pixels} x {max_pixels})."
+            f"Maximum image dimensions supported are ({max_pixels_width or element_width} x {max_pixels_height or element_height})."
         )
         logger.warning("The image will be resized.")
 
-    element_pixel, param = (
-        (element_width, "max_width")
-        if element_width > element_height
-        else (element_height, "max_height")
-    )
+    # No limitations provided, we keep the image initial dimensions
+    if max_pixels_width is None and max_pixels_height is None:
+        open_image_param, max_value = (
+            ("max_height", element_height)
+            if portrait_format
+            else ("max_width", element_width)
+        )
+    # A limitation is only given for the height, we resize it
+    elif max_pixels_width is None:
+        open_image_param, max_value = ("max_height", max_pixels_height)
+    # A limitation is only given for the width, we resize it
+    elif max_pixels_height is None:
+        open_image_param, max_value = ("max_width", max_pixels_width)
+    # Limitations are provided for both sides:
+    # - we resize only the one with the biggest scale factor
+    # - the remaining one will automatically fall within the other limitation
+    else:
+        width_rescaling_factor = element_width / max_pixels_width
+        height_rescaling_factor = element_height / max_pixels_height
+        open_image_param, max_value = (
+            ("max_height", max_pixels_height)
+            if height_rescaling_factor > width_rescaling_factor
+            else ("max_width", max_pixels_width)
+        )
 
-    for resized_pixel in sorted(
-        set(
-            min(round(ratio * element_pixel), max_pixels or element_pixel)
-            for ratio in IMAGE_RATIO
-        ),
-        reverse=True,
-    ):
+    resized_pixels = set(
+        min(round(ratio * max_value), max_value) for ratio in IMAGE_RATIOS
+    )
+    for resized_pixel in sorted(resized_pixels, reverse=True):
         with element.open_image_tempfile(
-            *args, **{**kwargs, param: resized_pixel}
+            *args, **{**kwargs, open_image_param: resized_pixel}
         ) as image:
             pillow_image = Image.open(image)
             if (
@@ -254,8 +296,12 @@ def resized_images(
                     f"The image was resized to ({pillow_image.width} x {pillow_image.height})."
                 )
 
-            # The image is still too large
             image_size = Path(image.name).stat().st_size
+            if use_base64:
+                image = base64.b64encode(Path(image.name).read_bytes()).decode("utf-8")
+                image_size = len(image)
+
+            # The image is still too heavy
             if max_bytes and image_size > max_bytes:
                 logger.warning(f"The image size is {humanize.naturalsize(image_size)}.")
                 logger.warning(
@@ -283,20 +329,26 @@ def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
 def _retry_log(retry_state, *args, **kwargs):
     logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
-        f'retrying in {retry_state.idle_for} {pluralize("second", retry_state.idle_for)}'
+        f"retrying in {retry_state.idle_for} {pluralize('second', retry_state.idle_for)}"
     )
 
 
 @retry(
     stop=stop_after_attempt(3),
-    wait=wait_exponential(multiplier=2),
+    # In the event of `requests.RequestException` errors, the call will be retried after 5 seconds, 10 seconds and finally 90 seconds before failing.
+    wait=wait_chain(wait_fixed(5), wait_fixed(10), wait_fixed(90)),
     retry=retry_if_exception_type(requests.RequestException),
     before_sleep=_retry_log,
     reraise=True,
 )
 def _retried_request(url, *args, method=requests.get, **kwargs):
     resp = method(
-        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+        url,
+        *args,
+        headers={"User-Agent": IIIF_USER_AGENT},
+        timeout=DOWNLOAD_TIMEOUT,
+        verify=should_verify_cert(url),
+        **kwargs,
     )
     resp.raise_for_status()
     return resp
@@ -316,9 +368,9 @@ def download_tiles(url: str) -> Image:
 
     image_width, image_height = info.get("width"), info.get("height")
     assert image_width and image_height, "Missing image dimensions in info.json"
-    assert info.get(
-        "tiles"
-    ), "Image cannot be retrieved at full size and tiles are not supported"
+    assert info.get("tiles"), (
+        "Image cannot be retrieved at full size and tiles are not supported"
+    )
 
     # Take the biggest available tile size
     tile = sorted(info["tiles"], key=lambda tile: tile.get("width", 0), reverse=True)[0]
@@ -385,22 +437,20 @@ def trim_polygon(
     :param image_width: Width of the image.
     :param image_height: Height of the image.
     :returns: A polygon trimmed to the image's bounds.
-       Some points may appear as missing, as the trimming can deduplicate points.
-       The first and last point are always equal, to reproduce the behavior
-       of the Arkindex backend.
     :raises AssertionError: When argument types are invalid or when the trimmed polygon
        is entirely outside of the image's bounds.
     """
 
-    assert isinstance(
-        polygon, list | tuple
-    ), "Input polygon must be a valid list or tuple of points."
-    assert all(
-        isinstance(point, list | tuple) for point in polygon
-    ), "Polygon points must be tuples or lists."
-    assert all(
-        len(point) == 2 for point in polygon
-    ), "Polygon points must be tuples or lists of 2 elements."
+    assert isinstance(polygon, list | tuple), (
+        "Polygon must be a valid list or tuple of points."
+    )
+    assert len(polygon) >= 3, "Polygon should have at least three points."
+    assert all(isinstance(point, list | tuple) for point in polygon), (
+        "Polygon points must be tuples or lists."
+    )
+    assert all(len(point) == 2 for point in polygon), (
+        "Polygon points must be tuples or lists of 2 elements."
+    )
     assert all(
         isinstance(point[0], int) and isinstance(point[1], int) for point in polygon
     ), "Polygon point coordinates must be integers."
@@ -408,7 +458,7 @@ def trim_polygon(
         point[0] <= image_width and point[1] <= image_height for point in polygon
     ), "This polygon is entirely outside the image's bounds."
 
-    trimmed_polygon = [
+    return [
         [
             min(image_width, max(0, x)),
             min(image_height, max(0, y)),
@@ -416,17 +466,6 @@ def trim_polygon(
         for x, y in polygon
     ]
 
-    updated_polygon = []
-    for point in trimmed_polygon:
-        if point not in updated_polygon:
-            updated_polygon.append(point)
-
-    # Add back the matching last point, if it was present in the original polygon
-    if polygon[-1] == polygon[0]:
-        updated_polygon.append(updated_polygon[0])
-
-    return updated_polygon
-
 
 def revert_orientation(
     element: "Element | CachedElement",
@@ -451,22 +490,22 @@ def revert_orientation(
     from arkindex_worker.cache import CachedElement
     from arkindex_worker.models import Element
 
-    assert element and isinstance(
-        element, Element | CachedElement
-    ), "element shouldn't be null and should be an Element or CachedElement"
-    assert polygon and isinstance(
-        polygon, list
-    ), "polygon shouldn't be null and should be a list"
-    assert isinstance(reverse, bool), "Reverse should be a bool"
+    assert element and isinstance(element, Element | CachedElement), (
+        "element shouldn't be null and should be an Element or CachedElement"
+    )
+    assert polygon and isinstance(polygon, list), (
+        "polygon shouldn't be null and should be a list"
+    )
+    assert isinstance(reverse, bool), "reverse should be a bool"
     # Rotating with Pillow can cause it to move the image around, as the image cannot have negative coordinates
     # and must be a rectangle.  This means the origin point of any coordinates from an image is invalid, and the
     # center of the bounding box of the rotated image is different from the center of the element's bounding box.
     # To properly undo the mirroring and rotation implicitly applied by open_image, we first need to find the center
     # of the rotated bounding box.
     if isinstance(element, Element):
-        assert (
-            element.zone and element.zone.polygon
-        ), "element should have a zone and a polygon"
+        assert element.zone and element.zone.polygon, (
+            "element should have a zone and a polygon"
+        )
         parent_ring = LinearRing(element.zone.polygon)
     elif isinstance(element, CachedElement):
         assert element.polygon, "cached element should have a polygon"

{arkindex_base_worker-0.4.0rc5 → arkindex_base_worker-0.5.0}/arkindex_worker/utils.py

@@ -243,11 +243,12 @@ def batch_publication(func: Callable) -> Callable:
         bound_func.apply_defaults()
         batch_size = bound_func.arguments.get("batch_size")
         assert (
-            batch_size and isinstance(batch_size, int) and batch_size > 0
+            batch_size is not None and isinstance(batch_size, int) and batch_size > 0
         ), "batch_size shouldn't be null and should be a strictly positive integer"
 
         return func(self, *args, **kwargs)
 
+    wrapper.__name__ = func.__name__
     return wrapper