arkindex-base-worker 0.4.0b3__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

tests/test_elements_worker/test_worker.py

@@ -6,7 +6,6 @@ from apistar.exceptions import ErrorResponse
 
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.worker import ActivityState, ElementsWorker
-from tests import CORPUS_ID
 
 from . import BASE_API_CALLS
 
@@ -313,202 +312,3 @@ def test_start_activity_error(
     assert logger.error.call_args_list == [
         mocker.call("Ran on 1 element: 0 completed, 1 failed")
     ]
-
-
-@pytest.mark.parametrize(
-    (
-        "wk_version_config",
-        "wk_version_user_config",
-        "frontend_user_config",
-        "model_config",
-        "expected_config",
-    ),
-    [
-        ({}, {}, {}, {}, {}),
-        # Keep parameters from worker version configuration
-        ({"parameter": 0}, {}, {}, {}, {"parameter": 0}),
-        # Keep parameters from worker version configuration + user_config defaults
-        (
-            {"parameter": 0},
-            {
-                "parameter2": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 0,
-                    "required": False,
-                }
-            },
-            {},
-            {},
-            {"parameter": 0, "parameter2": 0},
-        ),
-        # Keep parameters from worker version configuration + user_config no defaults
-        (
-            {"parameter": 0},
-            {
-                "parameter2": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "required": False,
-                }
-            },
-            {},
-            {},
-            {"parameter": 0},
-        ),
-        # Keep parameters from worker version configuration but user_config defaults overrides
-        (
-            {"parameter": 0},
-            {
-                "parameter": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 1,
-                    "required": False,
-                }
-            },
-            {},
-            {},
-            {"parameter": 1},
-        ),
-        # Keep parameters from worker version configuration + frontend config
-        (
-            {"parameter": 0},
-            {},
-            {"parameter2": 0},
-            {},
-            {"parameter": 0, "parameter2": 0},
-        ),
-        # Keep parameters from worker version configuration + frontend config overrides
-        ({"parameter": 0}, {}, {"parameter": 1}, {}, {"parameter": 1}),
-        # Keep parameters from worker version configuration + model config
-        (
-            {"parameter": 0},
-            {},
-            {},
-            {"parameter2": 0},
-            {"parameter": 0, "parameter2": 0},
-        ),
-        # Keep parameters from worker version configuration + model config overrides
-        ({"parameter": 0}, {}, {}, {"parameter": 1}, {"parameter": 1}),
-        # Keep parameters from worker version configuration + user_config default + model config overrides
-        (
-            {"parameter": 0},
-            {
-                "parameter": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 1,
-                    "required": False,
-                }
-            },
-            {},
-            {"parameter": 2},
-            {"parameter": 2},
-        ),
-        # Keep parameters from worker version configuration + model config + frontend config overrides
-        ({"parameter": 0}, {}, {"parameter": 2}, {"parameter": 1}, {"parameter": 2}),
-        # Keep parameters from worker version configuration + user_config default + model config + frontend config overrides all
-        (
-            {"parameter": 0},
-            {
-                "parameter": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 1,
-                    "required": False,
-                }
-            },
-            {"parameter": 3},
-            {"parameter": 2},
-            {"parameter": 3},
-        ),
-    ],
-)
-def test_worker_config_multiple_source(
-    monkeypatch,
-    responses,
-    wk_version_config,
-    wk_version_user_config,
-    frontend_user_config,
-    model_config,
-    expected_config,
-):
-    # Compute WorkerRun info
-    payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "configuration": {
-                "docker": {"image": "python:3"},
-                "configuration": wk_version_config,
-                "secrets": [],
-                "user_configuration": wk_version_user_config,
-            },
-            "revision": {
-                "hash": "deadbeef1234",
-                "name": "some git revision",
-            },
-            "docker_image": "python:3",
-            "docker_image_name": "python:3",
-            "state": "created",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-        },
-        "configuration": {
-            "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
-            "name": "Configuration entered by user",
-            "configuration": frontend_user_config,
-        },
-        "model_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "name": "Model version 1337",
-            "configuration": model_config,
-            "model": {
-                "id": "hahahaha-haha-haha-haha-hahahahahaha",
-                "name": "My model",
-            },
-        },
-        "process": {
-            "name": None,
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "state": "running",
-            "mode": "workers",
-            "corpus": CORPUS_ID,
-            "use_cache": False,
-            "activity_state": "ready",
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
-        },
-        "summary": "Worker Fake worker @ 123412",
-    }
-
-    responses.add(
-        responses.GET,
-        "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
-        status=200,
-        body=json.dumps(payload),
-        content_type="application/json",
-    )
-
-    # Create and configure a worker
-    monkeypatch.setattr(sys, "argv", ["worker"])
-    worker = ElementsWorker()
-    worker.configure()
-
-    # Do what people do with a model configuration
-    if worker.model_configuration:
-        worker.config.update(worker.model_configuration)
-
-    if worker.user_configuration:
-        worker.config.update(worker.user_configuration)
-
-    # Check final config
-    assert worker.config == expected_config

tests/test_image.py

@@ -1,3 +1,4 @@
+import logging
 import math
 import unittest
 import uuid
@@ -18,21 +19,52 @@ from arkindex_worker.image import (
     download_tiles,
     open_image,
     polygon_bounding_box,
+    resized_images,
     revert_orientation,
     trim_polygon,
+    update_pillow_image_size_limit,
     upload_image,
 )
 from arkindex_worker.models import Element
+from tests import FIXTURES_DIR
 
-FIXTURES = Path(__file__).absolute().parent / "data"
-TILE = FIXTURES / "test_image.jpg"
-FULL_IMAGE = FIXTURES / "tiled_image.jpg"
-ROTATED_IMAGE = FIXTURES / "rotated_image.jpg"
-MIRRORED_IMAGE = FIXTURES / "mirrored_image.jpg"
-ROTATED_MIRRORED_IMAGE = FIXTURES / "rotated_mirrored_image.jpg"
+TILE = FIXTURES_DIR / "test_image.jpg"
+FULL_IMAGE = FIXTURES_DIR / "tiled_image.jpg"
+ROTATED_IMAGE = FIXTURES_DIR / "rotated_image.jpg"
+MIRRORED_IMAGE = FIXTURES_DIR / "mirrored_image.jpg"
+ROTATED_MIRRORED_IMAGE = FIXTURES_DIR / "rotated_mirrored_image.jpg"
 TEST_IMAGE = {"width": 800, "height": 300}
 
 
+@pytest.fixture()
+def mock_page():
+    class Page(Element):
+        def open_image(
+            self,
+            *args,
+            max_width: int | None = None,
+            max_height: int | None = None,
+            use_full_image: bool | None = False,
+            **kwargs,
+        ) -> Image.Image:
+            # Image from Socface (https://socface.site.ined.fr/) project (AD026)
+            image = Image.open(FIXTURES_DIR / "AD026_6M_00505_0001_0373.jpg")
+            image.thumbnail(size=(max_width or image.width, max_height or image.height))
+
+            return image
+
+    return Page(
+        id="page_id",
+        name="1",
+        zone={
+            "polygon": [[0, 0], [1000, 0], [1000, 3000], [0, 3000], [0, 0]],
+            "image": {"width": 1000, "height": 3000},
+        },
+        rotation_angle=0,
+        mirrored=False,
+    )
+
+
 def _root_mean_square(img_a, img_b):
     """
     Get the root-mean-square difference between two images for fuzzy matching
@@ -45,6 +77,30 @@ def _root_mean_square(img_a, img_b):
     )
 
 
+@pytest.mark.parametrize(
+    ("max_image_pixels", "expected_image_pixels"),
+    [
+        # Pillow Image size limit not updated
+        (None, Image.MAX_IMAGE_PIXELS),
+        # Pillow Image size limit set to None
+        ("0", None),
+        (0, None),
+        # Update Pillow Image size limit
+        ("1", 1),
+        (1, 1),
+    ],
+)
+def test_update_pillow_image_size_limit(max_image_pixels, expected_image_pixels):
+    MAX_IMAGE_PIXELS = Image.MAX_IMAGE_PIXELS
+
+    @update_pillow_image_size_limit
+    def function() -> int | None:
+        return Image.MAX_IMAGE_PIXELS
+
+    assert function(max_image_pixels=max_image_pixels) == expected_image_pixels
+    assert Image.MAX_IMAGE_PIXELS == MAX_IMAGE_PIXELS
+
+
 def test_download_tiles(responses):
     expected = Image.open(FULL_IMAGE).convert("RGB")
     tile_bytes = TILE.read_bytes()
@@ -582,3 +638,189 @@ def test_upload_image(responses):
 
     assert len(responses.calls) == 1
     assert list(map(attrgetter("request.url"), responses.calls)) == [dest_url]
+
+
+@pytest.mark.parametrize(
+    ("max_pixels", "max_bytes", "expected_sizes", "expected_logs"),
+    [
+        # No limits
+        (
+            None,
+            None,
+            [
+                (1992, 3000),
+                (1793, 2700),
+                (1694, 2550),
+                (1594, 2400),
+                (1494, 2250),
+                (1395, 2100),
+                (1195, 1800),
+                (996, 1500),
+                (797, 1200),
+                (598, 900),
+            ],
+            [
+                (logging.WARNING, "The image was resized to (1992 x 3000)."),
+                (logging.WARNING, "The image was resized to (1793 x 2700)."),
+                (logging.WARNING, "The image was resized to (1694 x 2550)."),
+                (logging.WARNING, "The image was resized to (1594 x 2400)."),
+                (logging.WARNING, "The image was resized to (1494 x 2250)."),
+                (logging.WARNING, "The image was resized to (1395 x 2100)."),
+                (logging.WARNING, "The image was resized to (1195 x 1800)."),
+                (logging.WARNING, "The image was resized to (996 x 1500)."),
+                (logging.WARNING, "The image was resized to (797 x 1200)."),
+                (logging.WARNING, "The image was resized to (598 x 900)."),
+            ],
+        ),
+        # Image already under the limits
+        (
+            10000,
+            4000000,  # 4MB
+            [
+                (1992, 3000),
+                (1793, 2700),
+                (1694, 2550),
+                (1594, 2400),
+                (1494, 2250),
+                (1395, 2100),
+                (1195, 1800),
+                (996, 1500),
+                (797, 1200),
+                (598, 900),
+            ],
+            [
+                (logging.WARNING, "The image was resized to (1992 x 3000)."),
+                (logging.WARNING, "The image was resized to (1793 x 2700)."),
+                (logging.WARNING, "The image was resized to (1694 x 2550)."),
+                (logging.WARNING, "The image was resized to (1594 x 2400)."),
+                (logging.WARNING, "The image was resized to (1494 x 2250)."),
+                (logging.WARNING, "The image was resized to (1395 x 2100)."),
+                (logging.WARNING, "The image was resized to (1195 x 1800)."),
+                (logging.WARNING, "The image was resized to (996 x 1500)."),
+                (logging.WARNING, "The image was resized to (797 x 1200)."),
+                (logging.WARNING, "The image was resized to (598 x 900)."),
+            ],
+        ),
+        # Image above the limits
+        (
+            None,
+            100000,  # 100kB
+            [(598, 900)],
+            [
+                (logging.WARNING, "The image was resized to (1992 x 3000)."),
+                (logging.WARNING, "The image size is 773.4 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1793 x 2700)."),
+                (logging.WARNING, "The image size is 616.0 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1694 x 2550)."),
+                (logging.WARNING, "The image size is 546.4 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1594 x 2400)."),
+                (logging.WARNING, "The image size is 479.4 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1494 x 2250)."),
+                (logging.WARNING, "The image size is 416.1 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1395 x 2100)."),
+                (logging.WARNING, "The image size is 360.5 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1195 x 1800)."),
+                (logging.WARNING, "The image size is 258.6 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (996 x 1500)."),
+                (logging.WARNING, "The image size is 179.0 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (797 x 1200)."),
+                (logging.WARNING, "The image size is 115.7 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (598 x 900)."),
+            ],
+        ),
+        # Image above the limits
+        (
+            2000,
+            None,
+            [(1328, 2000), (1195, 1800), (996, 1500), (797, 1200), (598, 900)],
+            [
+                (
+                    logging.WARNING,
+                    "Maximum image input size supported is (2000 x 2000).",
+                ),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1328 x 2000)."),
+                (logging.WARNING, "The image was resized to (1195 x 1800)."),
+                (logging.WARNING, "The image was resized to (996 x 1500)."),
+                (logging.WARNING, "The image was resized to (797 x 1200)."),
+                (logging.WARNING, "The image was resized to (598 x 900)."),
+            ],
+        ),
+        # Image above the limits
+        (
+            2000,
+            100000,  # 100kB
+            [(598, 900)],
+            [
+                (
+                    logging.WARNING,
+                    "Maximum image input size supported is (2000 x 2000).",
+                ),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1328 x 2000)."),
+                (logging.WARNING, "The image size is 325.5 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (1195 x 1800)."),
+                (logging.WARNING, "The image size is 258.6 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (996 x 1500)."),
+                (logging.WARNING, "The image size is 179.0 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (797 x 1200)."),
+                (logging.WARNING, "The image size is 115.7 kB."),
+                (logging.WARNING, "Maximum image input size supported is 100.0 kB."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (598 x 900)."),
+            ],
+        ),
+        # Image always above the limits
+        (
+            50,
+            50,  # 50B
+            [],
+            [
+                (logging.WARNING, "Maximum image input size supported is (50 x 50)."),
+                (logging.WARNING, "The image will be resized."),
+                (logging.WARNING, "The image was resized to (33 x 50)."),
+                (logging.WARNING, "The image size is 1.0 kB."),
+                (logging.WARNING, "Maximum image input size supported is 50 Bytes."),
+                (logging.WARNING, "The image will be resized."),
+            ],
+        ),
+    ],
+)
+def test_resized_images(
+    max_pixels, max_bytes, expected_sizes, expected_logs, mock_page, caplog
+):
+    caplog.set_level(logging.WARNING)
+
+    assert [
+        Image.open(image).size
+        for image in resized_images(
+            element=mock_page, max_pixels=max_pixels, max_bytes=max_bytes
+        )
+    ] == expected_sizes
+    assert [
+        (record.levelno, record.message) for record in caplog.records
+    ] == expected_logs

tests/test_merge.py

@@ -181,7 +181,6 @@ def test_merge_from_worker(
     (tmp_path / "my_task").mkdir()
     mock_base_worker_with_cache.args = mock_base_worker_with_cache.parser.parse_args()
     mock_base_worker_with_cache.configure()
-    mock_base_worker_with_cache.configure_cache()
     # Store parent tasks IDs as attribute
     assert mock_base_worker_with_cache.task_parents == ["first", "second"]
 

tests/test_utils.py

@@ -1,5 +1,3 @@
-from pathlib import Path
-
 import pytest
 
 from arkindex_worker.utils import (
@@ -9,9 +7,9 @@ from arkindex_worker.utils import (
     extract_tar_zst_archive,
     parse_source_id,
 )
+from tests import FIXTURES_DIR
 
-FIXTURES = Path(__file__).absolute().parent / "data"
-ARCHIVE = FIXTURES / "archive.tar.zst"
+ARCHIVE = FIXTURES_DIR / "archive.tar.zst"
 
 
 @pytest.mark.parametrize(