arkindex-base-worker 0.3.7rc2__py3-none-any.whl → 0.3.7rc4__py3-none-any.whl

tests/test_elements_worker/test_dataset.py

@@ -139,12 +139,26 @@ def test_list_dataset_elements_wrong_param_dataset(mock_dataset_worker, payload,
         mock_dataset_worker.list_dataset_elements(**payload)
 
 
+@pytest.mark.parametrize(
+    "sets",
+    [
+        ["set_1"],
+        ["set_1", "set_2", "set_3"],
+        ["set_1", "set_2", "set_3", "set_4"],
+    ],
+)
 def test_list_dataset_elements_api_error(
-    responses, mock_dataset_worker, default_dataset
+    responses, mock_dataset_worker, sets, default_dataset
 ):
+    default_dataset.selected_sets = sets
+    query_params = (
+        "?with_count=true"
+        if sets == default_dataset.sets
+        else "?set=set_1&with_count=true"
+    )
     responses.add(
         responses.GET,
-        f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/",
+        f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
         status=500,
     )
 
@@ -160,122 +174,107 @@ def test_list_dataset_elements_api_error(
         # The API call is retried 5 times
         (
             "GET",
-            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
         ),
         (
             "GET",
-            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
         ),
         (
             "GET",
-            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
         ),
         (
             "GET",
-            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
         ),
         (
             "GET",
-            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
         ),
     ]
 
 
+@pytest.mark.parametrize(
+    "sets",
+    [
+        ["set_1"],
+        ["set_1", "set_2", "set_3"],
+        ["set_1", "set_2", "set_3", "set_4"],
+    ],
+)
 def test_list_dataset_elements(
     responses,
     mock_dataset_worker,
+    sets,
     default_dataset,
 ):
-    expected_results = [
-        {
-            "set": "set_1",
-            "element": {
-                "id": "0000",
-                "type": "page",
-                "name": "Test",
-                "corpus": {},
-                "thumbnail_url": None,
-                "zone": {},
-                "best_classes": None,
-                "has_children": None,
-                "worker_version_id": None,
-                "worker_run_id": None,
-            },
-        },
-        {
-            "set": "set_1",
-            "element": {
-                "id": "1111",
-                "type": "page",
-                "name": "Test 2",
-                "corpus": {},
-                "thumbnail_url": None,
-                "zone": {},
-                "best_classes": None,
-                "has_children": None,
-                "worker_version_id": None,
-                "worker_run_id": None,
-            },
-        },
-        {
-            "set": "set_2",
-            "element": {
-                "id": "2222",
-                "type": "page",
-                "name": "Test 3",
-                "corpus": {},
-                "thumbnail_url": None,
-                "zone": {},
-                "best_classes": None,
-                "has_children": None,
-                "worker_version_id": None,
-                "worker_run_id": None,
-            },
-        },
-        {
-            "set": "set_3",
-            "element": {
-                "id": "3333",
-                "type": "page",
-                "name": "Test 4",
-                "corpus": {},
-                "thumbnail_url": None,
-                "zone": {},
-                "best_classes": None,
-                "has_children": None,
-                "worker_version_id": None,
-                "worker_run_id": None,
+    default_dataset.selected_sets = sets
+
+    dataset_elements = []
+    for split in default_dataset.sets:
+        index = split[-1]
+        dataset_elements.append(
+            {
+                "set": split,
+                "element": {
+                    "id": str(index) * 4,
+                    "type": "page",
+                    "name": f"Test {index}",
+                    "corpus": {},
+                    "thumbnail_url": None,
+                    "zone": {},
+                    "best_classes": None,
+                    "has_children": None,
+                    "worker_version_id": None,
+                    "worker_run_id": None,
+                },
+            }
+        )
+        if split == "set_1":
+            dataset_elements.append({**dataset_elements[-1]})
+            dataset_elements[-1]["element"]["name"] = f"Test {index} (bis)"
+
+    # All sets are selected, we call the unfiltered endpoint once
+    if default_dataset.sets == default_dataset.selected_sets:
+        expected_results = dataset_elements
+        responses.add(
+            responses.GET,
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
+            status=200,
+            json={
+                "count": len(expected_results),
+                "next": None,
+                "results": expected_results,
             },
-        },
-    ]
-    responses.add(
-        responses.GET,
-        f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/",
-        status=200,
-        json={
-            "count": 4,
-            "next": None,
-            "results": expected_results
-            # `set_4` is not in `default_dataset.selected_sets`
-            + [
-                {
-                    "set": "set_4",
-                    "element": {
-                        "id": "4444",
-                        "type": "page",
-                        "name": "Test 5",
-                        "corpus": {},
-                        "thumbnail_url": None,
-                        "zone": {},
-                        "best_classes": None,
-                        "has_children": None,
-                        "worker_version_id": None,
-                        "worker_run_id": None,
-                    },
-                }
-            ],
-        },
-    )
+        )
+        expected_calls = [
+            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true"
+        ]
+
+    # Not all sets are selected, we call the filtered endpoint multiple times, once per set
+    else:
+        expected_results, expected_calls = [], []
+        for selected_set in default_dataset.selected_sets:
+            partial_results = [
+                element
+                for element in dataset_elements
+                if element["set"] == selected_set
+            ]
+            expected_results += partial_results
+            responses.add(
+                responses.GET,
+                f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?set={selected_set}&with_count=true",
+                status=200,
+                json={
+                    "count": len(partial_results),
+                    "next": None,
+                    "results": partial_results,
+                },
+            )
+            expected_calls += [
+                f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?set={selected_set}&with_count=true"
+            ]
 
     for idx, element in enumerate(
         mock_dataset_worker.list_dataset_elements(dataset=default_dataset)
@@ -285,15 +284,10 @@ def test_list_dataset_elements(
             expected_results[idx]["element"],
         )
 
-    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert len(responses.calls) == len(BASE_API_CALLS) + len(expected_calls)
     assert [
         (call.request.method, call.request.url) for call in responses.calls
-    ] == BASE_API_CALLS + [
-        (
-            "GET",
-            f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/?with_count=true",
-        ),
-    ]
+    ] == BASE_API_CALLS + [("GET", expected_call) for expected_call in expected_calls]
 
 
 @pytest.mark.parametrize(

tests/test_elements_worker/test_elements.py

@@ -1797,9 +1797,12 @@ def test_list_element_children_wrong_worker_version(
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
 
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
-    ), pytest.raises(AssertionError, match=f"{param} should be of type str or bool"):
+    with (
+        pytest.deprecated_call(
+            match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
+        ),
+        pytest.raises(AssertionError, match=f"{param} should be of type str or bool"),
+    ):
         mock_elements_worker.list_element_children(
             element=elt,
             **{param: value},
@@ -1838,10 +1841,13 @@ def test_list_element_children_wrong_bool_worker_version(
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
 
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
-    ), pytest.raises(
-        AssertionError, match=f"if of type bool, {param} can only be set to False"
+    with (
+        pytest.deprecated_call(
+            match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
+        ),
+        pytest.raises(
+            AssertionError, match=f"if of type bool, {param} can only be set to False"
+        ),
     ):
         mock_elements_worker.list_element_children(
             element=elt,
@@ -2323,9 +2329,12 @@ def test_list_element_parents_wrong_worker_version(
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
 
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
-    ), pytest.raises(AssertionError, match=f"{param} should be of type str or bool"):
+    with (
+        pytest.deprecated_call(
+            match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
+        ),
+        pytest.raises(AssertionError, match=f"{param} should be of type str or bool"),
+    ):
         mock_elements_worker.list_element_parents(
             element=elt,
             **{param: value},
@@ -2364,10 +2373,13 @@ def test_list_element_parents_wrong_bool_worker_version(
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
 
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
-    ), pytest.raises(
-        AssertionError, match=f"if of type bool, {param} can only be set to False"
+    with (
+        pytest.deprecated_call(
+            match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
+        ),
+        pytest.raises(
+            AssertionError, match=f"if of type bool, {param} can only be set to False"
+        ),
     ):
         mock_elements_worker.list_element_parents(
             element=elt,

tests/test_elements_worker/test_transcriptions.py

@@ -1727,10 +1727,13 @@ def test_list_transcriptions_wrong_worker_version(mock_elements_worker):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
 
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match="`worker_version` usage is deprecated. Consider using `worker_run` instead."
-    ), pytest.raises(
-        AssertionError, match="worker_version should be of type str or bool"
+    with (
+        pytest.deprecated_call(
+            match="`worker_version` usage is deprecated. Consider using `worker_run` instead."
+        ),
+        pytest.raises(
+            AssertionError, match="worker_version should be of type str or bool"
+        ),
     ):
         mock_elements_worker.list_transcriptions(
             element=elt,
@@ -1754,10 +1757,14 @@ def test_list_transcriptions_wrong_bool_worker_version(mock_elements_worker):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
 
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match="`worker_version` usage is deprecated. Consider using `worker_run` instead."
-    ), pytest.raises(
-        AssertionError, match="if of type bool, worker_version can only be set to False"
+    with (
+        pytest.deprecated_call(
+            match="`worker_version` usage is deprecated. Consider using `worker_run` instead."
+        ),
+        pytest.raises(
+            AssertionError,
+            match="if of type bool, worker_version can only be set to False",
+        ),
     ):
         mock_elements_worker.list_transcriptions(
             element=elt,

tests/test_elements_worker/test_worker.py

@@ -62,9 +62,10 @@ def test_get_worker_version_slug(mocker, fake_dummy_worker):
 
 def test_get_worker_version_slug_none(fake_dummy_worker):
     # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
-    with pytest.deprecated_call(
-        match="WorkerVersion usage is deprecated."
-    ), pytest.raises(ValueError, match="No worker version ID"):
+    with (
+        pytest.deprecated_call(match="WorkerVersion usage is deprecated."),
+        pytest.raises(ValueError, match="No worker version ID"),
+    ):
         fake_dummy_worker.get_worker_version_slug(None)
 
 
@@ -309,7 +310,7 @@ def test_start_activity_error(
         ),
     ]
     assert logger.error.call_args_list == [
-        mocker.call("Ran on 1 elements: 0 completed, 1 failed")
+        mocker.call("Ran on 1 element: 0 completed, 1 failed")
     ]
 
 
@@ -467,6 +468,10 @@ def test_worker_config_multiple_source(
             "id": "12341234-1234-1234-1234-123412341234",
             "name": "Model version 1337",
             "configuration": model_config,
+            "model": {
+                "id": "hahahaha-haha-haha-haha-hahahahahaha",
+                "name": "My model",
+            },
         },
         "process": {
             "name": None,

tests/test_image.py

@@ -7,6 +7,7 @@ from pathlib import Path
 
 import pytest
 from PIL import Image, ImageChops, ImageOps
+from requests import HTTPError
 
 from arkindex_worker.cache import CachedElement, create_tables, init_cache_db
 from arkindex_worker.image import (
@@ -19,6 +20,7 @@ from arkindex_worker.image import (
     polygon_bounding_box,
     revert_orientation,
     trim_polygon,
+    upload_image,
 )
 from arkindex_worker.models import Element
 
@@ -547,3 +549,38 @@ def test_download_image_retry_with_max(responses):
     assert list(map(attrgetter("request.url"), responses.calls)) == [full_url] * 3 + [
         max_url
     ]
+
+
+def test_upload_image_retries(responses):
+    dest_url = "https://blabla.com/iiif/2/image_path.jpg/full/full/0/default.jpg"
+    responses.add(
+        responses.PUT,
+        dest_url,
+        status=400,
+    )
+
+    image = Image.open(FULL_IMAGE).convert("RGB")
+    with pytest.raises(
+        HTTPError, match=f"400 Client Error: Bad Request for url: {dest_url}"
+    ):
+        upload_image(image, dest_url)
+
+    # We try 3 times
+    assert len(responses.calls) == 3
+    assert list(map(attrgetter("request.url"), responses.calls)) == [dest_url] * 3
+
+
+def test_upload_image(responses):
+    dest_url = "https://blabla.com/iiif/2/image_path.jpg/full/full/0/default.jpg"
+    responses.add(
+        responses.PUT,
+        dest_url,
+        status=200,
+    )
+
+    image = Image.open(FULL_IMAGE).convert("RGB")
+    resp = upload_image(image, dest_url)
+    assert resp
+
+    assert len(responses.calls) == 1
+    assert list(map(attrgetter("request.url"), responses.calls)) == [dest_url]

arkindex_base_worker-0.3.7rc2.dist-info/RECORD

@@ -1,41 +0,0 @@
-arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,162
-arkindex_worker/cache.py,sha256=FTlB0coXofn5zTNRTcVIvh709mcw4a1bPGqkwWjKs3w,11248
-arkindex_worker/image.py,sha256=uwfUE9hy0Iw-e3vU7OHmLSqouxbznWq08SykXmPD1Cs,14107
-arkindex_worker/models.py,sha256=DVrZPIurSiOoHvj3t_Szwd0j1t6pnwBx_dqwhNakzN0,9528
-arkindex_worker/utils.py,sha256=_lC1-RYvNWXEkK-AuF4_FraoggP1tYPdalNFSj4jDb4,6885
-arkindex_worker/worker/__init__.py,sha256=ev_EI7AA4qrQhH4B5WJf2LClcMVWPcxmaz_6L6Fp1V8,19383
-arkindex_worker/worker/base.py,sha256=qPcf-oOFtf7fLPFsQOukcLpkdeXBW4tpw99wCbio6Hw,19599
-arkindex_worker/worker/classification.py,sha256=0OiwxV9lb97Zs3kODm3hzyk0V7IxBTiW5SL6AYgRH1M,10351
-arkindex_worker/worker/dataset.py,sha256=IaXQXO9Zj-u7H0fvuDXOzf9FRga22eub_X-LYQHEpYY,3026
-arkindex_worker/worker/element.py,sha256=4f893hTSbk8QBD48xUKgHIzpr4wMxyxy634MqSwqebg,33847
-arkindex_worker/worker/entity.py,sha256=YT2Ttdn-L5TRoDdhOI3Z4GE1vtkWl7tKZqbYrtxZ2Ug,14630
-arkindex_worker/worker/metadata.py,sha256=XEKkwFeOflw0EsNwJhcCfv32iCCZwvQqYIyaqSEMNQc,6670
-arkindex_worker/worker/task.py,sha256=cz3wJNPgogZv1lm_3lm7WScitQtYQtL6H6I7Xokq208,1475
-arkindex_worker/worker/training.py,sha256=rhg4TPFo0ignnCkgbekUYmNXX8u2iZGyeM1VCs1R5kI,10140
-arkindex_worker/worker/transcription.py,sha256=6R7ofcGnNqX4rjT0kRKIE-G9FHq2TJ1tfztNM5sTqYE,20464
-arkindex_worker/worker/version.py,sha256=ogEpGdJ9GVA-eer6GMt6tiIvnuwfad6lDjdLNa6fiUQ,1926
-tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/conftest.py,sha256=wzKXRnS7OKQCNHrlDaQhMC8EXlsQTY_S4L9U_hXbjpM,22004
-tests/test_base_worker.py,sha256=Uq6_MpLW23gmKFXkU-SyDUaA_4dlViLBGG4e3gpBBz0,24512
-tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
-tests/test_dataset_worker.py,sha256=Y-g354MroQ1aYAZnqM5dhqgZZoDgpbHxpEZrcaN7SJY,29305
-tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
-tests/test_image.py,sha256=nIT0NhmuHtD9I1rcMSvqSfjQqvTE5Ko8tAQGLIkm_zo,15232
-tests/test_merge.py,sha256=Q4zCbtZbe0wBfqE56gvAD06c6pDuhqnjKaioFqIgAQw,8331
-tests/test_utils.py,sha256=pFXegcBvIuy1tJDDSgQtCbC_tRaoLjd2055R5lu3hS0,1236
-tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
-tests/test_elements_worker/test_classifications.py,sha256=0_6vryoQc2-s3CQWANsEvajkyC3aub34cxb3r97pRsk,32027
-tests/test_elements_worker/test_cli.py,sha256=BsFTswLti63WAZ2pf6ipiZKWJJyCQuSfuKnSlESuK8g,2878
-tests/test_elements_worker/test_dataset.py,sha256=uqgu5Jpazp2mH8p26jCn7RkJ7jdWpYKdJimWpeqaK1o,13251
-tests/test_elements_worker/test_elements.py,sha256=DOw1LKgJeCCFtvtQssBytIvE21mgiikhbkPglcpqRtA,84823
-tests/test_elements_worker/test_entities.py,sha256=yi1mXzvKvNwUNMzo0UZ56YOIJstYHcLyeepPJ8f10MQ,34557
-tests/test_elements_worker/test_metadata.py,sha256=b9CNv4W31TRJqYauvX_pRIN2SvnybaLqF-FWoFwa2Vc,18672
-tests/test_elements_worker/test_task.py,sha256=FCpxE9UpouKXgjGvWgNHEai_Hiy2d1YmqRG-_v2s27s,6312
-tests/test_elements_worker/test_training.py,sha256=WeG-cDuJ-YhPgfKH47TtXBxyargtLuk7c8tsik2WnL8,8414
-tests/test_elements_worker/test_transcriptions.py,sha256=w-Fs4dS8RNDChp-9BgC087LiKGJYlIWyI8OreWEj-E4,73373
-tests/test_elements_worker/test_worker.py,sha256=3T4f4SkZbyk_lRPikINKnzToNnLVGHcYxc3kylGSae4,17017
-arkindex_base_worker-0.3.7rc2.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
-arkindex_base_worker-0.3.7rc2.dist-info/METADATA,sha256=6iGFf8SnxJmqnaBHh9UCmCEV1sPxzhwuHApKOng_lVQ,3411
-arkindex_base_worker-0.3.7rc2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-arkindex_base_worker-0.3.7rc2.dist-info/top_level.txt,sha256=TtagLI8LSv7GE7nG8MQqDFAJ5bNDPJn7Z5vizOgrWkA,22
-arkindex_base_worker-0.3.7rc2.dist-info/RECORD,,