arkindex-base-worker 0.3.7rc4__py3-none-any.whl → 0.5.0a1__py3-none-any.whl

tests/test_elements_worker/test_metadata.py

@@ -2,11 +2,12 @@ import json
 import re
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex.mock import MockApiClient
 from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Element
+from arkindex_worker.utils import DEFAULT_BATCH_SIZE
 from arkindex_worker.worker import MetaType
 
 from . import BASE_API_CALLS
@@ -136,7 +137,7 @@ def test_create_metadata_api_error(responses, mock_elements_worker):
     responses.add(
         responses.POST,
         "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/",
-        status=500,
+        status=418,
     )
 
     with pytest.raises(ErrorResponse):
@@ -147,31 +148,14 @@ def test_create_metadata_api_error(responses, mock_elements_worker):
             value="La Turbine, Grenoble 38000",
         )
 
-    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
     assert [
         (call.request.method, call.request.url) for call in responses.calls
     ] == BASE_API_CALLS + [
-        # We retry 5 times the API call
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/",
-        ),
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/",
-        ),
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/",
-        ),
         (
             "POST",
             "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/",
-        ),
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/",
-        ),
+        )
     ]
 
 
@@ -245,148 +229,272 @@ def test_create_metadata_cached_element(responses, mock_elements_worker_with_cac
     assert metadata_id == "12345678-1234-1234-1234-123456789123"
 
 
-@pytest.mark.parametrize(
-    "metadata_list",
-    [
-        [{"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}],
-        [
-            {
-                "type": MetaType.Text,
-                "name": "fake_name",
-                "value": "fake_value",
-                "entity_id": "fake_entity_id",
-            }
-        ],
-    ],
-)
-def test_create_metadatas(responses, mock_elements_worker, metadata_list):
+@pytest.mark.parametrize("batch_size", [DEFAULT_BATCH_SIZE, 1])
+def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
-    responses.add(
-        responses.POST,
-        "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        status=201,
-        json={
-            "worker_run_id": "56785678-5678-5678-5678-567856785678",
-            "metadata_list": [
-                {
-                    "id": "fake_metadata_id",
-                    "type": metadata_list[0]["type"].value,
-                    "name": metadata_list[0]["name"],
-                    "value": metadata_list[0]["value"],
-                    "dates": [],
-                    "entity_id": metadata_list[0].get("entity_id"),
-                }
-            ],
-        },
-    )
 
-    created_metadata_list = mock_elements_worker.create_metadatas(
-        element, metadata_list
+    metadata_list = [
+        {"type": MetaType.Text, "name": "fake_name", "value": "fake_value"},
+        {
+            "type": MetaType.Text,
+            "name": "Year",
+            "value": "2024",
+            "entity_id": "entity_id",
+        },
+    ]
+    if batch_size > 1:
+        responses.add(
+            responses.POST,
+            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
+            status=201,
+            json={
+                "worker_run_id": "56785678-5678-5678-5678-567856785678",
+                "metadata_list": [
+                    {
+                        "id": "fake_metadata_id1",
+                        "type": metadata_list[0]["type"].value,
+                        "name": metadata_list[0]["name"],
+                        "value": metadata_list[0]["value"],
+                        "dates": [],
+                        "entity_id": None,
+                    },
+                    {
+                        "id": "fake_metadata_id2",
+                        "type": metadata_list[1]["type"].value,
+                        "name": metadata_list[1]["name"],
+                        "value": metadata_list[1]["value"],
+                        "dates": [],
+                        "entity_id": metadata_list[1]["entity_id"],
+                    },
+                ],
+            },
+        )
+    else:
+        for idx, meta in enumerate(metadata_list, start=1):
+            responses.add(
+                responses.POST,
+                "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
+                status=201,
+                json={
+                    "worker_run_id": "56785678-5678-5678-5678-567856785678",
+                    "metadata_list": [
+                        {
+                            "id": f"fake_metadata_id{idx}",
+                            "type": meta["type"].value,
+                            "name": meta["name"],
+                            "value": meta["value"],
+                            "dates": [],
+                            "entity_id": meta.get("entity_id"),
+                        }
+                    ],
+                },
+            )
+
+    created_metadata_list = mock_elements_worker.create_metadata_bulk(
+        element, metadata_list, batch_size=batch_size
     )
 
-    assert len(responses.calls) == len(BASE_API_CALLS) + 1
-    assert [
-        (call.request.method, call.request.url) for call in responses.calls
-    ] == BASE_API_CALLS + [
+    bulk_api_calls = [
         (
             "POST",
             "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
-    ]
-    assert json.loads(responses.calls[-1].request.body)["metadata_list"] == [
-        {
-            "type": metadata_list[0]["type"].value,
-            "name": metadata_list[0]["name"],
-            "value": metadata_list[0]["value"],
-            "entity_id": metadata_list[0].get("entity_id"),
-        }
+        )
     ]
+    if batch_size != DEFAULT_BATCH_SIZE:
+        bulk_api_calls.append(
+            (
+                "POST",
+                "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
+            )
+        )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + len(bulk_api_calls)
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + bulk_api_calls
+
+    first_meta = {
+        **metadata_list[0],
+        "type": metadata_list[0]["type"].value,
+        "entity_id": None,
+    }
+    second_meta = {**metadata_list[1], "type": metadata_list[1]["type"].value}
+    empty_payload = {
+        "worker_run_id": "56785678-5678-5678-5678-567856785678",
+        "metadata_list": [],
+    }
+
+    bodies = []
+    first_call_idx = None
+    if batch_size > 1:
+        first_call_idx = -1
+        bodies.append({**empty_payload, "metadata_list": [first_meta, second_meta]})
+    else:
+        first_call_idx = -2
+        bodies.append({**empty_payload, "metadata_list": [first_meta]})
+        bodies.append({**empty_payload, "metadata_list": [second_meta]})
+
+    assert [
+        json.loads(bulk_call.request.body)
+        for bulk_call in responses.calls[first_call_idx:]
+    ] == bodies
+
     assert created_metadata_list == [
         {
-            "id": "fake_metadata_id",
+            "id": "fake_metadata_id1",
             "type": metadata_list[0]["type"].value,
             "name": metadata_list[0]["name"],
             "value": metadata_list[0]["value"],
             "dates": [],
-            "entity_id": metadata_list[0].get("entity_id"),
-        }
+            "entity_id": None,
+        },
+        {
+            "id": "fake_metadata_id2",
+            "type": metadata_list[1]["type"].value,
+            "name": metadata_list[1]["name"],
+            "value": metadata_list[1]["value"],
+            "dates": [],
+            "entity_id": metadata_list[1]["entity_id"],
+        },
     ]
 
 
-@pytest.mark.parametrize(
-    "metadata_list",
-    [
-        [{"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}],
-        [
-            {
-                "type": MetaType.Text,
-                "name": "fake_name",
-                "value": "fake_value",
-                "entity_id": "fake_entity_id",
-            }
-        ],
-    ],
-)
-def test_create_metadatas_cached_element(
-    responses, mock_elements_worker_with_cache, metadata_list
+@pytest.mark.parametrize("batch_size", [DEFAULT_BATCH_SIZE, 1])
+def test_create_metadata_bulk_cached_element(
+    batch_size, responses, mock_elements_worker_with_cache
 ):
     element = CachedElement.create(
         id="12341234-1234-1234-1234-123412341234", type="thing"
     )
-    responses.add(
-        responses.POST,
-        "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        status=201,
-        json={
-            "worker_run_id": "56785678-5678-5678-5678-567856785678",
-            "metadata_list": [
-                {
-                    "id": "fake_metadata_id",
-                    "type": metadata_list[0]["type"].value,
-                    "name": metadata_list[0]["name"],
-                    "value": metadata_list[0]["value"],
-                    "dates": [],
-                    "entity_id": metadata_list[0].get("entity_id"),
-                }
-            ],
-        },
-    )
 
-    created_metadata_list = mock_elements_worker_with_cache.create_metadatas(
-        element, metadata_list
+    metadata_list = [
+        {"type": MetaType.Text, "name": "fake_name", "value": "fake_value"},
+        {
+            "type": MetaType.Text,
+            "name": "Year",
+            "value": "2024",
+            "entity_id": "entity_id",
+        },
+    ]
+    if batch_size > 1:
+        responses.add(
+            responses.POST,
+            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
+            status=201,
+            json={
+                "worker_run_id": "56785678-5678-5678-5678-567856785678",
+                "metadata_list": [
+                    {
+                        "id": "fake_metadata_id1",
+                        "type": metadata_list[0]["type"].value,
+                        "name": metadata_list[0]["name"],
+                        "value": metadata_list[0]["value"],
+                        "dates": [],
+                        "entity_id": None,
+                    },
+                    {
+                        "id": "fake_metadata_id2",
+                        "type": metadata_list[1]["type"].value,
+                        "name": metadata_list[1]["name"],
+                        "value": metadata_list[1]["value"],
+                        "dates": [],
+                        "entity_id": metadata_list[1]["entity_id"],
+                    },
+                ],
+            },
+        )
+    else:
+        for idx, meta in enumerate(metadata_list, start=1):
+            responses.add(
+                responses.POST,
+                "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
+                status=201,
+                json={
+                    "worker_run_id": "56785678-5678-5678-5678-567856785678",
+                    "metadata_list": [
+                        {
+                            "id": f"fake_metadata_id{idx}",
+                            "type": meta["type"].value,
+                            "name": meta["name"],
+                            "value": meta["value"],
+                            "dates": [],
+                            "entity_id": meta.get("entity_id"),
+                        }
+                    ],
+                },
+            )
+
+    created_metadata_list = mock_elements_worker_with_cache.create_metadata_bulk(
+        element, metadata_list, batch_size=batch_size
     )
 
-    assert len(responses.calls) == len(BASE_API_CALLS) + 1
-    assert [
-        (call.request.method, call.request.url) for call in responses.calls
-    ] == BASE_API_CALLS + [
+    bulk_api_calls = [
         (
             "POST",
             "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
-    ]
-    assert json.loads(responses.calls[-1].request.body)["metadata_list"] == [
-        {
-            "type": metadata_list[0]["type"].value,
-            "name": metadata_list[0]["name"],
-            "value": metadata_list[0]["value"],
-            "entity_id": metadata_list[0].get("entity_id"),
-        }
+        )
     ]
+    if batch_size != DEFAULT_BATCH_SIZE:
+        bulk_api_calls.append(
+            (
+                "POST",
+                "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
+            )
+        )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + len(bulk_api_calls)
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + bulk_api_calls
+
+    first_meta = {
+        **metadata_list[0],
+        "type": metadata_list[0]["type"].value,
+        "entity_id": None,
+    }
+    second_meta = {**metadata_list[1], "type": metadata_list[1]["type"].value}
+    empty_payload = {
+        "worker_run_id": "56785678-5678-5678-5678-567856785678",
+        "metadata_list": [],
+    }
+
+    bodies = []
+    first_call_idx = None
+    if batch_size > 1:
+        first_call_idx = -1
+        bodies.append({**empty_payload, "metadata_list": [first_meta, second_meta]})
+    else:
+        first_call_idx = -2
+        bodies.append({**empty_payload, "metadata_list": [first_meta]})
+        bodies.append({**empty_payload, "metadata_list": [second_meta]})
+
+    assert [
+        json.loads(bulk_call.request.body)
+        for bulk_call in responses.calls[first_call_idx:]
+    ] == bodies
+
     assert created_metadata_list == [
         {
-            "id": "fake_metadata_id",
+            "id": "fake_metadata_id1",
             "type": metadata_list[0]["type"].value,
             "name": metadata_list[0]["name"],
             "value": metadata_list[0]["value"],
             "dates": [],
-            "entity_id": metadata_list[0].get("entity_id"),
-        }
+            "entity_id": None,
+        },
+        {
+            "id": "fake_metadata_id2",
+            "type": metadata_list[1]["type"].value,
+            "name": metadata_list[1]["name"],
+            "value": metadata_list[1]["value"],
+            "dates": [],
+            "entity_id": metadata_list[1]["entity_id"],
+        },
     ]
 
 
 @pytest.mark.parametrize("wrong_element", [None, "not_element_type", 1234, 12.5])
-def test_create_metadatas_wrong_element(mock_elements_worker, wrong_element):
+def test_create_metadata_bulk_wrong_element(mock_elements_worker, wrong_element):
     wrong_metadata_list = [
         {"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}
     ]
@@ -394,13 +502,13 @@ def test_create_metadatas_wrong_element(mock_elements_worker, wrong_element):
         AssertionError,
         match="element shouldn't be null and should be of type Element or CachedElement",
     ):
-        mock_elements_worker.create_metadatas(
-            element=wrong_element, metadatas=wrong_metadata_list
+        mock_elements_worker.create_metadata_bulk(
+            element=wrong_element, metadata_list=wrong_metadata_list
         )
 
 
 @pytest.mark.parametrize("wrong_type", [None, "not_metadata_type", 1234, 12.5])
-def test_create_metadatas_wrong_type(mock_elements_worker, wrong_type):
+def test_create_metadata_bulk_wrong_type(mock_elements_worker, wrong_type):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
     wrong_metadata_list = [
         {"type": wrong_type, "name": "fake_name", "value": "fake_value"}
@@ -408,13 +516,13 @@ def test_create_metadatas_wrong_type(mock_elements_worker, wrong_type):
     with pytest.raises(
         AssertionError, match="type shouldn't be null and should be of type MetaType"
     ):
-        mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadata_list
+        mock_elements_worker.create_metadata_bulk(
+            element=element, metadata_list=wrong_metadata_list
         )
 
 
 @pytest.mark.parametrize("wrong_name", [None, 1234, 12.5, [1, 2, 3, 4]])
-def test_create_metadatas_wrong_name(mock_elements_worker, wrong_name):
+def test_create_metadata_bulk_wrong_name(mock_elements_worker, wrong_name):
     element = Element({"id": "fake_element_id"})
     wrong_metadata_list = [
         {"type": MetaType.Text, "name": wrong_name, "value": "fake_value"}
@@ -422,13 +530,13 @@ def test_create_metadatas_wrong_name(mock_elements_worker, wrong_name):
     with pytest.raises(
         AssertionError, match="name shouldn't be null and should be of type str"
     ):
-        mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadata_list
+        mock_elements_worker.create_metadata_bulk(
+            element=element, metadata_list=wrong_metadata_list
         )
 
 
 @pytest.mark.parametrize("wrong_value", [None, [1, 2, 3, 4]])
-def test_create_metadatas_wrong_value(mock_elements_worker, wrong_value):
+def test_create_metadata_bulk_wrong_value(mock_elements_worker, wrong_value):
     element = Element({"id": "fake_element_id"})
     wrong_metadata_list = [
         {"type": MetaType.Text, "name": "fake_name", "value": wrong_value}
@@ -439,13 +547,13 @@ def test_create_metadatas_wrong_value(mock_elements_worker, wrong_value):
             "value shouldn't be null and should be of type (str or float or int)"
         ),
     ):
-        mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadata_list
+        mock_elements_worker.create_metadata_bulk(
+            element=element, metadata_list=wrong_metadata_list
         )
 
 
 @pytest.mark.parametrize("wrong_entity", [[1, 2, 3, 4], 1234, 12.5])
-def test_create_metadatas_wrong_entity(mock_elements_worker, wrong_entity):
+def test_create_metadata_bulk_wrong_entity(mock_elements_worker, wrong_entity):
     element = Element({"id": "fake_element_id"})
     wrong_metadata_list = [
         {
@@ -456,12 +564,12 @@ def test_create_metadatas_wrong_entity(mock_elements_worker, wrong_entity):
         }
     ]
     with pytest.raises(AssertionError, match="entity_id should be None or a str"):
-        mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadata_list
+        mock_elements_worker.create_metadata_bulk(
+            element=element, metadata_list=wrong_metadata_list
         )
 
 
-def test_create_metadatas_api_error(responses, mock_elements_worker):
+def test_create_metadata_bulk_api_error(responses, mock_elements_worker):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
     metadata_list = [
         {
@@ -474,37 +582,20 @@ def test_create_metadatas_api_error(responses, mock_elements_worker):
     responses.add(
         responses.POST,
         "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        status=500,
+        status=418,
     )
 
     with pytest.raises(ErrorResponse):
-        mock_elements_worker.create_metadatas(element, metadata_list)
+        mock_elements_worker.create_metadata_bulk(element, metadata_list)
 
-    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
     assert [
         (call.request.method, call.request.url) for call in responses.calls
     ] == BASE_API_CALLS + [
-        # We retry 5 times the API call
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
         (
             "POST",
             "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
-        (
-            "POST",
-            "http://testserver/api/v1/element/12341234-1234-1234-1234-123412341234/metadata/bulk/",
-        ),
+        )
     ]
 
 

tests/test_elements_worker/test_process.py

@@ -0,0 +1,89 @@
+import pytest
+
+from tests import PROCESS_ID
+
+
+@pytest.mark.parametrize(
+    ("with_image", "elements"),
+    [
+        (
+            False,
+            [
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 1",
+                    "confidence": 1,
+                    "image_id": None,
+                    "image_width": None,
+                    "image_height": None,
+                    "image_url": None,
+                    "polygon": None,
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa0",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 2",
+                    "confidence": 1,
+                    "image_id": None,
+                    "image_width": None,
+                    "image_height": None,
+                    "image_url": None,
+                    "polygon": None,
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+            ],
+        ),
+        (
+            True,
+            [
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 1",
+                    "confidence": 1,
+                    "image_id": "aaa2aaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "image_width": 76,
+                    "image_height": 138,
+                    "image_url": "http://somewhere.com/iiif/image.jpeg",
+                    "polygon": [[0, 0], [0, 40], [20, 40], [20, 0]],
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+                {
+                    "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa0",
+                    "type_id": "baaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "name": "element 2",
+                    "confidence": 1,
+                    "image_id": "aaa2aaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "image_width": 138,
+                    "image_height": 76,
+                    "image_url": "http://somewhere.com/iiif/image.jpeg",
+                    "polygon": [[0, 0], [0, 40], [20, 40], [20, 0]],
+                    "rotation_angle": 0,
+                    "mirrored": False,
+                },
+            ],
+        ),
+    ],
+)
+def test_list_process_elements_with_image(
+    responses, mock_elements_worker, with_image, elements
+):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/process/{PROCESS_ID}/elements/?page_size=500&with_count=true&with_image={with_image}",
+        status=200,
+        json={
+            "count": 2,
+            "next": None,
+            "results": elements,
+        },
+    )
+    assert (
+        list(mock_elements_worker.list_process_elements(with_image=with_image))
+        == elements
+    )