arkindex-base-worker 0.4.0b2__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

tests/test_elements_worker/test_elements.py

@@ -109,7 +109,7 @@ def test_create_missing_types(responses, mock_elements_worker):
     )
 
 
-def test_list_elements_elements_list_arg_wrong_type(
+def test_get_elements_elements_list_arg_wrong_type(
     monkeypatch, tmp_path, mock_elements_worker
 ):
     elements_path = tmp_path / "elements.json"
@@ -120,10 +120,10 @@ def test_list_elements_elements_list_arg_wrong_type(
     worker.configure()
 
     with pytest.raises(AssertionError, match="Elements list must be a list"):
-        worker.list_elements()
+        worker.get_elements()
 
 
-def test_list_elements_elements_list_arg_empty_list(
+def test_get_elements_elements_list_arg_empty_list(
     monkeypatch, tmp_path, mock_elements_worker
 ):
     elements_path = tmp_path / "elements.json"
@@ -134,10 +134,10 @@ def test_list_elements_elements_list_arg_empty_list(
     worker.configure()
 
     with pytest.raises(AssertionError, match="No elements in elements list"):
-        worker.list_elements()
+        worker.get_elements()
 
 
-def test_list_elements_elements_list_arg_missing_id(
+def test_get_elements_elements_list_arg_missing_id(
     monkeypatch, tmp_path, mock_elements_worker
 ):
     elements_path = tmp_path / "elements.json"
@@ -147,12 +147,12 @@ def test_list_elements_elements_list_arg_missing_id(
     worker = ElementsWorker()
     worker.configure()
 
-    elt_list = worker.list_elements()
+    elt_list = worker.get_elements()
 
     assert elt_list == []
 
 
-def test_list_elements_elements_list_arg_not_uuid(
+def test_get_elements_elements_list_arg_not_uuid(
     monkeypatch, tmp_path, mock_elements_worker
 ):
     elements_path = tmp_path / "elements.json"
@@ -175,10 +175,10 @@ def test_list_elements_elements_list_arg_not_uuid(
         Exception,
         match="These element IDs are invalid: volumeid, pageid, actid, surfaceid",
     ):
-        worker.list_elements()
+        worker.get_elements()
 
 
-def test_list_elements_elements_list_arg(monkeypatch, tmp_path, mock_elements_worker):
+def test_get_elements_elements_list_arg(monkeypatch, tmp_path, mock_elements_worker):
     elements_path = tmp_path / "elements.json"
     elements_path.write_text(
         json.dumps(
@@ -194,7 +194,7 @@ def test_list_elements_elements_list_arg(monkeypatch, tmp_path, mock_elements_wo
     worker = ElementsWorker()
     worker.configure()
 
-    elt_list = worker.list_elements()
+    elt_list = worker.get_elements()
 
     assert elt_list == [
         "11111111-1111-1111-1111-111111111111",
@@ -203,7 +203,7 @@ def test_list_elements_elements_list_arg(monkeypatch, tmp_path, mock_elements_wo
     ]
 
 
-def test_list_elements_element_arg_not_uuid(mocker, mock_elements_worker):
+def test_get_elements_element_arg_not_uuid(mocker, mock_elements_worker):
     mocker.patch(
         "arkindex_worker.worker.base.argparse.ArgumentParser.parse_args",
         return_value=Namespace(
@@ -221,10 +221,10 @@ def test_list_elements_element_arg_not_uuid(mocker, mock_elements_worker):
     with pytest.raises(
         Exception, match="These element IDs are invalid: volumeid, pageid"
     ):
-        worker.list_elements()
+        worker.get_elements()
 
 
-def test_list_elements_element_arg(mocker, mock_elements_worker):
+def test_get_elements_element_arg(mocker, mock_elements_worker):
     mocker.patch(
         "arkindex_worker.worker.base.argparse.ArgumentParser.parse_args",
         return_value=Namespace(
@@ -242,7 +242,7 @@ def test_list_elements_element_arg(mocker, mock_elements_worker):
     worker = ElementsWorker()
     worker.configure()
 
-    elt_list = worker.list_elements()
+    elt_list = worker.get_elements()
 
     assert elt_list == [
         "11111111-1111-1111-1111-111111111111",
@@ -250,7 +250,7 @@ def test_list_elements_element_arg(mocker, mock_elements_worker):
     ]
 
 
-def test_list_elements_both_args_error(mocker, mock_elements_worker, tmp_path):
+def test_get_elements_both_args_error(mocker, mock_elements_worker, tmp_path):
     elements_path = tmp_path / "elements.json"
     elements_path.write_text(
         json.dumps(
@@ -279,7 +279,7 @@ def test_list_elements_both_args_error(mocker, mock_elements_worker, tmp_path):
     with pytest.raises(
         AssertionError, match="elements-list and element CLI args shouldn't be both set"
     ):
-        worker.list_elements()
+        worker.get_elements()
 
 
 def test_database_arg(mocker, mock_elements_worker, tmp_path):
@@ -1500,6 +1500,139 @@ def test_create_element_parent(responses, mock_elements_worker):
     }
 
 
+@pytest.mark.parametrize(
+    ("arg_name", "data", "error_message"),
+    [
+        (
+            "parent",
+            None,
+            "parent shouldn't be null and should be of type Element",
+        ),
+        (
+            "parent",
+            "not element type",
+            "parent shouldn't be null and should be of type Element",
+        ),
+        (
+            "children",
+            None,
+            "children shouldn't be null and should be of type list",
+        ),
+        (
+            "children",
+            "not a list",
+            "children shouldn't be null and should be of type list",
+        ),
+        (
+            "children",
+            [
+                Element({"id": "11111111-1111-1111-1111-111111111111"}),
+                "not element type",
+            ],
+            "Child at index 1 in children: Should be of type Element",
+        ),
+    ],
+)
+def test_create_element_children_wrong_params(
+    arg_name, data, error_message, mock_elements_worker
+):
+    with pytest.raises(AssertionError, match=error_message):
+        mock_elements_worker.create_element_children(
+            **{
+                "parent": Element({"id": "12341234-1234-1234-1234-123412341234"}),
+                "children": [
+                    Element({"id": "11111111-1111-1111-1111-111111111111"}),
+                    Element({"id": "22222222-2222-2222-2222-222222222222"}),
+                ],
+                # Overwrite with wrong data
+                arg_name: data,
+            },
+        )
+
+
+def test_create_element_children_api_error(responses, mock_elements_worker):
+    parent = Element({"id": "12341234-1234-1234-1234-123412341234"})
+    responses.add(
+        responses.POST,
+        f"http://testserver/api/v1/element/parent/{parent.id}/",
+        status=418,
+    )
+
+    with pytest.raises(ErrorResponse):
+        mock_elements_worker.create_element_children(
+            parent=parent,
+            children=[
+                Element({"id": "11111111-1111-1111-1111-111111111111"}),
+                Element({"id": "22222222-2222-2222-2222-222222222222"}),
+            ],
+        )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "POST",
+            f"http://testserver/api/v1/element/parent/{parent.id}/",
+        )
+    ]
+
+
+@pytest.mark.parametrize("batch_size", [DEFAULT_BATCH_SIZE, 1])
+def test_create_element_children(batch_size, responses, mock_elements_worker):
+    parent = Element({"id": "12341234-1234-1234-1234-123412341234"})
+
+    first_child = Element({"id": "11111111-1111-1111-1111-111111111111"})
+    second_child = Element({"id": "22222222-2222-2222-2222-222222222222"})
+
+    responses.add(
+        responses.POST,
+        f"http://testserver/api/v1/element/parent/{parent.id}/",
+        status=200,
+        json={"children": []},
+    )
+
+    mock_elements_worker.create_element_children(
+        parent=parent,
+        children=[first_child, second_child],
+        batch_size=batch_size,
+    )
+
+    bulk_api_calls = [
+        (
+            "POST",
+            f"http://testserver/api/v1/element/parent/{parent.id}/",
+        )
+    ]
+    if batch_size != DEFAULT_BATCH_SIZE:
+        bulk_api_calls.append(
+            (
+                "POST",
+                f"http://testserver/api/v1/element/parent/{parent.id}/",
+            )
+        )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + len(bulk_api_calls)
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + bulk_api_calls
+
+    bodies = []
+    first_call_idx = None
+    if batch_size > 1:
+        first_call_idx = -1
+        bodies.append({"children": [first_child.id, second_child.id]})
+    else:
+        first_call_idx = -2
+        bodies.append({"children": [first_child.id]})
+        bodies.append({"children": [second_child.id]})
+
+    assert [
+        json.loads(bulk_call.request.body)
+        for bulk_call in responses.calls[first_call_idx:]
+    ] == bodies
+
+
 @pytest.mark.parametrize(
     ("payload", "error"),
     [
@@ -1825,6 +1958,433 @@ def test_partial_update_element_confidence(
     assert cached_element.confidence == confidence
 
 
+def test_list_elements_wrong_folder(mock_elements_worker):
+    with pytest.raises(AssertionError, match="folder should be of type bool"):
+        mock_elements_worker.list_elements(folder="not bool")
+
+
+def test_list_elements_wrong_name(mock_elements_worker):
+    with pytest.raises(AssertionError, match="name should be of type str"):
+        mock_elements_worker.list_elements(name=1234)
+
+
+def test_list_elements_wrong_top_level(mock_elements_worker):
+    with pytest.raises(AssertionError, match="top_level should be of type bool"):
+        mock_elements_worker.list_elements(top_level="not bool")
+
+
+def test_list_elements_wrong_type(mock_elements_worker):
+    with pytest.raises(AssertionError, match="type should be of type str"):
+        mock_elements_worker.list_elements(type=1234)
+
+
+def test_list_elements_wrong_with_classes(mock_elements_worker):
+    with pytest.raises(AssertionError, match="with_classes should be of type bool"):
+        mock_elements_worker.list_elements(with_classes="not bool")
+
+
+def test_list_elements_wrong_with_corpus(mock_elements_worker):
+    with pytest.raises(AssertionError, match="with_corpus should be of type bool"):
+        mock_elements_worker.list_elements(with_corpus="not bool")
+
+
+def test_list_elements_wrong_with_has_children(mock_elements_worker):
+    with pytest.raises(
+        AssertionError, match="with_has_children should be of type bool"
+    ):
+        mock_elements_worker.list_elements(with_has_children="not bool")
+
+
+def test_list_elements_wrong_with_zone(mock_elements_worker):
+    with pytest.raises(AssertionError, match="with_zone should be of type bool"):
+        mock_elements_worker.list_elements(with_zone="not bool")
+
+
+def test_list_elements_wrong_with_metadata(mock_elements_worker):
+    with pytest.raises(AssertionError, match="with_metadata should be of type bool"):
+        mock_elements_worker.list_elements(with_metadata="not bool")
+
+
+@pytest.mark.parametrize(
+    ("param", "value"),
+    [
+        ("worker_run", 1234),
+        ("transcription_worker_run", 1234),
+    ],
+)
+def test_list_elements_wrong_worker_run(mock_elements_worker, param, value):
+    with pytest.raises(AssertionError, match=f"{param} should be of type str or bool"):
+        mock_elements_worker.list_elements(**{param: value})
+
+
+@pytest.mark.parametrize(
+    ("param", "alternative", "value"),
+    [
+        ("worker_version", "worker_run", 1234),
+        ("transcription_worker_version", "transcription_worker_run", 1234),
+    ],
+)
+def test_list_elements_wrong_worker_version(
+    mock_elements_worker, param, alternative, value
+):
+    # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
+    with (
+        pytest.deprecated_call(
+            match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
+        ),
+        pytest.raises(AssertionError, match=f"{param} should be of type str or bool"),
+    ):
+        mock_elements_worker.list_elements(**{param: value})
+
+
+@pytest.mark.parametrize(
+    "param",
+    [
+        "worker_run",
+        "transcription_worker_run",
+    ],
+)
+def test_list_elements_wrong_bool_worker_run(mock_elements_worker, param):
+    with pytest.raises(
+        AssertionError, match=f"if of type bool, {param} can only be set to False"
+    ):
+        mock_elements_worker.list_elements(**{param: True})
+
+
+@pytest.mark.parametrize(
+    ("param", "alternative"),
+    [
+        ("worker_version", "worker_run"),
+        ("transcription_worker_version", "transcription_worker_run"),
+    ],
+)
+def test_list_elements_wrong_bool_worker_version(
+    mock_elements_worker, param, alternative
+):
+    # WARNING: pytest.deprecated_call must be placed BEFORE pytest.raises, otherwise `match` argument won't be checked
+    with (
+        pytest.deprecated_call(
+            match=f"`{param}` usage is deprecated. Consider using `{alternative}` instead."
+        ),
+        pytest.raises(
+            AssertionError, match=f"if of type bool, {param} can only be set to False"
+        ),
+    ):
+        mock_elements_worker.list_elements(**{param: True})
+
+
+def test_list_elements_api_error(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        status=418,
+    )
+
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        next(mock_elements_worker.list_elements())
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        # We do 5 retries
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        ),
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        ),
+    ]
+
+
+def test_list_elements(responses, mock_elements_worker):
+    expected_children = [
+        {
+            "id": "0000",
+            "type": "page",
+            "name": "Test",
+            "corpus": {},
+            "thumbnail_url": None,
+            "zone": {},
+            "best_classes": None,
+            "has_children": None,
+            "worker_version_id": None,
+            "worker_run_id": None,
+        },
+        {
+            "id": "1111",
+            "type": "page",
+            "name": "Test 2",
+            "corpus": {},
+            "thumbnail_url": None,
+            "zone": {},
+            "best_classes": None,
+            "has_children": None,
+            "worker_version_id": None,
+            "worker_run_id": None,
+        },
+        {
+            "id": "2222",
+            "type": "page",
+            "name": "Test 3",
+            "corpus": {},
+            "thumbnail_url": None,
+            "zone": {},
+            "best_classes": None,
+            "has_children": None,
+            "worker_version_id": None,
+            "worker_run_id": None,
+        },
+    ]
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        status=200,
+        json={
+            "count": 3,
+            "next": None,
+            "results": expected_children,
+        },
+    )
+
+    for idx, child in enumerate(mock_elements_worker.list_elements()):
+        assert child == expected_children[idx]
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/",
+        ),
+    ]
+
+
+def test_list_elements_manual_worker_version(responses, mock_elements_worker):
+    expected_children = [
+        {
+            "id": "0000",
+            "type": "page",
+            "name": "Test",
+            "corpus": {},
+            "thumbnail_url": None,
+            "zone": {},
+            "best_classes": None,
+            "has_children": None,
+            "worker_version_id": None,
+            "worker_run_id": None,
+        }
+    ]
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/?worker_version=False",
+        status=200,
+        json={
+            "count": 1,
+            "next": None,
+            "results": expected_children,
+        },
+    )
+
+    with pytest.deprecated_call(
+        match="`worker_version` usage is deprecated. Consider using `worker_run` instead."
+    ):
+        for idx, child in enumerate(
+            mock_elements_worker.list_elements(worker_version=False)
+        ):
+            assert child == expected_children[idx]
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/?worker_version=False",
+        ),
+    ]
+
+
+def test_list_elements_manual_worker_run(responses, mock_elements_worker):
+    expected_children = [
+        {
+            "id": "0000",
+            "type": "page",
+            "name": "Test",
+            "corpus": {},
+            "thumbnail_url": None,
+            "zone": {},
+            "best_classes": None,
+            "has_children": None,
+            "worker_version_id": None,
+            "worker_run_id": None,
+        }
+    ]
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/?worker_run=False",
+        status=200,
+        json={
+            "count": 1,
+            "next": None,
+            "results": expected_children,
+        },
+    )
+
+    for idx, child in enumerate(mock_elements_worker.list_elements(worker_run=False)):
+        assert child == expected_children[idx]
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            f"http://testserver/api/v1/corpus/{mock_elements_worker.corpus_id}/elements/?worker_run=False",
+        ),
+    ]
+
+
+def test_list_elements_with_cache_unhandled_param(mock_elements_worker_with_cache):
+    with pytest.raises(
+        AssertionError,
+        match="When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'",
+    ):
+        mock_elements_worker_with_cache.list_elements(with_corpus=True)
+
+
+@pytest.mark.usefixtures("_mock_cached_elements")
+@pytest.mark.parametrize(
+    ("filters", "expected_ids"),
+    [
+        # Filter on element should give all elements inserted
+        (
+            {},
+            (
+                "99999999-9999-9999-9999-999999999999",
+                "12341234-1234-1234-1234-123412341234",
+                "11111111-1111-1111-1111-111111111111",
+                "22222222-2222-2222-2222-222222222222",
+                "33333333-3333-3333-3333-333333333333",
+            ),
+        ),
+        # Filter on element and page should give the second element
+        (
+            {"type": "page"},
+            ("22222222-2222-2222-2222-222222222222",),
+        ),
+        # Filter on element and worker run should give second
+        (
+            {
+                "worker_run": "56785678-5678-5678-5678-567856785678",
+            },
+            (
+                "12341234-1234-1234-1234-123412341234",
+                "22222222-2222-2222-2222-222222222222",
+            ),
+        ),
+        # Filter on element, manual worker run should give first and third
+        (
+            {"worker_run": False},
+            (
+                "99999999-9999-9999-9999-999999999999",
+                "11111111-1111-1111-1111-111111111111",
+                "33333333-3333-3333-3333-333333333333",
+            ),
+        ),
+    ],
+)
+def test_list_elements_with_cache(
+    responses, mock_elements_worker_with_cache, filters, expected_ids
+):
+    # Check we have 5 elements already present in database
+    assert CachedElement.select().count() == 5
+
+    # Query database through cache
+    elements = mock_elements_worker_with_cache.list_elements(**filters)
+    assert elements.count() == len(expected_ids)
+    for child, expected_id in zip(elements.order_by("id"), expected_ids, strict=True):
+        assert child.id == UUID(expected_id)
+
+    # Check the worker never hits the API for elements
+    assert len(responses.calls) == len(BASE_API_CALLS)
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS
+
+
+@pytest.mark.usefixtures("_mock_cached_elements")
+@pytest.mark.parametrize(
+    ("filters", "expected_ids"),
+    [
+        # Filter on element and worker version
+        (
+            {
+                "worker_version": "56785678-5678-5678-5678-567856785678",
+            },
+            (
+                "12341234-1234-1234-1234-123412341234",
+                "11111111-1111-1111-1111-111111111111",
+                "22222222-2222-2222-2222-222222222222",
+            ),
+        ),
+        # Filter on element, type double_page and worker version
+        (
+            {"type": "page", "worker_version": "56785678-5678-5678-5678-567856785678"},
+            ("22222222-2222-2222-2222-222222222222",),
+        ),
+        # Filter on element, manual worker version
+        (
+            {"worker_version": False},
+            (
+                "99999999-9999-9999-9999-999999999999",
+                "33333333-3333-3333-3333-333333333333",
+            ),
+        ),
+    ],
+)
+def test_list_elements_with_cache_deprecation(
+    responses,
+    mock_elements_worker_with_cache,
+    filters,
+    expected_ids,
+):
+    # Check we have 5 elements already present in database
+    assert CachedElement.select().count() == 5
+
+    with pytest.deprecated_call(
+        match="`worker_version` usage is deprecated. Consider using `worker_run` instead."
+    ):
+        # Query database through cache
+        elements = mock_elements_worker_with_cache.list_elements(**filters)
+    assert elements.count() == len(expected_ids)
+    for child, expected_id in zip(elements.order_by("id"), expected_ids, strict=True):
+        assert child.id == UUID(expected_id)
+
+    # Check the worker never hits the API for elements
+    assert len(responses.calls) == len(BASE_API_CALLS)
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS
+
+
 def test_list_element_children_wrong_element(mock_elements_worker):
     with pytest.raises(
         AssertionError,