arkindex-base-worker 0.3.7rc10__py3-none-any.whl → 0.4.0__py3-none-any.whl

tests/test_elements_worker/test_cli.py

@@ -2,7 +2,6 @@ import json
 import sys
 import tempfile
 from pathlib import Path
-from uuid import UUID
 
 import pytest
 
@@ -58,13 +57,6 @@ def test_cli_arg_elements_list_given(mocker):
     path.unlink()
 
 
-def test_cli_arg_element_one_given_not_uuid(mocker):
-    mocker.patch.object(sys, "argv", ["worker", "--element", "1234"])
-    worker = ElementsWorker()
-    with pytest.raises(SystemExit):
-        worker.configure()
-
-
 @pytest.mark.usefixtures("_mock_worker_run_api")
 def test_cli_arg_element_one_given(mocker):
     mocker.patch.object(
@@ -73,7 +65,7 @@ def test_cli_arg_element_one_given(mocker):
     worker = ElementsWorker()
     worker.configure()
 
-    assert worker.args.element == [UUID("12341234-1234-1234-1234-123412341234")]
+    assert worker.args.element == ["12341234-1234-1234-1234-123412341234"]
     # elements_list is None because TASK_ELEMENTS environment variable isn't set
     assert not worker.args.elements_list
 
@@ -94,8 +86,8 @@ def test_cli_arg_element_many_given(mocker):
     worker.configure()
 
     assert worker.args.element == [
-        UUID("12341234-1234-1234-1234-123412341234"),
-        UUID("43214321-4321-4321-4321-432143214321"),
+        "12341234-1234-1234-1234-123412341234",
+        "43214321-4321-4321-4321-432143214321",
     ]
     # elements_list is None because TASK_ELEMENTS environment variable isn't set
     assert not worker.args.elements_list

tests/test_elements_worker/test_corpus.py

@@ -0,0 +1,168 @@
+import re
+import uuid
+
+import pytest
+
+from arkindex.exceptions import ErrorResponse
+from arkindex_worker.worker.corpus import CorpusExportState
+from tests import CORPUS_ID
+from tests.test_elements_worker import BASE_API_CALLS
+
+
+def test_download_export_not_a_uuid(responses, mock_elements_worker):
+    with pytest.raises(ValueError, match="export_id is not a valid uuid."):
+        mock_elements_worker.download_export("mon export")
+
+
+def test_download_export(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        status=302,
+        body=b"some SQLite export",
+        content_type="application/x-sqlite3",
+        stream=True,
+    )
+
+    export = mock_elements_worker.download_export(
+        "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+    )
+    assert export.name == "/tmp/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            "http://testserver/api/v1/export/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff/",
+        ),
+    ]
+
+
+def mock_list_exports_call(responses, export_id):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/",
+        status=200,
+        json={
+            "count": len(CorpusExportState),
+            "next": None,
+            "results": [
+                {
+                    "id": str(uuid.uuid4())
+                    if state != CorpusExportState.Done
+                    else export_id,
+                    "created": "2019-08-24T14:15:22Z",
+                    "updated": "2019-08-24T14:15:22Z",
+                    "corpus_id": CORPUS_ID,
+                    "user": {
+                        "id": 0,
+                        "email": "user@example.com",
+                        "display_name": "User",
+                    },
+                    "state": state.value,
+                    "source": "default",
+                }
+                for state in CorpusExportState
+            ],
+        },
+    )
+
+
+def test_download_latest_export_list_error(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/",
+        status=418,
+    )
+
+    with pytest.raises(
+        Exception, match="Stopping pagination as data will be incomplete"
+    ):
+        mock_elements_worker.download_latest_export()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        # The API call is retried 5 times
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+    ]
+
+
+def test_download_latest_export_no_available_exports(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/",
+        status=200,
+        json={
+            "count": 0,
+            "next": None,
+            "results": [],
+        },
+    )
+
+    with pytest.raises(
+        AssertionError,
+        match=re.escape(
+            f'No available exports found for the corpus ({CORPUS_ID}) with state "Done".'
+        ),
+    ):
+        mock_elements_worker.download_latest_export()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+    ]
+
+
+def test_download_latest_export_download_error(responses, mock_elements_worker):
+    export_id = str(uuid.uuid4())
+    mock_list_exports_call(responses, export_id)
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/export/{export_id}/",
+        status=418,
+    )
+
+    with pytest.raises(ErrorResponse):
+        mock_elements_worker.download_latest_export()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 2
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+        ("GET", f"http://testserver/api/v1/export/{export_id}/"),
+    ]
+
+
+def test_download_latest_export(responses, mock_elements_worker):
+    export_id = str(uuid.uuid4())
+    mock_list_exports_call(responses, export_id)
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/export/{export_id}/",
+        status=302,
+        body=b"some SQLite export",
+        content_type="application/x-sqlite3",
+        stream=True,
+    )
+
+    export = mock_elements_worker.download_latest_export()
+    assert export.name == f"/tmp/{export_id}"
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 2
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/export/"),
+        ("GET", f"http://testserver/api/v1/export/{export_id}/"),
+    ]

tests/test_elements_worker/test_dataset.py

@@ -2,11 +2,11 @@ import json
 import logging
 
 import pytest
-from apistar.exceptions import ErrorResponse
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Element, Set
 from arkindex_worker.worker.dataset import DatasetState
-from tests.conftest import PROCESS_ID
+from tests import PROCESS_ID
 from tests.test_elements_worker import BASE_API_CALLS
 
 
@@ -25,7 +25,7 @@ def test_list_process_sets_api_error(responses, mock_dataset_worker):
     responses.add(
         responses.GET,
         f"http://testserver/api/v1/process/{PROCESS_ID}/sets/",
-        status=500,
+        status=418,
     )
 
     with pytest.raises(
@@ -152,7 +152,7 @@ def test_list_set_elements_api_error(
     responses.add(
         responses.GET,
         f"http://testserver/api/v1/datasets/{default_dataset.id}/elements/{query_params}",
-        status=500,
+        status=418,
     )
 
     with pytest.raises(
@@ -321,7 +321,7 @@ def test_update_dataset_state_api_error(
     responses.add(
         responses.PATCH,
         f"http://testserver/api/v1/datasets/{default_dataset.id}/",
-        status=500,
+        status=418,
     )
 
     with pytest.raises(ErrorResponse):
@@ -330,16 +330,11 @@ def test_update_dataset_state_api_error(
             state=DatasetState.Building,
         )
 
-    assert len(responses.calls) == len(BASE_API_CALLS) + 5
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
     assert [
         (call.request.method, call.request.url) for call in responses.calls
     ] == BASE_API_CALLS + [
-        # We retry 5 times the API call
-        ("PATCH", f"http://testserver/api/v1/datasets/{default_dataset.id}/"),
-        ("PATCH", f"http://testserver/api/v1/datasets/{default_dataset.id}/"),
-        ("PATCH", f"http://testserver/api/v1/datasets/{default_dataset.id}/"),
-        ("PATCH", f"http://testserver/api/v1/datasets/{default_dataset.id}/"),
-        ("PATCH", f"http://testserver/api/v1/datasets/{default_dataset.id}/"),
+        ("PATCH", f"http://testserver/api/v1/datasets/{default_dataset.id}/")
     ]
 
 

tests/test_elements_worker/test_element.py

@@ -0,0 +1,427 @@
+import json
+import re
+
+import pytest
+from responses import matchers
+
+from arkindex.exceptions import ErrorResponse
+from arkindex_worker.cache import (
+    CachedElement,
+    CachedImage,
+)
+from arkindex_worker.models import Element
+from arkindex_worker.worker.element import MissingTypeError
+from tests import CORPUS_ID
+
+from . import BASE_API_CALLS
+
+
+def test_list_corpus_types(responses, mock_elements_worker):
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/",
+        json={
+            "id": CORPUS_ID,
+            "types": [{"slug": "folder"}, {"slug": "page"}],
+        },
+    )
+
+    mock_elements_worker.list_corpus_types()
+
+    assert mock_elements_worker.corpus_types == {
+        "folder": {"slug": "folder"},
+        "page": {"slug": "page"},
+    }
+
+
+def test_check_required_types_argument_types(mock_elements_worker):
+    with pytest.raises(
+        AssertionError, match="At least one element type slug is required."
+    ):
+        mock_elements_worker.check_required_types()
+
+    with pytest.raises(AssertionError, match="Element type slugs must be strings."):
+        mock_elements_worker.check_required_types("lol", 42)
+
+
+def test_check_required_types(mock_elements_worker):
+    mock_elements_worker.corpus_types = {
+        "folder": {"slug": "folder"},
+        "page": {"slug": "page"},
+    }
+
+    assert mock_elements_worker.check_required_types("page")
+    assert mock_elements_worker.check_required_types("page", "folder")
+
+    with pytest.raises(
+        MissingTypeError,
+        match=re.escape(
+            "Element types act, text_line were not found in corpus (11111111-1111-1111-1111-111111111111)."
+        ),
+    ):
+        assert mock_elements_worker.check_required_types("page", "text_line", "act")
+
+
+def test_check_required_types_create_missing(responses, mock_elements_worker):
+    mock_elements_worker.corpus_types = {
+        "folder": {"slug": "folder"},
+        "page": {"slug": "page"},
+    }
+
+    responses.add(
+        responses.POST,
+        "http://testserver/api/v1/elements/type/",
+        match=[
+            matchers.json_params_matcher(
+                {
+                    "slug": "text_line",
+                    "display_name": "text_line",
+                    "folder": False,
+                    "corpus": CORPUS_ID,
+                }
+            )
+        ],
+    )
+    responses.add(
+        responses.POST,
+        "http://testserver/api/v1/elements/type/",
+        match=[
+            matchers.json_params_matcher(
+                {
+                    "slug": "act",
+                    "display_name": "act",
+                    "folder": False,
+                    "corpus": CORPUS_ID,
+                }
+            )
+        ],
+    )
+
+    assert mock_elements_worker.check_required_types(
+        "page", "text_line", "act", create_missing=True
+    )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Element
+        (
+            {"element": None},
+            "element shouldn't be null and should be an Element or CachedElement",
+        ),
+        (
+            {"element": "not element type"},
+            "element shouldn't be null and should be an Element or CachedElement",
+        ),
+    ],
+)
+def test_partial_update_element_wrong_param_element(
+    mock_elements_worker, payload, error
+):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Type
+        ({"type": 1234}, "type should be a str"),
+        ({"type": None}, "type should be a str"),
+    ],
+)
+def test_partial_update_element_wrong_param_type(mock_elements_worker, payload, error):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Name
+        ({"name": 1234}, "name should be a str"),
+        ({"name": None}, "name should be a str"),
+    ],
+)
+def test_partial_update_element_wrong_param_name(mock_elements_worker, payload, error):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Polygon
+        ({"polygon": "not a polygon"}, "polygon should be a list"),
+        ({"polygon": None}, "polygon should be a list"),
+        ({"polygon": [[1, 1], [2, 2]]}, "polygon should have at least three points"),
+        (
+            {"polygon": [[1, 1, 1], [2, 2, 1], [2, 1, 1], [1, 2, 1]]},
+            "polygon points should be lists of two items",
+        ),
+        (
+            {"polygon": [[1], [2], [2], [1]]},
+            "polygon points should be lists of two items",
+        ),
+        (
+            {"polygon": [["not a coord", 1], [2, 2], [2, 1], [1, 2]]},
+            "polygon points should be lists of two numbers",
+        ),
+    ],
+)
+def test_partial_update_element_wrong_param_polygon(
+    mock_elements_worker, payload, error
+):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Confidence
+        ({"confidence": "lol"}, "confidence should be None or a float in [0..1] range"),
+        ({"confidence": "0.2"}, "confidence should be None or a float in [0..1] range"),
+        ({"confidence": -1.0}, "confidence should be None or a float in [0..1] range"),
+        ({"confidence": 1.42}, "confidence should be None or a float in [0..1] range"),
+        (
+            {"confidence": float("inf")},
+            "confidence should be None or a float in [0..1] range",
+        ),
+    ],
+)
+def test_partial_update_element_wrong_param_conf(mock_elements_worker, payload, error):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=re.escape(error)):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Rotation angle
+        ({"rotation_angle": "lol"}, "rotation_angle should be a positive integer"),
+        ({"rotation_angle": -1}, "rotation_angle should be a positive integer"),
+        ({"rotation_angle": 0.5}, "rotation_angle should be a positive integer"),
+        ({"rotation_angle": None}, "rotation_angle should be a positive integer"),
+    ],
+)
+def test_partial_update_element_wrong_param_rota(mock_elements_worker, payload, error):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Mirrored
+        ({"mirrored": "lol"}, "mirrored should be a boolean"),
+        ({"mirrored": 1234}, "mirrored should be a boolean"),
+        ({"mirrored": None}, "mirrored should be a boolean"),
+    ],
+)
+def test_partial_update_element_wrong_param_mir(mock_elements_worker, payload, error):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+@pytest.mark.parametrize(
+    ("payload", "error"),
+    [
+        # Image
+        ({"image": "lol"}, "image should be a UUID"),
+        ({"image": 1234}, "image should be a UUID"),
+        ({"image": None}, "image should be a UUID"),
+    ],
+)
+def test_partial_update_element_wrong_param_image(mock_elements_worker, payload, error):
+    api_payload = {
+        "element": Element({"zone": None}),
+        **payload,
+    }
+
+    with pytest.raises(AssertionError, match=error):
+        mock_elements_worker.partial_update_element(
+            **api_payload,
+        )
+
+
+def test_partial_update_element_api_error(responses, mock_elements_worker):
+    elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
+    responses.add(
+        responses.PATCH,
+        f"http://testserver/api/v1/element/{elt.id}/",
+        status=418,
+    )
+
+    with pytest.raises(ErrorResponse):
+        mock_elements_worker.partial_update_element(
+            element=elt,
+            type="something",
+            name="0",
+            polygon=[[1, 1], [2, 2], [2, 1], [1, 2]],
+        )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [("PATCH", f"http://testserver/api/v1/element/{elt.id}/")]
+
+
+@pytest.mark.usefixtures("_mock_cached_elements", "_mock_cached_images")
+@pytest.mark.parametrize(
+    "payload",
+    [
+        (
+            {
+                "polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
+                "confidence": None,
+            }
+        ),
+        (
+            {
+                "rotation_angle": 45,
+                "mirrored": False,
+            }
+        ),
+        (
+            {
+                "polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
+                "confidence": None,
+                "rotation_angle": 45,
+                "mirrored": False,
+            }
+        ),
+    ],
+)
+def test_partial_update_element(responses, mock_elements_worker_with_cache, payload):
+    elt = CachedElement.select().first()
+    new_image = CachedImage.select().first()
+
+    elt_response = {
+        "image": str(new_image.id),
+        **payload,
+    }
+    responses.add(
+        responses.PATCH,
+        f"http://testserver/api/v1/element/{elt.id}/",
+        status=200,
+        # UUID not allowed in JSON
+        json=elt_response,
+    )
+
+    element_update_response = mock_elements_worker_with_cache.partial_update_element(
+        element=elt,
+        **{**elt_response, "image": new_image.id},
+    )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "PATCH",
+            f"http://testserver/api/v1/element/{elt.id}/",
+        ),
+    ]
+    assert json.loads(responses.calls[-1].request.body) == elt_response
+    assert element_update_response == elt_response
+
+    cached_element = CachedElement.get(CachedElement.id == elt.id)
+    # Always present in payload
+    assert str(cached_element.image_id) == elt_response["image"]
+    # Optional params
+    if "polygon" in payload:
+        # Cast to string as this is the only difference compared to model
+        elt_response["polygon"] = str(elt_response["polygon"])
+
+    for param in payload:
+        assert getattr(cached_element, param) == elt_response[param]
+
+
+@pytest.mark.usefixtures("_mock_cached_elements")
+@pytest.mark.parametrize("confidence", [None, 0.42])
+def test_partial_update_element_confidence(
+    responses, mock_elements_worker_with_cache, confidence
+):
+    elt = CachedElement.select().first()
+    elt_response = {
+        "polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
+        "confidence": confidence,
+    }
+    responses.add(
+        responses.PATCH,
+        f"http://testserver/api/v1/element/{elt.id}/",
+        status=200,
+        json=elt_response,
+    )
+
+    element_update_response = mock_elements_worker_with_cache.partial_update_element(
+        element=elt,
+        **elt_response,
+    )
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "PATCH",
+            f"http://testserver/api/v1/element/{elt.id}/",
+        ),
+    ]
+    assert json.loads(responses.calls[-1].request.body) == elt_response
+    assert element_update_response == elt_response
+
+    cached_element = CachedElement.get(CachedElement.id == elt.id)
+    assert cached_element.polygon == str(elt_response["polygon"])
+    assert cached_element.confidence == confidence