arkindex-base-worker 0.3.5rc6__py3-none-any.whl → 0.3.6rc2__py3-none-any.whl

tests/test_dataset_worker.py

@@ -44,7 +44,7 @@ def test_download_dataset_artifact_download_api_error(
     expected_results = [
         {
             "id": "artifact_1",
-            "path": "dataset_id.zstd",
+            "path": "dataset_id.tar.zst",
             "size": 42,
             "content_type": "application/zstd",
             "s3_put_url": None,
@@ -69,7 +69,7 @@ def test_download_dataset_artifact_download_api_error(
     )
     responses.add(
         responses.GET,
-        f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd",
+        f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst",
         status=500,
     )
 
@@ -82,11 +82,11 @@ def test_download_dataset_artifact_download_api_error(
     ] == BASE_API_CALLS + [
         ("GET", f"http://testserver/api/v1/task/{task_id}/artifacts/"),
         # The API call is retried 5 times
-        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd"),
+        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst"),
     ]
 
 
@@ -132,7 +132,10 @@ def test_download_dataset_artifact(
 ):
     task_id = default_dataset.task_id
     archive_path = (
-        FIXTURES_DIR / "extract_parent_archives" / "first_parent" / "arkindex_data.zstd"
+        FIXTURES_DIR
+        / "extract_parent_archives"
+        / "first_parent"
+        / "arkindex_data.tar.zst"
     )
     mocker.patch(
         "arkindex_worker.worker.base.BaseWorker.find_extras_directory",
@@ -142,7 +145,7 @@ def test_download_dataset_artifact(
     expected_results = [
         {
             "id": "artifact_1",
-            "path": "dataset_id.zstd",
+            "path": "dataset_id.tar.zst",
             "size": 42,
             "content_type": "application/zstd",
             "s3_put_url": None,
@@ -167,14 +170,14 @@ def test_download_dataset_artifact(
     )
     responses.add(
         responses.GET,
-        f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd",
+        f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst",
         status=200,
         body=archive_path.read_bytes(),
         content_type="application/zstd",
     )
 
     archive = mock_dataset_worker.download_dataset_artifact(default_dataset)
-    assert archive == tmp_path / "dataset_id.zstd"
+    assert archive == tmp_path / "dataset_id.tar.zst"
     assert archive.read_bytes() == archive_path.read_bytes()
     archive.unlink()
 
@@ -183,7 +186,7 @@ def test_download_dataset_artifact(
         (call.request.method, call.request.url) for call in responses.calls
     ] == BASE_API_CALLS + [
         ("GET", f"http://testserver/api/v1/task/{task_id}/artifacts/"),
-        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.zstd"),
+        ("GET", f"http://testserver/api/v1/task/{task_id}/artifact/dataset_id.tar.zst"),
     ]
 
 
@@ -410,7 +413,7 @@ def test_list_datasets(responses, mock_dataset_worker):
     ]
 
 
-@pytest.mark.parametrize("generator", (True, False))
+@pytest.mark.parametrize("generator", [True, False])
 def test_run_no_datasets(mocker, caplog, mock_dataset_worker, generator):
     mocker.patch("arkindex_worker.worker.DatasetWorker.list_datasets", return_value=[])
     mock_dataset_worker.generator = generator
@@ -425,7 +428,7 @@ def test_run_no_datasets(mocker, caplog, mock_dataset_worker, generator):
 
 
 @pytest.mark.parametrize(
-    "generator, error",
+    ("generator", "error"),
     [
         (True, "When generating a new dataset, its state should be Open."),
         (False, "When processing an existing dataset, its state should be Complete."),
@@ -654,7 +657,7 @@ def test_run_no_downloaded_artifact_error(
 
 
 @pytest.mark.parametrize(
-    "generator, state", [(True, DatasetState.Open), (False, DatasetState.Complete)]
+    ("generator", "state"), [(True, DatasetState.Open), (False, DatasetState.Complete)]
 )
 def test_run(
     mocker,
@@ -701,7 +704,7 @@ def test_run(
             FIXTURES_DIR
             / "extract_parent_archives"
             / "first_parent"
-            / "arkindex_data.zstd"
+            / "arkindex_data.tar.zst"
         )
         responses.add(
             responses.GET,
@@ -711,7 +714,7 @@ def test_run(
         )
         responses.add(
             responses.GET,
-            f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.zstd",
+            f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.tar.zst",
             status=200,
             body=archive_path.read_bytes(),
             content_type="application/zstd",
@@ -723,7 +726,7 @@ def test_run(
             ),
             (
                 "GET",
-                f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.zstd",
+                f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.tar.zst",
             ),
         ]
         extra_logs += [
@@ -746,7 +749,7 @@ def test_run(
 
 
 @pytest.mark.parametrize(
-    "generator, state", [(True, DatasetState.Open), (False, DatasetState.Complete)]
+    ("generator", "state"), [(True, DatasetState.Open), (False, DatasetState.Complete)]
 )
 def test_run_read_only(
     mocker,
@@ -799,7 +802,7 @@ def test_run_read_only(
             FIXTURES_DIR
             / "extract_parent_archives"
             / "first_parent"
-            / "arkindex_data.zstd"
+            / "arkindex_data.tar.zst"
         )
         responses.add(
             responses.GET,
@@ -809,7 +812,7 @@ def test_run_read_only(
         )
         responses.add(
             responses.GET,
-            f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.zstd",
+            f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.tar.zst",
             status=200,
             body=archive_path.read_bytes(),
             content_type="application/zstd",
@@ -821,7 +824,7 @@ def test_run_read_only(
             ),
             (
                 "GET",
-                f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.zstd",
+                f"http://testserver/api/v1/task/{default_dataset.task_id}/artifact/dataset_id.tar.zst",
             ),
         ]
         extra_logs += [

tests/test_element.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pytest
 from requests import HTTPError
 

tests/test_elements_worker/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # API calls during worker configuration
 BASE_API_CALLS = [
     (

tests/test_elements_worker/test_classifications.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 import re
 from uuid import UUID, uuid4

tests/test_elements_worker/test_cli.py

@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
 import json
-import os
 import sys
 import tempfile
+from pathlib import Path
 from uuid import UUID
 
 import pytest
@@ -10,49 +9,53 @@ import pytest
 from arkindex_worker.worker import ElementsWorker
 
 
-def test_cli_default(monkeypatch, mock_worker_run_api):
+@pytest.mark.usefixtures("_mock_worker_run_api")
+def test_cli_default(monkeypatch):
     _, path = tempfile.mkstemp()
-    with open(path, "w") as f:
-        json.dump(
+    path = Path(path)
+    path.write_text(
+        json.dumps(
             [
                 {"id": "volumeid", "type": "volume"},
                 {"id": "pageid", "type": "page"},
                 {"id": "actid", "type": "act"},
                 {"id": "surfaceid", "type": "surface"},
             ],
-            f,
         )
+    )
 
     monkeypatch.setenv("TASK_ELEMENTS", path)
     monkeypatch.setattr(sys, "argv", ["worker"])
     worker = ElementsWorker()
     worker.configure()
 
-    assert worker.args.elements_list.name == path
+    assert worker.args.elements_list.name == str(path)
     assert not worker.args.element
-    os.unlink(path)
+    path.unlink()
 
 
-def test_cli_arg_elements_list_given(mocker, mock_worker_run_api):
+@pytest.mark.usefixtures("_mock_worker_run_api")
+def test_cli_arg_elements_list_given(mocker):
     _, path = tempfile.mkstemp()
-    with open(path, "w") as f:
-        json.dump(
+    path = Path(path)
+    path.write_text(
+        json.dumps(
             [
                 {"id": "volumeid", "type": "volume"},
                 {"id": "pageid", "type": "page"},
                 {"id": "actid", "type": "act"},
                 {"id": "surfaceid", "type": "surface"},
             ],
-            f,
         )
+    )
 
-    mocker.patch.object(sys, "argv", ["worker", "--elements-list", path])
+    mocker.patch.object(sys, "argv", ["worker", "--elements-list", str(path)])
     worker = ElementsWorker()
     worker.configure()
 
-    assert worker.args.elements_list.name == path
+    assert worker.args.elements_list.name == str(path)
     assert not worker.args.element
-    os.unlink(path)
+    path.unlink()
 
 
 def test_cli_arg_element_one_given_not_uuid(mocker):
@@ -62,7 +65,8 @@ def test_cli_arg_element_one_given_not_uuid(mocker):
         worker.configure()
 
 
-def test_cli_arg_element_one_given(mocker, mock_worker_run_api):
+@pytest.mark.usefixtures("_mock_worker_run_api")
+def test_cli_arg_element_one_given(mocker):
     mocker.patch.object(
         sys, "argv", ["worker", "--element", "12341234-1234-1234-1234-123412341234"]
     )
@@ -74,7 +78,8 @@ def test_cli_arg_element_one_given(mocker, mock_worker_run_api):
     assert not worker.args.elements_list
 
 
-def test_cli_arg_element_many_given(mocker, mock_worker_run_api):
+@pytest.mark.usefixtures("_mock_worker_run_api")
+def test_cli_arg_element_many_given(mocker):
     mocker.patch.object(
         sys,
         "argv",

tests/test_elements_worker/test_dataset.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 import logging
 
@@ -107,8 +106,8 @@ def test_list_process_datasets(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Dataset
         (
             {"dataset": None},
@@ -118,7 +117,7 @@ def test_list_process_datasets(
             {"dataset": "not Dataset type"},
             "dataset shouldn't be null and should be a Dataset",
         ),
-    ),
+    ],
 )
 def test_list_dataset_elements_wrong_param_dataset(mock_dataset_worker, payload, error):
     with pytest.raises(AssertionError, match=error):
@@ -265,8 +264,8 @@ def test_list_dataset_elements(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Dataset
         (
             {"dataset": None},
@@ -276,7 +275,7 @@ def test_list_dataset_elements(
             {"dataset": "not dataset type"},
             "dataset shouldn't be null and should be a Dataset",
         ),
-    ),
+    ],
 )
 def test_update_dataset_state_wrong_param_dataset(
     mock_dataset_worker, default_dataset, payload, error
@@ -292,8 +291,8 @@ def test_update_dataset_state_wrong_param_dataset(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # DatasetState
         (
             {"state": None},
@@ -303,7 +302,7 @@ def test_update_dataset_state_wrong_param_dataset(
             {"state": "not dataset type"},
             "state shouldn't be null and should be a str from DatasetState",
         ),
-    ),
+    ],
 )
 def test_update_dataset_state_wrong_param_state(
     mock_dataset_worker, default_dataset, payload, error

tests/test_elements_worker/test_elements.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 import re
 from argparse import Namespace
@@ -1219,8 +1218,8 @@ def test_create_elements_integrity_error(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Element
         (
             {"element": None},
@@ -1230,7 +1229,7 @@ def test_create_elements_integrity_error(
             {"element": "not element type"},
             "element shouldn't be null and should be an Element or CachedElement",
         ),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_element(
     mock_elements_worker, payload, error
@@ -1247,12 +1246,12 @@ def test_partial_update_element_wrong_param_element(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Type
         ({"type": 1234}, "type should be a str"),
         ({"type": None}, "type should be a str"),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_type(mock_elements_worker, payload, error):
     api_payload = {
@@ -1267,12 +1266,12 @@ def test_partial_update_element_wrong_param_type(mock_elements_worker, payload,
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Name
         ({"name": 1234}, "name should be a str"),
         ({"name": None}, "name should be a str"),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_name(mock_elements_worker, payload, error):
     api_payload = {
@@ -1287,8 +1286,8 @@ def test_partial_update_element_wrong_param_name(mock_elements_worker, payload,
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Polygon
         ({"polygon": "not a polygon"}, "polygon should be a list"),
         ({"polygon": None}, "polygon should be a list"),
@@ -1305,7 +1304,7 @@ def test_partial_update_element_wrong_param_name(mock_elements_worker, payload,
             {"polygon": [["not a coord", 1], [2, 2], [2, 1], [1, 2]]},
             "polygon points should be lists of two numbers",
         ),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_polygon(
     mock_elements_worker, payload, error
@@ -1322,8 +1321,8 @@ def test_partial_update_element_wrong_param_polygon(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Confidence
         ({"confidence": "lol"}, "confidence should be None or a float in [0..1] range"),
         ({"confidence": "0.2"}, "confidence should be None or a float in [0..1] range"),
@@ -1333,7 +1332,7 @@ def test_partial_update_element_wrong_param_polygon(
             {"confidence": float("inf")},
             "confidence should be None or a float in [0..1] range",
         ),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_conf(mock_elements_worker, payload, error):
     api_payload = {
@@ -1348,14 +1347,14 @@ def test_partial_update_element_wrong_param_conf(mock_elements_worker, payload,
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Rotation angle
         ({"rotation_angle": "lol"}, "rotation_angle should be a positive integer"),
         ({"rotation_angle": -1}, "rotation_angle should be a positive integer"),
         ({"rotation_angle": 0.5}, "rotation_angle should be a positive integer"),
         ({"rotation_angle": None}, "rotation_angle should be a positive integer"),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_rota(mock_elements_worker, payload, error):
     api_payload = {
@@ -1370,13 +1369,13 @@ def test_partial_update_element_wrong_param_rota(mock_elements_worker, payload,
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Mirrored
         ({"mirrored": "lol"}, "mirrored should be a boolean"),
         ({"mirrored": 1234}, "mirrored should be a boolean"),
         ({"mirrored": None}, "mirrored should be a boolean"),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_mir(mock_elements_worker, payload, error):
     api_payload = {
@@ -1391,13 +1390,13 @@ def test_partial_update_element_wrong_param_mir(mock_elements_worker, payload, e
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Image
         ({"image": "lol"}, "image should be a UUID"),
         ({"image": 1234}, "image should be a UUID"),
         ({"image": None}, "image should be a UUID"),
-    ),
+    ],
 )
 def test_partial_update_element_wrong_param_image(mock_elements_worker, payload, error):
     api_payload = {
@@ -1440,9 +1439,10 @@ def test_partial_update_element_api_error(responses, mock_elements_worker):
     ]
 
 
+@pytest.mark.usefixtures("_mock_cached_elements", "_mock_cached_images")
 @pytest.mark.parametrize(
     "payload",
-    (
+    [
         (
             {
                 "polygon": [[10, 10], [20, 20], [20, 10], [10, 20]],
@@ -1463,15 +1463,9 @@ def test_partial_update_element_api_error(responses, mock_elements_worker):
                 "mirrored": False,
             }
         ),
-    ),
+    ],
 )
-def test_partial_update_element(
-    responses,
-    mock_elements_worker_with_cache,
-    mock_cached_elements,
-    mock_cached_images,
-    payload,
-):
+def test_partial_update_element(responses, mock_elements_worker_with_cache, payload):
     elt = CachedElement.select().first()
     new_image = CachedImage.select().first()
 
@@ -1516,9 +1510,10 @@ def test_partial_update_element(
         assert getattr(cached_element, param) == elt_response[param]
 
 
-@pytest.mark.parametrize("confidence", (None, 0.42))
+@pytest.mark.usefixtures("_mock_cached_elements")
+@pytest.mark.parametrize("confidence", [None, 0.42])
 def test_partial_update_element_confidence(
-    responses, mock_elements_worker_with_cache, mock_cached_elements, confidence
+    responses, mock_elements_worker_with_cache, confidence
 ):
     elt = CachedElement.select().first()
     elt_response = {
@@ -1661,13 +1656,13 @@ def test_list_element_children_wrong_with_metadata(mock_elements_worker):
 
 
 @pytest.mark.parametrize(
-    "param, value",
-    (
+    ("param", "value"),
+    [
         ("worker_version", 1234),
         ("worker_run", 1234),
         ("transcription_worker_version", 1234),
         ("transcription_worker_run", 1234),
-    ),
+    ],
 )
 def test_list_element_children_wrong_worker_version(mock_elements_worker, param, value):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
@@ -1681,12 +1676,12 @@ def test_list_element_children_wrong_worker_version(mock_elements_worker, param,
 
 @pytest.mark.parametrize(
     "param",
-    (
-        ("worker_version"),
-        ("worker_run"),
-        ("transcription_worker_version"),
-        ("transcription_worker_run"),
-    ),
+    [
+        "worker_version",
+        "worker_run",
+        "transcription_worker_version",
+        "transcription_worker_run",
+    ],
 )
 def test_list_element_children_wrong_bool_worker_version(mock_elements_worker, param):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
@@ -1908,9 +1903,10 @@ def test_list_element_children_with_cache_unhandled_param(
         )
 
 
+@pytest.mark.usefixtures("_mock_cached_elements")
 @pytest.mark.parametrize(
-    "filters, expected_ids",
-    (
+    ("filters", "expected_ids"),
+    [
         # Filter on element should give all elements inserted
         (
             {
@@ -1977,12 +1973,11 @@ def test_list_element_children_with_cache_unhandled_param(
                 "33333333-3333-3333-3333-333333333333",
             ),
         ),
-    ),
+    ],
 )
 def test_list_element_children_with_cache(
     responses,
     mock_elements_worker_with_cache,
-    mock_cached_elements,
     filters,
     expected_ids,
 ):
@@ -1992,7 +1987,7 @@ def test_list_element_children_with_cache(
     # Query database through cache
     elements = mock_elements_worker_with_cache.list_element_children(**filters)
     assert elements.count() == len(expected_ids)
-    for child, expected_id in zip(elements.order_by("id"), expected_ids):
+    for child, expected_id in zip(elements.order_by("id"), expected_ids, strict=True):
         assert child.id == UUID(expected_id)
 
     # Check the worker never hits the API for elements
@@ -2109,13 +2104,13 @@ def test_list_element_parents_wrong_with_metadata(mock_elements_worker):
 
 
 @pytest.mark.parametrize(
-    "param, value",
-    (
+    ("param", "value"),
+    [
         ("worker_version", 1234),
         ("worker_run", 1234),
         ("transcription_worker_version", 1234),
         ("transcription_worker_run", 1234),
-    ),
+    ],
 )
 def test_list_element_parents_wrong_worker_version(mock_elements_worker, param, value):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
@@ -2129,12 +2124,12 @@ def test_list_element_parents_wrong_worker_version(mock_elements_worker, param,
 
 @pytest.mark.parametrize(
     "param",
-    (
-        ("worker_version"),
-        ("worker_run"),
-        ("transcription_worker_version"),
-        ("transcription_worker_run"),
-    ),
+    [
+        "worker_version",
+        "worker_run",
+        "transcription_worker_version",
+        "transcription_worker_run",
+    ],
 )
 def test_list_element_parents_wrong_bool_worker_version(mock_elements_worker, param):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
@@ -2356,9 +2351,10 @@ def test_list_element_parents_with_cache_unhandled_param(
         )
 
 
+@pytest.mark.usefixtures("_mock_cached_elements")
 @pytest.mark.parametrize(
-    "filters, expected_id",
-    (
+    ("filters", "expected_id"),
+    [
         # Filter on element
         (
             {
@@ -2415,12 +2411,11 @@ def test_list_element_parents_with_cache_unhandled_param(
             },
             "99999999-9999-9999-9999-999999999999",
         ),
-    ),
+    ],
 )
 def test_list_element_parents_with_cache(
     responses,
     mock_elements_worker_with_cache,
-    mock_cached_elements,
     filters,
     expected_id,
 ):