arkindex-base-worker 0.3.5rc6__py3-none-any.whl → 0.3.6rc2__py3-none-any.whl

tests/test_elements_worker/test_entities.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 import re
 from uuid import UUID
@@ -56,7 +55,7 @@ def test_create_entity_wrong_type(mock_elements_worker):
         )
 
 
-def test_create_entity_wrong_corpus(monkeypatch, mock_elements_worker):
+def test_create_entity_wrong_corpus(mock_elements_worker):
     # Triggering an error on metas param, not giving corpus should work since
     # ARKINDEX_CORPUS_ID environment variable is set on mock_elements_worker
     with pytest.raises(AssertionError, match="metas should be of type dict"):
@@ -760,22 +759,13 @@ def test_list_corpus_entities(responses, mock_elements_worker):
     ]
 
 
-@pytest.mark.parametrize(
-    "wrong_name",
-    [
-        1234,
-        12.5,
-    ],
-)
+@pytest.mark.parametrize("wrong_name", [1234, 12.5])
 def test_list_corpus_entities_wrong_name(mock_elements_worker, wrong_name):
     with pytest.raises(AssertionError, match="name should be of type str"):
         mock_elements_worker.list_corpus_entities(name=wrong_name)
 
 
-@pytest.mark.parametrize(
-    "wrong_parent",
-    [{"id": "element_id"}, 12.5, "blabla"],
-)
+@pytest.mark.parametrize("wrong_parent", [{"id": "element_id"}, 12.5, "blabla"])
 def test_list_corpus_entities_wrong_parent(mock_elements_worker, wrong_parent):
     with pytest.raises(AssertionError, match="parent should be of type Element"):
         mock_elements_worker.list_corpus_entities(parent=wrong_parent)
@@ -850,7 +840,7 @@ def test_check_required_entity_types_no_creation_allowed(
     ] == BASE_API_CALLS
 
 
-@pytest.mark.parametrize("transcription", (None, "not a transcription", 1))
+@pytest.mark.parametrize("transcription", [None, "not a transcription", 1])
 def test_create_transcription_entities_wrong_transcription(
     mock_elements_worker, transcription
 ):
@@ -865,8 +855,8 @@ def test_create_transcription_entities_wrong_transcription(
 
 
 @pytest.mark.parametrize(
-    "entities, error",
-    (
+    ("entities", "error"),
+    [
         (None, "entities shouldn't be null and should be of type list"),
         (
             "not a list of entities",
@@ -886,7 +876,7 @@ def test_create_transcription_entities_wrong_transcription(
             * 2,
             "entities should be unique",
         ),
-    ),
+    ],
 )
 def test_create_transcription_entities_wrong_entities(
     mock_elements_worker, entities, error
@@ -909,8 +899,8 @@ def test_create_transcription_entities_wrong_entities_subtype(mock_elements_work
 
 
 @pytest.mark.parametrize(
-    "entity, error",
-    (
+    ("entity", "error"),
+    [
         (
             {
                 "name": None,
@@ -989,7 +979,7 @@ def test_create_transcription_entities_wrong_entities_subtype(mock_elements_work
             },
             "Entity at index 0 in entities: confidence should be None or a float in [0..1] range",
         ),
-    ),
+    ],
 )
 def test_create_transcription_entities_wrong_entity(
     mock_elements_worker, entity, error

tests/test_elements_worker/test_metadata.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 import re
 
@@ -247,22 +246,20 @@ def test_create_metadata_cached_element(responses, mock_elements_worker_with_cac
 
 
 @pytest.mark.parametrize(
-    "metadatas",
+    "metadata_list",
     [
-        ([{"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}]),
-        (
-            [
-                {
-                    "type": MetaType.Text,
-                    "name": "fake_name",
-                    "value": "fake_value",
-                    "entity_id": "fake_entity_id",
-                }
-            ]
-        ),
+        [{"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}],
+        [
+            {
+                "type": MetaType.Text,
+                "name": "fake_name",
+                "value": "fake_value",
+                "entity_id": "fake_entity_id",
+            }
+        ],
     ],
 )
-def test_create_metadatas(responses, mock_elements_worker, metadatas):
+def test_create_metadatas(responses, mock_elements_worker, metadata_list):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
     responses.add(
         responses.POST,
@@ -273,17 +270,19 @@ def test_create_metadatas(responses, mock_elements_worker, metadatas):
             "metadata_list": [
                 {
                     "id": "fake_metadata_id",
-                    "type": metadatas[0]["type"].value,
-                    "name": metadatas[0]["name"],
-                    "value": metadatas[0]["value"],
+                    "type": metadata_list[0]["type"].value,
+                    "name": metadata_list[0]["name"],
+                    "value": metadata_list[0]["value"],
                     "dates": [],
-                    "entity_id": metadatas[0].get("entity_id"),
+                    "entity_id": metadata_list[0].get("entity_id"),
                 }
             ],
         },
     )
 
-    created_metadatas = mock_elements_worker.create_metadatas(element, metadatas)
+    created_metadata_list = mock_elements_worker.create_metadatas(
+        element, metadata_list
+    )
 
     assert len(responses.calls) == len(BASE_API_CALLS) + 1
     assert [
@@ -296,42 +295,40 @@ def test_create_metadatas(responses, mock_elements_worker, metadatas):
     ]
     assert json.loads(responses.calls[-1].request.body)["metadata_list"] == [
         {
-            "type": metadatas[0]["type"].value,
-            "name": metadatas[0]["name"],
-            "value": metadatas[0]["value"],
-            "entity_id": metadatas[0].get("entity_id"),
+            "type": metadata_list[0]["type"].value,
+            "name": metadata_list[0]["name"],
+            "value": metadata_list[0]["value"],
+            "entity_id": metadata_list[0].get("entity_id"),
         }
     ]
-    assert created_metadatas == [
+    assert created_metadata_list == [
         {
             "id": "fake_metadata_id",
-            "type": metadatas[0]["type"].value,
-            "name": metadatas[0]["name"],
-            "value": metadatas[0]["value"],
+            "type": metadata_list[0]["type"].value,
+            "name": metadata_list[0]["name"],
+            "value": metadata_list[0]["value"],
             "dates": [],
-            "entity_id": metadatas[0].get("entity_id"),
+            "entity_id": metadata_list[0].get("entity_id"),
         }
     ]
 
 
 @pytest.mark.parametrize(
-    "metadatas",
+    "metadata_list",
     [
-        ([{"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}]),
-        (
-            [
-                {
-                    "type": MetaType.Text,
-                    "name": "fake_name",
-                    "value": "fake_value",
-                    "entity_id": "fake_entity_id",
-                }
-            ]
-        ),
+        [{"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}],
+        [
+            {
+                "type": MetaType.Text,
+                "name": "fake_name",
+                "value": "fake_value",
+                "entity_id": "fake_entity_id",
+            }
+        ],
     ],
 )
 def test_create_metadatas_cached_element(
-    responses, mock_elements_worker_with_cache, metadatas
+    responses, mock_elements_worker_with_cache, metadata_list
 ):
     element = CachedElement.create(
         id="12341234-1234-1234-1234-123412341234", type="thing"
@@ -345,18 +342,18 @@ def test_create_metadatas_cached_element(
             "metadata_list": [
                 {
                     "id": "fake_metadata_id",
-                    "type": metadatas[0]["type"].value,
-                    "name": metadatas[0]["name"],
-                    "value": metadatas[0]["value"],
+                    "type": metadata_list[0]["type"].value,
+                    "name": metadata_list[0]["name"],
+                    "value": metadata_list[0]["value"],
                     "dates": [],
-                    "entity_id": metadatas[0].get("entity_id"),
+                    "entity_id": metadata_list[0].get("entity_id"),
                 }
             ],
         },
     )
 
-    created_metadatas = mock_elements_worker_with_cache.create_metadatas(
-        element, metadatas
+    created_metadata_list = mock_elements_worker_with_cache.create_metadatas(
+        element, metadata_list
     )
 
     assert len(responses.calls) == len(BASE_API_CALLS) + 1
@@ -370,35 +367,27 @@ def test_create_metadatas_cached_element(
     ]
     assert json.loads(responses.calls[-1].request.body)["metadata_list"] == [
         {
-            "type": metadatas[0]["type"].value,
-            "name": metadatas[0]["name"],
-            "value": metadatas[0]["value"],
-            "entity_id": metadatas[0].get("entity_id"),
+            "type": metadata_list[0]["type"].value,
+            "name": metadata_list[0]["name"],
+            "value": metadata_list[0]["value"],
+            "entity_id": metadata_list[0].get("entity_id"),
         }
     ]
-    assert created_metadatas == [
+    assert created_metadata_list == [
         {
             "id": "fake_metadata_id",
-            "type": metadatas[0]["type"].value,
-            "name": metadatas[0]["name"],
-            "value": metadatas[0]["value"],
+            "type": metadata_list[0]["type"].value,
+            "name": metadata_list[0]["name"],
+            "value": metadata_list[0]["value"],
             "dates": [],
-            "entity_id": metadatas[0].get("entity_id"),
+            "entity_id": metadata_list[0].get("entity_id"),
         }
     ]
 
 
-@pytest.mark.parametrize(
-    "wrong_element",
-    [
-        None,
-        "not_element_type",
-        1234,
-        12.5,
-    ],
-)
+@pytest.mark.parametrize("wrong_element", [None, "not_element_type", 1234, 12.5])
 def test_create_metadatas_wrong_element(mock_elements_worker, wrong_element):
-    wrong_metadatas = [
+    wrong_metadata_list = [
         {"type": MetaType.Text, "name": "fake_name", "value": "fake_value"}
     ]
     with pytest.raises(
@@ -406,48 +395,42 @@ def test_create_metadatas_wrong_element(mock_elements_worker, wrong_element):
         match="element shouldn't be null and should be of type Element or CachedElement",
     ):
         mock_elements_worker.create_metadatas(
-            element=wrong_element, metadatas=wrong_metadatas
+            element=wrong_element, metadatas=wrong_metadata_list
         )
 
 
-@pytest.mark.parametrize(
-    "wrong_type",
-    [
-        None,
-        "not_metadata_type",
-        1234,
-        12.5,
-    ],
-)
+@pytest.mark.parametrize("wrong_type", [None, "not_metadata_type", 1234, 12.5])
 def test_create_metadatas_wrong_type(mock_elements_worker, wrong_type):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
-    wrong_metadatas = [{"type": wrong_type, "name": "fake_name", "value": "fake_value"}]
+    wrong_metadata_list = [
+        {"type": wrong_type, "name": "fake_name", "value": "fake_value"}
+    ]
     with pytest.raises(
         AssertionError, match="type shouldn't be null and should be of type MetaType"
     ):
         mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadatas
+            element=element, metadatas=wrong_metadata_list
         )
 
 
-@pytest.mark.parametrize("wrong_name", [(None), (1234), (12.5), ([1, 2, 3, 4])])
+@pytest.mark.parametrize("wrong_name", [None, 1234, 12.5, [1, 2, 3, 4]])
 def test_create_metadatas_wrong_name(mock_elements_worker, wrong_name):
     element = Element({"id": "fake_element_id"})
-    wrong_metadatas = [
+    wrong_metadata_list = [
         {"type": MetaType.Text, "name": wrong_name, "value": "fake_value"}
     ]
     with pytest.raises(
         AssertionError, match="name shouldn't be null and should be of type str"
     ):
         mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadatas
+            element=element, metadatas=wrong_metadata_list
         )
 
 
-@pytest.mark.parametrize("wrong_value", [(None), ([1, 2, 3, 4])])
+@pytest.mark.parametrize("wrong_value", [None, [1, 2, 3, 4]])
 def test_create_metadatas_wrong_value(mock_elements_worker, wrong_value):
     element = Element({"id": "fake_element_id"})
-    wrong_metadatas = [
+    wrong_metadata_list = [
         {"type": MetaType.Text, "name": "fake_name", "value": wrong_value}
     ]
     with pytest.raises(
@@ -457,21 +440,14 @@ def test_create_metadatas_wrong_value(mock_elements_worker, wrong_value):
         ),
     ):
         mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadatas
+            element=element, metadatas=wrong_metadata_list
         )
 
 
-@pytest.mark.parametrize(
-    "wrong_entity",
-    [
-        [1, 2, 3, 4],
-        1234,
-        12.5,
-    ],
-)
+@pytest.mark.parametrize("wrong_entity", [[1, 2, 3, 4], 1234, 12.5])
 def test_create_metadatas_wrong_entity(mock_elements_worker, wrong_entity):
     element = Element({"id": "fake_element_id"})
-    wrong_metadatas = [
+    wrong_metadata_list = [
         {
             "type": MetaType.Text,
             "name": "fake_name",
@@ -481,13 +457,13 @@ def test_create_metadatas_wrong_entity(mock_elements_worker, wrong_entity):
     ]
     with pytest.raises(AssertionError, match="entity_id should be None or a str"):
         mock_elements_worker.create_metadatas(
-            element=element, metadatas=wrong_metadatas
+            element=element, metadatas=wrong_metadata_list
         )
 
 
 def test_create_metadatas_api_error(responses, mock_elements_worker):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
-    metadatas = [
+    metadata_list = [
         {
             "type": MetaType.Text,
             "name": "fake_name",
@@ -502,7 +478,7 @@ def test_create_metadatas_api_error(responses, mock_elements_worker):
     )
 
     with pytest.raises(ErrorResponse):
-        mock_elements_worker.create_metadatas(element, metadatas)
+        mock_elements_worker.create_metadatas(element, metadata_list)
 
     assert len(responses.calls) == len(BASE_API_CALLS) + 5
     assert [

tests/test_elements_worker/test_task.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import uuid
 
 import pytest
@@ -12,8 +11,8 @@ TASK_ID = uuid.UUID("cafecafe-cafe-cafe-cafe-cafecafecafe")
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Task ID
         (
             {"task_id": None},
@@ -23,7 +22,7 @@ TASK_ID = uuid.UUID("cafecafe-cafe-cafe-cafe-cafecafecafe")
             {"task_id": "12341234-1234-1234-1234-123412341234"},
             "task_id shouldn't be null and should be an UUID",
         ),
-    ),
+    ],
 )
 def test_list_artifacts_wrong_param_task_id(mock_dataset_worker, payload, error):
     with pytest.raises(AssertionError, match=error):
@@ -60,7 +59,7 @@ def test_list_artifacts(
     expected_results = [
         {
             "id": "artifact_1",
-            "path": "dataset_id.zstd",
+            "path": "dataset_id.tar.zst",
             "size": 42,
             "content_type": "application/zstd",
             "s3_put_url": None,
@@ -97,8 +96,8 @@ def test_list_artifacts(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Task ID
         (
             {"task_id": None},
@@ -108,7 +107,7 @@ def test_list_artifacts(
             {"task_id": "12341234-1234-1234-1234-123412341234"},
             "task_id shouldn't be null and should be an UUID",
         ),
-    ),
+    ],
 )
 def test_download_artifact_wrong_param_task_id(
     mock_dataset_worker, default_artifact, payload, error
@@ -124,8 +123,8 @@ def test_download_artifact_wrong_param_task_id(
 
 
 @pytest.mark.parametrize(
-    "payload, error",
-    (
+    ("payload", "error"),
+    [
         # Artifact
         (
             {"artifact": None},
@@ -135,7 +134,7 @@ def test_download_artifact_wrong_param_task_id(
             {"artifact": "not artifact type"},
             "artifact shouldn't be null and should be an Artifact",
         ),
-    ),
+    ],
 )
 def test_download_artifact_wrong_param_artifact(
     mock_dataset_worker, default_artifact, payload, error
@@ -153,7 +152,7 @@ def test_download_artifact_wrong_param_artifact(
 def test_download_artifact_api_error(responses, mock_dataset_worker, default_artifact):
     responses.add(
         responses.GET,
-        f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd",
+        f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst",
         status=500,
     )
 
@@ -167,11 +166,11 @@ def test_download_artifact_api_error(responses, mock_dataset_worker, default_art
         (call.request.method, call.request.url) for call in responses.calls
     ] == BASE_API_CALLS + [
         # The API call is retried 5 times
-        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd"),
-        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd"),
+        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst"),
+        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst"),
     ]
 
 
@@ -181,11 +180,14 @@ def test_download_artifact(
     default_artifact,
 ):
     archive_path = (
-        FIXTURES_DIR / "extract_parent_archives" / "first_parent" / "arkindex_data.zstd"
+        FIXTURES_DIR
+        / "extract_parent_archives"
+        / "first_parent"
+        / "arkindex_data.tar.zst"
     )
     responses.add(
         responses.GET,
-        f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd",
+        f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst",
         status=200,
         body=archive_path.read_bytes(),
         content_type="application/zstd",
@@ -202,5 +204,5 @@ def test_download_artifact(
     assert [
         (call.request.method, call.request.url) for call in responses.calls
     ] == BASE_API_CALLS + [
-        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.zstd"),
+        ("GET", f"http://testserver/api/v1/task/{TASK_ID}/artifact/dataset_id.tar.zst"),
     ]

tests/test_elements_worker/test_training.py

@@ -1,16 +1,14 @@
-# -*- coding: utf-8 -*-
 import logging
 import sys
 
 import pytest
-import responses
 
 from arkindex.mock import MockApiClient
 from arkindex_worker.worker import BaseWorker
 from arkindex_worker.worker.training import TrainingMixin, create_archive
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_training_worker(monkeypatch):
     class TrainingWorker(BaseWorker, TrainingMixin):
         """
@@ -24,7 +22,7 @@ def mock_training_worker(monkeypatch):
     return training_worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def default_model_version():
     return {
         "id": "model_version_id",
@@ -79,23 +77,32 @@ def test_create_archive_with_subfolder(model_file_dir_with_subfolder):
     assert not zst_archive_path.exists(), "Auto removal failed"
 
 
-def test_handle_s3_uploading_errors(mock_training_worker, model_file_dir):
+def test_handle_s3_uploading_errors(responses, mock_training_worker, model_file_dir):
     s3_endpoint_url = "http://s3.localhost.com"
     responses.add_passthru(s3_endpoint_url)
-    responses.add(responses.Response(method="PUT", url=s3_endpoint_url, status=400))
+    responses.add(responses.PUT, s3_endpoint_url, status=400)
+
+    mock_training_worker.model_version = {
+        "state": "Created",
+        "s3_put_url": s3_endpoint_url,
+    }
+
     file_path = model_file_dir / "model_file.pth"
-    with pytest.raises(Exception):
-        mock_training_worker.upload_to_s3(file_path, {"s3_put_url": s3_endpoint_url})
+    with pytest.raises(
+        Exception,
+        match="400 Client Error: Bad Request for url: http://s3.localhost.com/",
+    ):
+        mock_training_worker.upload_to_s3(file_path)
 
 
 @pytest.mark.parametrize(
     "method",
     [
-        ("publish_model_version"),
-        ("create_model_version"),
-        ("update_model_version"),
-        ("upload_to_s3"),
-        ("validate_model_version"),
+        "publish_model_version",
+        "create_model_version",
+        "update_model_version",
+        "upload_to_s3",
+        "validate_model_version",
     ],
 )
 def test_training_mixin_read_only(mock_training_worker, method, caplog):

tests/test_elements_worker/test_transcriptions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 import re
 from uuid import UUID
@@ -1867,9 +1866,10 @@ def test_list_transcriptions_manual_worker_version(responses, mock_elements_work
     ]
 
 
+@pytest.mark.usefixtures("_mock_cached_transcriptions")
 @pytest.mark.parametrize(
-    "filters, expected_ids",
-    (
+    ("filters", "expected_ids"),
+    [
         # Filter on element should give first and sixth transcription
         (
             {
@@ -1963,14 +1963,10 @@ def test_list_transcriptions_manual_worker_version(responses, mock_elements_work
             },
             ("66666666-6666-6666-6666-666666666666",),
         ),
-    ),
+    ],
 )
 def test_list_transcriptions_with_cache(
-    responses,
-    mock_elements_worker_with_cache,
-    mock_cached_transcriptions,
-    filters,
-    expected_ids,
+    responses, mock_elements_worker_with_cache, filters, expected_ids
 ):
     # Check we have 5 elements already present in database
     assert CachedTranscription.select().count() == 6
@@ -1979,7 +1975,7 @@ def test_list_transcriptions_with_cache(
     transcriptions = mock_elements_worker_with_cache.list_transcriptions(**filters)
     assert transcriptions.count() == len(expected_ids)
     for transcription, expected_id in zip(
-        transcriptions.order_by(CachedTranscription.id), expected_ids
+        transcriptions.order_by(CachedTranscription.id), expected_ids, strict=True
     ):
         assert transcription.id == UUID(expected_id)