arkindex-base-worker 0.5.0b3__py3-none-any.whl → 0.5.1__py3-none-any.whl

tests/test_elements_worker/test_metadata.py

@@ -119,19 +119,6 @@ def test_create_metadata_wrong_value(mock_elements_worker):
         )
 
 
-def test_create_metadata_wrong_entity(mock_elements_worker):
-    elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
-
-    with pytest.raises(AssertionError, match="entity should be of type str"):
-        mock_elements_worker.create_metadata(
-            element=elt,
-            type=MetaType.Location,
-            name="Teklia",
-            value="La Turbine, Grenoble 38000",
-            entity=1234,
-        )
-
-
 def test_create_metadata_api_error(responses, mock_elements_worker):
     elt = Element({"id": "12341234-1234-1234-1234-123412341234"})
     responses.add(
@@ -188,7 +175,6 @@ def test_create_metadata(responses, mock_elements_worker):
         "type": "location",
         "name": "Teklia",
         "value": "La Turbine, Grenoble 38000",
-        "entity_id": None,
         "worker_run_id": "56785678-5678-5678-5678-567856785678",
     }
     assert metadata_id == "12345678-1234-1234-1234-123456789123"
@@ -223,7 +209,6 @@ def test_create_metadata_cached_element(responses, mock_elements_worker_with_cac
         "type": "location",
         "name": "Teklia",
         "value": "La Turbine, Grenoble 38000",
-        "entity_id": None,
         "worker_run_id": "56785678-5678-5678-5678-567856785678",
     }
     assert metadata_id == "12345678-1234-1234-1234-123456789123"
@@ -239,7 +224,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
             "type": MetaType.Text,
             "name": "Year",
             "value": "2024",
-            "entity_id": "entity_id",
         },
     ]
     if batch_size > 1:
@@ -256,7 +240,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
                         "name": metadata_list[0]["name"],
                         "value": metadata_list[0]["value"],
                         "dates": [],
-                        "entity_id": None,
                     },
                     {
                         "id": "fake_metadata_id2",
@@ -264,7 +247,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
                         "name": metadata_list[1]["name"],
                         "value": metadata_list[1]["value"],
                         "dates": [],
-                        "entity_id": metadata_list[1]["entity_id"],
                     },
                 ],
             },
@@ -284,7 +266,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
                             "name": meta["name"],
                             "value": meta["value"],
                             "dates": [],
-                            "entity_id": meta.get("entity_id"),
                         }
                     ],
                 },
@@ -316,7 +297,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
     first_meta = {
         **metadata_list[0],
         "type": metadata_list[0]["type"].value,
-        "entity_id": None,
     }
     second_meta = {**metadata_list[1], "type": metadata_list[1]["type"].value}
     empty_payload = {
@@ -346,7 +326,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
             "name": metadata_list[0]["name"],
             "value": metadata_list[0]["value"],
             "dates": [],
-            "entity_id": None,
         },
         {
             "id": "fake_metadata_id2",
@@ -354,7 +333,6 @@ def test_create_metadata_bulk(batch_size, responses, mock_elements_worker):
             "name": metadata_list[1]["name"],
             "value": metadata_list[1]["value"],
             "dates": [],
-            "entity_id": metadata_list[1]["entity_id"],
         },
     ]
 
@@ -373,7 +351,6 @@ def test_create_metadata_bulk_cached_element(
             "type": MetaType.Text,
             "name": "Year",
             "value": "2024",
-            "entity_id": "entity_id",
         },
     ]
     if batch_size > 1:
@@ -390,7 +367,6 @@ def test_create_metadata_bulk_cached_element(
                         "name": metadata_list[0]["name"],
                         "value": metadata_list[0]["value"],
                         "dates": [],
-                        "entity_id": None,
                     },
                     {
                         "id": "fake_metadata_id2",
@@ -398,7 +374,6 @@ def test_create_metadata_bulk_cached_element(
                         "name": metadata_list[1]["name"],
                         "value": metadata_list[1]["value"],
                         "dates": [],
-                        "entity_id": metadata_list[1]["entity_id"],
                     },
                 ],
             },
@@ -418,7 +393,6 @@ def test_create_metadata_bulk_cached_element(
                             "name": meta["name"],
                             "value": meta["value"],
                             "dates": [],
-                            "entity_id": meta.get("entity_id"),
                         }
                     ],
                 },
@@ -450,7 +424,6 @@ def test_create_metadata_bulk_cached_element(
     first_meta = {
         **metadata_list[0],
         "type": metadata_list[0]["type"].value,
-        "entity_id": None,
     }
     second_meta = {**metadata_list[1], "type": metadata_list[1]["type"].value}
     empty_payload = {
@@ -480,7 +453,6 @@ def test_create_metadata_bulk_cached_element(
             "name": metadata_list[0]["name"],
             "value": metadata_list[0]["value"],
             "dates": [],
-            "entity_id": None,
         },
         {
             "id": "fake_metadata_id2",
@@ -488,7 +460,6 @@ def test_create_metadata_bulk_cached_element(
             "name": metadata_list[1]["name"],
             "value": metadata_list[1]["value"],
             "dates": [],
-            "entity_id": metadata_list[1]["entity_id"],
         },
     ]
 
@@ -552,23 +523,6 @@ def test_create_metadata_bulk_wrong_value(mock_elements_worker, wrong_value):
         )
 
 
-@pytest.mark.parametrize("wrong_entity", [[1, 2, 3, 4], 1234, 12.5])
-def test_create_metadata_bulk_wrong_entity(mock_elements_worker, wrong_entity):
-    element = Element({"id": "fake_element_id"})
-    wrong_metadata_list = [
-        {
-            "type": MetaType.Text,
-            "name": "fake_name",
-            "value": "fake_value",
-            "entity_id": wrong_entity,
-        }
-    ]
-    with pytest.raises(AssertionError, match="entity_id should be None or a str"):
-        mock_elements_worker.create_metadata_bulk(
-            element=element, metadata_list=wrong_metadata_list
-        )
-
-
 def test_create_metadata_bulk_api_error(responses, mock_elements_worker):
     element = Element({"id": "12341234-1234-1234-1234-123412341234"})
     metadata_list = [
@@ -576,7 +530,6 @@ def test_create_metadata_bulk_api_error(responses, mock_elements_worker):
             "type": MetaType.Text,
             "name": "fake_name",
             "value": "fake_value",
-            "entity_id": "fake_entity_id",
         }
     ]
     responses.add(

tests/test_elements_worker/test_worker.py

@@ -685,6 +685,112 @@ def test_run_cache(monkeypatch, mocker, mock_elements_worker_with_cache):
     ]
 
 
+def test_run_consuming_worker_activities(
+    monkeypatch,
+    mock_elements_worker_consume_wa,
+    responses,
+    caplog,
+):
+    """Check the consuming worker activities runtime uses StartWorkerActivity + UpdateWorkerActivity"""
+    # Disable second configure call from run()
+    monkeypatch.setattr(mock_elements_worker_consume_wa, "configure", lambda: None)
+
+    assert mock_elements_worker_consume_wa.is_read_only is False
+
+    # Provide 2 worker activities to run and the corresponding update call
+    # and 2 element details response
+    for i, elt_id in enumerate(("page_1", "page_2"), 1):
+        responses.add(
+            responses.POST,
+            "http://testserver/api/v1/process/start-activity/",
+            status=200,
+            json={
+                "id": elt_id,
+                "type_id": "page-aaaa-aaaa-aaaa-aaaaaaaaaaaa",  # Element type provided by mock corpus
+                "name": f"Page n°{i}",
+            },
+        )
+        responses.add(
+            responses.PUT,
+            "http://testserver/api/v1/workers/versions/56785678-5678-5678-5678-567856785678/activity/",
+            status=200,
+        )
+        responses.add(
+            responses.GET,
+            f"http://testserver/api/v1/element/{elt_id}/",
+            status=200,
+            json={
+                "id": elt_id,
+                "type": "page",
+                "name": f"Page n°{i}",
+            },
+        )
+
+    # Then a 404 to stop iterating
+    responses.add(
+        responses.POST,
+        "http://testserver/api/v1/process/start-activity/",
+        status=404,
+    )
+
+    # Simply run the process
+    mock_elements_worker_consume_wa.run()
+
+    # We call twice configure in the conftest
+    assert len(responses.calls) == len(BASE_API_CALLS) * 2 + 7
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS * 2 + [
+        (
+            "POST",
+            "http://testserver/api/v1/process/start-activity/",
+        ),
+        (
+            "GET",
+            "http://testserver/api/v1/element/page_1/",
+        ),
+        (
+            "PUT",
+            "http://testserver/api/v1/workers/versions/56785678-5678-5678-5678-567856785678/activity/",
+        ),
+        (
+            "POST",
+            "http://testserver/api/v1/process/start-activity/",
+        ),
+        (
+            "GET",
+            "http://testserver/api/v1/element/page_2/",
+        ),
+        (
+            "PUT",
+            "http://testserver/api/v1/workers/versions/56785678-5678-5678-5678-567856785678/activity/",
+        ),
+        (
+            "POST",
+            "http://testserver/api/v1/process/start-activity/",
+        ),
+    ]
+
+    assert [(record.levelno, record.message) for record in caplog.records] == [
+        (
+            logging.INFO,
+            "Using StartWorkerActivity instead of reading init_elements JSON file",
+        ),
+        (
+            logging.INFO,
+            "Processing page Page n°1 (page_1) (n°1)",
+        ),
+        (
+            logging.INFO,
+            "Processing page Page n°2 (page_2) (n°2)",
+        ),
+        (
+            logging.INFO,
+            "Ran on 2 elements: 2 completed, 0 failed",
+        ),
+    ]
+
+
 def test_start_activity_conflict(
     monkeypatch, responses, mock_elements_worker_with_list, caplog
 ):

tests/test_image.py

@@ -113,21 +113,37 @@ def test_update_pillow_image_size_limit(max_image_pixels, expected_image_pixels)
     assert Image.MAX_IMAGE_PIXELS == MAX_IMAGE_PIXELS
 
 
-def test_download_tiles(responses):
+@pytest.mark.parametrize(
+    ("id_key", "resize"),
+    [
+        # IIIF version 2
+        ("@id", "full"),
+        # IIIF version 3
+        ("id", "max"),
+    ],
+)
+def test_download_tiles(responses, id_key, resize):
     expected = Image.open(FULL_IMAGE).convert("RGB")
     tile_bytes = TILE.read_bytes()
 
     responses.add(
         responses.GET,
         "http://nowhere/info.json",
-        json={"width": 543, "height": 720, "tiles": [{"width": 181, "height": 240}]},
+        json={
+            id_key: "http://nowhere",
+            "width": 543,
+            "height": 720,
+            "tiles": [
+                {"width": 181, "height": 240},
+            ],
+        },
     )
 
     for x in (0, 181, 362):
         for y in (0, 240, 480):
             responses.add(
                 responses.GET,
-                f"http://nowhere/{x},{y},181,240/full/0/default.jpg",
+                f"http://nowhere/{x},{y},181,240/{resize}/0/default.jpg",
                 body=tile_bytes,
             )
 

tests/test_merge.py

@@ -7,7 +7,6 @@ from arkindex_worker.cache import (
     SQL_VERSION,
     CachedClassification,
     CachedElement,
-    CachedEntity,
     CachedImage,
     CachedTranscription,
     CachedTranscriptionEntity,
@@ -85,7 +84,6 @@ def test_merge_databases(
         assert CachedElement.select().count() == 0
         assert CachedTranscription.select().count() == 0
         assert CachedClassification.select().count() == 0
-        assert CachedEntity.select().count() == 0
         assert CachedTranscriptionEntity.select().count() == 0
 
     # Retrieve parents databases paths
@@ -103,7 +101,6 @@ def test_merge_databases(
         assert CachedElement.select().count() == len(expected_elements)
         assert CachedTranscription.select().count() == len(expected_transcriptions)
         assert CachedClassification.select().count() == 0
-        assert CachedEntity.select().count() == 0
         assert CachedTranscriptionEntity.select().count() == 0
         assert [
             e.id for e in CachedElement.select().order_by("id")
@@ -124,7 +121,6 @@ def test_merge_chunk(mock_databases, tmp_path):
         assert CachedElement.select().count() == 0
         assert CachedTranscription.select().count() == 0
         assert CachedClassification.select().count() == 0
-        assert CachedEntity.select().count() == 0
         assert CachedTranscriptionEntity.select().count() == 0
 
     # Check filenames
@@ -144,7 +140,6 @@ def test_merge_chunk(mock_databases, tmp_path):
         assert CachedElement.select().count() == 3
         assert CachedTranscription.select().count() == 0
         assert CachedClassification.select().count() == 0
-        assert CachedEntity.select().count() == 0
         assert CachedTranscriptionEntity.select().count() == 0
         assert [e.id for e in CachedElement.select().order_by("id")] == [
             UUID("42424242-4242-4242-4242-424242424242"),
@@ -171,7 +166,6 @@ def test_merge_from_worker(
         assert CachedElement.select().count() == 0
         assert CachedTranscription.select().count() == 0
         assert CachedClassification.select().count() == 0
-        assert CachedEntity.select().count() == 0
         assert CachedTranscriptionEntity.select().count() == 0
 
     # Configure worker with a specific data directory
@@ -191,7 +185,6 @@ def test_merge_from_worker(
     assert CachedElement.select().count() == 3
     assert CachedTranscription.select().count() == 1
     assert CachedClassification.select().count() == 0
-    assert CachedEntity.select().count() == 0
     assert CachedTranscriptionEntity.select().count() == 0
     assert [e.id for e in CachedElement.select().order_by("id")] == [
         UUID("12341234-1234-1234-1234-123412341234"),

tests/test_modern_config.py

@@ -0,0 +1,81 @@
+def test_simple_configuration(mock_base_worker_modern_conf, responses):
+    # Provide the full configuration directly from the worker run
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=200,
+        json={"configuration": [{"key": "some_key", "value": "test", "secret": False}]},
+    )
+
+    mock_base_worker_modern_conf.configure()
+
+    assert mock_base_worker_modern_conf.config == {"some_key": "test"}
+    assert (
+        mock_base_worker_modern_conf.user_configuration
+        == mock_base_worker_modern_conf.config
+    )
+    assert mock_base_worker_modern_conf.secrets == {}
+
+
+def test_empty(mock_base_worker_modern_conf, responses):
+    # Provide the full configuration directly from the worker run
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=200,
+        json={"configuration": []},
+    )
+
+    mock_base_worker_modern_conf.configure()
+
+    assert mock_base_worker_modern_conf.config == {}
+    assert (
+        mock_base_worker_modern_conf.user_configuration
+        == mock_base_worker_modern_conf.config
+    )
+    assert mock_base_worker_modern_conf.secrets == {}
+
+
+def test_with_secrets(mock_base_worker_modern_conf, responses):
+    # Provide the full configuration directly from the worker run
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=200,
+        json={
+            "configuration": [
+                {"key": "some_key", "value": "test", "secret": False},
+                {
+                    "key": "a_secret",
+                    "value": "471b9e64-29af-48dc-8bda-1a64a2da0c12",
+                    "secret": True,
+                },
+            ]
+        },
+    )
+
+    # Provide a secret value
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/secret/471b9e64-29af-48dc-8bda-1a64a2da0c12",
+        status=200,
+        json={
+            "id": "471b9e64-29af-48dc-8bda-1a64a2da0c12",
+            "name": "a_secret",
+            "content": "My super duper secret value",
+        },
+    )
+
+    mock_base_worker_modern_conf.configure()
+
+    assert mock_base_worker_modern_conf.config == {
+        "a_secret": "My super duper secret value",
+        "some_key": "test",
+    }
+    assert (
+        mock_base_worker_modern_conf.user_configuration
+        == mock_base_worker_modern_conf.config
+    )
+    assert mock_base_worker_modern_conf.secrets == {
+        "a_secret": "My super duper secret value"
+    }

tests/test_utils.py

@@ -1,4 +1,5 @@
 import logging
+import zipfile
 
 import pytest
 
@@ -7,6 +8,7 @@ from arkindex_worker.utils import (
     DEFAULT_BATCH_SIZE,
     batch_publication,
     close_delete_file,
+    create_zip_archive,
     extract_tar_zst_archive,
     parse_source_id,
 )
@@ -118,3 +120,43 @@ def test_batch_publication_decorator_alongside_unsupported_cache(caplog):
             "This API helper `custom_publication_in_batches_without_cache` did not update the cache database",
         ),
     ]
+
+
+def test_zip_archive():
+    # Create zip archive from fixtures
+    _, archive = create_zip_archive(FIXTURES_DIR / "extract_parent_archives/expected")
+
+    # Check the files in the archive
+    with zipfile.ZipFile(archive, mode="r") as f:
+        assert sorted(f.namelist()) == [
+            "test/",
+            "test/images/",
+            "test/images/f2649ce7-333e-44d2-ae73-387f18aad1f6.png",
+            "test/labels/",
+            "test/labels/f2649ce7-333e-44d2-ae73-387f18aad1f6.png",
+            "test/labels_json/",
+            "test/labels_json/f2649ce7-333e-44d2-ae73-387f18aad1f6.json",
+            "train/",
+            "train/images/",
+            "train/images/98115546-df07-448c-a2f0-34aa24789b77.png",
+            "train/images/ebeaa451-9287-4df7-9c40-07eb25cadb78.png",
+            "train/labels/",
+            "train/labels/98115546-df07-448c-a2f0-34aa24789b77.png",
+            "train/labels/ebeaa451-9287-4df7-9c40-07eb25cadb78.png",
+            "train/labels_json/",
+            "train/labels_json/98115546-df07-448c-a2f0-34aa24789b77.json",
+            "train/labels_json/ebeaa451-9287-4df7-9c40-07eb25cadb78.json",
+            "val/",
+            "val/images/",
+            "val/images/2987176d-4338-40f2-90d9-6d2cb4fd4a00.png",
+            "val/images/e3f91312-9201-45b7-9c32-e04a97ff1334.png",
+            "val/labels/",
+            "val/labels/2987176d-4338-40f2-90d9-6d2cb4fd4a00.png",
+            "val/labels/e3f91312-9201-45b7-9c32-e04a97ff1334.png",
+            "val/labels_json/",
+            "val/labels_json/2987176d-4338-40f2-90d9-6d2cb4fd4a00.json",
+            "val/labels_json/e3f91312-9201-45b7-9c32-e04a97ff1334.json",
+        ]
+
+    # Cleanup
+    archive.unlink()