arkindex-base-worker 0.4.0rc5__py3-none-any.whl → 0.5.0__py3-none-any.whl

tests/conftest.py

@@ -3,7 +3,6 @@ import json
 import os
 import sys
 import time
-from pathlib import Path
 from uuid import UUID
 
 import pytest
@@ -46,7 +45,7 @@ def _disable_sleep(monkeypatch):
     monkeypatch.setattr(time, "sleep", lambda x: None)
 
 
-@pytest.fixture()
+@pytest.fixture
 def _cache_yaml(monkeypatch):
     """
     Cache all calls to yaml.safe_load in order to speedup
@@ -78,19 +77,6 @@ def _cache_yaml(monkeypatch):
 def _setup_api(responses, monkeypatch, _cache_yaml):
     # Always use the environment variable first
     schema_url = os.environ.get("ARKINDEX_API_SCHEMA_URL")
-    if schema_url is None:
-        # Try to load a local schema as the current developer of base-worker
-        # may also work on the backend nearby
-        paths = [
-            "~/dev/ark/backend/schema.yml",
-            "~/dev/ark/backend/output/schema.yml",
-        ]
-        for path in paths:
-            path = Path(path).expanduser().absolute()
-            if path.exists():
-                monkeypatch.setenv("ARKINDEX_API_SCHEMA_URL", str(path))
-                schema_url = str(path)
-                break
 
     # Fallback to prod environment
     if schema_url is None:
@@ -111,7 +97,7 @@ def _give_env_variable(monkeypatch):
     monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", "56785678-5678-5678-5678-567856785678")
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_worker_run_api(responses):
     """Provide a mock API response to get worker run information"""
     payload = {
@@ -180,7 +166,7 @@ def _mock_worker_run_api(responses):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_worker_run_no_revision_api(responses):
     """Provide a mock API response to get worker run not linked to a revision information"""
     payload = {
@@ -247,7 +233,7 @@ def _mock_worker_run_no_revision_api(responses):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_activity_calls(responses):
     """
     Mock responses when updating the activity state for multiple element of the same version
@@ -259,7 +245,7 @@ def _mock_activity_calls(responses):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker(monkeypatch, _mock_worker_run_api):
     """Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker"])
@@ -268,7 +254,7 @@ def mock_elements_worker(monkeypatch, _mock_worker_run_api):
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker_read_only(monkeypatch):
     """Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker", "--dev"])
@@ -277,7 +263,7 @@ def mock_elements_worker_read_only(monkeypatch):
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker):
     """
     Mock a worker instance to list and retrieve a single element
@@ -296,7 +282,7 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
     return mock_elements_worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_cache_db(tmp_path):
     cache_path = tmp_path / "db.sqlite"
 
@@ -307,7 +293,7 @@ def mock_cache_db(tmp_path):
     return cache_path
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_base_worker_with_cache(monkeypatch, _mock_worker_run_api):
     """Build a BaseWorker using SQLite cache, also mocking a PONOS_TASK"""
     monkeypatch.setattr(sys, "argv", ["worker"])
@@ -318,7 +304,7 @@ def mock_base_worker_with_cache(monkeypatch, _mock_worker_run_api):
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run_api):
     """Build and configure an ElementsWorker using SQLite cache with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker", "-d", str(mock_cache_db)])
@@ -329,17 +315,17 @@ def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def model_file_dir():
     return SAMPLES_DIR / "model_files"
 
 
-@pytest.fixture()
+@pytest.fixture
 def model_file_dir_with_subfolder():
     return SAMPLES_DIR / "root_folder"
 
 
-@pytest.fixture()
+@pytest.fixture
 def fake_dummy_worker():
     api_client = MockApiClient()
     worker = ElementsWorker()
@@ -347,7 +333,7 @@ def fake_dummy_worker():
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_cached_elements(mock_cache_db):
     """Insert few elements in local cache"""
     CachedElement.create(
@@ -392,7 +378,7 @@ def _mock_cached_elements(mock_cache_db):
     assert CachedElement.select().count() == 5
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_cached_images(mock_cache_db):
     """Insert few elements in local cache"""
     CachedImage.create(
@@ -404,7 +390,7 @@ def _mock_cached_images(mock_cache_db):
     assert CachedImage.select().count() == 1
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_cached_transcriptions(mock_cache_db):
     """Insert few transcriptions in local cache, on a shared element"""
     CachedElement.create(
@@ -493,7 +479,7 @@ def _mock_cached_transcriptions(mock_cache_db):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_databases(tmp_path):
     """
     Initialize several temporary databases
@@ -576,7 +562,7 @@ def mock_databases(tmp_path):
     return out
 
 
-@pytest.fixture()
+@pytest.fixture
 def default_dataset():
     return Dataset(
         {
@@ -594,12 +580,12 @@ def default_dataset():
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def default_train_set(default_dataset):
     return Set(name="train", dataset=default_dataset)
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
     monkeypatch.setenv("PONOS_TASK", "my_task")
     mocker.patch.object(sys, "argv", ["worker"])
@@ -615,7 +601,7 @@ def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
     return dataset_worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_dev_dataset_worker(mocker):
     mocker.patch.object(
         sys,
@@ -640,7 +626,7 @@ def mock_dev_dataset_worker(mocker):
     return dataset_worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def default_artifact():
     return Artifact(
         **{

tests/test_base_worker.py

@@ -803,7 +803,7 @@ def test_corpus_id_set_read_only_mode(
             },
             {},
             {},
-            {"parameter": 0},
+            {"parameter": 0, "parameter2": None},
         ),
         # Keep parameters from worker version configuration but user_config defaults overrides
         (

tests/test_cache.py

@@ -60,9 +60,8 @@ def test_create_tables(tmp_path):
 CREATE TABLE "dataset_elements" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "dataset_id" TEXT NOT NULL, "set_name" VARCHAR(255) NOT NULL, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"), FOREIGN KEY ("dataset_id") REFERENCES "datasets" ("id"))
 CREATE TABLE "datasets" ("id" TEXT NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, "state" VARCHAR(255) NOT NULL DEFAULT 'open', "sets" TEXT NOT NULL)
 CREATE TABLE "elements" ("id" TEXT NOT NULL PRIMARY KEY, "parent_id" TEXT, "type" VARCHAR(50) NOT NULL, "image_id" TEXT, "polygon" text, "rotation_angle" INTEGER NOT NULL, "mirrored" INTEGER NOT NULL, "initial" INTEGER NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, FOREIGN KEY ("image_id") REFERENCES "images" ("id"))
-CREATE TABLE "entities" ("id" TEXT NOT NULL PRIMARY KEY, "type" VARCHAR(50) NOT NULL, "name" TEXT NOT NULL, "validated" INTEGER NOT NULL, "metas" text, "worker_run_id" TEXT)
 CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL)
-CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "entity_id" TEXT NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "entity_id"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"), FOREIGN KEY ("entity_id") REFERENCES "entities" ("id"))
+CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "type" VARCHAR(50) NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "type"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"))
 CREATE TABLE "transcriptions" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "text" TEXT NOT NULL, "confidence" REAL, "orientation" VARCHAR(50) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))"""
 
     actual_schema = "\n".join(

tests/test_dataset_worker.py

@@ -17,7 +17,7 @@ from tests.test_elements_worker import BASE_API_CALLS
 RANDOM_UUID = uuid.uuid4()
 
 
-@pytest.fixture()
+@pytest.fixture
 def tmp_archive(tmp_path):
     archive = tmp_path / "test_archive.tar.zst"
     archive.touch()

tests/test_elements_worker/test_element.py

@@ -10,7 +10,7 @@ from arkindex_worker.cache import (
     CachedImage,
 )
 from arkindex_worker.models import Element
-from arkindex_worker.worker.element import MissingTypeError
+from arkindex_worker.worker.element import MissingElementType
 from tests import CORPUS_ID
 
 from . import BASE_API_CALLS
@@ -34,73 +34,247 @@ def test_list_corpus_types(responses, mock_elements_worker):
     }
 
 
-def test_check_required_types_argument_types(mock_elements_worker):
+def test_create_element_type_wrong_slug(mock_elements_worker):
     with pytest.raises(
-        AssertionError, match="At least one element type slug is required."
+        AssertionError, match="slug shouldn't be null and should be of type str"
     ):
-        mock_elements_worker.check_required_types()
+        mock_elements_worker.create_element_type(slug=None, name="page")
 
-    with pytest.raises(AssertionError, match="Element type slugs must be strings."):
-        mock_elements_worker.check_required_types("lol", 42)
+    with pytest.raises(
+        AssertionError, match="slug shouldn't be null and should be of type str"
+    ):
+        mock_elements_worker.create_element_type(slug=1234, name="page")
 
 
-def test_check_required_types(mock_elements_worker):
-    mock_elements_worker.corpus_types = {
-        "folder": {"slug": "folder"},
-        "page": {"slug": "page"},
-    }
+def test_create_element_type_wrong_name(mock_elements_worker):
+    with pytest.raises(
+        AssertionError, match="name shouldn't be null and should be of type str"
+    ):
+        mock_elements_worker.create_element_type(slug="page", name=None)
+
+    with pytest.raises(
+        AssertionError, match="name shouldn't be null and should be of type str"
+    ):
+        mock_elements_worker.create_element_type(slug="page", name=1234)
 
-    assert mock_elements_worker.check_required_types("page")
-    assert mock_elements_worker.check_required_types("page", "folder")
 
+def test_create_element_type_wrong_is_folder(mock_elements_worker):
     with pytest.raises(
-        MissingTypeError,
-        match=re.escape(
-            "Element types act, text_line were not found in corpus (11111111-1111-1111-1111-111111111111)."
-        ),
+        AssertionError, match="is_folder shouldn't be null and should be of type bool"
+    ):
+        mock_elements_worker.create_element_type(
+            slug="page", name="page", is_folder=None
+        )
+
+    with pytest.raises(
+        AssertionError, match="is_folder shouldn't be null and should be of type bool"
     ):
-        assert mock_elements_worker.check_required_types("page", "text_line", "act")
+        mock_elements_worker.create_element_type(
+            slug="page", name="page", is_folder=1234
+        )
+
+
+def test_create_element_type_api_error(responses, mock_elements_worker):
+    responses.add(
+        responses.POST,
+        "http://testserver/api/v1/elements/type/",
+        status=418,
+    )
+
+    with pytest.raises(ErrorResponse):
+        mock_elements_worker.create_element_type(slug="page", name="page")
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [("POST", "http://testserver/api/v1/elements/type/")]
+
 
+def test_create_element_type_already_exists(responses, mock_elements_worker):
+    assert mock_elements_worker.corpus_types == {}
 
-def test_check_required_types_create_missing(responses, mock_elements_worker):
-    mock_elements_worker.corpus_types = {
+    responses.add(
+        responses.POST,
+        "http://testserver/api/v1/elements/type/",
+        status=400,
+        match=[
+            matchers.json_params_matcher(
+                {
+                    "slug": "page",
+                    "display_name": "page",
+                    "folder": False,
+                    "corpus": CORPUS_ID,
+                }
+            )
+        ],
+    )
+    responses.add(
+        responses.GET,
+        f"http://testserver/api/v1/corpus/{CORPUS_ID}/",
+        status=200,
+        json={
+            "id": CORPUS_ID,
+            "types": [{"slug": "folder"}, {"slug": "page"}],
+        },
+    )
+
+    mock_elements_worker.create_element_type(slug="page", name="page")
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 2
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        ("POST", "http://testserver/api/v1/elements/type/"),
+        ("GET", f"http://testserver/api/v1/corpus/{CORPUS_ID}/"),
+    ]
+
+    # Make sure the corpus_types attribute has been updated
+    assert mock_elements_worker.corpus_types == {
         "folder": {"slug": "folder"},
         "page": {"slug": "page"},
     }
 
+
+def test_create_element_type(responses, mock_elements_worker):
+    assert mock_elements_worker.corpus_types == {}
+
     responses.add(
         responses.POST,
         "http://testserver/api/v1/elements/type/",
+        status=200,
         match=[
             matchers.json_params_matcher(
                 {
-                    "slug": "text_line",
-                    "display_name": "text_line",
+                    "slug": "page",
+                    "display_name": "page",
                     "folder": False,
                     "corpus": CORPUS_ID,
                 }
             )
         ],
+        json={"id": "page-id", "slug": "page", "display_name": "page", "folder": False},
     )
+
+    mock_elements_worker.create_element_type(slug="page", name="page")
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        ("POST", "http://testserver/api/v1/elements/type/"),
+    ]
+
+    # Make sure the corpus_types attribute has been updated
+    assert mock_elements_worker.corpus_types == {
+        "page": {
+            "id": "page-id",
+            "slug": "page",
+            "display_name": "page",
+            "folder": False,
+        }
+    }
+
+
+def test_check_required_types_wrong_type_slugs(mock_elements_worker):
+    with pytest.raises(
+        AssertionError, match="type_slugs shouldn't be null and should be of type list"
+    ):
+        mock_elements_worker.check_required_types(type_slugs=None)
+
+    with pytest.raises(
+        AssertionError, match="type_slugs shouldn't be null and should be of type list"
+    ):
+        mock_elements_worker.check_required_types(type_slugs=1234)
+
+    with pytest.raises(
+        AssertionError,
+        match="Element type at index 1 in type_slugs: Should be of type str",
+    ):
+        mock_elements_worker.check_required_types(type_slugs=["page", 1234])
+
+
+def test_check_required_types_wrong_create_missing(mock_elements_worker):
+    with pytest.raises(
+        AssertionError,
+        match="create_missing shouldn't be null and should be of type bool",
+    ):
+        mock_elements_worker.check_required_types(
+            type_slugs=["page"], create_missing=None
+        )
+
+    with pytest.raises(
+        AssertionError,
+        match="create_missing shouldn't be null and should be of type bool",
+    ):
+        mock_elements_worker.check_required_types(
+            type_slugs=["page"], create_missing=1234
+        )
+
+
+def test_check_required_types_do_not_create_missing(responses, mock_elements_worker):
+    # Set one element type
+    mock_elements_worker.corpus_types = {"folder": {"slug": "folder"}}
+
+    with pytest.raises(
+        MissingElementType, match="Element type `page` was not in the corpus."
+    ):
+        mock_elements_worker.check_required_types(
+            type_slugs=["folder", "page"], create_missing=False
+        )
+
+    assert len(responses.calls) == len(BASE_API_CALLS)
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS
+
+
+def test_check_required_types(responses, mock_elements_worker):
+    # Set one element type
+    mock_elements_worker.corpus_types = {"folder": {"slug": "folder"}}
+
+    # Call to create a new element type
     responses.add(
         responses.POST,
         "http://testserver/api/v1/elements/type/",
+        status=200,
         match=[
             matchers.json_params_matcher(
                 {
-                    "slug": "act",
-                    "display_name": "act",
+                    "slug": "page",
+                    "display_name": "page",
                     "folder": False,
                     "corpus": CORPUS_ID,
                 }
             )
         ],
+        json={"id": "page-id", "slug": "page", "display_name": "page", "folder": False},
     )
 
-    assert mock_elements_worker.check_required_types(
-        "page", "text_line", "act", create_missing=True
+    mock_elements_worker.check_required_types(
+        type_slugs=["folder", "page"], create_missing=True
     )
 
+    assert len(responses.calls) == len(BASE_API_CALLS) + 1
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "POST",
+            "http://testserver/api/v1/elements/type/",
+        ),
+    ]
+
+    # Make sure the element_types attribute has been updated
+    assert mock_elements_worker.corpus_types == {
+        "folder": {"slug": "folder"},
+        "page": {
+            "id": "page-id",
+            "slug": "page",
+            "display_name": "page",
+            "folder": False,
+        },
+    }
+
 
 @pytest.mark.parametrize(
     ("payload", "error"),