arkindex-base-worker 0.4.0b3__py3-none-any.whl → 0.4.0rc2__py3-none-any.whl

tests/conftest.py

@@ -23,10 +23,15 @@ from arkindex_worker.cache import (
     init_cache_db,
 )
 from arkindex_worker.models import Artifact, Dataset, Set
-from arkindex_worker.worker import BaseWorker, DatasetWorker, ElementsWorker
+from arkindex_worker.worker import (
+    BaseWorker,
+    DatasetWorker,
+    ElementsWorker,
+    ProcessMode,
+)
 from arkindex_worker.worker.dataset import DatasetState
 from arkindex_worker.worker.transcription import TextOrientation
-from tests import CORPUS_ID, FIXTURES_DIR, PROCESS_ID, SAMPLES_DIR
+from tests import CORPUS_ID, SAMPLES_DIR
 
 __yaml_cache = {}
 
@@ -277,9 +282,7 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
     """
     Mock a worker instance to list and retrieve a single element
     """
-    monkeypatch.setattr(
-        mock_elements_worker, "list_elements", lambda: ["1234-deadbeef"]
-    )
+    monkeypatch.setattr(mock_elements_worker, "get_elements", lambda: ["1234-deadbeef"])
     responses.add(
         responses.GET,
         "http://testserver/api/v1/element/1234-deadbeef/",
@@ -326,23 +329,6 @@ def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run
     return worker
 
 
-@pytest.fixture()
-def fake_page_element():
-    return json.loads((FIXTURES_DIR / "page_element.json").read_text())
-
-
-@pytest.fixture()
-def fake_ufcn_worker_version():
-    return json.loads(
-        (FIXTURES_DIR / "ufcn_line_historical_worker_version.json").read_text()
-    )
-
-
-@pytest.fixture()
-def fake_transcriptions_small():
-    return json.loads((FIXTURES_DIR / "line_transcriptions_small.json").read_text())
-
-
 @pytest.fixture()
 def model_file_dir():
     return SAMPLES_DIR / "model_files"
@@ -620,7 +606,9 @@ def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
 
     dataset_worker = DatasetWorker()
     dataset_worker.configure()
-    dataset_worker.process_information = {"id": PROCESS_ID}
+
+    # Update process mode
+    dataset_worker.process_information["mode"] = ProcessMode.Dataset
 
     assert not dataset_worker.is_read_only
 

tests/test_base_worker.py

@@ -206,6 +206,7 @@ def test_configure_worker_run(mocker, responses, caplog):
             "Loaded Worker Fake worker @ 123412 from API",
         ),
         ("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
+        ("arkindex_worker", logging.INFO, "User configuration retrieved"),
     ]
 
     assert worker.user_configuration == {"a": "b"}
@@ -284,12 +285,21 @@ def test_configure_user_configuration_defaults(mocker, responses):
 
     worker.configure()
 
-    assert worker.config == {"param_1": "/some/path/file.pth", "param_2": 12}
     assert worker.user_configuration == {
         "integer_parameter": 0,
         "param_3": "Animula vagula blandula",
         "param_5": True,
     }
+    # All configurations are merged
+    assert worker.config == {
+        # Default config
+        "param_1": "/some/path/file.pth",
+        "param_2": 12,
+        # User config
+        "integer_parameter": 0,
+        "param_3": "Animula vagula blandula",
+        "param_5": True,
+    }
 
 
 @pytest.mark.parametrize("debug", [True, False])
@@ -676,7 +686,6 @@ def test_find_parents_file_paths(responses, mock_base_worker_with_cache, tmp_pat
     mock_base_worker_with_cache.args = mock_base_worker_with_cache.parser.parse_args()
 
     mock_base_worker_with_cache.configure()
-    mock_base_worker_with_cache.configure_cache()
 
     assert mock_base_worker_with_cache.find_parents_file_paths(filename) == [
         tmp_path / "first" / filename,
@@ -753,3 +762,195 @@ def test_corpus_id_set_read_only_mode(
     mock_elements_worker_read_only.configure()
 
     assert mock_elements_worker_read_only.corpus_id == corpus_id
+
+
+@pytest.mark.parametrize(
+    (
+        "wk_version_config",
+        "wk_version_user_config",
+        "frontend_user_config",
+        "model_config",
+        "expected_config",
+    ),
+    [
+        ({}, {}, {}, {}, {}),
+        # Keep parameters from worker version configuration
+        ({"parameter": 0}, {}, {}, {}, {"parameter": 0}),
+        # Keep parameters from worker version configuration + user_config defaults
+        (
+            {"parameter": 0},
+            {
+                "parameter2": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 0,
+                    "required": False,
+                }
+            },
+            {},
+            {},
+            {"parameter": 0, "parameter2": 0},
+        ),
+        # Keep parameters from worker version configuration + user_config no defaults
+        (
+            {"parameter": 0},
+            {
+                "parameter2": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "required": False,
+                }
+            },
+            {},
+            {},
+            {"parameter": 0},
+        ),
+        # Keep parameters from worker version configuration but user_config defaults overrides
+        (
+            {"parameter": 0},
+            {
+                "parameter": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 1,
+                    "required": False,
+                }
+            },
+            {},
+            {},
+            {"parameter": 1},
+        ),
+        # Keep parameters from worker version configuration + frontend config
+        (
+            {"parameter": 0},
+            {},
+            {"parameter2": 0},
+            {},
+            {"parameter": 0, "parameter2": 0},
+        ),
+        # Keep parameters from worker version configuration + frontend config overrides
+        ({"parameter": 0}, {}, {"parameter": 1}, {}, {"parameter": 1}),
+        # Keep parameters from worker version configuration + model config
+        (
+            {"parameter": 0},
+            {},
+            {},
+            {"parameter2": 0},
+            {"parameter": 0, "parameter2": 0},
+        ),
+        # Keep parameters from worker version configuration + model config overrides
+        ({"parameter": 0}, {}, {}, {"parameter": 1}, {"parameter": 1}),
+        # Keep parameters from worker version configuration + user_config default + model config overrides
+        (
+            {"parameter": 0},
+            {
+                "parameter": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 1,
+                    "required": False,
+                }
+            },
+            {},
+            {"parameter": 2},
+            {"parameter": 2},
+        ),
+        # Keep parameters from worker version configuration + model config + frontend config overrides
+        ({"parameter": 0}, {}, {"parameter": 2}, {"parameter": 1}, {"parameter": 2}),
+        # Keep parameters from worker version configuration + user_config default + model config + frontend config overrides all
+        (
+            {"parameter": 0},
+            {
+                "parameter": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 1,
+                    "required": False,
+                }
+            },
+            {"parameter": 3},
+            {"parameter": 2},
+            {"parameter": 3},
+        ),
+    ],
+)
+def test_worker_config_multiple_source(
+    monkeypatch,
+    responses,
+    wk_version_config,
+    wk_version_user_config,
+    frontend_user_config,
+    model_config,
+    expected_config,
+):
+    # Compute WorkerRun info
+    payload = {
+        "id": "56785678-5678-5678-5678-567856785678",
+        "parents": [],
+        "worker_version": {
+            "id": "12341234-1234-1234-1234-123412341234",
+            "configuration": {
+                "docker": {"image": "python:3"},
+                "configuration": wk_version_config,
+                "secrets": [],
+                "user_configuration": wk_version_user_config,
+            },
+            "revision": {
+                "hash": "deadbeef1234",
+                "name": "some git revision",
+            },
+            "docker_image": "python:3",
+            "docker_image_name": "python:3",
+            "state": "created",
+            "worker": {
+                "id": "deadbeef-1234-5678-1234-worker",
+                "name": "Fake worker",
+                "slug": "fake_worker",
+                "type": "classifier",
+            },
+        },
+        "configuration": {
+            "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
+            "name": "Configuration entered by user",
+            "configuration": frontend_user_config,
+        },
+        "model_version": {
+            "id": "12341234-1234-1234-1234-123412341234",
+            "name": "Model version 1337",
+            "configuration": model_config,
+            "model": {
+                "id": "hahahaha-haha-haha-haha-hahahahahaha",
+                "name": "My model",
+            },
+        },
+        "process": {
+            "name": None,
+            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+            "state": "running",
+            "mode": "workers",
+            "corpus": CORPUS_ID,
+            "use_cache": False,
+            "activity_state": "ready",
+            "model_id": None,
+            "train_folder_id": None,
+            "validation_folder_id": None,
+            "test_folder_id": None,
+        },
+        "summary": "Worker Fake worker @ 123412",
+    }
+
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
+        status=200,
+        body=json.dumps(payload),
+        content_type="application/json",
+    )
+
+    # Create and configure a worker
+    monkeypatch.setattr(sys, "argv", ["worker"])
+    worker = BaseWorker()
+    worker.configure()
+
+    # Check final config
+    assert worker.config == expected_config

tests/test_dataset_worker.py

@@ -6,8 +6,11 @@ import pytest
 from apistar.exceptions import ErrorResponse
 
 from arkindex_worker.models import Dataset, Set
-from arkindex_worker.worker import MissingDatasetArchive, check_dataset_set
-from arkindex_worker.worker.dataset import DatasetState
+from arkindex_worker.worker.dataset import (
+    DatasetState,
+    MissingDatasetArchive,
+    check_dataset_set,
+)
 from tests import FIXTURES_DIR, PROCESS_ID
 from tests.test_elements_worker import BASE_API_CALLS