arkindex-base-worker 0.3.7rc5__py3-none-any.whl → 0.5.0a1__py3-none-any.whl

tests/conftest.py

@@ -22,15 +22,16 @@ from arkindex_worker.cache import (
     create_version_table,
     init_cache_db,
 )
-from arkindex_worker.models import Artifact, Dataset
-from arkindex_worker.worker import BaseWorker, DatasetWorker, ElementsWorker
+from arkindex_worker.models import Artifact, Dataset, Set
+from arkindex_worker.worker import (
+    BaseWorker,
+    DatasetWorker,
+    ElementsWorker,
+    ProcessMode,
+)
 from arkindex_worker.worker.dataset import DatasetState
 from arkindex_worker.worker.transcription import TextOrientation
-
-FIXTURES_DIR = Path(__file__).resolve().parent / "data"
-SAMPLES_DIR = Path(__file__).resolve().parent / "samples"
-
-PROCESS_ID = "cafecafe-cafe-cafe-cafe-cafecafecafe"
+from tests import CORPUS_ID, SAMPLES_DIR
 
 __yaml_cache = {}
 
@@ -45,7 +46,7 @@ def _disable_sleep(monkeypatch):
     monkeypatch.setattr(time, "sleep", lambda x: None)
 
 
-@pytest.fixture()
+@pytest.fixture
 def _cache_yaml(monkeypatch):
     """
     Cache all calls to yaml.safe_load in order to speedup
@@ -93,7 +94,7 @@ def _setup_api(responses, monkeypatch, _cache_yaml):
 
     # Fallback to prod environment
     if schema_url is None:
-        schema_url = "https://arkindex.teklia.com/api/v1/openapi/?format=openapi-json"
+        schema_url = "https://arkindex.teklia.com/api/v1/openapi/?format=json"
         monkeypatch.setenv("ARKINDEX_API_SCHEMA_URL", schema_url)
 
     # Allow accessing remote API schemas
@@ -110,7 +111,7 @@ def _give_env_variable(monkeypatch):
     monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", "56785678-5678-5678-5678-567856785678")
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_worker_run_api(responses):
     """Provide a mock API response to get worker run information"""
     payload = {
@@ -159,7 +160,7 @@ def _mock_worker_run_api(responses):
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
             "state": "running",
             "mode": "workers",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
             "use_cache": False,
             "activity_state": "ready",
             "model_id": None,
@@ -179,7 +180,7 @@ def _mock_worker_run_api(responses):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_worker_run_no_revision_api(responses):
     """Provide a mock API response to get worker run not linked to a revision information"""
     payload = {
@@ -226,7 +227,7 @@ def _mock_worker_run_no_revision_api(responses):
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
             "state": "running",
             "mode": "workers",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
             "use_cache": False,
             "activity_state": "ready",
             "model_id": None,
@@ -246,7 +247,7 @@ def _mock_worker_run_no_revision_api(responses):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_activity_calls(responses):
     """
     Mock responses when updating the activity state for multiple element of the same version
@@ -258,7 +259,7 @@ def _mock_activity_calls(responses):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker(monkeypatch, _mock_worker_run_api):
     """Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker"])
@@ -267,7 +268,7 @@ def mock_elements_worker(monkeypatch, _mock_worker_run_api):
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker_read_only(monkeypatch):
     """Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker", "--dev"])
@@ -276,14 +277,12 @@ def mock_elements_worker_read_only(monkeypatch):
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker):
     """
     Mock a worker instance to list and retrieve a single element
     """
-    monkeypatch.setattr(
-        mock_elements_worker, "list_elements", lambda: ["1234-deadbeef"]
-    )
+    monkeypatch.setattr(mock_elements_worker, "get_elements", lambda: ["1234-deadbeef"])
     responses.add(
         responses.GET,
         "http://testserver/api/v1/element/1234-deadbeef/",
@@ -297,7 +296,7 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
     return mock_elements_worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_cache_db(tmp_path):
     cache_path = tmp_path / "db.sqlite"
 
@@ -308,7 +307,7 @@ def mock_cache_db(tmp_path):
     return cache_path
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_base_worker_with_cache(monkeypatch, _mock_worker_run_api):
     """Build a BaseWorker using SQLite cache, also mocking a PONOS_TASK"""
     monkeypatch.setattr(sys, "argv", ["worker"])
@@ -319,7 +318,7 @@ def mock_base_worker_with_cache(monkeypatch, _mock_worker_run_api):
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run_api):
     """Build and configure an ElementsWorker using SQLite cache with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker", "-d", str(mock_cache_db)])
@@ -330,34 +329,17 @@ def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run
     return worker
 
 
-@pytest.fixture()
-def fake_page_element():
-    return json.loads((FIXTURES_DIR / "page_element.json").read_text())
-
-
-@pytest.fixture()
-def fake_ufcn_worker_version():
-    return json.loads(
-        (FIXTURES_DIR / "ufcn_line_historical_worker_version.json").read_text()
-    )
-
-
-@pytest.fixture()
-def fake_transcriptions_small():
-    return json.loads((FIXTURES_DIR / "line_transcriptions_small.json").read_text())
-
-
-@pytest.fixture()
+@pytest.fixture
 def model_file_dir():
     return SAMPLES_DIR / "model_files"
 
 
-@pytest.fixture()
+@pytest.fixture
 def model_file_dir_with_subfolder():
     return SAMPLES_DIR / "root_folder"
 
 
-@pytest.fixture()
+@pytest.fixture
 def fake_dummy_worker():
     api_client = MockApiClient()
     worker = ElementsWorker()
@@ -365,7 +347,7 @@ def fake_dummy_worker():
     return worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_cached_elements(mock_cache_db):
     """Insert few elements in local cache"""
     CachedElement.create(
@@ -410,7 +392,7 @@ def _mock_cached_elements(mock_cache_db):
     assert CachedElement.select().count() == 5
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_cached_images(mock_cache_db):
     """Insert few elements in local cache"""
     CachedImage.create(
@@ -422,7 +404,7 @@ def _mock_cached_images(mock_cache_db):
     assert CachedImage.select().count() == 1
 
 
-@pytest.fixture()
+@pytest.fixture
 def _mock_cached_transcriptions(mock_cache_db):
     """Insert few transcriptions in local cache, on a shared element"""
     CachedElement.create(
@@ -511,7 +493,7 @@ def _mock_cached_transcriptions(mock_cache_db):
     )
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_databases(tmp_path):
     """
     Initialize several temporary databases
@@ -594,10 +576,10 @@ def mock_databases(tmp_path):
     return out
 
 
-@pytest.fixture()
+@pytest.fixture
 def default_dataset():
     return Dataset(
-        **{
+        {
             "id": "dataset_id",
             "name": "My dataset",
             "description": "A super dataset built by me",
@@ -608,26 +590,32 @@ def default_dataset():
             "task_id": "11111111-1111-1111-1111-111111111111",
             "created": "2000-01-01T00:00:00Z",
             "updated": "2000-01-01T00:00:00Z",
-        },
-        selected_sets=["set_1", "set_2", "set_3"],
+        }
     )
 
 
-@pytest.fixture()
+@pytest.fixture
+def default_train_set(default_dataset):
+    return Set(name="train", dataset=default_dataset)
+
+
+@pytest.fixture
 def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
     monkeypatch.setenv("PONOS_TASK", "my_task")
     mocker.patch.object(sys, "argv", ["worker"])
 
     dataset_worker = DatasetWorker()
     dataset_worker.configure()
-    dataset_worker.process_information = {"id": PROCESS_ID}
+
+    # Update process mode
+    dataset_worker.process_information["mode"] = ProcessMode.Dataset
 
     assert not dataset_worker.is_read_only
 
     return dataset_worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def mock_dev_dataset_worker(mocker):
     mocker.patch.object(
         sys,
@@ -635,9 +623,10 @@ def mock_dev_dataset_worker(mocker):
         [
             "worker",
             "--dev",
-            "--dataset",
-            "11111111-1111-1111-1111-111111111111",
-            "22222222-2222-2222-2222-222222222222",
+            "--set",
+            "11111111-1111-1111-1111-111111111111:train",
+            "11111111-1111-1111-1111-111111111111:val",
+            "22222222-2222-2222-2222-222222222222:my_set",
         ],
     )
 
@@ -651,7 +640,7 @@ def mock_dev_dataset_worker(mocker):
     return dataset_worker
 
 
-@pytest.fixture()
+@pytest.fixture
 def default_artifact():
     return Artifact(
         **{

tests/test_base_worker.py

@@ -11,7 +11,7 @@ from arkindex.mock import MockApiClient
 from arkindex_worker import logger
 from arkindex_worker.worker import BaseWorker, ElementsWorker
 from arkindex_worker.worker.base import ExtrasDirNotFoundError
-from tests.conftest import FIXTURES_DIR
+from tests import CORPUS_ID, FIXTURES_DIR
 
 
 def test_init_default_local_share():
@@ -178,7 +178,7 @@ def test_configure_worker_run(mocker, responses, caplog):
         "model_version": None,
         "process": {
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
         },
         "summary": "Worker Fake worker @ 123412",
     }
@@ -206,6 +206,7 @@ def test_configure_worker_run(mocker, responses, caplog):
             "Loaded Worker Fake worker @ 123412 from API",
         ),
         ("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
+        ("arkindex_worker", logging.INFO, "User configuration retrieved"),
     ]
 
     assert worker.user_configuration == {"a": "b"}
@@ -270,7 +271,7 @@ def test_configure_user_configuration_defaults(mocker, responses):
         "model_version": None,
         "process": {
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
         },
         "summary": "Worker Fake worker @ 123412",
     }
@@ -284,12 +285,21 @@ def test_configure_user_configuration_defaults(mocker, responses):
 
     worker.configure()
 
-    assert worker.config == {"param_1": "/some/path/file.pth", "param_2": 12}
     assert worker.user_configuration == {
         "integer_parameter": 0,
         "param_3": "Animula vagula blandula",
         "param_5": True,
     }
+    # All configurations are merged
+    assert worker.config == {
+        # Default config
+        "param_1": "/some/path/file.pth",
+        "param_2": 12,
+        # User config
+        "integer_parameter": 0,
+        "param_3": "Animula vagula blandula",
+        "param_5": True,
+    }
 
 
 @pytest.mark.parametrize("debug", [True, False])
@@ -319,7 +329,7 @@ def test_configure_user_config_debug(mocker, responses, debug):
         },
         "process": {
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
         },
         "summary": "Worker Fake worker @ 123412",
     }
@@ -367,7 +377,7 @@ def test_configure_worker_run_missing_conf(mocker, responses):
         "configuration": {"id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "BBB"},
         "process": {
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
         },
         "summary": "Worker Fake worker @ 123412",
     }
@@ -409,7 +419,7 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
         "configuration": None,
         "process": {
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
         },
         "summary": "Worker Fake worker @ 123412",
     }
@@ -458,7 +468,7 @@ def test_configure_load_model_configuration(mocker, responses):
         },
         "process": {
             "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": "11111111-1111-1111-1111-111111111111",
+            "corpus": CORPUS_ID,
         },
         "summary": "Worker Fake worker @ 123412",
     }
@@ -658,7 +668,7 @@ def test_find_extras_directory_not_found(monkeypatch, extras_path, exists, error
 def test_find_parents_file_paths(responses, mock_base_worker_with_cache, tmp_path):
     responses.add(
         responses.GET,
-        "http://testserver/api/v1/task/my_task/from-agent/",
+        "http://testserver/api/v1/task/my_task/",
         status=200,
         json={"parents": ["first", "second", "third"]},
     )
@@ -669,15 +679,13 @@ def test_find_parents_file_paths(responses, mock_base_worker_with_cache, tmp_pat
     ):
         (tmp_path / parent_id).mkdir()
         file_path = tmp_path / parent_id / filename
-        with file_path.open("w", encoding="utf-8") as f:
-            f.write(content)
+        file_path.write_text(content)
 
     # Configure worker with a specific data directory
     mock_base_worker_with_cache.task_data_dir = tmp_path
     mock_base_worker_with_cache.args = mock_base_worker_with_cache.parser.parse_args()
 
     mock_base_worker_with_cache.configure()
-    mock_base_worker_with_cache.configure_cache()
 
     assert mock_base_worker_with_cache.find_parents_file_paths(filename) == [
         tmp_path / "first" / filename,
@@ -754,3 +762,195 @@ def test_corpus_id_set_read_only_mode(
     mock_elements_worker_read_only.configure()
 
     assert mock_elements_worker_read_only.corpus_id == corpus_id
+
+
+@pytest.mark.parametrize(
+    (
+        "wk_version_config",
+        "wk_version_user_config",
+        "frontend_user_config",
+        "model_config",
+        "expected_config",
+    ),
+    [
+        ({}, {}, {}, {}, {}),
+        # Keep parameters from worker version configuration
+        ({"parameter": 0}, {}, {}, {}, {"parameter": 0}),
+        # Keep parameters from worker version configuration + user_config defaults
+        (
+            {"parameter": 0},
+            {
+                "parameter2": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 0,
+                    "required": False,
+                }
+            },
+            {},
+            {},
+            {"parameter": 0, "parameter2": 0},
+        ),
+        # Keep parameters from worker version configuration + user_config no defaults
+        (
+            {"parameter": 0},
+            {
+                "parameter2": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "required": False,
+                }
+            },
+            {},
+            {},
+            {"parameter": 0},
+        ),
+        # Keep parameters from worker version configuration but user_config defaults overrides
+        (
+            {"parameter": 0},
+            {
+                "parameter": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 1,
+                    "required": False,
+                }
+            },
+            {},
+            {},
+            {"parameter": 1},
+        ),
+        # Keep parameters from worker version configuration + frontend config
+        (
+            {"parameter": 0},
+            {},
+            {"parameter2": 0},
+            {},
+            {"parameter": 0, "parameter2": 0},
+        ),
+        # Keep parameters from worker version configuration + frontend config overrides
+        ({"parameter": 0}, {}, {"parameter": 1}, {}, {"parameter": 1}),
+        # Keep parameters from worker version configuration + model config
+        (
+            {"parameter": 0},
+            {},
+            {},
+            {"parameter2": 0},
+            {"parameter": 0, "parameter2": 0},
+        ),
+        # Keep parameters from worker version configuration + model config overrides
+        ({"parameter": 0}, {}, {}, {"parameter": 1}, {"parameter": 1}),
+        # Keep parameters from worker version configuration + user_config default + model config overrides
+        (
+            {"parameter": 0},
+            {
+                "parameter": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 1,
+                    "required": False,
+                }
+            },
+            {},
+            {"parameter": 2},
+            {"parameter": 2},
+        ),
+        # Keep parameters from worker version configuration + model config + frontend config overrides
+        ({"parameter": 0}, {}, {"parameter": 2}, {"parameter": 1}, {"parameter": 2}),
+        # Keep parameters from worker version configuration + user_config default + model config + frontend config overrides all
+        (
+            {"parameter": 0},
+            {
+                "parameter": {
+                    "type": "int",
+                    "title": "Lambda",
+                    "default": 1,
+                    "required": False,
+                }
+            },
+            {"parameter": 3},
+            {"parameter": 2},
+            {"parameter": 3},
+        ),
+    ],
+)
+def test_worker_config_multiple_source(
+    monkeypatch,
+    responses,
+    wk_version_config,
+    wk_version_user_config,
+    frontend_user_config,
+    model_config,
+    expected_config,
+):
+    # Compute WorkerRun info
+    payload = {
+        "id": "56785678-5678-5678-5678-567856785678",
+        "parents": [],
+        "worker_version": {
+            "id": "12341234-1234-1234-1234-123412341234",
+            "configuration": {
+                "docker": {"image": "python:3"},
+                "configuration": wk_version_config,
+                "secrets": [],
+                "user_configuration": wk_version_user_config,
+            },
+            "revision": {
+                "hash": "deadbeef1234",
+                "name": "some git revision",
+            },
+            "docker_image": "python:3",
+            "docker_image_name": "python:3",
+            "state": "created",
+            "worker": {
+                "id": "deadbeef-1234-5678-1234-worker",
+                "name": "Fake worker",
+                "slug": "fake_worker",
+                "type": "classifier",
+            },
+        },
+        "configuration": {
+            "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
+            "name": "Configuration entered by user",
+            "configuration": frontend_user_config,
+        },
+        "model_version": {
+            "id": "12341234-1234-1234-1234-123412341234",
+            "name": "Model version 1337",
+            "configuration": model_config,
+            "model": {
+                "id": "hahahaha-haha-haha-haha-hahahahahaha",
+                "name": "My model",
+            },
+        },
+        "process": {
+            "name": None,
+            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+            "state": "running",
+            "mode": "workers",
+            "corpus": CORPUS_ID,
+            "use_cache": False,
+            "activity_state": "ready",
+            "model_id": None,
+            "train_folder_id": None,
+            "validation_folder_id": None,
+            "test_folder_id": None,
+        },
+        "summary": "Worker Fake worker @ 123412",
+    }
+
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
+        status=200,
+        body=json.dumps(payload),
+        content_type="application/json",
+    )
+
+    # Create and configure a worker
+    monkeypatch.setattr(sys, "argv", ["worker"])
+    worker = BaseWorker()
+    worker.configure()
+
+    # Check final config
+    assert worker.config == expected_config