arkindex-base-worker 0.4.0__py3-none-any.whl → 0.4.0a1__py3-none-any.whl

tests/conftest.py

@@ -23,15 +23,10 @@ from arkindex_worker.cache import (
     init_cache_db,
 )
 from arkindex_worker.models import Artifact, Dataset, Set
-from arkindex_worker.worker import (
-    BaseWorker,
-    DatasetWorker,
-    ElementsWorker,
-    ProcessMode,
-)
+from arkindex_worker.worker import BaseWorker, DatasetWorker, ElementsWorker
 from arkindex_worker.worker.dataset import DatasetState
 from arkindex_worker.worker.transcription import TextOrientation
-from tests import CORPUS_ID, SAMPLES_DIR
+from tests import CORPUS_ID, FIXTURES_DIR, PROCESS_ID, SAMPLES_DIR
 
 __yaml_cache = {}
 
@@ -46,7 +41,7 @@ def _disable_sleep(monkeypatch):
     monkeypatch.setattr(time, "sleep", lambda x: None)
 
 
-@pytest.fixture
+@pytest.fixture()
 def _cache_yaml(monkeypatch):
     """
     Cache all calls to yaml.safe_load in order to speedup
@@ -111,7 +106,7 @@ def _give_env_variable(monkeypatch):
     monkeypatch.setenv("ARKINDEX_WORKER_RUN_ID", "56785678-5678-5678-5678-567856785678")
 
 
-@pytest.fixture
+@pytest.fixture()
 def _mock_worker_run_api(responses):
     """Provide a mock API response to get worker run information"""
     payload = {
@@ -180,7 +175,7 @@ def _mock_worker_run_api(responses):
     )
 
 
-@pytest.fixture
+@pytest.fixture()
 def _mock_worker_run_no_revision_api(responses):
     """Provide a mock API response to get worker run not linked to a revision information"""
     payload = {
@@ -247,7 +242,7 @@ def _mock_worker_run_no_revision_api(responses):
     )
 
 
-@pytest.fixture
+@pytest.fixture()
 def _mock_activity_calls(responses):
     """
     Mock responses when updating the activity state for multiple element of the same version
@@ -259,7 +254,7 @@ def _mock_activity_calls(responses):
     )
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_elements_worker(monkeypatch, _mock_worker_run_api):
     """Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker"])
@@ -268,7 +263,7 @@ def mock_elements_worker(monkeypatch, _mock_worker_run_api):
     return worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_elements_worker_read_only(monkeypatch):
     """Build and configure an ElementsWorker with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker", "--dev"])
@@ -277,12 +272,14 @@ def mock_elements_worker_read_only(monkeypatch):
     return worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker):
     """
     Mock a worker instance to list and retrieve a single element
     """
-    monkeypatch.setattr(mock_elements_worker, "get_elements", lambda: ["1234-deadbeef"])
+    monkeypatch.setattr(
+        mock_elements_worker, "list_elements", lambda: ["1234-deadbeef"]
+    )
     responses.add(
         responses.GET,
         "http://testserver/api/v1/element/1234-deadbeef/",
@@ -296,7 +293,7 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
     return mock_elements_worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_cache_db(tmp_path):
     cache_path = tmp_path / "db.sqlite"
 
@@ -307,7 +304,7 @@ def mock_cache_db(tmp_path):
     return cache_path
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_base_worker_with_cache(monkeypatch, _mock_worker_run_api):
     """Build a BaseWorker using SQLite cache, also mocking a PONOS_TASK"""
     monkeypatch.setattr(sys, "argv", ["worker"])
@@ -318,7 +315,7 @@ def mock_base_worker_with_cache(monkeypatch, _mock_worker_run_api):
     return worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run_api):
     """Build and configure an ElementsWorker using SQLite cache with fixed CLI parameters to avoid issues with pytest"""
     monkeypatch.setattr(sys, "argv", ["worker", "-d", str(mock_cache_db)])
@@ -329,17 +326,34 @@ def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run
     return worker
 
 
-@pytest.fixture
+@pytest.fixture()
+def fake_page_element():
+    return json.loads((FIXTURES_DIR / "page_element.json").read_text())
+
+
+@pytest.fixture()
+def fake_ufcn_worker_version():
+    return json.loads(
+        (FIXTURES_DIR / "ufcn_line_historical_worker_version.json").read_text()
+    )
+
+
+@pytest.fixture()
+def fake_transcriptions_small():
+    return json.loads((FIXTURES_DIR / "line_transcriptions_small.json").read_text())
+
+
+@pytest.fixture()
 def model_file_dir():
     return SAMPLES_DIR / "model_files"
 
 
-@pytest.fixture
+@pytest.fixture()
 def model_file_dir_with_subfolder():
     return SAMPLES_DIR / "root_folder"
 
 
-@pytest.fixture
+@pytest.fixture()
 def fake_dummy_worker():
     api_client = MockApiClient()
     worker = ElementsWorker()
@@ -347,7 +361,7 @@ def fake_dummy_worker():
     return worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def _mock_cached_elements(mock_cache_db):
     """Insert few elements in local cache"""
     CachedElement.create(
@@ -392,7 +406,7 @@ def _mock_cached_elements(mock_cache_db):
     assert CachedElement.select().count() == 5
 
 
-@pytest.fixture
+@pytest.fixture()
 def _mock_cached_images(mock_cache_db):
     """Insert few elements in local cache"""
     CachedImage.create(
@@ -404,7 +418,7 @@ def _mock_cached_images(mock_cache_db):
     assert CachedImage.select().count() == 1
 
 
-@pytest.fixture
+@pytest.fixture()
 def _mock_cached_transcriptions(mock_cache_db):
     """Insert few transcriptions in local cache, on a shared element"""
     CachedElement.create(
@@ -493,7 +507,7 @@ def _mock_cached_transcriptions(mock_cache_db):
     )
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_databases(tmp_path):
     """
     Initialize several temporary databases
@@ -576,7 +590,7 @@ def mock_databases(tmp_path):
     return out
 
 
-@pytest.fixture
+@pytest.fixture()
 def default_dataset():
     return Dataset(
         {
@@ -594,28 +608,26 @@ def default_dataset():
     )
 
 
-@pytest.fixture
+@pytest.fixture()
 def default_train_set(default_dataset):
     return Set(name="train", dataset=default_dataset)
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_dataset_worker(monkeypatch, mocker, _mock_worker_run_api):
     monkeypatch.setenv("PONOS_TASK", "my_task")
     mocker.patch.object(sys, "argv", ["worker"])
 
     dataset_worker = DatasetWorker()
     dataset_worker.configure()
-
-    # Update process mode
-    dataset_worker.process_information["mode"] = ProcessMode.Dataset
+    dataset_worker.process_information = {"id": PROCESS_ID}
 
     assert not dataset_worker.is_read_only
 
     return dataset_worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def mock_dev_dataset_worker(mocker):
     mocker.patch.object(
         sys,
@@ -640,7 +652,7 @@ def mock_dev_dataset_worker(mocker):
     return dataset_worker
 
 
-@pytest.fixture
+@pytest.fixture()
 def default_artifact():
     return Artifact(
         **{

tests/test_base_worker.py

@@ -206,7 +206,6 @@ def test_configure_worker_run(mocker, responses, caplog):
             "Loaded Worker Fake worker @ 123412 from API",
         ),
         ("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
-        ("arkindex_worker", logging.INFO, "User configuration retrieved"),
     ]
 
     assert worker.user_configuration == {"a": "b"}
@@ -285,21 +284,12 @@ def test_configure_user_configuration_defaults(mocker, responses):
 
     worker.configure()
 
+    assert worker.config == {"param_1": "/some/path/file.pth", "param_2": 12}
     assert worker.user_configuration == {
         "integer_parameter": 0,
         "param_3": "Animula vagula blandula",
         "param_5": True,
     }
-    # All configurations are merged
-    assert worker.config == {
-        # Default config
-        "param_1": "/some/path/file.pth",
-        "param_2": 12,
-        # User config
-        "integer_parameter": 0,
-        "param_3": "Animula vagula blandula",
-        "param_5": True,
-    }
 
 
 @pytest.mark.parametrize("debug", [True, False])
@@ -668,7 +658,7 @@ def test_find_extras_directory_not_found(monkeypatch, extras_path, exists, error
 def test_find_parents_file_paths(responses, mock_base_worker_with_cache, tmp_path):
     responses.add(
         responses.GET,
-        "http://testserver/api/v1/task/my_task/",
+        "http://testserver/api/v1/task/my_task/from-agent/",
         status=200,
         json={"parents": ["first", "second", "third"]},
     )
@@ -686,6 +676,7 @@ def test_find_parents_file_paths(responses, mock_base_worker_with_cache, tmp_pat
     mock_base_worker_with_cache.args = mock_base_worker_with_cache.parser.parse_args()
 
     mock_base_worker_with_cache.configure()
+    mock_base_worker_with_cache.configure_cache()
 
     assert mock_base_worker_with_cache.find_parents_file_paths(filename) == [
         tmp_path / "first" / filename,
@@ -762,195 +753,3 @@ def test_corpus_id_set_read_only_mode(
     mock_elements_worker_read_only.configure()
 
     assert mock_elements_worker_read_only.corpus_id == corpus_id
-
-
-@pytest.mark.parametrize(
-    (
-        "wk_version_config",
-        "wk_version_user_config",
-        "frontend_user_config",
-        "model_config",
-        "expected_config",
-    ),
-    [
-        ({}, {}, {}, {}, {}),
-        # Keep parameters from worker version configuration
-        ({"parameter": 0}, {}, {}, {}, {"parameter": 0}),
-        # Keep parameters from worker version configuration + user_config defaults
-        (
-            {"parameter": 0},
-            {
-                "parameter2": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 0,
-                    "required": False,
-                }
-            },
-            {},
-            {},
-            {"parameter": 0, "parameter2": 0},
-        ),
-        # Keep parameters from worker version configuration + user_config no defaults
-        (
-            {"parameter": 0},
-            {
-                "parameter2": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "required": False,
-                }
-            },
-            {},
-            {},
-            {"parameter": 0},
-        ),
-        # Keep parameters from worker version configuration but user_config defaults overrides
-        (
-            {"parameter": 0},
-            {
-                "parameter": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 1,
-                    "required": False,
-                }
-            },
-            {},
-            {},
-            {"parameter": 1},
-        ),
-        # Keep parameters from worker version configuration + frontend config
-        (
-            {"parameter": 0},
-            {},
-            {"parameter2": 0},
-            {},
-            {"parameter": 0, "parameter2": 0},
-        ),
-        # Keep parameters from worker version configuration + frontend config overrides
-        ({"parameter": 0}, {}, {"parameter": 1}, {}, {"parameter": 1}),
-        # Keep parameters from worker version configuration + model config
-        (
-            {"parameter": 0},
-            {},
-            {},
-            {"parameter2": 0},
-            {"parameter": 0, "parameter2": 0},
-        ),
-        # Keep parameters from worker version configuration + model config overrides
-        ({"parameter": 0}, {}, {}, {"parameter": 1}, {"parameter": 1}),
-        # Keep parameters from worker version configuration + user_config default + model config overrides
-        (
-            {"parameter": 0},
-            {
-                "parameter": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 1,
-                    "required": False,
-                }
-            },
-            {},
-            {"parameter": 2},
-            {"parameter": 2},
-        ),
-        # Keep parameters from worker version configuration + model config + frontend config overrides
-        ({"parameter": 0}, {}, {"parameter": 2}, {"parameter": 1}, {"parameter": 2}),
-        # Keep parameters from worker version configuration + user_config default + model config + frontend config overrides all
-        (
-            {"parameter": 0},
-            {
-                "parameter": {
-                    "type": "int",
-                    "title": "Lambda",
-                    "default": 1,
-                    "required": False,
-                }
-            },
-            {"parameter": 3},
-            {"parameter": 2},
-            {"parameter": 3},
-        ),
-    ],
-)
-def test_worker_config_multiple_source(
-    monkeypatch,
-    responses,
-    wk_version_config,
-    wk_version_user_config,
-    frontend_user_config,
-    model_config,
-    expected_config,
-):
-    # Compute WorkerRun info
-    payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "configuration": {
-                "docker": {"image": "python:3"},
-                "configuration": wk_version_config,
-                "secrets": [],
-                "user_configuration": wk_version_user_config,
-            },
-            "revision": {
-                "hash": "deadbeef1234",
-                "name": "some git revision",
-            },
-            "docker_image": "python:3",
-            "docker_image_name": "python:3",
-            "state": "created",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-        },
-        "configuration": {
-            "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
-            "name": "Configuration entered by user",
-            "configuration": frontend_user_config,
-        },
-        "model_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "name": "Model version 1337",
-            "configuration": model_config,
-            "model": {
-                "id": "hahahaha-haha-haha-haha-hahahahahaha",
-                "name": "My model",
-            },
-        },
-        "process": {
-            "name": None,
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "state": "running",
-            "mode": "workers",
-            "corpus": CORPUS_ID,
-            "use_cache": False,
-            "activity_state": "ready",
-            "model_id": None,
-            "train_folder_id": None,
-            "validation_folder_id": None,
-            "test_folder_id": None,
-        },
-        "summary": "Worker Fake worker @ 123412",
-    }
-
-    responses.add(
-        responses.GET,
-        "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
-        status=200,
-        body=json.dumps(payload),
-        content_type="application/json",
-    )
-
-    # Create and configure a worker
-    monkeypatch.setattr(sys, "argv", ["worker"])
-    worker = BaseWorker()
-    worker.configure()
-
-    # Check final config
-    assert worker.config == expected_config

tests/test_dataset_worker.py

@@ -3,21 +3,18 @@ import uuid
 from argparse import ArgumentTypeError
 
 import pytest
+from apistar.exceptions import ErrorResponse
 
-from arkindex.exceptions import ErrorResponse
 from arkindex_worker.models import Dataset, Set
-from arkindex_worker.worker.dataset import (
-    DatasetState,
-    MissingDatasetArchive,
-    check_dataset_set,
-)
+from arkindex_worker.worker import MissingDatasetArchive, check_dataset_set
+from arkindex_worker.worker.dataset import DatasetState
 from tests import FIXTURES_DIR, PROCESS_ID
 from tests.test_elements_worker import BASE_API_CALLS
 
 RANDOM_UUID = uuid.uuid4()
 
 
-@pytest.fixture
+@pytest.fixture()
 def tmp_archive(tmp_path):
     archive = tmp_path / "test_archive.tar.zst"
     archive.touch()