arkindex-base-worker 0.5.1b4__tar.gz → 0.5.1b5__tar.gz

{arkindex_base_worker-0.5.1b4 → arkindex_base_worker-0.5.1b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.1b4
+Version: 0.5.1b5
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>

{arkindex_base_worker-0.5.1b4 → arkindex_base_worker-0.5.1b5}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.1b4
+Version: 0.5.1b5
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>

{arkindex_base_worker-0.5.1b4 → arkindex_base_worker-0.5.1b5}/arkindex_worker/worker/base.py

@@ -265,6 +265,11 @@ class BaseWorker:
             if not item["secret"]:
                 return (item["key"], item["value"])
 
+            # The secret may not be picked by the user
+            if item["value"] is None:
+                logger.info(f"Skipping optional secret {item['key']}")
+                return (item["key"], None)
+
             # Load secret, only available in Arkindex EE
             try:
                 secret = self.load_secret(Path(item["value"]))
@@ -276,6 +281,19 @@ class BaseWorker:
 
             return (item["key"], secret)
 
+        # Load model version configuration when available
+        # Workers will use model version ID and details to download the model
+        model_version = worker_run.get("model_version")
+        if model_version:
+            logger.info("Loaded model version configuration from WorkerRun")
+            self.model_configuration.update(model_version["configuration"])
+
+            # Set model_version ID as worker attribute
+            self.model_version_id = model_version["id"]
+
+            # Set model details as worker attribute
+            self.model_details = model_version["model"]
+
         # Load worker run information
         try:
             config = self.api_client.request(
@@ -295,6 +313,9 @@ class BaseWorker:
             }
             logger.info("Using modern configuration")
 
+            # Reset the model configuration to make sure workers rely on the single new source
+            self.model_configuration = {}
+
             return  # Stop here once we have modern configuration
 
         except ErrorResponse as e:
@@ -303,18 +324,6 @@ class BaseWorker:
             logger.info("Modern configuration is not available")
 
         # Use old-style configuration with local merge
-        # Load model version configuration when available
-        model_version = worker_run.get("model_version")
-        if model_version:
-            logger.info("Loaded model version configuration from WorkerRun")
-            self.model_configuration.update(model_version["configuration"])
-
-            # Set model_version ID as worker attribute
-            self.model_version_id = model_version["id"]
-
-            # Set model details as worker attribute
-            self.model_details = model_version["model"]
-
         # Retrieve initial configuration from API
         self.config = worker_version["configuration"].get("configuration", {})
         if "user_configuration" in worker_version["configuration"]:

{arkindex_base_worker-0.5.1b4 → arkindex_base_worker-0.5.1b5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "arkindex-base-worker"
-version = "0.5.1b4"
+version = "0.5.1b5"
 description = "Base Worker to easily build Arkindex ML workflows"
 license = { file = "LICENSE" }
 dependencies = [

{arkindex_base_worker-0.5.1b4 → arkindex_base_worker-0.5.1b5}/tests/conftest.py

@@ -103,12 +103,6 @@ def _mock_worker_run_api(responses):
     payload = {
         "id": "56785678-5678-5678-5678-567856785678",
         "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "configuration": {
@@ -180,12 +174,6 @@ def _mock_worker_run_no_revision_api(responses):
     payload = {
         "id": "56785678-5678-5678-5678-567856785678",
         "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "configuration": {
@@ -355,12 +343,6 @@ def mock_elements_worker_consume_wa(monkeypatch, responses, mock_elements_worker
         json={
             "id": "56785678-5678-5678-5678-567856785678",
             "parents": [],
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
             "worker_version": {
                 "id": "12341234-1234-1234-1234-123412341234",
                 "configuration": {

{arkindex_base_worker-0.5.1b4 → arkindex_base_worker-0.5.1b5}/tests/test_base_worker.py

@@ -13,6 +13,29 @@ from arkindex_worker.worker import BaseWorker, ElementsWorker
 from arkindex_worker.worker.base import ExtrasDirNotFoundError
 from tests import CORPUS_ID, FIXTURES_DIR
 
+SIMPLE_PAYLOAD = {
+    "id": "56785678-5678-5678-5678-567856785678",
+    "parents": [],
+    "worker_version": {
+        "id": "12341234-1234-1234-1234-123412341234",
+        "worker": {
+            "id": "deadbeef-1234-5678-1234-worker",
+            "name": "Fake worker",
+            "slug": "fake_worker",
+            "type": "classifier",
+        },
+        "revision": {"hash": "deadbeef1234"},
+        "configuration": {"configuration": {}},
+    },
+    "configuration": None,
+    "model_version": None,
+    "process": {
+        "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+        "corpus": CORPUS_ID,
+    },
+    "summary": "Worker Fake worker @ 123412",
+}
+
 
 def test_init_default_local_share():
     worker = BaseWorker()
@@ -149,38 +172,13 @@ def test_configure_worker_run(mocker, responses, caplog):
 
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
-    user_configuration = {
-        "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
-        "name": "BBB",
-        "configuration": {"a": "b"},
-    }
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": user_configuration,
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
+        **SIMPLE_PAYLOAD,
+        "configuration": {
+            "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+            "name": "BBB",
+            "configuration": {"a": "b"},
         },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(
@@ -262,8 +260,7 @@ def test_configure_user_configuration_defaults(mocker, responses):
     worker.args = worker.parser.parse_args()
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
+        **SIMPLE_PAYLOAD,
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "worker": {
@@ -293,12 +290,6 @@ def test_configure_user_configuration_defaults(mocker, responses):
                 "param_5": True,
             },
         },
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -340,30 +331,12 @@ def test_configure_user_config_debug(mocker, responses, debug):
     mocker.patch.object(sys, "argv", ["worker"])
     assert logger.level == logging.NOTSET
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {
             "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
             "name": "BBB",
             "configuration": {"debug": debug},
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -393,32 +366,8 @@ def test_configure_worker_run_missing_conf(mocker, responses):
     mocker.patch.object(sys, "argv", ["worker"])
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {"id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "BBB"},
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -446,28 +395,7 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
 
-    payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {},
-        },
-        "model_version": None,
-        "configuration": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
-    }
+    payload = SIMPLE_PAYLOAD
     responses.add(
         responses.GET,
         "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
@@ -491,20 +419,7 @@ def test_configure_load_model_configuration(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": None,
+        **SIMPLE_PAYLOAD,
         "model_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "model": {
@@ -517,11 +432,6 @@ def test_configure_load_model_configuration(mocker, responses):
                 "param3": None,
             },
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(