arkindex-base-worker 0.5.1b4__py3-none-any.whl → 0.5.1b5__py3-none-any.whl

{arkindex_base_worker-0.5.1b4.dist-info → arkindex_base_worker-0.5.1b5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.1b4
+Version: 0.5.1b5
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>

{arkindex_base_worker-0.5.1b4.dist-info → arkindex_base_worker-0.5.1b5.dist-info}/RECORD

@@ -1,11 +1,11 @@
-arkindex_base_worker-0.5.1b4.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.5.1b5.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
 arkindex_worker/__init__.py,sha256=Sdt5KXn8EgURb2MurYVrUWaHbH3iFA1XLRo0Lc5AJ44,250
 arkindex_worker/cache.py,sha256=x1d1oVF297ItLoZnPkZQoEefa39ZigrwRoHC_6az94k,10731
 arkindex_worker/image.py,sha256=GvIpW7LNSalVw3Obt9nySDWnW7-NbC0__SWREEQqVCk,20696
 arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
 arkindex_worker/utils.py,sha256=MbbJT8oh8DMHHR-vidFeXdUH0TSXGWm7ZDGWzrRXoEY,9933
 arkindex_worker/worker/__init__.py,sha256=SzD0s1_m6gMV02EUF-NeciqZdVPA4dpXI84tSj-g494,17869
-arkindex_worker/worker/base.py,sha256=fbRJ5vDON3DfSQfwxFqto85HY8Dw2_YgOmnm5cxbQ2g,21725
+arkindex_worker/worker/base.py,sha256=-ASOgLzemHOAz7EnPZ9hT6wON_9GJ7uadxx7oZLgUD4,22149
 arkindex_worker/worker/classification.py,sha256=qvykymkgd4nGywHCxL8obo4egstoGsmWNS4Ztc1qNWQ,11024
 arkindex_worker/worker/corpus.py,sha256=MeIMod7jkWyX0frtD0a37rhumnMV3p9ZOC1xwAoXrAA,2291
 arkindex_worker/worker/dataset.py,sha256=tVaPx43vaH-KTtx4w5V06e26ha8XPfiJTRzBXlu928Y,5273
@@ -21,8 +21,8 @@ examples/standalone/python/worker.py,sha256=Zr4s4pHvgexEjlkixLFYZp1UuwMLeoTxjyNG
 examples/tooled/python/worker.py,sha256=kIYlHLsO5UpwX4XtERRq4tf2qTsvqKK30C-w8t0yyhA,1821
 hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
 tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
-tests/conftest.py,sha256=OaajGB0FS0Jh3rFhL8a3HdaN84XzIxNbRORvzI5W2ns,24928
-tests/test_base_worker.py,sha256=3YjhjxSWVjEWFYS8m8pYYoaVAhHFkJLNTs0QPQIkBDM,32651
+tests/conftest.py,sha256=Tp7YFK17NATwF2yAcBwi0QFNyKSXtLS0VhZ-zZngsQI,24343
+tests/test_base_worker.py,sha256=lwS4X3atS2ktEKd1XdogmN3mbzq-tO206-k_0EDITlw,29302
 tests/test_cache.py,sha256=nnEFfAAqtYHk2ymOwN0spXJd8nrRiwp3voj0tOmIbQ8,10407
 tests/test_dataset_worker.py,sha256=iDJM2C4PfQNH0r4_QqSWoPt8BcM0geUUdODtWY0Z9PA,22412
 tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
@@ -55,7 +55,7 @@ worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.5.1b4.dist-info/METADATA,sha256=hLR30IODhVUqKVq_4qXv_AcNswb0afCA-2Nce13UPno,3137
-arkindex_base_worker-0.5.1b4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-arkindex_base_worker-0.5.1b4.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
-arkindex_base_worker-0.5.1b4.dist-info/RECORD,,
+arkindex_base_worker-0.5.1b5.dist-info/METADATA,sha256=pVX35tc38qCuTCXwhXS2veoEJpTMJ3KWh3a1CUZQ3yg,3137
+arkindex_base_worker-0.5.1b5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+arkindex_base_worker-0.5.1b5.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
+arkindex_base_worker-0.5.1b5.dist-info/RECORD,,

arkindex_worker/worker/base.py

@@ -265,6 +265,11 @@ class BaseWorker:
             if not item["secret"]:
                 return (item["key"], item["value"])
 
+            # The secret may not be picked by the user
+            if item["value"] is None:
+                logger.info(f"Skipping optional secret {item['key']}")
+                return (item["key"], None)
+
             # Load secret, only available in Arkindex EE
             try:
                 secret = self.load_secret(Path(item["value"]))
@@ -276,6 +281,19 @@ class BaseWorker:
 
             return (item["key"], secret)
 
+        # Load model version configuration when available
+        # Workers will use model version ID and details to download the model
+        model_version = worker_run.get("model_version")
+        if model_version:
+            logger.info("Loaded model version configuration from WorkerRun")
+            self.model_configuration.update(model_version["configuration"])
+
+            # Set model_version ID as worker attribute
+            self.model_version_id = model_version["id"]
+
+            # Set model details as worker attribute
+            self.model_details = model_version["model"]
+
         # Load worker run information
         try:
             config = self.api_client.request(
@@ -295,6 +313,9 @@ class BaseWorker:
             }
             logger.info("Using modern configuration")
 
+            # Reset the model configuration to make sure workers rely on the single new source
+            self.model_configuration = {}
+
             return  # Stop here once we have modern configuration
 
         except ErrorResponse as e:
@@ -303,18 +324,6 @@ class BaseWorker:
             logger.info("Modern configuration is not available")
 
         # Use old-style configuration with local merge
-        # Load model version configuration when available
-        model_version = worker_run.get("model_version")
-        if model_version:
-            logger.info("Loaded model version configuration from WorkerRun")
-            self.model_configuration.update(model_version["configuration"])
-
-            # Set model_version ID as worker attribute
-            self.model_version_id = model_version["id"]
-
-            # Set model details as worker attribute
-            self.model_details = model_version["model"]
-
         # Retrieve initial configuration from API
         self.config = worker_version["configuration"].get("configuration", {})
         if "user_configuration" in worker_version["configuration"]:

tests/conftest.py

@@ -103,12 +103,6 @@ def _mock_worker_run_api(responses):
     payload = {
         "id": "56785678-5678-5678-5678-567856785678",
         "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "configuration": {
@@ -180,12 +174,6 @@ def _mock_worker_run_no_revision_api(responses):
     payload = {
         "id": "56785678-5678-5678-5678-567856785678",
         "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "configuration": {
@@ -355,12 +343,6 @@ def mock_elements_worker_consume_wa(monkeypatch, responses, mock_elements_worker
         json={
             "id": "56785678-5678-5678-5678-567856785678",
             "parents": [],
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
             "worker_version": {
                 "id": "12341234-1234-1234-1234-123412341234",
                 "configuration": {

tests/test_base_worker.py

@@ -13,6 +13,29 @@ from arkindex_worker.worker import BaseWorker, ElementsWorker
 from arkindex_worker.worker.base import ExtrasDirNotFoundError
 from tests import CORPUS_ID, FIXTURES_DIR
 
+SIMPLE_PAYLOAD = {
+    "id": "56785678-5678-5678-5678-567856785678",
+    "parents": [],
+    "worker_version": {
+        "id": "12341234-1234-1234-1234-123412341234",
+        "worker": {
+            "id": "deadbeef-1234-5678-1234-worker",
+            "name": "Fake worker",
+            "slug": "fake_worker",
+            "type": "classifier",
+        },
+        "revision": {"hash": "deadbeef1234"},
+        "configuration": {"configuration": {}},
+    },
+    "configuration": None,
+    "model_version": None,
+    "process": {
+        "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+        "corpus": CORPUS_ID,
+    },
+    "summary": "Worker Fake worker @ 123412",
+}
+
 
 def test_init_default_local_share():
     worker = BaseWorker()
@@ -149,38 +172,13 @@ def test_configure_worker_run(mocker, responses, caplog):
 
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
-    user_configuration = {
-        "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
-        "name": "BBB",
-        "configuration": {"a": "b"},
-    }
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": user_configuration,
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
+        **SIMPLE_PAYLOAD,
+        "configuration": {
+            "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+            "name": "BBB",
+            "configuration": {"a": "b"},
         },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(
@@ -262,8 +260,7 @@ def test_configure_user_configuration_defaults(mocker, responses):
     worker.args = worker.parser.parse_args()
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
+        **SIMPLE_PAYLOAD,
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "worker": {
@@ -293,12 +290,6 @@ def test_configure_user_configuration_defaults(mocker, responses):
                 "param_5": True,
             },
         },
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -340,30 +331,12 @@ def test_configure_user_config_debug(mocker, responses, debug):
     mocker.patch.object(sys, "argv", ["worker"])
     assert logger.level == logging.NOTSET
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {
             "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
             "name": "BBB",
             "configuration": {"debug": debug},
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -393,32 +366,8 @@ def test_configure_worker_run_missing_conf(mocker, responses):
     mocker.patch.object(sys, "argv", ["worker"])
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {"id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "BBB"},
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -446,28 +395,7 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
 
-    payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {},
-        },
-        "model_version": None,
-        "configuration": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
-    }
+    payload = SIMPLE_PAYLOAD
     responses.add(
         responses.GET,
         "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
@@ -491,20 +419,7 @@ def test_configure_load_model_configuration(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": None,
+        **SIMPLE_PAYLOAD,
         "model_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "model": {
@@ -517,11 +432,6 @@ def test_configure_load_model_configuration(mocker, responses):
                 "param3": None,
             },
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(