arkindex-base-worker 0.5.1b1__py3-none-any.whl → 0.5.1b2__py3-none-any.whl

{arkindex_base_worker-0.5.1b1.dist-info → arkindex_base_worker-0.5.1b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.1b1
+Version: 0.5.1b2
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>

{arkindex_base_worker-0.5.1b1.dist-info → arkindex_base_worker-0.5.1b2.dist-info}/RECORD

@@ -1,11 +1,11 @@
-arkindex_base_worker-0.5.1b1.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.5.1b2.dist-info/licenses/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
 arkindex_worker/__init__.py,sha256=Sdt5KXn8EgURb2MurYVrUWaHbH3iFA1XLRo0Lc5AJ44,250
 arkindex_worker/cache.py,sha256=x1d1oVF297ItLoZnPkZQoEefa39ZigrwRoHC_6az94k,10731
 arkindex_worker/image.py,sha256=GvIpW7LNSalVw3Obt9nySDWnW7-NbC0__SWREEQqVCk,20696
 arkindex_worker/models.py,sha256=bPQzGZNs5a6z6DEcygsa8T33VOqPlMUbwKzHqlKzwbw,9923
 arkindex_worker/utils.py,sha256=MbbJT8oh8DMHHR-vidFeXdUH0TSXGWm7ZDGWzrRXoEY,9933
 arkindex_worker/worker/__init__.py,sha256=Ho4rzGDtpFwd13cl5tL45yAVZJuyGCOIOroLPGZsvmk,18399
-arkindex_worker/worker/base.py,sha256=03WLu7R8vca54LI00g-S0EqIbFiNaIsgZjN6zmfHisw,20126
+arkindex_worker/worker/base.py,sha256=fbRJ5vDON3DfSQfwxFqto85HY8Dw2_YgOmnm5cxbQ2g,21725
 arkindex_worker/worker/classification.py,sha256=qvykymkgd4nGywHCxL8obo4egstoGsmWNS4Ztc1qNWQ,11024
 arkindex_worker/worker/corpus.py,sha256=MeIMod7jkWyX0frtD0a37rhumnMV3p9ZOC1xwAoXrAA,2291
 arkindex_worker/worker/dataset.py,sha256=tVaPx43vaH-KTtx4w5V06e26ha8XPfiJTRzBXlu928Y,5273
@@ -21,15 +21,16 @@ examples/standalone/python/worker.py,sha256=Zr4s4pHvgexEjlkixLFYZp1UuwMLeoTxjyNG
 examples/tooled/python/worker.py,sha256=kIYlHLsO5UpwX4XtERRq4tf2qTsvqKK30C-w8t0yyhA,1821
 hooks/pre_gen_project.py,sha256=xQJERv3vv9VzIqcBHI281eeWLWREXUF4mMw7PvJHHXM,269
 tests/__init__.py,sha256=DG--S6IpGl399rzSAjDdHL76CkOIeZIjajCcyUSDhOQ,241
-tests/conftest.py,sha256=NwYfoWO5ZTRfK5REYLGExt64CS1OPTcTpJ4LfbvuouE,22048
-tests/test_base_worker.py,sha256=dA00oxauTSCwnFX3ZFBl-RI71HN6GmK48FBBW_oYN-k,30627
+tests/conftest.py,sha256=jRFykK7cQm2AsZduVp78oEvKtHyzm3wr95IVkF4XpyY,23825
+tests/test_base_worker.py,sha256=3YjhjxSWVjEWFYS8m8pYYoaVAhHFkJLNTs0QPQIkBDM,32651
 tests/test_cache.py,sha256=nnEFfAAqtYHk2ymOwN0spXJd8nrRiwp3voj0tOmIbQ8,10407
-tests/test_dataset_worker.py,sha256=z8ydliUlwW2j-irgLAotJMacgJXkVvF5TgsWLyCn1Jo,22087
+tests/test_dataset_worker.py,sha256=iDJM2C4PfQNH0r4_QqSWoPt8BcM0geUUdODtWY0Z9PA,22412
 tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
 tests/test_image.py,sha256=NEIp5evr6QoTWgJ-_fze19IEFm_hG6YEcuW1kxnxS_I,28013
 tests/test_merge.py,sha256=REpZ13jkq_qm_4L5URQgFy5lxvPZtXxQEiWfYLMdmF0,7956
+tests/test_modern_config.py,sha256=Bm-a4LYQXgLZWQX7AmVyfJW0LNoLy1wj2d2GjzDkcBk,2683
 tests/test_utils.py,sha256=nYL1s2ViZoLoMiNpLGDaWwxf8dJ1D8aT522AO-PVaEQ,3607
-tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
+tests/test_elements_worker/__init__.py,sha256=2t3NciCIOun_N-Wv63FWGsTm5W9N3mbwAWVuFORlMg8,308
 tests/test_elements_worker/test_classification.py,sha256=nya7veSPR_O9G41Enodp2-o6AifMBcaSTWJP2vXSSJ4,30133
 tests/test_elements_worker/test_cli.py,sha256=a23i1pUDbXi23MUtbWwGEcLLrmc_YlrbDgOG3h66wLM,2620
 tests/test_elements_worker/test_corpus.py,sha256=kscJyM8k1njYJJFGuvliVzn89lWh41mEyDCCawnp3W8,5483
@@ -54,7 +55,7 @@ worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.5.1b1.dist-info/METADATA,sha256=LZx_PDuHE_v64WNubUcSPXN2FCWBwnHxEIxogiprVtU,3137
-arkindex_base_worker-0.5.1b1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-arkindex_base_worker-0.5.1b1.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
-arkindex_base_worker-0.5.1b1.dist-info/RECORD,,
+arkindex_base_worker-0.5.1b2.dist-info/METADATA,sha256=oAv0lAu9iGzHWuW8YL_i7uf1dqppslQ7iRznUJYLiGw,3137
+arkindex_base_worker-0.5.1b2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+arkindex_base_worker-0.5.1b2.dist-info/top_level.txt,sha256=-vNjP2VfROx0j83mdi9aIqRZ88eoJjxeWz-R_gPgyXU,49
+arkindex_base_worker-0.5.1b2.dist-info/RECORD,,

arkindex_worker/worker/base.py

@@ -9,12 +9,13 @@ import os
 import shutil
 from pathlib import Path
 from tempfile import mkdtemp
+from typing import Any
 
 import gnupg
 import yaml
 
 from arkindex import options_from_env
-from arkindex.exceptions import ErrorResponse
+from arkindex.exceptions import ClientError, ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     check_version,
@@ -260,6 +261,48 @@ class BaseWorker:
 
         logger.info(f"Loaded {worker_run['summary']} from API")
 
+        def _process_config_item(item: dict) -> tuple[str, Any]:
+            if not item["secret"]:
+                return (item["key"], item["value"])
+
+            # Load secret, only available in Arkindex EE
+            try:
+                secret = self.load_secret(Path(item["value"]))
+            except ClientError as e:
+                logger.error(
+                    f"Failed to retrieve the secret {item['value']}, probably an Arkindex Community Edition: {e}"
+                )
+                return (item["key"], None)
+
+            return (item["key"], secret)
+
+        # Load worker run information
+        try:
+            config = self.api_client.request(
+                "RetrieveWorkerRunConfiguration", id=self.worker_run_id
+            )
+
+            # Provide the same configuration through all previous attributes
+            self.config = self.user_configuration = dict(
+                map(_process_config_item, config["configuration"])
+            )
+
+            # Provide secret values through the previous attribute
+            self.secrets = {
+                item["key"]: self.config[item["key"]]
+                for item in config["configuration"]
+                if item["secret"]
+            }
+            logger.info("Using modern configuration")
+
+            return  # Stop here once we have modern configuration
+
+        except ErrorResponse as e:
+            if e.status_code != 400:
+                raise
+            logger.info("Modern configuration is not available")
+
+        # Use old-style configuration with local merge
         # Load model version configuration when available
         model_version = worker_run.get("model_version")
         if model_version:

tests/conftest.py

@@ -165,6 +165,13 @@ def _mock_worker_run_api(responses):
         content_type="application/json",
     )
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
 
 @pytest.fixture
 def _mock_worker_run_no_revision_api(responses):
@@ -233,6 +240,56 @@ def _mock_worker_run_no_revision_api(responses):
     )
 
 
+@pytest.fixture
+def mock_base_worker_modern_conf(mocker, responses):
+    """
+    Provide a base worker to test modern configuration with (not provided in the fixture)
+    """
+    worker = BaseWorker()
+    mocker.patch.object(sys, "argv")
+    worker.args = worker.parser.parse_args()
+
+    payload = {
+        "id": "56785678-5678-5678-5678-567856785678",
+        "parents": [],
+        "worker_version": {
+            "id": "12341234-1234-1234-1234-123412341234",
+            "worker": {
+                "id": "deadbeef-1234-5678-1234-worker",
+                "name": "Fake worker",
+                "slug": "fake_worker",
+                "type": "classifier",
+            },
+            "revision": {"hash": "deadbeef1234"},
+            "configuration": {
+                "configuration": {"extra_key1": "not showing up"},
+                "user_configuration": {"extra_key2": "not showing up"},
+            },
+        },
+        "configuration": {
+            "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
+            "name": "my-userconfig",
+            "configuration": {
+                "extra_key3": "not showing up",
+            },
+        },
+        "model_version": None,
+        "process": {
+            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+            "corpus": CORPUS_ID,
+        },
+        "summary": "Worker Fake worker @ 123412",
+    }
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
+        status=200,
+        json=payload,
+    )
+
+    return worker
+
+
 @pytest.fixture
 def _mock_activity_calls(responses):
     """

tests/test_base_worker.py

@@ -190,6 +190,14 @@ def test_configure_worker_run(mocker, responses, caplog):
         body=json.dumps(payload),
         content_type="application/json",
     )
+
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     worker.args = worker.parser.parse_args()
     assert worker.is_read_only is False
     assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
@@ -205,6 +213,11 @@ def test_configure_worker_run(mocker, responses, caplog):
             logging.INFO,
             "Loaded Worker Fake worker @ 123412 from API",
         ),
+        (
+            "arkindex_worker",
+            logging.INFO,
+            "Modern configuration is not available",
+        ),
         ("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
         ("arkindex_worker", logging.INFO, "User configuration retrieved"),
     ]
@@ -213,9 +226,16 @@ def test_configure_worker_run(mocker, responses, caplog):
 
 
 @pytest.mark.usefixtures("_mock_worker_run_no_revision_api")
-def test_configure_worker_run_no_revision(mocker, caplog):
+def test_configure_worker_run_no_revision(mocker, caplog, responses):
     worker = BaseWorker()
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     mocker.patch.object(sys, "argv", ["worker"])
     worker.args = worker.parser.parse_args()
     assert worker.is_read_only is False
@@ -227,7 +247,12 @@ def test_configure_worker_run_no_revision(mocker, caplog):
         worker.configure()
 
     assert caplog.record_tuples == [
-        ("arkindex_worker", logging.INFO, "Loaded Worker Fake worker @ 1 from API")
+        ("arkindex_worker", logging.INFO, "Loaded Worker Fake worker @ 1 from API"),
+        (
+            "arkindex_worker",
+            logging.INFO,
+            "Modern configuration is not available",
+        ),
     ]
 
 
@@ -283,6 +308,13 @@ def test_configure_user_configuration_defaults(mocker, responses):
         content_type="application/json",
     )
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     worker.configure()
 
     assert worker.user_configuration == {
@@ -340,6 +372,13 @@ def test_configure_user_config_debug(mocker, responses, debug):
         body=json.dumps(payload),
         content_type="application/json",
     )
+
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     worker.configure()
 
@@ -388,6 +427,12 @@ def test_configure_worker_run_missing_conf(mocker, responses):
         body=json.dumps(payload),
         content_type="application/json",
     )
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     worker.configure()
 
@@ -430,6 +475,12 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
         body=json.dumps(payload),
         content_type="application/json",
     )
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     worker.configure()
 
@@ -480,6 +531,12 @@ def test_configure_load_model_configuration(mocker, responses):
         body=json.dumps(payload),
         content_type="application/json",
     )
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     assert worker.is_read_only is False
     assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
@@ -947,6 +1004,13 @@ def test_worker_config_multiple_source(
         content_type="application/json",
     )
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     # Create and configure a worker
     monkeypatch.setattr(sys, "argv", ["worker"])
     worker = BaseWorker()

tests/test_dataset_worker.py

@@ -430,6 +430,7 @@ def test_run_no_sets(mocker, caplog, mock_dataset_worker):
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (logging.WARNING, "No sets to process, stopping."),
     ]
 
@@ -453,6 +454,7 @@ def test_run_initial_dataset_state_error(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.WARNING,
             "Failed running worker on Set (train) from Dataset (dataset_id): AssertionError('When processing a set, its dataset state should be Complete.')",
@@ -497,6 +499,7 @@ def test_run_download_dataset_artifact_api_error(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.INFO,
             "Retrieving data for Set (train) from Dataset (dataset_id) (1/1)",
@@ -550,6 +553,7 @@ def test_run_no_downloaded_dataset_artifact_error(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.INFO,
             "Retrieving data for Set (train) from Dataset (dataset_id) (1/1)",
@@ -626,6 +630,7 @@ def test_run(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.INFO,
             "Retrieving data for Set (train) from Dataset (dataset_id) (1/1)",

tests/test_elements_worker/__init__.py

@@ -4,4 +4,8 @@ BASE_API_CALLS = [
         "GET",
         "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
     ),
+    (
+        "GET",
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+    ),
 ]

tests/test_modern_config.py

@@ -0,0 +1,81 @@
+def test_simple_configuration(mock_base_worker_modern_conf, responses):
+    # Provide the full configuration directly from the worker run
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=200,
+        json={"configuration": [{"key": "some_key", "value": "test", "secret": False}]},
+    )
+
+    mock_base_worker_modern_conf.configure()
+
+    assert mock_base_worker_modern_conf.config == {"some_key": "test"}
+    assert (
+        mock_base_worker_modern_conf.user_configuration
+        == mock_base_worker_modern_conf.config
+    )
+    assert mock_base_worker_modern_conf.secrets == {}
+
+
+def test_empty(mock_base_worker_modern_conf, responses):
+    # Provide the full configuration directly from the worker run
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=200,
+        json={"configuration": []},
+    )
+
+    mock_base_worker_modern_conf.configure()
+
+    assert mock_base_worker_modern_conf.config == {}
+    assert (
+        mock_base_worker_modern_conf.user_configuration
+        == mock_base_worker_modern_conf.config
+    )
+    assert mock_base_worker_modern_conf.secrets == {}
+
+
+def test_with_secrets(mock_base_worker_modern_conf, responses):
+    # Provide the full configuration directly from the worker run
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=200,
+        json={
+            "configuration": [
+                {"key": "some_key", "value": "test", "secret": False},
+                {
+                    "key": "a_secret",
+                    "value": "471b9e64-29af-48dc-8bda-1a64a2da0c12",
+                    "secret": True,
+                },
+            ]
+        },
+    )
+
+    # Provide a secret value
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/secret/471b9e64-29af-48dc-8bda-1a64a2da0c12",
+        status=200,
+        json={
+            "id": "471b9e64-29af-48dc-8bda-1a64a2da0c12",
+            "name": "a_secret",
+            "content": "My super duper secret value",
+        },
+    )
+
+    mock_base_worker_modern_conf.configure()
+
+    assert mock_base_worker_modern_conf.config == {
+        "a_secret": "My super duper secret value",
+        "some_key": "test",
+    }
+    assert (
+        mock_base_worker_modern_conf.user_configuration
+        == mock_base_worker_modern_conf.config
+    )
+    assert mock_base_worker_modern_conf.secrets == {
+        "a_secret": "My super duper secret value"
+    }