vantage6-node 5.0.3rc6__tar.gz → 5.1.0__tar.gz

{vantage6_node-5.0.3rc6 → vantage6_node-5.1.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.5
 Name: vantage6-node
-Version: 5.0.3rc6
+Version: 5.1.0
 Summary: vantage6 node
 Author: Vantage6 Team
 Maintainer-email: Frank Martin <f.martin@iknl.nl>, Bart van Beusekom <b.vanbeusekom@iknl.nl>
-License: MIT
+License: Apache-2.0
 Requires-Python: >=3.13
 Requires-Dist: click==8.1.3
 Requires-Dist: flask==3.1.3
@@ -12,17 +12,12 @@ Requires-Dist: gevent==25.4.2
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: kubernetes==36.0.2
 Requires-Dist: nvidia-ml-py==12.535.133
-Requires-Dist: parquet-tools==0.2.16
 Requires-Dist: psutil==7.0.0
 Requires-Dist: python-socketio==5.16.2
 Requires-Dist: requests==2.33.1
-Requires-Dist: vantage6-client==5.0.3rc6
-Requires-Dist: vantage6-common==5.0.3rc6
-Requires-Dist: vantage6==5.0.3rc6
-Provides-Extra: dev
-Requires-Dist: black; extra == 'dev'
-Requires-Dist: coverage==7.10.2; extra == 'dev'
-Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: vantage6-client==5.1.0
+Requires-Dist: vantage6-common==5.1.0
+Requires-Dist: vantage6==5.1.0
 Description-Content-Type: text/markdown
 
 <h1 align="center">

{vantage6_node-5.0.3rc6 → vantage6_node-5.1.0}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "hatchling.build"
 
 [project]
 name = "vantage6-node"
-version = "5.0.3rc6"
+version = "5.1.0"
 description = "vantage6 node"
 readme = "README.md"
-license = {text = "MIT"}
+license = {text = "Apache-2.0"}
 authors = [
     {name = "Vantage6 Team"}
 ]
@@ -23,20 +23,12 @@ dependencies = [
     "jinja2==3.1.6",
     "kubernetes==36.0.2",
     "nvidia-ml-py==12.535.133",
-    "parquet-tools==0.2.16",
     "psutil==7.0.0",
     "python-socketio==5.16.2",
     "requests==2.33.1",
-    "vantage6==5.0.3rc6",
-    "vantage6-client==5.0.3rc6",
-    "vantage6-common==5.0.3rc6",
-]
-
-[project.optional-dependencies]
-dev = [
-    "coverage==7.10.2",
-    "black",
-    "pre-commit",
+    "vantage6==5.1.0",
+    "vantage6-client==5.1.0",
+    "vantage6-common==5.1.0",
 ]
 
 [project.scripts]

vantage6_node-5.1.0/tests_node/test_algorithm_policies.py

@@ -0,0 +1,80 @@
+import unittest
+from unittest.mock import MagicMock, patch
+
+from vantage6.common.globals import NodePolicy
+
+from vantage6.node import Node
+
+
+class TestCheckAlgorithmPolicies(unittest.TestCase):
+    """
+    Tests for Node.check_algorithm_policies().
+
+    Node.__init__ connects to HQ and Kubernetes, so these tests bypass it (via __new__)
+    and set up only the attributes the check reads: `config` and `log`.
+    """
+
+    def _node(self, config: dict) -> Node:
+        node = Node.__new__(Node)
+        node.config = config
+        node.log = MagicMock()
+        return node
+
+    def _assert_exits(self, config: dict):
+        node = self._node(config)
+        with (
+            patch("vantage6.node.sys.exit", side_effect=SystemExit(1)) as mock_exit,
+            self.assertRaises(SystemExit),
+        ):
+            node.check_algorithm_policies()
+        mock_exit.assert_called_once_with(1)
+        node.log.error.assert_called_once()
+
+    def _assert_starts(self, config: dict) -> Node:
+        node = self._node(config)
+        with patch("vantage6.node.sys.exit") as mock_exit:
+            node.check_algorithm_policies()
+        mock_exit.assert_not_called()
+        return node
+
+    def test_production_without_policies_exits(self):
+        self._assert_exits({"production": True})
+
+    def test_production_is_default(self):
+        self._assert_exits({})
+
+    def test_production_with_empty_policies_exits(self):
+        self._assert_exits(
+            {
+                "policies": {
+                    NodePolicy.ALLOWED_ALGORITHMS.value: [],
+                    NodePolicy.ALLOWED_ALGORITHM_STORES.value: [],
+                    NodePolicy.ALLOWED_USERS.value: [1],
+                }
+            }
+        )
+
+    def test_production_with_allowed_algorithms_starts(self):
+        node = self._assert_starts(
+            {
+                "production": True,
+                "policies": {NodePolicy.ALLOWED_ALGORITHMS.value: ["^some/image$"]},
+            }
+        )
+        node.log.warning.assert_not_called()
+
+    def test_production_with_allowed_stores_starts(self):
+        node = self._assert_starts(
+            {
+                "production": True,
+                "policies": {
+                    NodePolicy.ALLOWED_ALGORITHM_STORES.value: ["https://store.example"]
+                },
+            }
+        )
+        node.log.warning.assert_not_called()
+
+    def test_non_production_without_policies_warns(self):
+        node = self._assert_starts({"production": False})
+        node.log.warning.assert_called()
+        node.log.error.assert_not_called()

{vantage6_node-5.0.3rc6 → vantage6_node-5.1.0}/tests_node/test_container_manager.py

@@ -1,9 +1,12 @@
 import logging
 import os
+import tempfile
 import unittest
+from pathlib import Path
 from unittest.mock import MagicMock, patch
 
-from vantage6.common.globals import NodePolicy
+from vantage6.common.enum import AlgorithmStepType, TaskDatabaseType
+from vantage6.common.globals import ContainerEnvNames, NodePolicy
 
 from vantage6.node.k8s.container_manager import ContainerManager
 
@@ -215,5 +218,155 @@ class TestGetImagePullPolicy:
         assert manager._get_image_pull_policy() == "Always"
 
 
+class TestDatabaseMountMode(unittest.TestCase):
+    """
+    Tests that the configured database mount mode is applied to algorithm pods
+    (GHSA-27gf-8rcw-jqgh): 'ro' (default) databases are mounted read-only and file
+    databases in 'copy' mode are copied, so algorithms never get write access to the
+    original.
+    """
+
+    HOST_DATA_DIR = "/host/tasks"
+
+    def setUp(self):
+        tmp_dir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp_dir.cleanup)
+        self.db_dir = Path(tmp_dir.name) / "databases"
+        self.tasks_dir = Path(tmp_dir.name) / "tasks"
+        self.db_dir.mkdir()
+        (self.db_dir / "file_db.csv").write_text("a,b\n1,2\n")
+        (self.db_dir / "dir_db.parquet").mkdir()
+
+        for name, value in [
+            ("DATABASE_BASE_PATH", str(self.db_dir)),
+            ("TASK_FILES_ROOT", str(self.tasks_dir)),
+        ]:
+            patcher = patch(f"vantage6.node.k8s.container_manager.{name}", value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+
+        self.manager = ContainerManager.__new__(ContainerManager)
+        self.manager.log = _get_null_logger()
+        self.manager.host_data_dir = self.HOST_DATA_DIR
+
+    def _get_databases(self, env: dict) -> dict:
+        with patch.dict(os.environ, env, clear=True):
+            return self.manager._get_database_metadata()
+
+    @staticmethod
+    def _file_db_env(mount_mode: str | None = None) -> dict:
+        env = {
+            "DATABASE_LABELS": "file_db",
+            "DATABASE_FILE_DB_URI": "/data/file_db.csv",
+            "DATABASE_FILE_DB_TYPE": "csv",
+        }
+        if mount_mode is not None:
+            env["DATABASE_FILE_DB_MOUNT_MODE"] = mount_mode
+        return env
+
+    def test_read_only_file_is_not_copied(self):
+        db = self._get_databases(self._file_db_env("ro"))["file_db"]
+
+        self.assertEqual(db.mount_mode, "ro")
+        self.assertEqual(db.uri, "/data/file_db.csv")
+        self.assertEqual(db.local_uri, str(self.db_dir / "file_db.csv"))
+        self.assertFalse((self.tasks_dir / "databases").exists())
+
+    def test_mount_mode_defaults_to_read_only(self):
+        db = self._get_databases(self._file_db_env())["file_db"]
+
+        self.assertEqual(db.mount_mode, "ro")
+        self.assertEqual(db.uri, "/data/file_db.csv")
+        self.assertFalse((self.tasks_dir / "databases").exists())
+
+    def test_copy_mode_copies_file(self):
+        db = self._get_databases(self._file_db_env("copy"))["file_db"]
+
+        self.assertEqual(db.mount_mode, "copy")
+        self.assertEqual(db.uri, f"{self.HOST_DATA_DIR}/databases/file_db.csv")
+        self.assertEqual(db.local_uri, str(self.db_dir / "file_db.csv"))
+        self.assertEqual(
+            (self.tasks_dir / "databases" / "file_db.csv").read_text(), "a,b\n1,2\n"
+        )
+
+    def test_mount_mode_is_case_insensitive(self):
+        db = self._get_databases(self._file_db_env("RO"))["file_db"]
+        self.assertEqual(db.mount_mode, "ro")
+
+    def test_copy_mode_directory_is_not_copied(self):
+        env = {
+            "DATABASE_LABELS": "dir_db",
+            "DATABASE_DIR_DB_URI": "/data/dir_db",
+            "DATABASE_DIR_DB_TYPE": "parquet",
+            "DATABASE_DIR_DB_MOUNT_MODE": "copy",
+        }
+        db = self._get_databases(env)["dir_db"]
+
+        self.assertEqual(db.mount_mode, "copy")
+        self.assertEqual(db.uri, "/data/dir_db")
+        self.assertFalse((self.tasks_dir / "databases").exists())
+
+    def test_invalid_mount_mode_raises(self):
+        with self.assertRaises(ValueError):
+            self._get_databases(self._file_db_env("rw"))
+
+    def test_mount_mode_not_passed_to_algorithm_env(self):
+        env = self._file_db_env("ro")
+        env["DATABASE_FILE_DB_SOME_KEY"] = "some_value"
+        db = self._get_databases(env)["file_db"]
+
+        self.assertEqual(db.env, {"SOME_KEY": "some_value"})
+
+    def _create_extraction_mounts(self, env: dict, label: str):
+        self.manager.databases = self._get_databases(env)
+        run_io = MagicMock()
+        run_io.action = AlgorithmStepType.DATA_EXTRACTION
+        run_io.run_id = 1
+        run_io.create_files.return_value = ("1/input", "1/output")
+        run_io.session_file_manager.session_folder = "session"
+        run_io.session_file_manager.session_state_file_name = "state.parquet"
+        databases_to_use = [{"label": label, "type": TaskDatabaseType.SOURCE}]
+
+        with patch.object(self.manager, "_validate_source_database"):
+            return self.manager._create_volume_mounts(run_io, b"", databases_to_use)
+
+    def _get_extraction_db_mount(self, mount_mode: str):
+        volumes, vol_mounts, _, _ = self._create_extraction_mounts(
+            self._file_db_env(mount_mode), "file_db"
+        )
+
+        volume = next(v for v in volumes if v.name == "task-1-db-file_db")
+        vol_mount = next(m for m in vol_mounts if m.name == "task-1-db-file_db")
+        return volume, vol_mount
+
+    def test_read_only_database_mounted_read_only(self):
+        volume, vol_mount = self._get_extraction_db_mount("ro")
+
+        self.assertTrue(vol_mount.read_only)
+        self.assertEqual(volume.host_path.path, "/data/file_db.csv")
+
+    def test_copy_database_mounts_copy_read_write(self):
+        volume, vol_mount = self._get_extraction_db_mount("copy")
+
+        self.assertFalse(vol_mount.read_only)
+        self.assertEqual(
+            volume.host_path.path, f"{self.HOST_DATA_DIR}/databases/file_db.csv"
+        )
+        self.assertEqual(vol_mount.mount_path, str(self.db_dir / "file_db.csv"))
+
+    def test_service_database_ignores_default_read_only(self):
+        env = {
+            "DATABASE_LABELS": "pg_db",
+            "DATABASE_PG_DB_URI": "postgresql://host:5432/db",
+            "DATABASE_PG_DB_TYPE": "sql",
+        }
+        volumes, _, env_vars, _ = self._create_extraction_mounts(env, "pg_db")
+
+        self.assertFalse(any(v.name == "task-1-db-pg_db" for v in volumes))
+        self.assertEqual(
+            env_vars[ContainerEnvNames.DATABASE_URI.value], "postgresql://host:5432/db"
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

{vantage6_node-5.0.3rc6 → vantage6_node-5.1.0}/tests_node/test_network_isolation.py

@@ -3,6 +3,8 @@ from unittest.mock import MagicMock, patch
 import pytest
 from kubernetes import client as k8s_client
 
+from vantage6.common.globals import NodePolicy
+
 from vantage6.node import Node
 from vantage6.node.k8s.network_isolation import (
     _probe_pod_reached_target,
@@ -69,10 +71,13 @@ class TestValidateAlgorithmIsolation:
         assert "verified" in message
 
 
+_ALGORITHM_POLICY = {NodePolicy.ALLOWED_ALGORITHMS.value: ["^some/image$"]}
+
+
 class TestNodeProductionGating:
     def test_production_true_exits_on_isolation_failure(self):
         ctx = MagicMock()
-        ctx.config = {"production": True}
+        ctx.config = {"production": True, "policies": _ALGORITHM_POLICY}
 
         with (
             patch("vantage6.node.validate_required_env_vars"),
@@ -84,7 +89,9 @@ class TestNodeProductionGating:
                 False,
                 "probe failed",
             )
-            with patch("vantage6.node.exit", side_effect=SystemExit(1)) as mock_exit:
+            with patch(
+                "vantage6.node.sys.exit", side_effect=SystemExit(1)
+            ) as mock_exit:
                 with (
                     patch.object(Node, "_setup_node_client", return_value=MagicMock()),
                     pytest.raises(SystemExit),
@@ -94,7 +101,11 @@ class TestNodeProductionGating:
 
     def test_production_false_warns_on_isolation_failure(self, caplog):
         ctx = MagicMock()
-        ctx.config = {"production": False, "debug": {}}
+        ctx.config = {
+            "production": False,
+            "debug": {},
+            "policies": _ALGORITHM_POLICY,
+        }
 
         def _connect_socket(self):
             self.socketIO = MagicMock()
@@ -109,7 +120,7 @@ class TestNodeProductionGating:
                 False,
                 "probe failed",
             )
-            with patch("vantage6.node.exit") as mock_exit:
+            with patch("vantage6.node.sys.exit") as mock_exit:
                 with (
                     patch.object(Node, "_setup_node_client", return_value=MagicMock()),
                     patch.object(Node, "authenticate"),

vantage6_node-5.1.0/tests_node/test_node_status_emit.py

@@ -0,0 +1,51 @@
+import unittest
+from unittest.mock import Mock, patch
+
+from socketio.exceptions import BadNamespaceError
+
+from vantage6.common.enum import RunStatus
+
+from vantage6.node import Node
+
+
+class TestEmitAlgorithmStatusChange(unittest.TestCase):
+    def setUp(self):
+        self.node = Node.__new__(Node)
+        self.node.log = Mock()
+        self.node.socketIO = Mock()
+        self.node.socketIO.namespaces = {"/tasks": "sid"}
+
+    def _emit(self):
+        with patch("vantage6.node.time.sleep"):
+            self.node._Node__emit_algorithm_status_change(
+                run_id=20, status=RunStatus.ACTIVE
+            )
+
+    def test_emit_when_connected(self):
+        self._emit()
+
+        self.node.socketIO.emit.assert_called_once_with(
+            "algorithm_status_change",
+            data={"run_id": 20, "status": RunStatus.ACTIVE.value},
+            namespace="/tasks",
+        )
+        self.node.log.warning.assert_not_called()
+
+    def test_skip_when_namespace_never_connects(self):
+        self.node.socketIO.namespaces = {}
+
+        self._emit()
+
+        self.node.socketIO.emit.assert_not_called()
+        self.node.log.warning.assert_called_once()
+
+    def test_no_exception_when_namespace_drops_during_emit(self):
+        self.node.socketIO.emit.side_effect = BadNamespaceError("/tasks")
+
+        self._emit()
+
+        self.node.log.warning.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

{vantage6_node-5.0.3rc6 → vantage6_node-5.1.0}/tests_node/test_pod_status_mapping.py

@@ -96,6 +96,32 @@ class TestPodStatus(unittest.TestCase):
             RunStatus.NO_DOCKER_IMAGE,
         )
 
+    def test_image_being_pulled(self):
+        self.mock_pod.status.phase = "Pending"
+        self.mock_pod.status.container_statuses[0].state = self.waiting_container_state
+        self.mock_pod.status.container_statuses[
+            0
+        ].state.waiting.reason = "ContainerCreating"
+        self.assertEqual(
+            compute_run_pod_status(
+                log=self.silent_logger, task_namespace="", label="", pod=self.mock_pod
+            ),
+            RunStatus.PULLING_IMAGE,
+        )
+
+    def test_pod_still_initializing(self):
+        self.mock_pod.status.phase = "Pending"
+        self.mock_pod.status.container_statuses[0].state = self.waiting_container_state
+        self.mock_pod.status.container_statuses[
+            0
+        ].state.waiting.reason = "PodInitializing"
+        self.assertEqual(
+            compute_run_pod_status(
+                log=self.silent_logger, task_namespace="", label="", pod=self.mock_pod
+            ),
+            RunStatus.INITIALIZING,
+        )
+
     def test_waiting_reason_not_ready_err(self):
         self.mock_pod.status.phase = "Pending"
         self.mock_pod.status.container_statuses[0].state = self.waiting_container_state

{vantage6_node-5.0.3rc6 → vantage6_node-5.1.0}/tests_node/test_share_algorithm_logs.py

@@ -33,7 +33,7 @@ class TestContainerManagerLogSharing(unittest.TestCase):
         with patch(
             "vantage6.node.k8s.container_manager.watch.Watch", return_value=fake_watch
         ):
-            manager._ContainerManager__log_stream(run_io=run_io, task_id=99)
+            manager._ContainerManager__log_stream(run_io=run_io)
 
         manager.socket_io.emit.assert_not_called()
 
@@ -46,16 +46,11 @@ class TestContainerManagerLogSharing(unittest.TestCase):
         with patch(
             "vantage6.node.k8s.container_manager.watch.Watch", return_value=fake_watch
         ):
-            manager._ContainerManager__log_stream(run_io=run_io, task_id=99)
+            manager._ContainerManager__log_stream(run_io=run_io)
 
         manager.socket_io.emit.assert_called_once_with(
             "algorithm_log",
-            {
-                "collaboration_id": 7,
-                "run_id": 42,
-                "task_id": 99,
-                "log": "hello from algorithm",
-            },
+            {"run_id": 42, "log": "hello from algorithm"},
             namespace="/tasks",
         )
 

vantage6_node-5.1.0/tests_node/test_socket_transport.py

@@ -0,0 +1,46 @@
+import unittest
+from unittest.mock import Mock, patch
+
+from engineio.payload import Payload
+
+from vantage6.common.globals import SOCKETIO_MAX_DECODE_PACKETS
+
+from vantage6.node import Node
+
+
+class TestSocketTransport(unittest.TestCase):
+    def setUp(self):
+        # the packet limit is a class attribute, so reset it to the library default
+        self.addCleanup(
+            setattr, Payload, "max_decode_packets", Payload.max_decode_packets
+        )
+        Payload.max_decode_packets = 16
+
+    def _make_node(self, config: dict) -> Node:
+        node = Node.__new__(Node)
+        node.log = Mock()
+        node.config = config
+        node.debug = {}
+        node.client = Mock()
+        node.client.hq_url = "http://hq.example"
+        node.client.headers = {"Authorization": "Bearer token"}
+        return node
+
+    def _connect(self, config: dict) -> Mock:
+        node = self._make_node(config)
+        with patch("vantage6.node.SocketIO") as socketio_cls:
+            socketio_cls.return_value.connected = True
+            node.connect_to_socket()
+        return socketio_cls.return_value.connect
+
+    def test_websocket_only_by_default(self):
+        connect = self._connect({})
+        self.assertEqual(connect.call_args.kwargs["transports"], ["websocket"])
+
+    def test_http_polling_allowed(self):
+        connect = self._connect({"allow_http_polling": True})
+        self.assertIsNone(connect.call_args.kwargs["transports"])
+
+    def test_max_decode_packets_raised(self):
+        self._connect({})
+        self.assertEqual(Payload.max_decode_packets, SOCKETIO_MAX_DECODE_PACKETS)