panda-server-source 0.8.0__py3-none-any.whl

pandaserver/api/v1/tests/credential_management_api_tests.py

@@ -0,0 +1,116 @@
+# TODO: the result of the tests depend on the cert/token used for the call. These particular results are for running with user pandasv1
+
+import unittest
+from datetime import datetime, timezone
+
+from pandaserver.api.v1.http_client import HttpClient, api_url, api_url_ssl
+
+# Get current UTC time with microseconds. The format needs to be compatible with the one used in the database
+now_utc = datetime.now(timezone.utc)
+formatted_time = now_utc.strftime("%d.%m.%y %H:%M:%S") + f".{now_utc.microsecond // 1000:02d}"
+
+KEY = "test_key"
+VALUE = "test_value"
+
+NO_SSL_RESPONSE = {"success": False, "message": "SSL secure connection is required", "data": None}
+
+
+class TestSecretManagementAPI(unittest.TestCase):
+    def setUp(self):
+        self.http_client = HttpClient()
+        self.urls = [api_url, api_url_ssl]  # Run tests with both base URLs
+
+    def test_set_user_secrets(self):
+        for url in self.urls:
+            with self.subTest(base_url=url):
+                full_url = f"{url}/creds/set_user_secrets"
+                print(f"Testing URL: {full_url}")
+                data = {"key": KEY, "value": VALUE}
+                status, output = self.http_client.post(full_url, data)
+                print(output)
+                if url.startswith(api_url):
+                    expected_response = NO_SSL_RESPONSE
+                else:
+                    expected_response = {"success": True, "message": "OK", "data": None}
+                self.assertEqual(output, expected_response)
+
+    def test_get_user_secrets(self):
+        for url in self.urls:
+            with self.subTest(base_url=url):
+                full_url = f"{url}/creds/get_user_secrets"
+                print(f"Testing URL: {full_url}")
+                data = {"keys": KEY}
+                status, output = self.http_client.get(full_url, data)
+                print(output)
+                if url.startswith(api_url):
+                    expected_response = NO_SSL_RESPONSE
+                else:
+                    expected_response = {"success": True, "message": "", "data": '{"test_key": "test_value"}'}
+                self.assertEqual(output, expected_response)
+
+    def test_get_key_pair(self):
+        for url in self.urls:
+            with self.subTest(base_url=url):
+                full_url = f"{url}/creds/get_key_pair"
+                print(f"Testing URL: {full_url}")
+                data = {"public_key_name": "a", "private_key_name": "b"}
+                status, output = self.http_client.get(full_url, data)
+                print(output)
+                if url.startswith(api_url):
+                    expected_response = NO_SSL_RESPONSE
+                else:
+                    expected_response = {
+                        "success": False,
+                        "message": "Failed since 'pandasv1' not authorized with 'k' in ATLAS_PANDAMETA.USERS.GRIDPREF",
+                        "data": None,
+                    }
+                self.assertEqual(output, expected_response)
+
+    def test_get_proxy(self):
+        for url in self.urls:
+            with self.subTest(base_url=url):
+                full_url = f"{url}/creds/get_proxy"
+                print(f"Testing URL: {full_url}")
+                data = {"role": "atlas", "dn": "atlpilo2"}
+                status, output = self.http_client.get(full_url, data)
+                print(output)
+                if url.startswith(api_url):
+                    expected_response = NO_SSL_RESPONSE
+                else:
+                    expected_response = {"success": False, "message": "'proxy' not found for atlpilo2", "data": None}
+                self.assertEqual(output, expected_response)
+
+    def test_get_access_token(self):
+        for url in self.urls:
+            with self.subTest(base_url=url):
+                full_url = f"{url}/creds/get_access_token"
+                print(f"Testing URL: {full_url}")
+                data = {"client_name": "pilot_server"}
+                status, output = self.http_client.get(full_url, data)
+                print(output)
+                if url.startswith(api_url):
+                    expected_response = NO_SSL_RESPONSE
+                else:
+                    expected_response = {"success": False, "message": "failed since token key is invalid for pilot_server", "data": None}
+                self.assertEqual(output, expected_response)
+
+    def test_get_token_key(self):
+        for url in self.urls:
+            with self.subTest(base_url=url):
+                full_url = f"{url}/creds/get_token_key"
+                print(f"Testing URL: {full_url}")
+                data = {"client_name": "pilot_server"}
+                status, output = self.http_client.get(full_url, data)
+                print(output)
+                if url.startswith(api_url):
+                    expected_response = NO_SSL_RESPONSE
+                else:
+                    del output["data"]
+                    del output["message"]
+                    expected_response = {"success": True}
+                self.assertEqual(output, expected_response)
+
+
+# Run tests
+if __name__ == "__main__":
+    unittest.main()

pandaserver/api/v1/tests/data_carousel_api_tests.py

@@ -0,0 +1,82 @@
+# Description: Unit tests for the Data Carousel API methods
+import os
+import unittest
+from datetime import datetime, timedelta
+
+from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
+
+# to run the tests with a real Data Carousel request ID or dataset name by setting the environment variable
+REQUEST_ID = int(os.environ.get("REQUEST_ID", None))
+DATASET = str(os.environ.get("DATASET", None))
+
+
+class TestDataCarouselAPI(unittest.TestCase):
+    def setUp(self):
+        self.http_client = HttpClient()
+
+    def test_change_staging_destination_by_request_id(self):
+        url = f"{api_url_ssl}/data_carousel/change_staging_destination"
+        print(f"Testing URL: {url}")
+        data = {"request_id": REQUEST_ID}
+        status, output = self.http_client.post(url, data)
+        print(status, output)
+        if status == 0 and output.get("data"):
+            self.assertEqual(output["data"].get("request_id"), REQUEST_ID)
+            self.assertIsInstance(output["data"].get("new_request_id"), int)
+
+    def test_change_staging_destination_by_dataset(self):
+        url = f"{api_url_ssl}/data_carousel/change_staging_destination"
+        print(f"Testing URL: {url}")
+        data = {"dataset": DATASET}
+        status, output = self.http_client.post(url, data)
+        print(status, output)
+        if status == 0 and output.get("data"):
+            self.assertEqual(output["data"].get("dataset"), DATASET)
+            self.assertIsInstance(output["data"].get("new_request_id"), int)
+
+    def test_change_staging_source_by_request_id(self):
+        url = f"{api_url_ssl}/data_carousel/change_staging_source"
+        print(f"Testing URL: {url}")
+        data = {"request_id": REQUEST_ID}
+        status, output = self.http_client.post(url, data)
+        print(status, output)
+        if status == 0 and output.get("data"):
+            self.assertEqual(output["data"].get("request_id"), REQUEST_ID)
+            self.assertIsInstance(output["data"].get("ddm_rule_id"), str)
+
+    def test_change_staging_source_by_dataset(self):
+        url = f"{api_url_ssl}/data_carousel/change_staging_source"
+        print(f"Testing URL: {url}")
+        data = {"dataset": DATASET}
+        status, output = self.http_client.post(url, data)
+        print(status, output)
+        if status == 0 and output.get("data"):
+            self.assertEqual(output["data"].get("dataset"), DATASET)
+            self.assertIsInstance(output["data"].get("ddm_rule_id"), str)
+
+    def test_force_to_staging_by_request_id(self):
+        url = f"{api_url_ssl}/data_carousel/force_to_staging"
+        print(f"Testing URL: {url}")
+        data = {"request_id": REQUEST_ID}
+        status, output = self.http_client.post(url, data)
+        print(status, output)
+        if status == 0 and output.get("data"):
+            self.assertEqual(output["data"].get("request_id"), REQUEST_ID)
+            self.assertEqual(output["data"].get("status"), "staging")
+            self.assertIsInstance(output["data"].get("ddm_rule_id"), str)
+
+    def test_force_to_staging_by_dataset(self):
+        url = f"{api_url_ssl}/data_carousel/force_to_staging"
+        print(f"Testing URL: {url}")
+        data = {"dataset": DATASET}
+        status, output = self.http_client.post(url, data)
+        print(status, output)
+        if status == 0 and output.get("data"):
+            self.assertEqual(output["data"].get("dataset"), DATASET)
+            self.assertEqual(output["data"].get("status"), "staging")
+            self.assertIsInstance(output["data"].get("ddm_rule_id"), str)
+
+
+# Run tests
+if __name__ == "__main__":
+    unittest.main()

pandaserver/api/v1/tests/harvester_api_tests.py

@@ -0,0 +1,188 @@
+import json
+import unittest
+from datetime import datetime, timezone
+
+from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
+
+# Get current UTC time with microseconds. The format needs to be compatible with the one used in the database
+now_utc = datetime.now(timezone.utc)
+formatted_time = now_utc.strftime("%d.%m.%y %H:%M:%S") + f".{now_utc.microsecond // 1000:02d}"
+
+HARVESTER_ID = "test_fbarreir"
+HARVESTER_HOST = "test_fbarreir.cern.ch"
+PANDA_QUEUE = "test_queue"
+
+
+class TestHarvesterAPI(unittest.TestCase):
+    def setUp(self):
+        self.http_client = HttpClient()
+
+    def test_update_service_metrics(self):
+        url = f"{api_url_ssl}/harvester/update_service_metrics"
+        print(f"Testing URL: {url}")
+        harvester_id = HARVESTER_ID
+        harvester_host = HARVESTER_HOST
+        creation_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
+        metric = {
+            "rss_mib": 2737.36,
+            "memory_pc": 39.19,
+            "cpu_pc": 15.23,
+            "volume_data_pc": 20.0,
+            "cert_lifetime": {
+                "/data/atlpan/proxy/x509up_u25606_prod": 81,
+                "/data/atlpan/proxy/x509up_u25606_pilot": 81,
+                "/cephfs/atlpan/harvester/proxy/x509up_u25606_prod": 96,
+                "/cephfs/atlpan/harvester/proxy/x509up_u25606_pilot": 96,
+            },
+        }
+
+        # DBProxy expects the metrics in json format and stores them directly in the database
+        metrics = [[creation_time, harvester_host, json.dumps(metric)]]
+
+        data = {"harvester_id": harvester_id, "metrics": metrics}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        expected_response = {"success": True, "message": "", "data": [True]}
+        self.assertEqual(output, expected_response)
+
+    def test_add_dialogs(self):
+        url = f"{api_url_ssl}/harvester/add_dialogs"
+        print(f"Testing URL: {url}")
+        harvester_id = HARVESTER_ID
+
+        dialog = {
+            "diagID": 1,
+            "moduleName": "test_module",
+            "identifier": "test identifier",
+            "creationTime": datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
+            "messageLevel": "INFO",
+            "diagMessage": "test message",
+        }
+
+        dialogs = [dialog]
+
+        data = {"harvester_id": harvester_id, "dialogs": dialogs}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        expected_response = {"success": True, "message": "", "data": None}
+        self.assertEqual(output, expected_response)
+
+    def test_heartbeat(self):
+        url = f"{api_url_ssl}/harvester/heartbeat"
+        print(f"Testing URL: {url}")
+        harvester_id = HARVESTER_ID
+        data = {"harvester_id": harvester_id, "data": []}
+        status, output = self.http_client.post(url, data)
+
+        expected_response = {"success": True, "message": "", "data": None}
+        self.assertEqual(output, expected_response)
+
+    def test_get_worker_statistics(self):
+        url = f"{api_url_ssl}/harvester/get_worker_statistics"
+        print(f"Testing URL: {url}")
+        data = {}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        # the statistics can't be predicted, so we just check the type of the response
+        self.assertEqual(True, output["success"])
+        self.assertEqual(dict, type(output["data"]))
+
+    def test_get_current_worker_id(self):
+        url = f"{api_url_ssl}/harvester/get_current_worker_id"
+        print(f"Testing URL: {url}")
+        data = {"harvester_id": HARVESTER_ID}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        # the current/max worker id can't be predicted, so we just check the type of the response
+        self.assertEqual(True, output["success"])
+        self.assertEqual(int, type(output["data"]))
+
+    def test_report_worker_statistics(self):
+        url = f"{api_url_ssl}/harvester/report_worker_statistics"
+        print(f"Testing URL: {url}")
+        harvester_id = HARVESTER_ID
+        panda_queue = PANDA_QUEUE
+        # the json loads is done in DBProxy
+        statistics = json.dumps({"user": {"SCORE": {"running": 1, "submitted": 1}}, "managed": {"MCORE": {"running": 1, "submitted": 1}}})
+        data = {"harvester_id": harvester_id, "panda_queue": panda_queue, "statistics": statistics}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        expected_response = {"success": True, "message": "OK", "data": None}
+        self.assertEqual(output, expected_response)
+
+    def test_update_workers(self):
+        url = f"{api_url_ssl}/harvester/update_workers"
+        print(f"Testing URL: {url}")
+        worker = {
+            "workerID": 1,
+            "batchID": 1,
+            "queueName": "queue1",
+            "status": "running",
+            "computingSite": "site1",
+            "nCore": 1,
+            "nodeID": None,
+            "submitTime": "02-NOV-24 00:02:18",
+            "startTime": "02-NOV-24 00:02:18",
+            "endTime": None,
+            "jobType": "managed",
+            "resourceType": "SCORE",
+            "nativeExitCode": None,
+            "nativeStatus": None,
+            "diagMessage": None,
+            "nJobs": 1,
+            "computingElement": "ce1",
+            "syncLevel": 0,
+            "submissionHost": "submissionhost1",
+            "harvesterHost": "harvesterhost1",
+            "errorCode": None,
+            "minRamCount": 2000,
+        }
+        workers = [worker]
+
+        harvester_id = HARVESTER_ID
+        data = {"harvester_id": harvester_id, "workers": workers}
+
+        status, output = self.http_client.post(url, data)
+        print(output)
+        expected_response = {"success": True, "message": "", "data": [True]}
+
+        self.assertEqual(output, expected_response)
+
+    def test_acquire_commands(self):
+        url = f"{api_url_ssl}/harvester/acquire_commands"
+        print(f"Testing URL: {url}")
+
+        harvester_id = HARVESTER_ID
+        n_commands = 1
+        data = {"harvester_id": harvester_id, "n_commands": n_commands}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        # the commands can't be predicted, so we just check the type of the response
+        self.assertEqual(True, output["success"])
+        self.assertEqual(list, type(output["data"]))
+
+    def test_acknowledge_commands(self):
+        url = f"{api_url_ssl}/harvester/acknowledge_commands"
+        print(f"Testing URL: {url}")
+        command_ids = [1]
+        data = {"command_ids": command_ids}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        expected_response = {"success": True, "message": "", "data": None}
+        self.assertEqual(output, expected_response)
+
+    def test_add_target_slots(self):
+        url = f"{api_url_ssl}/harvester/add_target_slots"
+        print(f"Testing URL: {url}")
+        panda_queue = PANDA_QUEUE
+        slots = 0
+        data = {"panda_queue": panda_queue, "slots": slots}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        self.assertEqual(True, output["success"])
+        self.assertEqual(None, output["data"])
+
+
+# Run tests
+if __name__ == "__main__":
+    unittest.main()

pandaserver/api/v1/tests/idds_api_tests.py

@@ -0,0 +1,45 @@
+# Description: Unit tests for the IDDS API methods
+# TODO: These tests are placeholders and need to be completed
+import unittest
+
+from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
+
+
+class TestIDDSAPI(unittest.TestCase):
+    def setUp(self):
+        self.http_client = HttpClient()
+
+    def test_relay_idds_command(self):
+        url = f"{api_url_ssl}/idds/relay_idds_command"
+        print(f"Testing URL: {url}")
+        data = {}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        output["status"] = status
+
+        # Remove the data field from the response for comparison
+        del output["data"]
+
+        expected_response = {"success": True, "message": "", "status": 0}
+
+        self.assertEqual(output, expected_response)
+
+    def test_execute_idds_workflow_command(self):
+        url = f"{api_url_ssl}/statistics/execute_idds_workflow_command"
+        print(f"Testing URL: {url}")
+        data = {}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        output["status"] = status
+
+        # Remove the data field from the response for comparison
+        del output["data"]
+
+        expected_response = {"success": True, "message": "", "status": 0}
+
+        self.assertEqual(output, expected_response)
+
+
+# Run tests
+if __name__ == "__main__":
+    unittest.main()

pandaserver/api/v1/tests/job_api_tests.py

@@ -0,0 +1,112 @@
+# Description: Unit tests for the Job API methods
+import os
+import unittest
+
+from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
+
+# To run tests with real IDs, set environment variables:
+#   PANDA_JOB_ID_TEST=<job_id>  JEDI_TASK_ID_TEST=<task_id>
+PANDA_JOB_ID = int(os.environ.get("PANDA_JOB_ID_TEST", -1))
+JEDI_TASK_ID = int(os.environ.get("JEDI_TASK_ID_TEST", -1))
+
+
+class TestJobAPI(unittest.TestCase):
+    def setUp(self):
+        self.http_client = HttpClient()
+
+    def test_get_status(self):
+        # def get_status(req: PandaRequest, job_ids: List[int], timeout: int = 60) -> Dict:
+        url = f"{api_url_ssl}/job/get_status"
+        print(f"Testing URL: {url}")
+        data = {"job_ids": [PANDA_JOB_ID]}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        output["status"] = status
+        del output["data"]
+        del output["message"]
+
+        # Returns success=True regardless of whether job exists (returns empty or None entries)
+        expected_response = {"status": 0, "success": True}
+        self.assertEqual(output, expected_response)
+
+    def test_get_description(self):
+        # def get_description(req: PandaRequest, job_ids: List[int]) -> Dict:
+        url = f"{api_url_ssl}/job/get_description"
+        print(f"Testing URL: {url}")
+        data = {"job_ids": [PANDA_JOB_ID]}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        output["status"] = status
+        del output["data"]
+        del output["message"]
+
+        # Returns success=True regardless (returns [None] for unknown job IDs)
+        expected_response = {"status": 0, "success": True}
+        self.assertEqual(output, expected_response)
+
+    def test_get_description_incl_archive(self):
+        # def get_description_incl_archive(req: PandaRequest, job_ids: List[int]) -> Dict:
+        url = f"{api_url_ssl}/job/get_description_incl_archive"
+        print(f"Testing URL: {url}")
+        data = {"job_ids": [PANDA_JOB_ID]}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        output["status"] = status
+        del output["data"]
+        del output["message"]
+
+        # Returns success=True regardless (returns [None] for unknown job IDs)
+        expected_response = {"status": 0, "success": True}
+        self.assertEqual(output, expected_response)
+
+    def test_get_metadata_for_analysis_jobs(self):
+        # def get_metadata_for_analysis_jobs(req: PandaRequest, task_id: int) -> Dict:
+        url = f"{api_url_ssl}/job/get_metadata_for_analysis_jobs"
+        print(f"Testing URL: {url}")
+        data = {"task_id": JEDI_TASK_ID}
+        status, output = self.http_client.get(url, data)
+        print(output)
+        output["status"] = status
+        del output["data"]
+
+        if JEDI_TASK_ID == -1:
+            # Fake task returns success=True with "No metadata found" message
+            expected_response = {"status": 0, "success": True, "message": "No metadata found"}
+        else:
+            # Real task
+            expected_response = {"status": 0, "success": True, "message": ""}
+            output["message"] = ""
+        self.assertEqual(output, expected_response)
+
+    def test_kill(self):
+        # def kill(req, job_ids: List[int], code: int = None, ...) -> Dict:
+        url = f"{api_url_ssl}/job/kill"
+        print(f"Testing URL: {url}")
+        data = {"job_ids": [PANDA_JOB_ID]}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        output["status"] = status
+        del output["data"]
+        del output["message"]
+
+        # killJobs always returns success=True
+        expected_response = {"status": 0, "success": True}
+        self.assertEqual(output, expected_response)
+
+    def test_reassign(self):
+        # def reassign(req: PandaRequest, job_ids: List[int]) -> Dict:
+        url = f"{api_url_ssl}/job/reassign"
+        print(f"Testing URL: {url}")
+        data = {"job_ids": [PANDA_JOB_ID]}
+        status, output = self.http_client.post(url, data)
+        print(output)
+        output["status"] = status
+        del output["message"]
+
+        # reassignJobs always returns success=True
+        expected_response = {"status": 0, "success": True, "data": None}
+        self.assertEqual(output, expected_response)
+
+
+if __name__ == "__main__":
+    unittest.main()