ciocore 7.0.2b4__py2.py3-none-any.whl → 8.0.0__py2.py3-none-any.whl

tests/test_downloader.py

@@ -0,0 +1,118 @@
+
+from ciocore.downloader.job_downloader import JobDownloader, flatten
+import unittest
+from unittest import mock
+from ciocore.downloader.base_downloader import temp_file
+import tempfile
+import shutil
+import os
+import stat
+import contextlib
+
+class TempFileTests(unittest.TestCase):
+
+    def test_temp_file_creation(self):
+        # Create a temporary file and test if it exists
+        with temp_file("tests/files/file.txt") as temp_path:
+            self.assertTrue(os.path.exists(temp_path))
+            self.assertTrue(os.access(temp_path, os.W_OK & os.R_OK))
+
+    def test_temp_file_overwrite_existing(self):
+        # Test if existing file is overwritten by the temporary file
+        # Create a dummy file at the specified path
+        os.makedirs("tests/files", exist_ok=True)
+        with open("tests/files/file.txt", "w") as f:
+            f.write("dummy content")
+
+        # Replace the file with a temporary file and check the content
+        with temp_file("tests/files/file.txt") as temp_path:
+            with open(temp_path, "r") as tf:
+                content = tf.read()
+            self.assertEqual(content, "")
+
+    def test_temp_file_cleanup(self):
+        with temp_file("tests/files/file.txt") as temp_path:
+            pass  # Do nothing
+        self.assertFalse(os.path.exists(temp_path))
+        
+class TestJobDownloaderFlatten(unittest.TestCase):
+
+    def test_pad_job_id(self):
+        input = ("1234",)
+        result = flatten(input)
+        self.assertEqual(result, [{"job_id": "01234", "task_ids":None}])
+
+    def test_several_job_ids(self):
+        input = ("1234","1235","1236")
+        result = flatten(input)
+        self.assertEqual(result, [
+            {"job_id": "01234", "task_ids":None},
+            {"job_id": "01235", "task_ids":None},
+            {"job_id": "01236", "task_ids":None}
+            ])
+
+    def test_job_and_tasks(self):
+        input = ("1234:1-7x2,10",)
+        result = flatten(input)
+        self.assertEqual(result, [{"job_id": "01234", "task_ids":["001","003","005","007","010"]}])
+
+    def test_several_job_and_tasks(self):
+        input = ("1234:1-7x2,10","1235:12-15")
+        result = flatten(input)
+        self.assertEqual(result, [
+            {"job_id": "01234", "task_ids":["001","003","005","007","010"]},
+            {"job_id": "01235", "task_ids":["012","013","014","015"]}
+            ])
+
+    def test_mix_job_and_job_with_tasks(self):
+        input = ("1234","1235:12-15")
+        result = flatten(input)
+        self.assertEqual(result, [
+            {"job_id": "01234", "task_ids":None},
+            {"job_id": "01235", "task_ids":["012","013","014","015"]}
+            ])
+
+    def test_invalid_range_downloads_whole_job(self):
+        # Someone might have a bunch of stuff queued up and made a mistake and left for the night.
+        # We should download the whole job in this case so they don't have to restart the dl in the
+        # morning.
+        input = ("1234:badrange",)
+        result = flatten(input)
+        self.assertEqual(result, [
+            {"job_id": "01234", "task_ids":None}
+            ])
+
+class TestJobDownloaderGetSomeTasks(unittest.TestCase):
+    def setUp(self):
+        # Create a mock instance of JobDownloader
+        self.obj = JobDownloader(jobs=["00000"])
+
+    @mock.patch("ciocore.downloader.job_downloader.logger.error")
+    def test_get_some_tasks_error(self, mock_error):
+        # Mock the api client to return an error
+        mock_api_client = mock.Mock()
+        response = '{"error": "some error message"}'
+        code = 404
+        mock_api_client.make_request.return_value = (response, code)
+        self.obj.client = mock_api_client
+        # call get_some_tasks
+        tasks, locator = self.obj.get_some_tasks({})
+
+        self.assertEqual(tasks, [])
+        self.assertIsNone(locator)
+        mock_error.assert_called_with(
+            'Error fetching download info for job ID: %s : %s : %s', '00000', '/jobs/00000/downloads', mock.ANY)
+
+    def test_get_some_tasks_success(self):
+        mock_api_client = mock.Mock()
+        # Mock the api client to return a batch of tasks and a next cursor
+        response = '{"downloads": [{"id": 1, "name": "task1"}, {"id": 2, "name": "task2"}], "next_cursor": "1234"}'
+        code = 201
+        mock_api_client.make_request.return_value = (response, code)
+        self.obj.client = mock_api_client
+        self.obj.jobs = [{"job_id": "00000", "task_ids": [1, 2]}]
+        # call get_some_tasks with a locator
+        tasks, locator = self.obj.get_some_tasks({"job_index": 0, "cursor": None})
+
+        self.assertEqual(tasks, [{"id": 1, "name": "task1"}, {"id": 2, "name": "task2"}])
+        self.assertEqual(locator, {"job_index": 0, "cursor": "1234"})

tests/test_hardware_set.py

@@ -3,7 +3,7 @@
    isort:skip_file
 """
 import unittest
-from unittest.mock import patch
+
 
 from ciocore.hardware_set import HardwareSet
 
@@ -15,7 +15,12 @@ PROJECTS = [
 ]
 
 from package_fixtures import *
-from instance_type_fixtures import CW_INSTANCE_TYPES, CW_INSTANCE_TYPES_WITH_GPUS, ALL_INSTANCE_TYPES
+from instance_type_fixtures import (
+    CW_INSTANCE_TYPES,
+    AWS_INSTANCE_TYPES,
+    CW_INSTANCE_TYPES_WITH_GPUS,
+    ALL_INSTANCE_TYPES,
+)
 
 
 class TestCategorizedInstanceTypes(unittest.TestCase):
@@ -31,20 +36,29 @@ class TestCategorizedInstanceTypes(unittest.TestCase):
 
     def test_content_count(self):
         low_category_values = [c["value"] for c in self.hs.get_model()[0]["content"]]
-        self.assertEqual(low_category_values, ["a-4-16", "b-8-16"])
+        self.assertEqual(low_category_values, ["cw-a-4-16", "cw-b-8-16"])
 
     def test_in_several_categories(self):
         low_category_values = [c["value"] for c in self.hs.get_model()[0]["content"]]
         extra_category_values = [c["value"] for c in self.hs.get_model()[3]["content"]]
-        self.assertIn("a-4-16", low_category_values)
-        self.assertIn("a-4-16", extra_category_values)
+        self.assertIn("cw-a-4-16", low_category_values)
+        self.assertIn("cw-a-4-16", extra_category_values)
+
+    def test_category_names(self):
+        names = self.hs.labels()
+        self.assertEqual(names, ["low", "mid", "high", "extra"])
+
 
 class TestRecategorizeInstanceTypes(unittest.TestCase):
     def setUp(self):
         self.hs = HardwareSet(CW_INSTANCE_TYPES_WITH_GPUS)
-        
+
     def test_recategorize_cpu_gpu(self):
-        test_func = lambda x: [{'label': 'GPU', 'order': 2}] if "gpu" in x and x["gpu"] else [{'label': 'CPU', 'order': 1}]
+        test_func = (
+            lambda x: [{"label": "GPU", "order": 2}]
+            if "gpu" in x and x["gpu"]
+            else [{"label": "CPU", "order": 1}]
+        )
         self.hs.recategorize(test_func)
         self.assertEqual(self.hs.number_of_categories(), 2)
 
@@ -53,18 +67,40 @@ class TestRecategorizeInstanceTypes(unittest.TestCase):
         result = self.hs.find_all(test_func)
         self.assertEqual(len(result), 2)
         print(self.hs)
-        
+
     def test_find_first_by_condition(self):
-        test_func = lambda x: x[ "memory"] < 32
+        test_func = lambda x: x["memory"] < 32
         result = self.hs.find_first(test_func)
         self.assertTrue(result["memory"] < 32)
-        
+
     def test_find_category(self):
         label = "mid"
         result = self.hs.find_category(label)
-        self.assertEqual(result["label"] , label)
-        self.assertEqual(result["order"] , 2)
-        
+        self.assertEqual(result["label"], label)
+        self.assertEqual(result["order"], 2)
+
+
+class TestFind(unittest.TestCase):
+    def setUp(self):
+        self.hs = HardwareSet(CW_INSTANCE_TYPES_WITH_GPUS)
+
+    def test_find_unspecified_category(self):
+        result = self.hs.find("cw-e-4-32")
+        self.assertIsNotNone(result)
+
+    def test_category_that_exists(self):
+        result = self.hs.find("cw-e-4-32", category="high")
+        self.assertIsNotNone(result)
+
+    def test_returns_none_if_not_in_category(self):
+        result = self.hs.find("cw-e-4-32", category="low")
+        self.assertIsNone(result)
+
+    def test_returns_none_if_non_existent_category(self):
+        result = self.hs.find("cw-e-4-32", category="foo")
+        self.assertIsNone(result)
+
+
 class TestUncategorizedInstanceTypes(unittest.TestCase):
     def setUp(self):
         self.hs = HardwareSet(ALL_INSTANCE_TYPES)
@@ -78,12 +114,26 @@ class TestUncategorizedInstanceTypes(unittest.TestCase):
         self.assertEqual(
             result,
             [
-                'windows 4 core 26.0GB Mem',
-                'linux 4 core 27.0GB Mem',
-                'linux 8 core 30.0GB Mem',
-                'windows 32 core 208.0GB Mem',
-                'linux 32 core 208.0GB Mem',
-                'windows 64 core 416.0GB Mem',
-                'linux 64 core 416.0GB Mem'
+                "windows 4 core 26.0GB Mem",
+                "linux 4 core 27.0GB Mem",
+                "linux 8 core 30.0GB Mem",
+                "windows 32 core 208.0GB Mem",
+                "linux 32 core 208.0GB Mem",
+                "windows 64 core 416.0GB Mem",
+                "linux 64 core 416.0GB Mem",
             ],
         )
+
+
+class TestProvider(unittest.TestCase):
+    def test_gcp_provider(self):
+        hs = HardwareSet(ALL_INSTANCE_TYPES)
+        self.assertEqual(hs.provider, "gcp")
+
+    def test_cw_provider(self):
+        hs = HardwareSet(CW_INSTANCE_TYPES_WITH_GPUS)
+        self.assertEqual(hs.provider, "cw")
+
+    def test_aws_provider(self):
+        hs = HardwareSet(AWS_INSTANCE_TYPES)
+        self.assertEqual(hs.provider, "aws")

tests/test_job_downloader.py

@@ -0,0 +1,213 @@
+from ciocore.downloader.job_downloader import JobDownloader, flatten
+import unittest
+from unittest import mock
+import re
+import json
+
+DOWNLOAD_BATCHES = {
+    "00000": [
+        {
+            "next_cursor": "00000_1",
+            "downloads": [
+                {
+                    "download_id": 1,
+                    "files": [
+                        {"url": "https://example.com/file1", "size": 100},
+                        {"url": "https://example.com/file2", "size": 200},
+                    ],
+                },
+                {
+                    "download_id": 2,
+                    "files": [
+                        {"url": "https://example.com/file3", "size": 100},
+                        {"url": "https://example.com/file4", "size": 200},
+                    ],
+                },
+            ],
+        },
+        {
+            "next_cursor": "00000_2",
+            "downloads": [
+                {
+                    "download_id": 3,
+                    "files": [
+                        {"url": "https://example.com/file5", "size": 100},
+                        {"url": "https://example.com/file6", "size": 200},
+                    ],
+                },
+                {
+                    "download_id": 4,
+                    "files": [
+                        {"url": "https://example.com/file7", "size": 100},
+                        {"url": "https://example.com/file8", "size": 200},
+                    ],
+                },
+            ],
+        },
+        {
+            "next_cursor": None,
+            "downloads": [
+                {
+                    "download_id": 5,
+                    "files": [
+                        {"url": "https://example.com/file9", "size": 100},
+                        {"url": "https://example.com/file10", "size": 200},
+                    ],
+                }
+            ],
+        },
+    ],
+    "00001": [
+        {
+            "next_cursor": "00001_1",
+            "downloads": [
+                {
+                    "download_id": 101,
+                    "files": [
+                        {"url": "https://example.com/file101", "size": 100},
+                        {"url": "https://example.com/file102", "size": 200},
+                    ],
+                },
+                {
+                    "download_id": 102,
+                    "files": [
+                        {"url": "https://example.com/file103", "size": 100},
+                        {"url": "https://example.com/file104", "size": 200},
+                    ],
+                },
+            ],
+        },
+        {
+            "next_cursor": "00001_2",
+            "downloads": [
+                {
+                    "download_id": 103,
+                    "files": [
+                        {"url": "https://example.com/file105", "size": 100},
+                        {"url": "https://example.com/file106", "size": 200},
+                    ],
+                },
+                {
+                    "download_id": 104,
+                    "files": [
+                        {"url": "https://example.com/file107", "size": 100},
+                        {"url": "https://example.com/file108", "size": 200},
+                    ],
+                },
+            ],
+        },
+        {
+            "next_cursor": None,
+            "downloads": [
+                {
+                    "download_id": 105,
+                    "files": [
+                        {"url": "https://example.com/file109", "size": 100},
+                        {"url": "https://example.com/file110", "size": 200},
+                    ],
+                }
+            ],
+        },
+    ],
+}
+
+class MockedApiClient:
+    def _get_batch(self, cursor):
+        if not cursor:
+            return json.dumps(DOWNLOAD_BATCHES["00000"][0])
+        
+        job_id, index = cursor.split("_")
+        index = int(index)
+        batch = json.dumps(DOWNLOAD_BATCHES[job_id][index])
+        return batch
+
+    def make_request(self, url, verb="POST", params=None, data=None, use_api_key=True):
+        m = re.match("/jobs/(\d{5})/downloads", url)
+        if m:
+            job_id = m.group(1)
+            # This is a request for a download batch
+            if not job_id in DOWNLOAD_BATCHES:
+                print(f"Job {job_id} not found")
+                return None, 204
+            else:
+                print(f"Job {job_id} found")
+            
+            cursor = params.get("start_cursor", None)
+            batch = self._get_batch(cursor)
+            return batch, 201
+        else:
+            # This is not a request for a download batch
+            raise NotImplementedError("This is not a request for a download batch")
+
+
+class TestJobDownloader(unittest.TestCase):
+    
+    
+    def setUp(self):
+        download_tasks_patcher = mock.patch.object(JobDownloader,"download_tasks")
+        self.mock_download_tasks = download_tasks_patcher.start()
+        self.addCleanup(download_tasks_patcher.stop)
+    
+
+    def test_batch_all_batches(self):
+        mocked_jobs = DOWNLOAD_BATCHES.keys()
+
+        # instantiate
+        jdl = JobDownloader(mocked_jobs)
+
+        # set the mocked api client
+        jdl.client = MockedApiClient()
+
+        jdl.run()
+        self.assertEqual(self.mock_download_tasks.call_count, 6)
+        
+#     def test_no_calls_if_non_existent_job(self):
+#         mocked_jobs = ["00002"]
+# ``
+#         # instantiate
+#         jdl = JobDownloader(mocked_jobs)
+
+#         # set the mocked api client
+#         jdl.client = MockedApiClient()
+
+#         jdl.run()
+#         self.assertEqual(self.mock_download_tasks.call_count, 0)
+
+
+        # mock_get_some_tasks.side_effect = get_some_tasks_side_effect
+
+        # tasks, locator = jdl.get_some_tasks(None)
+        # self.assertEqual(tasks, ["t1,t2,t3"])
+        # self.assertEqual(locator, {"job_index": 0, "cursor": "a"})
+
+        # print(jdl.get_some_tasks(2))
+        # print(jdl.get_some_tasks(3))
+
+        #  with mock.patch('requests.get') as mock_get:
+        #     # Construct the expected JSON response
+        #     json_response = {
+        #         "next_cursor": "AA",
+        #         "downloads": [
+        #             {
+        #                 "download_id": 1,
+        #                 "files": [
+        #                     {
+        #                         "url": "https://example.com/file1",
+        #                         "size": 100
+        #                     },
+        #                     {
+        #                         "url": "https://example.com/file2",
+        #                         "size": 200
+        #                     }
+        #                 ]
+        #             }
+        #         ],
+        #         "job_id": "00000",
+        #         "output_dir": "/some/path",
+        #         "size": 300,
+        #         "task_id": "001"
+        #     }
+        #     # Create a mock response object
+        #     mock_response = mock_get.return_value
+        #     # Set the JSON content of the response
+        #     mock_response.json.return_value = json_response

tests/test_submit.py

@@ -4,7 +4,14 @@
 """
 
 import unittest
-from unittest.mock import patch, MagicMock
+
+try:
+    from unittest.mock import MagicMock, patch
+except ImportError:
+    from mock import MagicMock, patch
+    
+
+
 def get_required_args():
     return {
         "instance_type": "inst",
@@ -136,4 +143,4 @@ class uploaderTest():
         up = uploader.Uploader(uploader_args)
         manager = up.create_manager("some_project")
         filemap = {"/a/b1": None,"/a/b2": None}
-        self.assertTrue(True)
+        # self.assertEqual(uploader_args["thread_count"], 4)

ciocore/__about__.py

@@ -1,10 +0,0 @@
-import os
-
-__all__ = ["__version__"]
-
-try:
-    with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'VERSION')) as version_file:
-        __version__ = version_file.read().strip()
-        
-except IOError:
-    __version__ = 'dev'

ciocore/cli/__init__.py

@@ -1,3 +0,0 @@
-"""
-Commandline tools.
-"""