ciocore 9.1.0b2__py2.py3-none-any.whl → 9.1.0rc2__py2.py3-none-any.whl

tests/test_data.py

@@ -1,163 +1,173 @@
-""" test data
-
-   isort:skip_file
-"""
-
 import unittest
+from unittest.mock import patch
+from ciocore import data as coredata
 
-try:
-    from unittest import mock
-except ImportError:
-    import mock
-
-from ciocore import data
-
-
-from project_fixtures import PROJECTS
-from package_fixtures import SOFTWARE_DATA
-from instance_type_fixtures import LIN_INSTANCE_TYPES, ALL_INSTANCE_TYPES
-
-class TestDataAllInstanceTypes(unittest.TestCase):
+class TestCoreData(unittest.TestCase):
     def setUp(self):
-        self.default_cache = {
-            "projects": PROJECTS,
-            "instance_types": ALL_INSTANCE_TYPES,
-            "packages": SOFTWARE_DATA,
+        """Set up test fixtures before each test method."""
+        self.mock_api_responses = {
+            'projects': ['ProjectA', 'ProjectB'],
+            'instance_types': [
+                {'operating_system': 'windows', 'name': 'win_machine', 'cores': 1, 'memory': '1024'},
+                {'operating_system': 'linux', 'name': 'linux_machine', 'cores': 1, 'memory': '1024'}
+            ],
+            'software': [  # Added package_id to each package
+                {
+                    'name': 'maya-io',
+                    'version': '2022',
+                    'platform': 'windows',
+                    'product': 'maya-io',
+                    'package_id': '123456789'
+                },
+                {
+                    'name': 'maya-io',
+                    'version': '2022',
+                    'platform': 'linux',
+                    'product': 'maya-io',
+                    'package_id': '123456780'
+                },
+                {
+                    'name': 'nuke',
+                    'version': '13.0',
+                    'platform': 'linux',
+                    'product': 'nuke',
+                    'package_id': '123456781'
+                }
+            ],
+            'extra_env': ['PATH=/custom/path', 'LICENSE_SERVER=1234']
         }
-
-        # self.get_data_patcher = mock.patch.object(DataCache, "get_data", return_value=self.default_cache)
-
-        # self.account_id_patcher = mock.patch(
-        #     "ciocore.data.get_account_id", return_value="1234"
-        # )
-
-        # self.mock_get_data = self.get_data_patcher.start()
-        # self.mock_account_id = self.account_id_patcher.start()
-
-        data.__data__ = {}
-        data.__products__ = None
+        # Clear data before each test
+        coredata.clear()
 
     def tearDown(self):
-        # self.get_data_patcher.stop()
-        # self.account_id_patcher.stop()
-        pass
-
-    # def override_default_cache(self, **kwargs):
-    #     return_val = {}
-    #     return_val.update(self.default_cache)
-    #     return_val.update(kwargs)
-        # self.mock_get_data.return_value = return_val
-
-    # def test_smoke(self):
-    #     self.assertEqual(1, 1)
-
-    # def test_init_empty_projects_global(self):
-    #     data.init()
-    #     self.assertEqual(data.products(), [])
-
-    # def test_init_stores_all_args(self):
-    #     data.init("a", "b", "c")
-    #     self.assertEqual(data.products(), ["a", "b", "c"])
-
-    # def test_data_raises_if_not_initialized(self):
-    #     with self.assertRaises(ValueError):
-    #         data.data()
-
-    # def test_valid(self):
-    #     self.assertEqual(data.valid(), False)
-    #     data.init()
-    #     data.data()
-    #     self.assertEqual(data.valid(), True)
-
-    # def test_clear(self):
-    #     data.init()
-    #     data.data()
-    #     self.assertEqual(data.valid(), True)
-    #     data.clear()
-    #     self.assertEqual(data.valid(), False)
-
-    # def test_does_not_refresh_if_not_force(self):
-    #     data.init()
-    #     p1 = data.data()["projects"]
-    #     self.assertEqual(len(p1), 4)
-    #     # merge the default cache with a new projects list containing only two projects
-    #     # self.override_default_cache(projects=["a", "b"])
-    #     p2 = data.data()["projects"]
-    #     self.assertEqual(p2, p1)
-
-    # def test_does_refresh_if_force_all(self):
-    #     data.init()
-    #     p1 = data.data()["projects"]
-    #     self.assertEqual(len(p1), 4)
-    #     # self.override_default_cache(projects=["a", "b"])
-    #     p2 = data.data(force=True)["projects"]
-    #     self.assertNotEqual(p2, p1)
-    #     self.assertEqual(len(p2), 2)
-
-    # def test_get_data_for_one_product(self):
-    #     data.init("c4d")
-    #     # inst = data.data()["instance_types"]
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 2)
-
-    # def test_several_products(self):
-    #     data.init("c4d", "maya")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 5)
-
-    # def test_one_product_can_be_a_plugin(self):
-    #     data.init("redshift")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 4)
-
-    # def test_several_products_ignore_plugins(self):
-    #     data.init("redshift", "c4d")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 2)
-
-    # def test_linux_only_instance_types(self):
-    #     # self.override_default_cache(instance_types=LIN_INSTANCE_TYPES)
-    #     data.init()
-    #     h = data.data()["instance_types"]
-    #     self.assertEqual(len(h.instance_types.keys()), 4)
-
-    # def test_linux_only_packages_when_linux_only_instance_types(self):
-    #     # self.override_default_cache(instance_types=LIN_INSTANCE_TYPES)
-    #     data.init("c4d")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 1)
-
-    # def test_platforms_method_only_linux(self):
-    #     self.override_default_cache(instance_types=LIN_INSTANCE_TYPES)/
-    #     data.init("c4d")
-    #     data.data()
-    #     self.assertEqual({"linux"}, data.platforms())
-
-    # def test_many_products(self):
-    #     self.override_default_cache(instance_types=LIN_INSTANCE_TYPES)
-    #     data.init("c4d", "maya")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 4)
-
-    # def test_product_keyword_translates_to_single_arg(self):
-    #     data.init(product="c4d")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 2)
-
-    # def test_product_keyword_all_translates_to_all_products(self):
-    #     data.init(product="all")
-    #     sw = data.data()["software"]
-    #     self.assertEqual(len(sw.supported_host_names()), 11)
-
-    # def test_data_raises_if_both_products_and_kwarg_given(self):
-    #     with self.assertRaises(ValueError):
-    #         data.init("maya", product="c4d")
-
-    # def test_data_raises_if_products_and_kwarg_given(self):
-    #     with self.assertRaises(ValueError):
-    #         data.init("maya", product="c4d")
-
-class TestDataSmoke(unittest.TestCase):
-    
-    def test_smoke(self):
-        self.assertTrue(True)
+        """Clean up after each test method."""
+        coredata.clear()
+
+    def test_init_with_single_product(self):
+        """Test initializing with a single product"""
+        coredata.init("maya-io")
+        self.assertEqual(coredata.products(), ["maya-io"])
+        self.assertEqual(coredata.platforms(), set(["windows", "linux"]))
+
+    def test_init_with_multiple_products(self):
+        """Test initializing with multiple products"""
+        coredata.init("maya-io", "nuke")
+        self.assertEqual(coredata.products(), ["maya-io", "nuke"])
+
+    def test_init_with_deprecated_product(self):
+        """Test initializing with deprecated product kwarg"""
+        coredata.init(product="maya-io")
+        self.assertEqual(coredata.products(), ["maya-io"])
+
+    def test_init_with_all_products(self):
+        """Test initializing with 'all' products"""
+        coredata.init(product="all")
+        self.assertEqual(coredata.products(), [])
+
+    def test_init_conflict_raises_error(self):
+        """Test that using both products arg and product kwarg raises error"""
+        with self.assertRaises(ValueError):
+            coredata.init("maya-io", product="nuke")
+
+    def test_data_requires_init(self):
+        """Test that data() raises error if not initialized"""
+        with self.assertRaises(ValueError):
+            coredata.data()
+
+    @patch('ciocore.data.api_client')
+    def test_data_fetching(self, mock_client):
+        """Test fetching data through the data() function"""
+        # Setup mock responses
+        mock_client.request_projects.return_value = self.mock_api_responses['projects']
+        mock_client.request_instance_types.return_value = self.mock_api_responses['instance_types']
+        mock_client.request_software_packages.return_value = self.mock_api_responses['software']
+        mock_client.request_extra_environment.return_value = self.mock_api_responses['extra_env']
+
+        coredata.init("maya-io")
+        result = coredata.data()
+        
+        self.assertIn("projects", result)
+        self.assertIn("instance_types", result)
+        self.assertIn("software", result)
+        self.assertIn("extra_environment", result)
+        
+        self.assertEqual(result["projects"], self.mock_api_responses["projects"])
+        self.assertIsInstance(result["instance_types"], coredata.HardwareSet)
+        self.assertIsInstance(result["software"], coredata.PackageTree)
+
+    @patch('ciocore.data.api_client')
+    def test_force_refresh(self, mock_client):
+        """Test forcing data refresh"""
+        # Setup mock responses
+        mock_client.request_projects.return_value = self.mock_api_responses['projects']
+        mock_client.request_instance_types.return_value = self.mock_api_responses['instance_types']
+        mock_client.request_software_packages.return_value = self.mock_api_responses['software']
+        mock_client.request_extra_environment.return_value = self.mock_api_responses['extra_env']
+
+        coredata.init("maya-io")
+        first_data = coredata.data()
+        
+        # Second call without force should not make new API calls
+        mock_client.request_projects.reset_mock()
+        second_data = coredata.data()
+        mock_client.request_projects.assert_not_called()
+        
+        # Call with force=True should make new API calls
+        third_data = coredata.data(force=True)
+        mock_client.request_projects.assert_called_once()
+
+    def test_valid_state(self):
+        """Test valid() function behavior"""
+        self.assertFalse(coredata.valid())  # Should be invalid before initialization
+        
+        coredata.init("maya-io")
+        self.assertFalse(coredata.valid())  # Should be invalid before data is fetched
+        
+        with patch('ciocore.data.api_client') as mock_client:
+            # Setup mock responses
+            mock_client.request_projects.return_value = self.mock_api_responses['projects']
+            mock_client.request_instance_types.return_value = self.mock_api_responses['instance_types']
+            mock_client.request_software_packages.return_value = self.mock_api_responses['software']
+            mock_client.request_extra_environment.return_value = self.mock_api_responses['extra_env']
+            
+            coredata.data()  # Fetch data
+            self.assertTrue(coredata.valid())  # Should be valid after data is fetched
+        
+        coredata.clear()
+        self.assertFalse(coredata.valid())  # Should be invalid after clear
+
+    @patch('ciocore.data.api_client')
+    def test_platforms_filtering(self, mock_client):
+        """Test platform filtering based on instance types and software"""
+        # Setup mock responses
+        mock_client.request_projects.return_value = self.mock_api_responses['projects']
+        mock_client.request_instance_types.return_value = self.mock_api_responses['instance_types']
+        mock_client.request_software_packages.return_value = self.mock_api_responses['software']
+        mock_client.request_extra_environment.return_value = self.mock_api_responses['extra_env']
+
+        coredata.init("nuke")  # nuke only supports linux
+        coredata.data()
+        
+        # Should only have linux instance types since nuke only supports linux
+        self.assertEqual(coredata.platforms(), {"linux"})
+
+    @patch('ciocore.data.api_client')
+    def test_instances_filter(self, mock_client):
+        """Testing that instance_types is passed correctly to api_client"""
+        # Set up mock response
+        mock_client.request_instance_types.return_value = self.mock_api_responses['instance_types']
+        coredata.init("maya-io")
+
+        # Test with no filter
+        coredata.data()
+        mock_client.request_instance_types.assert_called_with(filter_param="")
+        
+        # Test with filter and force=True
+        mock_client.request_instance_types.reset_mock()
+
+        instances_filter = "gpu.gpu_count=gte:1:int"
+        coredata.data(force=True, instances_filter=instances_filter)
+        mock_client.request_instance_types.assert_called_with(filter_param=instances_filter)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_submit.py

@@ -5,10 +5,7 @@
 
 import unittest
 
-try:
-    from unittest.mock import MagicMock, patch
-except ImportError:
-    from mock import MagicMock, patch
+from unittest.mock import MagicMock, patch
 
 
 def get_required_args():

tests/test_uploader.py

@@ -1,10 +1,7 @@
 import unittest
 import sys
 
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from unittest import mock
 
 PY3 = sys.version_info >= (3, 0)
 BUILTIN_OPEN = "builtins.open" if PY3 else "__builtin__.open"

ciocore/compat.py

@@ -1,15 +0,0 @@
-import sys
-if sys.version_info < (3,):
-    import codecs
-    def u(x):
-        return codecs.unicode_escape_decode(x)[0]
-
-    text_type = unicode
-    binary_type = str
-else:
-    def u(x):
-        return str(x)
-
-    text_type = str
-    binary_type = bytes
-

ciocore/uploader/thread_queue_job.py

@@ -1,101 +0,0 @@
-import logging
-from ciocore import loggeria
-
-logger = logging.getLogger("{}.uploader".format(loggeria.CONDUCTOR_LOGGER_NAME))
-
-class ThreadQueueJob():
-    pass
-
-class UploadThreadQueueJob(ThreadQueueJob):
-    
-    def __init__(self, path, file_size, presigned_url, file_md5=None, upload_id=None, part_size=None, total_parts=1, part_index=1, kms_key_name=None):
-
-        super().__init__()
-
-        self.path = path
-        self.file_size = file_size 
-        self.upload_id = upload_id
-        self.presigned_url = presigned_url
-        self.file_md5 = file_md5
-        self.part_size = part_size
-        self.part_index = part_index
-        self.total_parts = total_parts
-        self.kms_key_name = kms_key_name
-
-        logger.info("Creating %s (%s): %s", str(self.__class__), str(self), str(self.__dict__))
-
-    def is_multipart(self):
-        return self.total_parts != 1
-
-    def is_vendor_aws(self):
-        return "amazonaws" in self.presigned_url
-    
-    def is_vendor_cw(self):
-        return "coreweave" in self.presigned_url    
-
-    @classmethod
-    def create_from_response(cls, response):
-
-        new_thread_queue_jobs = []
-
-        for part_type, file_request_list in response.items():
-
-            for file_request in file_request_list:
-                if part_type == "multiPartURLs":
-                        
-                        for part in file_request["parts"]:
-                            new_tqj = cls( path=file_request['filePath'],
-                                           file_size = file_request['filePath'],
-                                           presigned_url = file_request['preSignedURL'],
-                                           file_md5  = file_request['preSignedURL'],
-                                           upload_id = file_request['preSignedURL'],
-                                           part_size = file_request['preSignedURL'],
-                                           part_index = file_request['preSignedURL'])
-                            
-
-                else:
-                    new_tqj = cls( path=file_request['filePath'],
-                                   file_size = file_request['filePath'],
-                                   presigned_url = file_request['preSignedURL'])
-                    
-        new_thread_queue_jobs.append(new_tqj)
-
-
-
-class MultiPartThreadQueueJob(ThreadQueueJob):
-    
-    def __init__(self, path, md5, total_parts=1, part_index=1):
-
-        super().__init__()
-
-        self.upload_id = None
-        self.md5 = md5
-        self.project = None
-        self.path = path
-        self.part_index = part_index
-        self.etag = None
-        self.total_parts = total_parts
-
-        logger.info("Creating %s (%s): %s", str(self.__class__), str(self), str(self.__dict__))
-
-    def is_multipart(self):
-        return self.total_parts != 1
-    
-    # def __str__(self):
-    #     return
-
-    @staticmethod
-    def aggregate_parts(parts):
-        """
-        Helper function to take all the parts of a multipart upload and put 
-        them into a format that's expected for the HTTP call.
-        """
-
-        completed_parts_payload = []
-        
-        for part in parts:
-            completed_parts_payload.append({'partNumber': part.part,
-                                                       'etag': part.etag}
-                                                      )
-            
-        return completed_parts_payload

tests/extra_env_fixtures.py

@@ -1,57 +0,0 @@
-
-
-EXTRA_ENV = [
-    {
-      "account_id": "6649535867387904",
-      "env": [],
-    },
-    {
-      "account_id": "5767615549800448",
-      "env": [
-        {
-          "merge_policy": "prepend",
-          "name": "test",
-          "value": "test"
-        }
-      ],
-    },
-    {
-      "account_id": "6649535867387904",
-      "env": [],
-    },
-    {
-      "account_id": "5669544198668288",
-      "env": [
-        {
-          "merge_policy": "append",
-          "name": "PATH",
-          "value": "/path/to/scripts"
-        },
-        {
-          "merge_policy": "exclusive",
-          "name": "RENDER_LOCATION",
-          "value": "cloud"
-        },
-        {
-          "merge_policy": "exclusive",
-          "name": "VARIABLE_USED_IN_SCRIPTS",
-          "value": "true"
-        },
-        {
-          "merge_policy": "exclusive",
-          "name": "testvar",
-          "value": "somevalue"
-        }
-      ],
-    },
-    {
-      "account_id": "6649535867387904",
-      "env": [
-        {
-          "merge_policy": "exclusive",
-          "name": "JMEYER",
-          "value": "JMEYER_ENV_VALUE"
-        }
-      ],
-    }
-  ]

tests/project_fixtures.py

@@ -1,8 +0,0 @@
-
-
-PROJECTS = [
-    "Deadpool",
-    "Harry Potter & the chamber of secrets",
-    "Captain Corelli's Mandolin",
-    "Gone with the Wind",
-]