ciocore 6.1.0__py2.py3-none-any.whl → 6.3.0__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-6.1.0
+6.3.0

ciocore/hardware_set.py

@@ -1,31 +1,46 @@
 
 import copy
+import logging
 
-MISC_CATEGORY_LABEL = "-- Misc --"
+logger = logging.getLogger(__name__)
 
 class HardwareSet(object):
     """A class to manage the categorized instance types.
     
     It takes flat instance types array and builds a nested structure where instance types exist in
     categories.
+    
+    If after categorization there is only one category and it is the misc category, then we
+    recategorize by cpu and gpu.
     """
 
     def __init__(self, instance_types):
+
         self.instance_types = self.build_unique(instance_types)
         self.categories = self.build_categories()
-        
+
     @staticmethod
     def build_unique(instance_types):
         """Build a dictionary of instance types using name as key.
         
         Remove any instance types whose description has already been seen. We can't have duplicate
         descriptions.
+        
+        If categories exist, then remove any instance types that don't have a category. Otherwise,
+        since there are no categories, we can't remove any instance types.
         """
+        categories = [category for it in instance_types for category in (it.get("categories") or [])]
         result = {}
         seen_descriptions = set() 
         for it in instance_types:
             if it["description"] in seen_descriptions:
                 continue
+            if categories:
+                if it.get("categories") in [[], None]:
+                    continue
+            else:
+                # make our own categories GPU/CPU
+                it["categories"] = [{'label': 'GPU', 'order': 2}] if "gpu" in it and it["gpu"] else [{'label': 'CPU', 'order': 1}]
             result[it["name"]] = it
             seen_descriptions.add(it["description"])
         return result
@@ -37,9 +52,7 @@ class HardwareSet(object):
         dikt = {}
         for key in self.instance_types:
             it = self.instance_types[key]
-            categories = it.get("categories") or []
-            if not categories:
-                categories.append({"label": MISC_CATEGORY_LABEL, "order": 999})
+            categories = it["categories"]
             for category in categories:
                 label = category["label"]
                 if label not in dikt:
@@ -53,29 +66,78 @@ class HardwareSet(object):
             result.append(category)
         return sorted(result, key=lambda k: k["order"])
  
- 
+    def recategorize(self, partitioner):
+        """Recategorize the instance types.
+
+        Partitioner is a function that takes an instance type and returns a list of categories.
+        
+        If for example you want to append to the existing categories, then you can use a function like this:
+        lambda x: x["categories"] + [{'label': 'Low cores', 'order': 10}] if  x["cores"] < 16 else [{'label': 'High cores', 'order': 20}]
+        
+        Rebuilds the categories structure after assigning the new categories.
+        """
+        for key in self.instance_types:
+            self.instance_types[key]["categories"] = partitioner(self.instance_types[key])
+        self.categories = self.build_categories()
+
+
     def get_model(self, with_misc=False):
-        """Returns the categories structure with filtering and renaming ready for some UI.
+        """Returns the categories structure with renaming ready for some UI.
 
-        with_misc: Include misc category. The misc category is always included if it's the only one.
+        NOTE: THIS METHOD WILL BE DEPRECATED. with_misc is no longer used, which means that this
+        function just renames a few keys. What's more, the init function ensures that every instance
+        type has a (non-misc) category. So this function is no longer needed. Submitters that use
+        it will work fine, but should be updated to use the categories structure directly. 
         """
+        if with_misc:
+            logger.warning("with_misc is no longer used")
         result = []
-        should_remove_misc = not with_misc and len( self.categories) > 1
-
         for category in self.categories:
-            if should_remove_misc and category["label"] == MISC_CATEGORY_LABEL:
-                continue
-
             result.append({
                 "label": category["label"],
-                "content": list(map(lambda k: {"label": k["description"], "value": k["name"]} , category["content"]))
+                "content": [{"label": k["description"], "value": k["name"]} for k in category["content"]]
             })
 
         return result
-   
+    
     def find(self, name):
+        """Find an instance type by name (sku).
+        
+        Example: find("p3.2xlarge")
+        
+        Returns and instance-type or None if not found.
+        """
         return self.instance_types.get(name)
     
+    def find_category(self, label):
+        """Find a category by label.
+        
+        Example: find_category("GPU")
+        
+        Returns the entire category and its contents or None if not found. 
+        """
+        return next((c for c in self.categories if c["label"] == label), None)
+    
+    
+    def find_all(self, condition):
+        """Find all instance types that match a condition.
+        
+        Example: find_all(lambda x: x["gpu"])
+        """
+        result = []
+        for key in self.instance_types:
+            if condition(self.instance_types[key]):
+                result.append(self.instance_types[key])
+        return result
+    
+    def find_first(self, condition):
+        """Find the first instance type that matches a condition.
+        
+        Example: find_first(lambda x: x["cores"] == 4)"])
+        """
+        return next(iter(self.find_all(condition)), None)
+    
     def number_of_categories(self):
+        """Return the number of categories in the data."""
         return len(self.categories)
     

{ciocore-6.1.0.dist-info → ciocore-6.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 6.1.0
+Version: 6.3.0
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -26,11 +26,6 @@ Use the command-line uploader and downloader or develop your own tools using the
 pip install --upgrade ciocore
 ```
 
-**To install a specific version, for example 0.1.0.**
-```bash
-pip install --upgrade ciocore==0.1.0
-```
-
 Run the conductor command to confirm the package was installed.
 ```bash
 conductor --help
@@ -48,13 +43,23 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
+## Version:6.3.0 -- 05 Jul 2023
+
+* In the absence of instance-type categories provided by the endpoint, the client generates default
+  categories CPU and GPU which at least splits the instance types into two groups, thereby making
+  dropdown menus more manageable, especially for CoreWeave's array of machines.
+* update circleci config and related documentation
+
 ## Version:6.1.0 -- 09 Jun 2023
 
-Add the ability to specify which platforms to get resources for. This helps consumers of the API to avoid fetching resources for Windows when the client is not ready.
+Add the ability to specify which platforms to get resources for. This helps consumers of the API to
+avoid fetching resources for Windows when the client is not ready.
 
 ## Version:6.0.0 -- 07 Jun 2023
 
-This is a breaking change due instance types being provided as an instance of HardwareSet with support for categories as opposed to a simple list. Consumers of the API will need to be updated to support this change.
+This is a breaking change due instance types being provided as an instance of HardwareSet with
+support for categories as opposed to a simple list. Consumers of the API will need to be updated to
+support this change.
 
 ## Version:6.0.0-rc.4 -- 06 Jun 2023
 

{ciocore-6.1.0.dist-info → ciocore-6.3.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ciocore/VERSION,sha256=ldru8b_Gwjy37St6Co1rJJjrYxhdPXqLQJkIrzMNpO8,5
+ciocore/VERSION,sha256=AcBmZzP8VI6iOQK01krLRTlT4Tu0al4tBEsVA7A-nUo,5
 ciocore/__about__.py,sha256=nTb4Xx0r9QtGROSFKgwDZ-Mr2LKjm2wVt1OkMQAkRAQ,241
 ciocore/__init__.py,sha256=zB_e7gBW2QloqbrGzcvccfpZhOhd6o9KACnScrwylm8,25
 ciocore/api_client.py,sha256=O9uvtnqgmE2IRkqlP9DoPn6CX_HAIuSLo9je5Om-800,17913
@@ -11,7 +11,7 @@ ciocore/data.py,sha256=RQZEYGkZrVFgyyPqzBat-L99sq49h-SQUnSdGIPDI18,8938
 ciocore/downloader.py,sha256=oRmRePdIx_GdQwM39ipl9_e2c2ZkcGJRanuHOZmqpTM,51147
 ciocore/exceptions.py,sha256=c1K48hWcD8VOORfNoSf5qlQkLLEqwjryH8FUwUYAYU4,1431
 ciocore/file_utils.py,sha256=bAlL31B4YkRgX-yT8kF8UXBFktQlsE1PvxbKqTeAeOU,17174
-ciocore/hardware_set.py,sha256=5qQg28OfuwG3NCfSjDxqcZPByW95PHwShsFhV2Qkb_Q,2771
+ciocore/hardware_set.py,sha256=txcSLrVSNewRGxKLye-wuM8szGMVloU29ktL8WHdUtM,5401
 ciocore/loggeria.py,sha256=dKKJC8ZtRZdghqD5R5XrA6eDoy8gKacfeTA-zNzXvDE,13482
 ciocore/package_environment.py,sha256=oEbNKXRtPSPzKR-yCoKtvgzu4OCmr-zaqAcNoLAN9Uk,7238
 ciocore/package_tree.py,sha256=kH03HVfjomj7nsaxJJtr-1KSQ_9ZSQY5msG_l9btvg8,16277
@@ -23,16 +23,16 @@ ciocore/auth/__init__.py,sha256=cdS-xZzMq41yXM5cz8sUlcYgo8CJYh8HcCCWmhbDgf0,606
 ciocore/auth/server.py,sha256=8btX9-EokUl6q55V8muDmEV2tvvbTBD0BHeWFbwkzUc,3892
 ciocore/cli/__init__.py,sha256=RmZKWJaMpzNyMdyYc2W3VXglaJiC8vyR2cgUlA-9Qmw,26
 ciocore/cli/conductor.py,sha256=snmlICkMgP94ZPKl6J7g299deB75QwGDWI0oCnZPfSI,10861
-ciocore-6.1.0.data/scripts/conductor,sha256=Nk3QsLQqbUUrtaKDp4b5mr9__4tz-xnssENpQe5vuIo,409
-ciocore-6.1.0.data/scripts/conductor.bat,sha256=T1_9ByheubBczgQZn8_LwfvMtWgE7Bt64EsEScnSXMs,447
+ciocore-6.3.0.data/scripts/conductor,sha256=Nk3QsLQqbUUrtaKDp4b5mr9__4tz-xnssENpQe5vuIo,409
+ciocore-6.3.0.data/scripts/conductor.bat,sha256=T1_9ByheubBczgQZn8_LwfvMtWgE7Bt64EsEScnSXMs,447
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/instance_type_fixtures.py,sha256=ajOfz_AO39Er1mKNN913oVcBVrW_El1ADBHQoYU6Pp4,2935
+tests/instance_type_fixtures.py,sha256=gR6ordfghp34I94oM2eDg6jfIe5HAwE644GKIwspuW8,3469
 tests/package_fixtures.py,sha256=CsJnhB7oYzIxJH7b1tCOPyvnnVSCqEbSPhtCnsHL-nA,5070
 tests/test_api_client.py,sha256=dWOmAIKmX0gLaDGsT9VfAq9Hcs9HIQf2P5IMM_Bt5dE,1284
 tests/test_common.py,sha256=lJpzRdL-7u4McXFbLuwPQQoUnuEOnCVQtZEt6e_dIYs,638
 tests/test_config.py,sha256=nSmpinX2SmDNAprIcxs9UHdB0VakJB0snXaZmAoKJSc,12863
 tests/test_data.py,sha256=YdP1kZJivQ6yb9z96UK6oMDaOfJAl4YMJqzKvlCQaes,5744
-tests/test_hardware_set.py,sha256=9MAXI2sTziTddfSnE4uIJmMfIAf5ijukPsO-q7XE4JY,2400
+tests/test_hardware_set.py,sha256=TcBh63rOxf1rKXxKlCPSnHueBFlz7rNP6BcoJjgVvPs,3065
 tests/test_imports_2and3.py,sha256=ehqpRYPVY7djBcb8OT_cnh86iCJJ9wuMWnfSR9RHxmY,507
 tests/test_package_environment.py,sha256=CdiC2PDVSnbcwTb4fsDTWqGYSzs1n5ca2KMoyISckGA,5893
 tests/test_package_tree.py,sha256=xCwNwYUmJrfmgCP2FGoHRFG-L0JPy8s4-66icxAls4o,6780
@@ -42,7 +42,7 @@ tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/api_client_mock.py,sha256=Wfv2JPFSZfyHftVqsqxcpXWJn136pEHx26I_esz567E,943
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-6.1.0.dist-info/METADATA,sha256=aU4R-bY3WcjkrDCqv08xp6PkfAKZ9FXtuNzZcPv-M4g,14026
-ciocore-6.1.0.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
-ciocore-6.1.0.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-6.1.0.dist-info/RECORD,,
+ciocore-6.3.0.dist-info/METADATA,sha256=Jf9qxoQUVb1qnmIgguLei22xyWZTXtoHIEYWBf9pe4w,14285
+ciocore-6.3.0.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
+ciocore-6.3.0.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-6.3.0.dist-info/RECORD,,

tests/instance_type_fixtures.py

@@ -105,7 +105,7 @@ CW_INSTANCE_TYPES = [
         "operating_system": "linux",
         "description": "Desc 8 C 32 M F",
     },
-        {
+    {
         "cores": 8,
         "memory": 64,
         "name": "g-8-32",
@@ -114,3 +114,24 @@ CW_INSTANCE_TYPES = [
         "description": "Desc 8 C 32 M G",
     },
 ]
+
+CW_INSTANCE_TYPES_WITH_GPUS = [
+    {
+        "cores": 8,
+        "memory": 64,
+        "name": "f-8-32-gpu",
+        "categories": [{"label": "high", "order": 3}],
+        "operating_system": "linux",
+        "description": "Desc 8 C 32 M F gpu",
+        "gpu": 1,
+    },
+    {
+        "cores": 8,
+        "memory": 64,
+        "name": "g-8-32-gpu",
+        "categories":  [{"label": "high", "order": 3}],
+        "operating_system": "linux",
+        "description": "Desc 8 C 32 M G gpu",
+        "gpu": 1,
+    },
+] + CW_INSTANCE_TYPES

tests/test_hardware_set.py

@@ -5,7 +5,7 @@
 import unittest
 from unittest.mock import patch
 
-from ciocore.hardware_set import MISC_CATEGORY_LABEL, HardwareSet
+from ciocore.hardware_set import HardwareSet
 
 PROJECTS = [
     "Deadpool",
@@ -15,7 +15,7 @@ PROJECTS = [
 ]
 
 from package_fixtures import *
-from instance_type_fixtures import *
+from instance_type_fixtures import CW_INSTANCE_TYPES, CW_INSTANCE_TYPES_WITH_GPUS, ALL_INSTANCE_TYPES
 
 
 class TestCategorizedInstanceTypes(unittest.TestCase):
@@ -23,7 +23,7 @@ class TestCategorizedInstanceTypes(unittest.TestCase):
         self.hs = HardwareSet(CW_INSTANCE_TYPES)
 
     def test_number_of_categories(self):
-        self.assertEqual(self.hs.number_of_categories(), 5)
+        self.assertEqual(self.hs.number_of_categories(), 4)
 
     def test_categories_sorted_on_order(self):
         labels = [i["label"] for i in self.hs.get_model()]
@@ -39,17 +39,31 @@ class TestCategorizedInstanceTypes(unittest.TestCase):
         self.assertIn("a-4-16", low_category_values)
         self.assertIn("a-4-16", extra_category_values)
 
-    def test_model_removes_misc_by_default(self):
-        model = self.hs.get_model()
-        labels = [c["label"] for c in model]
-        self.assertNotIn(MISC_CATEGORY_LABEL, labels)
-
-    def test_model_adds_misc_if_specified(self):
-        model = self.hs.get_model(with_misc=True)
-        labels = [c["label"] for c in model]
-        self.assertIn(MISC_CATEGORY_LABEL, labels)
-
-
+class TestRecategorizeInstanceTypes(unittest.TestCase):
+    def setUp(self):
+        self.hs = HardwareSet(CW_INSTANCE_TYPES_WITH_GPUS)
+        
+    def test_recategorize_cpu_gpu(self):
+        test_func = lambda x: [{'label': 'GPU', 'order': 2}] if "gpu" in x and x["gpu"] else [{'label': 'CPU', 'order': 1}]
+        self.hs.recategorize(test_func)
+        self.assertEqual(self.hs.number_of_categories(), 2)
+
+    def test_find_all_by_condition(self):
+        test_func = lambda x: "gpu" in x and x["gpu"]
+        result = self.hs.find_all(test_func)
+        self.assertEqual(len(result), 2)
+        
+    def test_find_first_by_condition(self):
+        test_func = lambda x: x[ "memory"] < 32
+        result = self.hs.find_first(test_func)
+        self.assertTrue(result["memory"] < 32)
+        
+    def test_find_category(self):
+        label = "mid"
+        result = self.hs.find_category(label)
+        self.assertEqual(result["label"] , label)
+        self.assertEqual(result["order"] , 2)
+        
 class TestUncategorizedInstanceTypes(unittest.TestCase):
     def setUp(self):
         self.hs = HardwareSet(ALL_INSTANCE_TYPES)