ciocore 6.3.2rc1__py2.py3-none-any.whl → 6.4.0__py2.py3-none-any.whl

ciocore/hardware_set.py

@@ -1,314 +1,37 @@
-"""
-This module contains the **HardwareSet** class. 
 
-It is designed to allow submission tools and other clients that consume instance types to be able to display them in categories. This is particularly useful for UIs that utilize combo-boxes, since they can be orgnaized into a nested structure and displayed in a tree-like fashion.
-
-        
-"""
 import copy
 import logging
 
 logger = logging.getLogger(__name__)
 
-
 class HardwareSet(object):
-    """A class to manage categorized instance types.
-
-    A HardwareSet encapsulates the instance types available to an account. It accepts a flat list of instance types and builds a nested structure where those instance types exist in categories.
-
-    It keeps a dictionary of instance types (`instance_types`) with the name field as key. This allows easy lookup by name.
-
-    In addition, it keeps the nested structure of categories (`categories`) that contain the instance types. Each category is a dictionary with keys: `label`, `content`, and `order`.
-
-    `content` is a list of instance types in the category. The order is used to sort the categories. The order of the instance types within a category is determined by the number of cores and memory.
-
-    If all instance_types have not been assigned any categories, then the structure is built with two default categories: CPU and GPU.
+    """A class to manage the categorized instance types.
+    
+    It takes flat instance types array and builds a nested structure where instance types exist in
+    categories.
+    
+    If after categorization there is only one category and it is the misc category, then we
+    recategorize by cpu and gpu.
     """
 
     def __init__(self, instance_types):
-        """Initialize the HardwareSet with a list of instance types.
-
-        Typically, you would access the HardwareSet through the ciocore.data.data() function, which initializes it for you. However, you can also initialize it directly with a list of instance types straight from ciocore.api_client. The difference being that the latter contains all instance types, whereas the former contains only the instance types compatible with the products you have specified, as well as being cached.
-
-        Args:
-            instance_types (list): A list of instance types.
-
-        Returns:
-            HardwareSet: The initialized HardwareSet.
-
-        Examples:
-            ### Initialize with a list of instance types
-            >>> from ciocore import api_client
-            >>> from ciocore.hardware_set import HardwareSet
-            >>> instance_types = api_client.request_instance_types()
-            >>> hardware_set = HardwareSet(instance_types)
-            <ciocore.hardware_set.HardwareSet object at 0x104c43d30>
-
-            ### Initialize implicitly with a list of instance types from ciocore.data (recommended).
-            >>> from ciocore import data as coredata
-            >>> coredata.init("cinema4d")
-            >>> hardware_set = coredata.data()["instance_types"]
-            <ciocore.hardware_set.HardwareSet object at 0x104c43ee0>
-
-            !!! note
-                To avoid repetition, we use the implicit initialization for the examples below.
-        """
-
-        self.instance_types = self._build_unique(instance_types)
-        self.categories = self._build_categories()
-        self.provider = self._get_provider()
-
-    def labels(self):
-        """Get the list of category labels.
-
-        Returns:
-            list: A list of category labels.
-
-        Example:
-            >>> from ciocore import data as coredata
-            >>> coredata.init()
-            >>> hardware_set = coredata.data()["instance_types"]
-            >>> hardware_set.labels()
-            ['CPU', 'GPU']
-
-        """
-        return [c["label"] for c in self.categories]
-
-    def number_of_categories(self):
-        """Get the number of categories in the data.
-
-        Returns:
-            int: The number of categories.
-
-        Example:
-            >>> from ciocore import data as coredata
-            >>> coredata.init()
-            >>> hardware_set = coredata.data()["instance_types"]
-            >>> hardware_set.number_of_categories()
-            2
-
-        """
-        return len(self.categories)
-
-    def recategorize(self, partitioner):
-        """Recategorize the instance types.
-
-        Args:
-            partitioner (function): A function that takes an instance type and returns a list of categories to assign to it. The function should return an empty list if the instance type should not be categorized.
-
-        Example:
-            # Confirm current categories
-            >>> from ciocore import data as coredata
-            >>> coredata.init()
-            >>> hardware_set = coredata.data()["instance_types"]
-            >>> print(hardware_set.labels()
-            ['CPU', 'GPU']
-
-            # Recategorize
-            >>> hardware_set.recategorize(lambda x: [{'label': 'Low cores', 'order': 10}] if  x["cores"] < 16 else [{'label': 'High cores', 'order': 20}])
-            >>> print(hardware_set.labels()
-            ['Low cores', 'High cores']
-        """
-        for key in self.instance_types:
-            self.instance_types[key]["categories"] = partitioner(
-                self.instance_types[key]
-            )
-        self.categories = self._build_categories()
-
-    def find(self, name, category=None):
-        """Find an instance type by it's name (sku).
-
-        Args:
-            name (str): The name of the instance type.
-
-        Returns:
-            dict: The instance type or None if not found.
-
-        Example:
-            >>> from ciocore import data as coredata
-            >>> coredata.init()
-            >>> hardware_set = coredata.data()["instance_types"]
-            >>> hardware_set.find("n2-highmem-80")
-            {
-                'cores': 80,
-                'description': '80 core, 640GB Mem',
-                'gpu': None,
-                'memory': '640',
-                'name': 'n2-highmem-80',
-                'operating_system': 'linux',
-                'categories': [
-                    {'label': 'High cores', 'order': 20}
-                ]
-            }
-
-        """
-        if not category:
-            return self.instance_types.get(name)
-        
-        return self.find_first(lambda x: x["name"] == name and category in [c["label"] for c in x["categories"]])
-
-    def find_category(self, label):
-        """Find a category by label.
-
-        Args:
-            label (str): The label of the category.
-
-        Returns:
-            dict: The category or None if not found.
-
-        Example:
-            >>> from ciocore import data as coredata
-            >>> coredata.init()
-            >>> hardware_set = coredata.data()["instance_types"]
-            >>> hardware_set.find_category("High cores")
-            {
-                "label": "Low cores",
-                "content": [
-                    {
-                        "cores": 8,
-                        "description": "8 core, 52GB Mem",
-                        "gpu": None,
-                        "memory": "52",
-                        "name": "n1-highmem-8",
-                        "operating_system": "linux",
-                        "categories": [{"label": "Low cores", "order": 10}],
-                    },
-                    {
-                        "cores": 8,
-                        "description": "8 core, 7.2GB Mem",
-                        "gpu": None,
-                        "memory": "7.2",
-                        "name": "n1-highcpu-8",
-                        "operating_system": "linux",
-                        "categories": [{"label": "Low cores", "order": 10}],
-                    },
-                    ...
-                ],
-                "order": 10
-            }
-        """
-        return next((c for c in self.categories if c["label"] == label), None)
-
-    def find_all(self, condition):
-        """Find all instance types that match a condition.
-
-        Args:
-            condition (function): A function that takes an instance type and returns True or False.
-
-        Returns:
-            list: A list of instance types that match the condition.
-
-        Example:
-            >>> from ciocore import data as coredata
-            >>> coredata.init()
-            >>> hardware_set = coredata.data()["instance_types"]
-            >>> hardware_set.find_all(lambda x: x["gpu"])
-            [
-                {
-                    "cores": 4,
-                    "description": "4 core, 15GB Mem (1 T4 Tensor GPU with 16GB Mem)",
-                    "gpu": {
-                        "gpu_architecture": "NVIDIA Turing",
-                        "gpu_count": 1,
-                        "gpu_cuda_cores": 2560,
-                        "gpu_memory": "16",
-                        "gpu_model": "T4 Tensor",
-                        "gpu_rt_cores": 0,
-                        "gpu_tensor_cores": 0,
-                        "total_gpu_cuda_cores": 2560,
-                        "total_gpu_memory": "16",
-                        "total_gpu_rt_cores": 0,
-                        "total_gpu_tensor_cores": 0,
-                    },
-                    "memory": "15",
-                    "name": "n1-standard-4-t4-1",
-                    "operating_system": "linux",
-                    "categories": [{"label": "Low cores", "order": 10}],
-                },
-                {
-                    "cores": 8,
-                    "description": "8 core, 30GB Mem (1 T4 Tensor GPU with 16GB Mem)",
-                    "gpu": {
-                        "gpu_architecture": "NVIDIA Turing",
-                        "gpu_count": 1,
-                        "gpu_cuda_cores": 2560,
-                        "gpu_memory": "16",
-                        "gpu_model": "T4 Tensor",
-                        "gpu_rt_cores": 0,
-                        "gpu_tensor_cores": 0,
-                        "total_gpu_cuda_cores": 2560,
-                        "total_gpu_memory": "16",
-                        "total_gpu_rt_cores": 0,
-                        "total_gpu_tensor_cores": 0,
-                    },
-                    "memory": "30",
-                    "name": "n1-standard-8-t4-1",
-                    "operating_system": "linux",
-                    "categories": [{"label": "Low cores", "order": 10}],
-                },
-                ...
-            ]
-        """
-        result = []
-        for key in self.instance_types:
-            if condition(self.instance_types[key]):
-                result.append(self.instance_types[key])
-        return result
-
-    def find_first(self, condition):
-        """Find the first instance type that matches a condition.
-
-        Please see find_all() above for more details. This method is just a convenience wrapper around find_all() that returns the first result or None if not found.
-
-        Args:
-            condition (function): A function that takes an instance type and returns True or False.
-
-        Returns:
-            dict: The first instance type that matches the condition or None if not found.
-        """
-        return next(iter(self.find_all(condition)), None)
-
-    # DEPRECATED
-    def get_model(self, with_misc=False):
-        """Get the categories structure with renaming ready for some specific widget, such as a Qt Combobox.
-
-        Deprecated:
-            The get_model() method is deprecated. The `with_misc` parameter is no longer used, which means that this function only serves to rename a few keys. What's more, the init function ensures that every instance type has a category. This function is no longer needed. Submitters that use it will work but should be updated to use the categories structure directly as it minimizes the levels of indirection necessary to work with it.
-        """
-        if with_misc:
-            logger.warning("with_misc is no longer used")
-        result = []
-        for category in self.categories:
-            result.append(
-                {
-                    "label": category["label"],
-                    "content": [
-                        {"label": k["description"], "value": k["name"]}
-                        for k in category["content"]
-                    ],
-                }
-            )
 
-        return result
+        self.instance_types = self.build_unique(instance_types)
+        self.categories = self.build_categories()
 
-    # PRIVATE METHODS
     @staticmethod
-    def _build_unique(instance_types):
-        """Build a dictionary of instance types using the name field as key. This allows fast lookup by name.
-
-        Args:
-            instance_types (list): A list of instance types.
-
-        Returns:
-            dict: A dictionary of instance types with the name field as key.
+    def build_unique(instance_types):
+        """Build a dictionary of instance types using name as key.
+        
+        Remove any instance types whose description has already been seen. We can't have duplicate
+        descriptions.
+        
+        If categories exist, then remove any instance types that don't have a category. Otherwise,
+        since there are no categories, we can't remove any instance types.
         """
-        categories = [
-            category
-            for it in instance_types
-            for category in (it.get("categories") or [])
-        ]
+        categories = [category for it in instance_types for category in (it.get("categories") or [])]
         result = {}
-        seen_descriptions = set()
+        seen_descriptions = set() 
         for it in instance_types:
             if it["description"] in seen_descriptions:
                 continue
@@ -317,22 +40,15 @@ class HardwareSet(object):
                     continue
             else:
                 # make our own categories GPU/CPU
-                it["categories"] = (
-                    [{"label": "GPU", "order": 2}]
-                    if "gpu" in it and it["gpu"]
-                    else [{"label": "CPU", "order": 1}]
-                )
+                it["categories"] = [{'label': 'GPU', 'order': 2}] if "gpu" in it and it["gpu"] else [{'label': 'CPU', 'order': 1}]
             result[it["name"]] = it
             seen_descriptions.add(it["description"])
         return result
-
-    def _build_categories(self):
-        """Build a sorted list of categories where each category contains a sorted list of machines.
-
-        Returns:
-            list: A list of categories where each category is a dictionary with keys: `label`, `content`, and `order`.
-        """
-
+    
+    
+    def build_categories(self):
+        """Build a sorted list of categories, each category containing a sorted list of machines"""
+ 
         dikt = {}
         for key in self.instance_types:
             it = self.instance_types[key]
@@ -340,11 +56,7 @@ class HardwareSet(object):
             for category in categories:
                 label = category["label"]
                 if label not in dikt:
-                    dikt[label] = {
-                        "label": label,
-                        "content": [],
-                        "order": category["order"],
-                    }
+                    dikt[label] = {"label": label, "content": [], "order": category["order"]}
                 dikt[label]["content"].append(it)
 
         result = []
@@ -353,18 +65,79 @@ class HardwareSet(object):
             category["content"].sort(key=lambda k: (k["cores"], k["memory"]))
             result.append(category)
         return sorted(result, key=lambda k: k["order"])
+ 
+    def recategorize(self, partitioner):
+        """Recategorize the instance types.
+
+        Partitioner is a function that takes an instance type and returns a list of categories.
+        
+        If for example you want to append to the existing categories, then you can use a function like this:
+        lambda x: x["categories"] + [{'label': 'Low cores', 'order': 10}] if  x["cores"] < 16 else [{'label': 'High cores', 'order': 20}]
+        
+        Rebuilds the categories structure after assigning the new categories.
+        """
+        for key in self.instance_types:
+            self.instance_types[key]["categories"] = partitioner(self.instance_types[key])
+        self.categories = self.build_categories()
+
 
-    def _get_provider(self):
-        """Get the provider from the first instance type.
+    def get_model(self, with_misc=False):
+        """Returns the categories structure with renaming ready for some UI.
+
+        NOTE: THIS METHOD WILL BE DEPRECATED. with_misc is no longer used, which means that this
+        function just renames a few keys. What's more, the init function ensures that every instance
+        type has a (non-misc) category. So this function is no longer needed. Submitters that use
+        it will work fine, but should be updated to use the categories structure directly. 
+        """
+        if with_misc:
+            logger.warning("with_misc is no longer used")
+        result = []
+        for category in self.categories:
+            result.append({
+                "label": category["label"],
+                "content": [{"label": k["description"], "value": k["name"]} for k in category["content"]]
+            })
 
-        Returns:
-            str: The provider.
+        return result
+    
+    def find(self, name):
+        """Find an instance type by name (sku).
+        
+        Example: find("p3.2xlarge")
+        
+        Returns and instance-type or None if not found.
+        """
+        return self.instance_types.get(name)
+    
+    def find_category(self, label):
+        """Find a category by label.
+        
+        Example: find_category("GPU")
+        
+        Returns the entire category and its contents or None if not found. 
+        """
+        return next((c for c in self.categories if c["label"] == label), None)
+    
+    
+    def find_all(self, condition):
+        """Find all instance types that match a condition.
+        
+        Example: find_all(lambda x: x["gpu"])
+        """
+        result = []
+        for key in self.instance_types:
+            if condition(self.instance_types[key]):
+                result.append(self.instance_types[key])
+        return result
+    
+    def find_first(self, condition):
+        """Find the first instance type that matches a condition.
+        
+        Example: find_first(lambda x: x["cores"] == 4)"])
         """
-        first_name = next(iter(self.instance_types))
-        if not first_name:
-            return None
-        if first_name.startswith("cw-"):
-            return "cw"
-        if "." in first_name:
-            return "aws"
-        return "gcp"
+        return next(iter(self.find_all(condition)), None)
+    
+    def number_of_categories(self):
+        """Return the number of categories in the data."""
+        return len(self.categories)
+    

ciocore/package_environment.py

@@ -7,15 +7,21 @@ class PackageEnvironment(object):
 
     def __init__(self, env_list=None, platform=None):
         """
-        Encapsulate a list of environment variables.
+        Create a list of environment variables.
 
-        Typically, one would initialize a PackageEnvironment with a package, and then modify by adding more packages or lists of variables. Extra variables can be added by the customer, or programmatically such as during asset scraping.
+        Typically, one would initialize a PackageEnvironment with a package, and then modify by
+        adding more packages or lists of variables. Extra variables can be added by the customer, or
+        programmatically such as during asset scraping.
+
+        Keyword Arguments:
+
+        * **`env_list`** -- Either a Package that contains an "environment" property, or a list of
+          environment objects. -- Defaults to `None`.
+        * **`platform`** -- If env_list is not a package, then the platform must be provided. --
+          Defaults to `None`.
+
+        On instantiation, delegete args to the [extend()](#extend) method.
 
-        Args:
-            env_list (object|list): An object that provides a list of dictionaries with properties: `name`, `value`, and `merge_policy`.
-            platform (str): If the env_list is a regular list, then this is required.
-            
-        Args are delegated to [extend()](/package_environment/#ciocore.package_environment.PackageEnvironment.extend).
         """
         self.platform = None
         self._env = {}
@@ -27,53 +33,73 @@ class PackageEnvironment(object):
         """
         Extend the Package environment with the given variable specifications.
 
-        Args:
-            env_list (object|list): Either:
-                * A list of dictionaries with properties: `name`, `value`, and `merge_policy`.
-                * An object with an `environment` key that contains a list of the dictionaries described above. The latter is the structure of a package. Therefore we can initialize or extend a PackageEnvironment with a package.
-            platform (str): Defaults to `None`. If env_list is a package, then the platform is taken from the package and the `platform` keyword is ignored. If env_list is a list, then if this is the first add, a platform should be specified, otherwise it will default to linux.
-                
-        The first time data is added to a PackageEnvironment, the platform is set in stone. Subsequent `adds` that try to change the platform are considered an error.
+        Arguments:
 
-        Each variable to be added specifies a merge_policy: `append` or `exclusive`. Once an individual variable has been initialized with a merge policy, it can't be changed. This means:
-    
-        1. It's not possible to overwrite variables that have been set as append.
-        2. Exclusive variables are always overwritten by subsequent adds.
+        * **`env_list`** --  Either:
+        1. A list of dictionaries with properties: `name`, `value`, and `merge_policy`.
+        2. An object with an `environment` key that contains a list of the dictionaries described
+           above. The latter is the structure of a package. Therefore we can initialize or extend a
+           PackageEnvironment with a package.
 
-        Raises:
-            ValueError: Either an attempt to change the platform once initialized, or an invalid merge policy.
-
-            
-        Example:
-            >>> from ciocore import api_client, package_tree, package_environment
-            >>> packages = api_client.request_software_packages()
-            >>> pt = package_tree.PackageTree(packages, product="cinema4d")
-            >>> one_dcc_name = pt.supported_host_names()[0]
-            cinema4d 21.209.RB305619 linux
+
+        Keyword Arguments:
+
+        * **`platform`** -- Defaults to `None`.
+            If env_list is a package, then the platform is taken from the package and the `platform`
+            keyword is ignored. If env_list is a list, then if this is the first add, a platform
+            should be specified, otherwise it willl default to linux.
         
-            >>> pkg = pt.find_by_name(one_dcc_name)
-            >>> pe = package_environment.PackageEnvironment(pkg)
-            >>> print(dict(pe))
-            {
-            "PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin",
-            "g_licenseServerRLM": "conductor-rlm:6112",
-            "LD_LIBRARY_PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/embree.module/libs/linux64",
-            "PYTHONPATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64/python2.7/lib-dynload",
-            }
-
-            >>> extra_env = [
+        The first time data is added to a PackageEnvironment, the platform is set in stone.
+        Subsequent `adds` that try to change the platform are considered an error.
+
+        Each variable to be added specifies a merge_policy: `append` or `exclusive`. Once an
+        individual variable has been initialized with a merge policy, it can't be changed. This
+        means: 1. It's not possible to overwrite variables that have been set as append. 2.
+        Exclusive variables are always overwritten by subsequent adds.
+
+        Raises:
+
+        * **`ValueError`** -- Attempt to change the platform once initialized.
+        * **`ValueError`** -- Unknown merge policy
+        .
+
+        ???+ example
+            ``` python
+
+            from ciocore import api_client, package_tree, package_environment
+            packages = api_client.request_software_packages()
+            pt = package_tree.PackageTree(packages, product="cinema4d")
+            one_dcc_name = pt.supported_host_names()[0]
+            # 'cinema4d 21.209.RB305619 linux'
+            pkg = pt.find_by_name(one_dcc_name)
+            pe = package_environment.PackageEnvironment(pkg)
+            print(dict(pe))
+
+            # Result:
+            # {
+            # "PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin",
+            # "g_licenseServerRLM": "conductor-rlm:6112",
+            # "LD_LIBRARY_PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/embree.module/libs/linux64",
+            # "PYTHONPATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64/python2.7/lib-dynload",
+            # }
+
+            extra_env = [
                 {"name":"PATH", "value": "/my/custom/scripts", "merge_policy":"append"},
                 {"name":"DEBUG_MODE", "value": "1", "merge_policy":"exclusive"}
             ]
-            >>> pe.extend(extra_env)
-            >>> print(dict(pe))
-            {
-                "PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin:/my/custom/scripts",
-                "g_licenseServerRLM": "conductor-rlm:6112",
-                "LD_LIBRARY_PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/embree.module/libs/linux64",
-                "PYTHONPATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64/python2.7/lib-dynload",
-                "DEBUG_MODE": "1",
-            }
+
+            pe.extend(extra_env)
+            print(dict(pe))
+
+            # Result:
+            # {
+            #     "PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin:/my/custom/scripts",
+            #     "g_licenseServerRLM": "conductor-rlm:6112",
+            #     "LD_LIBRARY_PATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64:/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/embree.module/libs/linux64",
+            #     "PYTHONPATH": "/opt/maxon/cinema4d/21/cinema4d21.209vRB305619/bin/resource/modules/python/libs/linux64/python.linux64.framework/lib64/python2.7/lib-dynload",
+            #     "DEBUG_MODE": "1",
+            # }
+            ```
         """
 
         if not env_list: