ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl

ciocore/hardware_set.py

@@ -0,0 +1,449 @@
+"""
+This module contains the **HardwareSet** class. 
+
+It is designed to allow submission tools and other clients that consume instance types to be able to display them in categories. This is particularly useful for UIs that utilize combo-boxes, since they can be orgnaized into a nested structure and displayed in a tree-like fashion.
+
+        
+"""
+import copy
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+#   {
+#     "cores": 32,
+#     "description": "32 core, 120GB Mem (8 V100 GPUs with 16GB Mem)",
+#     "gpu": {
+#       "gpu_architecture": "NVIDIA Volta",
+#       "gpu_count": 8,
+#       "gpu_cuda_cores": 5120,
+#       "gpu_memory": "16",
+#       "gpu_model": "V100",
+#       "total_gpu_cuda_cores": 40960,
+#       "total_gpu_memory": "128"
+#     },
+#     "memory": "120",
+#     "name": "n1-standard-32-v1-8",
+#     "operating_system": "linux"
+#   },
+
+
+DESCRIPTION_TEMPLATE_OS = {
+    "cpu": "{operating_system} {cores} core {memory}GB Mem",
+    "gpu": "{operating_system} {cores} core {memory}GB Mem ({gpu_count} {gpu_model} GPUs {gpu_memory}GB Mem)",
+}
+DESCRIPTION_TEMPLATE = {
+    "cpu": "{cores} core {memory}GB Mem",
+    "gpu": "{cores} core {memory}GB Mem ({gpu_count} {gpu_model} GPUs {gpu_memory}GB Mem)",
+}
+
+def flatten_dict(d):
+    flat_dict = {}
+    for key, value in d.items():
+        if isinstance(value, dict):
+            nested_dict = flatten_dict(value)
+            for nested_key, nested_value in nested_dict.items():
+                flat_dict[nested_key] = nested_value
+        else:
+            flat_dict[key] = value
+    return flat_dict
+
+
+class HardwareSet(object):
+    """A class to manage categorized instance types.
+
+    A HardwareSet encapsulates the instance types available to an account. It accepts a flat list of instance types and builds a nested structure where those instance types exist in categories.
+
+    It keeps a dictionary of instance types (`instance_types`) with the name field as key. This allows easy lookup by name.
+
+    In addition, it keeps the nested structure of categories (`categories`) that contain the instance types. Each category is a dictionary with keys: `label`, `content`, and `order`.
+
+    `content` is a list of instance types in the category. The order is used to sort the categories. The order of the instance types within a category is determined by the number of cores and memory.
+
+    If all instance_types have not been assigned any categories, then the structure is built with two default categories: CPU and GPU.
+    """
+
+    def __init__(self, instance_types):
+        """Initialize the HardwareSet with a list of instance types.
+        Typically, you would access the HardwareSet through the ciocore.data.data() function, which initializes it for you. However, you can also initialize it directly with a list of instance types straight from ciocore.api_client. The difference being that the latter contains all instance types, whereas the former contains only the instance types compatible with the products you have specified, as well as being cached.
+
+        Args:
+            instance_types (list): A list of instance types.
+
+        Returns:
+            HardwareSet: The initialized HardwareSet.
+
+        Examples:
+            ### Initialize with a list of instance types
+            >>> from ciocore import api_client
+            >>> from ciocore.hardware_set import HardwareSet
+            >>> instance_types = api_client.request_instance_types()
+            >>> hardware_set = HardwareSet(instance_types)
+            <ciocore.hardware_set.HardwareSet object at 0x104c43d30>
+
+            ### Initialize implicitly with a list of instance types from ciocore.data (recommended).
+            >>> from ciocore import data as coredata
+            >>> coredata.init("cinema4d")
+            >>> hardware_set = coredata.data()["instance_types"]
+            <ciocore.hardware_set.HardwareSet object at 0x104c43ee0>
+
+            !!! note
+                To avoid repetition, we use the implicit initialization for the examples below.
+        """
+        
+        self.instance_types = self._build_unique(instance_types)
+        self.categories = self._build_categories()
+        self.provider = self._get_provider()
+
+    def labels(self):
+        """Get the list of category labels.
+
+        Returns:
+            list: A list of category labels.
+
+        Example:
+            >>> from ciocore import data as coredata
+            >>> coredata.init()
+            >>> hardware_set = coredata.data()["instance_types"]
+            >>> hardware_set.labels()
+            ['CPU', 'GPU']
+
+        """
+        return [c["label"] for c in self.categories]
+
+    def number_of_categories(self):
+        """Get the number of categories in the data.
+
+        Returns:
+            int: The number of categories.
+
+        Example:
+            >>> from ciocore import data as coredata
+            >>> coredata.init()
+            >>> hardware_set = coredata.data()["instance_types"]
+            >>> hardware_set.number_of_categories()
+            2
+
+        """
+        return len(self.categories)
+
+    def recategorize(self, partitioner):
+        """Recategorize the instance types.
+
+        Args:
+            partitioner (function): A function that takes an instance type and returns a list of categories to assign to it. The function should return an empty list if the instance type should not be categorized.
+
+        Example:
+            # Confirm current categories
+            >>> from ciocore import data as coredata
+            >>> coredata.init()
+            >>> hardware_set = coredata.data()["instance_types"]
+            >>> print(hardware_set.labels()
+            ['CPU', 'GPU']
+
+            # Recategorize
+            >>> hardware_set.recategorize(lambda x: [{'label': 'Low cores', 'order': 10}] if  x["cores"] < 16 else [{'label': 'High cores', 'order': 20}])
+            >>> print(hardware_set.labels()
+            ['Low cores', 'High cores']
+        """
+        for key in self.instance_types:
+            self.instance_types[key]["categories"] = partitioner(
+                self.instance_types[key]
+            )
+        self.categories = self._build_categories()
+
+    def find(self, name, category=None):
+        """Find an instance type by its name (sku).
+
+        Args:
+            name (str): The name of the instance type.
+
+        Returns:
+            dict: The instance type or None if not found.
+        Example:
+            >>> from ciocore import data as coredata
+            >>> coredata.init()
+            >>> hardware_set = coredata.data()["instance_types"]
+            >>> hardware_set.find("n2-highmem-80")
+            {
+                'cores': 80,
+                'description': '80 core, 640GB Mem',
+                'gpu': None,
+                'memory': '640',
+                'name': 'n2-highmem-80',
+                'operating_system': 'linux',
+                'categories': [
+                    {'label': 'High cores', 'order': 20}
+                ]
+            }
+
+        """
+        if not category:
+            return self.instance_types.get(name)
+
+        return self.find_first(
+            lambda x: x["name"] == name
+            and category in [c["label"] for c in x["categories"]]
+        )
+
+    def find_category(self, label):
+        """Find a category by label.
+
+        Args:
+            label (str): The label of the category.
+
+        Returns:
+            dict: The category or None if not found.
+        Example:
+            >>> from ciocore import data as coredata
+            >>> coredata.init()
+            >>> hardware_set = coredata.data()["instance_types"]
+            >>> hardware_set.find_category("High cores")
+            {
+                "label": "Low cores",
+                "content": [
+                    {
+                        "cores": 8,
+                        "description": "8 core, 52GB Mem",
+                        "gpu": None,
+                        "memory": "52",
+                        "name": "n1-highmem-8",
+                        "operating_system": "linux",
+                        "categories": [{"label": "Low cores", "order": 10}],
+                    },
+                    {
+                        "cores": 8,
+                        "description": "8 core, 7.2GB Mem",
+                        "gpu": None,
+                        "memory": "7.2",
+                        "name": "n1-highcpu-8",
+                        "operating_system": "linux",
+                        "categories": [{"label": "Low cores", "order": 10}],
+                    },
+                    ...
+                ],
+                "order": 10
+            }
+        """
+        return next((c for c in self.categories if c["label"] == label), None)
+
+    def find_all(self, condition):
+        """Find all instance types that match a condition.
+
+        Args:
+            condition (function): A function that takes an instance type and returns True or False.
+
+        Returns:
+            list: A list of instance types that match the condition.
+
+        Example:
+            >>> from ciocore import data as coredata
+            >>> coredata.init()
+            >>> hardware_set = coredata.data()["instance_types"]
+            >>> hardware_set.find_all(lambda x: x["gpu"])
+            [
+                {
+                    "cores": 4,
+                    "description": "4 core, 15GB Mem (1 T4 Tensor GPU with 16GB Mem)",
+                    "gpu": {
+                        "gpu_architecture": "NVIDIA Turing",
+                        "gpu_count": 1,
+                        "gpu_cuda_cores": 2560,
+                        "gpu_memory": "16",
+                        "gpu_model": "T4 Tensor",
+                        "gpu_rt_cores": 0,
+                        "gpu_tensor_cores": 0,
+                        "total_gpu_cuda_cores": 2560,
+                        "total_gpu_memory": "16",
+                        "total_gpu_rt_cores": 0,
+                        "total_gpu_tensor_cores": 0,
+                    },
+                    "memory": "15",
+                    "name": "n1-standard-4-t4-1",
+                    "operating_system": "linux",
+                    "categories": [{"label": "Low cores", "order": 10}],
+                },
+                {
+                    "cores": 8,
+                    "description": "8 core, 30GB Mem (1 T4 Tensor GPU with 16GB Mem)",
+                    "gpu": {
+                        "gpu_architecture": "NVIDIA Turing",
+                        "gpu_count": 1,
+                        "gpu_cuda_cores": 2560,
+                        "gpu_memory": "16",
+                        "gpu_model": "T4 Tensor",
+                        "gpu_rt_cores": 0,
+                        "gpu_tensor_cores": 0,
+                        "total_gpu_cuda_cores": 2560,
+                        "total_gpu_memory": "16",
+                        "total_gpu_rt_cores": 0,
+                        "total_gpu_tensor_cores": 0,
+                    },
+                    "memory": "30",
+                    "name": "n1-standard-8-t4-1",
+                    "operating_system": "linux",
+                    "categories": [{"label": "Low cores", "order": 10}],
+                },
+                ...
+            ]
+        """
+        result = []
+        for key in self.instance_types:
+            if condition(self.instance_types[key]):
+                result.append(self.instance_types[key])
+        return result
+
+    def find_first(self, condition):
+        """Find the first instance type that matches a condition.
+
+        Please see find_all() above for more details. This method is just a convenience wrapper around find_all() that returns the first result or None if not found.
+
+        Args:
+            condition (function): A function that takes an instance type and returns True or False.
+
+        Returns:
+            dict: The first instance type that matches the condition or None if not found.
+        """
+        return next(iter(self.find_all(condition)), None)
+
+    # DEPRECATED
+    def get_model(self, with_misc=False):
+        """Get the categories structure with renaming ready for some specific widget, such as a Qt Combobox.
+
+        Deprecated:
+            The get_model() method is deprecated. The `with_misc` parameter is no longer used, which means that this function only serves to rename a few keys. What's more, the init function ensures that every instance type has a category. This function is no longer needed. Submitters that use it will work but should be updated to use the categories structure directly as it minimizes the levels of indirection necessary to work with it.
+        """
+        if with_misc:
+            logger.warning("with_misc is no longer used")
+        result = []
+        for category in self.categories:
+            result.append(
+                {
+                    "label": category["label"],
+                    "content": [
+                        {"label": k["description"], "value": k["name"]}
+                        for k in category["content"]
+                    ],
+                }
+            )
+
+        return result
+
+    # PRIVATE METHODS
+    @classmethod
+    def _build_unique(cls, instance_types):
+        """Build a dictionary of instance types using the name field as key. This allows fast lookup by name.
+
+        Args:
+            instance_types (list): A list of instance types.
+
+        Returns:
+            dict: A dictionary of instance types with the name field as key.
+        """
+
+        instance_types = cls._rewrite_descriptions(instance_types)
+        categories = [
+            category
+            for it in instance_types
+            for category in (it.get("categories") or [])
+        ]
+        result = {}
+        for it in instance_types:
+            is_gpu = it.get("gpu", False)
+            if categories:
+                if it.get("categories") in [[], None]:
+                    continue
+            else:
+                # make our own categories GPU/CPU
+                it["categories"] = (
+                    [{"label": "GPU", "order": 2}]
+                    if is_gpu
+                    else [{"label": "CPU", "order": 1}]
+                )
+            result[it["name"]] = it
+
+        return result
+
+
+
+
+    @classmethod
+    def _rewrite_descriptions(cls, instance_types):
+        """Rewrite the descriptions of the instance types.
+        
+        If there are both OS types, then the descriptions are prefixed with the OS type.
+
+        Args:
+            instance_types (list): A list of instance types.
+
+        Returns:
+            list: A list of instance types with rewritten descriptions.
+        """
+        if not instance_types:
+            return instance_types
+        
+        first_os = instance_types[0]["operating_system"]
+        dual_platforms = next((it for it in instance_types if it["operating_system"] != first_os), False)
+
+        if dual_platforms:
+            for it in instance_types:
+                flat_dict = flatten_dict(it)
+                is_gpu = "gpu_count" in flat_dict
+                if is_gpu:
+                    it["description"] = DESCRIPTION_TEMPLATE_OS["gpu"].format(**flat_dict)
+                else:
+                    it["description"] = DESCRIPTION_TEMPLATE_OS["cpu"].format(**flat_dict)
+        else:
+            for it in instance_types:
+                flat_dict = flatten_dict(it)
+                is_gpu = "gpu_count" in flat_dict
+                if is_gpu:
+                    it["description"] = DESCRIPTION_TEMPLATE["gpu"].format(**flat_dict)
+                else:
+                    it["description"] = DESCRIPTION_TEMPLATE["cpu"].format(**flat_dict)
+
+        return instance_types
+
+    def _build_categories(self):
+        """Build a sorted list of categories where each category contains a sorted list of machines.
+
+        Returns:
+            list: A list of categories where each category is a dictionary with keys: `label`, `content`, and `order`.
+        """
+
+        dikt = {}
+        for key in self.instance_types:
+            it = self.instance_types[key]
+            categories = it["categories"]
+            for category in categories:
+                label = category["label"]
+                if label not in dikt:
+                    dikt[label] = {
+                        "label": label,
+                        "content": [],
+                        "order": category["order"],
+                    }
+                dikt[label]["content"].append(it)
+
+        result = []
+        for label in dikt:
+            category = dikt[label]
+            category["content"].sort(key=lambda k: (k["cores"], k["memory"]))
+            result.append(category)
+        return sorted(result, key=lambda k: k["order"])
+
+    def _get_provider(self):
+        """Get the provider from the first instance type.
+
+        Returns:
+            str: The provider.
+        """
+        first_name = next(iter(self.instance_types))
+        if not first_name:
+            return None
+        if first_name.startswith("cw-"):
+            return "cw"
+        if "." in first_name:
+            return "aws"
+        return "gcp"

ciocore/loggeria.py

@@ -3,10 +3,11 @@ import logging.handlers
 import multiprocessing
 import os
 import sys
+import tempfile
 import threading
 import traceback
-from logging.handlers import TimedRotatingFileHandler
-from ciocore.common import CONDUCTOR_LOGGER_NAME
+
+from . import config
 
 LEVEL_CRITICAL = "CRITICAL"
 LEVEL_ERROR = "ERROR"
@@ -29,12 +30,15 @@ LEVEL_MAP = {
 
 
 FORMATTER_LIGHT = logging.Formatter("%(asctime)s %(name)s: %(message)s", "%Y-%m-%d %H:%M:%S")
-FORMATTER_VERBOSE = logging.Formatter("%(asctime)s %(name)s%(levelname)9s:  %(message)s")
+FORMATTER_VERBOSE = logging.Formatter("%(asctime)s  %(name)s%(levelname)9s %(filename)s-%(lineno)d %(threadName)s:  %(message)s")
 DEFAULT_LEVEL_CONSOLE = LEVEL_MAP[LEVEL_INFO]
 DEFAULT_LEVEL_FILE = LEVEL_MAP[LEVEL_DEBUG]
 DEFAULT_LEVEL_LOGGER = LEVEL_MAP[LEVEL_INFO]
 
 CONDUCTOR_LOGGER_NAME = "conductor"
+__logger__ = None
+
+LOG_PATH = None
 
 
 class LogLevelFilter(logging.Filter):
@@ -56,16 +60,19 @@ class LogLevelFilter(logging.Filter):
         return int(record.levelno < logging.ERROR)
 
 
-def setup_conductor_logging(
+def setup_conductor_logging(        
     logger_level=DEFAULT_LEVEL_LOGGER,
     console_level=None,
     console_formatter=FORMATTER_LIGHT,
-    log_filepath=None,
+    log_dirpath=None,
     file_level=None,
     file_formatter=FORMATTER_VERBOSE,
     multiproc=False,
     disable_console_logging = False,
-    propagate = True
+    propagate = True,
+    log_filename=None,
+    use_system_log=False
+
 ):
     """The is convenience function to help set up logging.
 
@@ -87,6 +94,11 @@ def setup_conductor_logging(
     multiproc: bool. If True, a custom file handler will be used that handles multiprocess logging
     correctly. This file handler creates an additional Process.
     """
+    global __logger__
+    
+    if __logger__:
+        return
+    
     # Get the top/parent conductor logger
     logger = get_conductor_logger()
     
@@ -118,38 +130,71 @@ def setup_conductor_logging(
             console_handler_out.setFormatter(console_formatter)
             console_handler_err.setFormatter(console_formatter)
 
-    # Create a file handler if a filepath was given
-    if log_filepath:
+    # Create a file handler. Use the given path or fallback to the default path.
+    if log_filename:
         if file_level:
             assert file_level in LEVEL_MAP.values(), "Not a valid log level: %s" % file_level
+
+        if use_system_log and not log_dirpath:
+            log_dirpath = get_system_log_dir()
+
+        log_dirpath = log_dirpath or get_user_log_dir()
+       
         # Rotating file handler. Rotates every day (24 hours). Stores 7 days at a time.
         file_handler = create_file_handler(
-            log_filepath, level=file_level, formatter=file_formatter, multiproc=multiproc
+            filepath=log_dirpath, 
+            filename=log_filename, 
+            level=file_level, 
+            formatter=file_formatter, 
+            multiproc=multiproc, 
+            use_fallback_path=True
         )
         logger.addHandler(file_handler)
 
+    __logger__ = logger
 
-def create_file_handler(filepath, level=None, formatter=None, multiproc=False):
-    """Create a file handler object for the given filepath.
+
+def create_file_handler(filepath, filename, level=None, formatter=None, multiproc=False, use_fallback_path=True):
+    """Create a file handler object in the given filepath with filename.
 
     This is a ROTATING file handler, which rotates every day (24 hours) and stores up to 7 days of
     logs at a time (equaling up to as many as 7 log files at a given time.
+
+    If writing to the given filepath fails due to permissions and use_fallback path is True, the
+    logger will try to use the users home folder.
     """
+    
     when = "h"  # rotate unit is "h" (hours)
     interval = 24  # rotate every  24 units (24 hours)
     backupCount = 7  # Retain up to 7 log files (7 days of log files)
+    
+    try:
+        if not os.path.exists(filepath):
+            os.makedirs(filepath)
+
+        else:
+            tempfile.TemporaryFile(dir=filepath)
+
+    except PermissionError as error_msg:
 
-    log_dirpath = os.path.dirname(filepath)
-    if not os.path.exists(log_dirpath):
-        os.makedirs(log_dirpath)
+        print("Unable to use {} for logs. ({})".format(filepath, error_msg))
+        
+        if filepath != get_user_log_dir and use_fallback_path:
+            return create_file_handler(filepath=get_user_log_dir(), 
+                                filename=filename, 
+                                formatter=formatter, 
+                                multiproc=multiproc)
+
+        else:
+            raise
 
     if multiproc:
         # Use custom rotating file handler that handles multiprocessing properly
-        handler = MPFileHandler(filepath, when=when, interval=interval, backupCount=backupCount)
+        handler = MPFileHandler(os.path.join(filepath, filename), when=when, interval=interval, backupCount=backupCount)
     else:
         # Rotating file handler. Rotates every day (24 hours). Stores 7 days at a time.
         handler = logging.handlers.TimedRotatingFileHandler(
-            filepath, when=when, interval=interval, backupCount=backupCount
+            os.path.join(filepath, filename), when=when, interval=interval, backupCount=backupCount
         )
     if formatter:
         handler.setFormatter(formatter)
@@ -157,6 +202,9 @@ def create_file_handler(filepath, level=None, formatter=None, multiproc=False):
     if level:
         handler.setLevel(level)
 
+    global LOG_PATH
+    LOG_PATH=os.path.join(filepath, filename)
+
     return handler
 
 
@@ -184,7 +232,7 @@ class MPFileHandler(logging.Handler):
     ):
         """See TimedRotatingFileHandler for arg docs."""
         logging.Handler.__init__(self)
-        self._handler = TimedRotatingFileHandler(
+        self._handler = logging.handlers.TimedRotatingFileHandler(
             filename,
             when=when,
             interval=interval,
@@ -359,9 +407,30 @@ class TableStr(object):
     def get_footer(self):
         return self.footer
     
-def get_default_log_dir(platform=None):
+def get_default_log_dir(platform=None, system_log=False):
+
+    if system_log:
+        log_dir = get_system_log_dir(platform)
+         
+        if os.access(log_dir, os.W_OK|os.X_OK):
+            return log_dir
+
+    return get_user_log_dir(platform=platform)
+
+def get_system_log_dir(platform=None):
+
+    platform = platform or sys.platform
+
+    if platform.startswith('win'):
+        return os.path.expandvars(os.path.join("%programdata%", "conductor", "logs"))
     
-    #platform = platform or sys.platform
+    else:
+        return "/var/log/conductor"
+
+def get_user_log_dir(platform=None):
     
-    return os.path.expanduser(os.path.join("~", ".conductor", "logs", "conductor.log"))
+    root_path = config.get()['user_dir']
+    return os.path.expanduser(os.path.join(root_path, "logs"))
+
+