ciocore 7.0.2b4__py2.py3-none-any.whl → 8.0.0__py2.py3-none-any.whl

ciocore/config.py

@@ -1,40 +1,51 @@
 """
-A configuration object implemented as a module-level singleton.
+Config is a configuration object implemented as a module-level singleton.
 
-Configuration variables can be shared by importing the module. If there are changes in environment variables or other sources, the config can be refreshed. See [config()](#config)
+Configuration variables can be shared by importing the module. If there are changes in environment variables or other sources, the config can be refreshed.
 """
 
 import logging
 import os
-import multiprocessing
+# import multiprocessing
 import base64
 import json
 import re
-from ciocore import loggeria
+import platform
 
 from ciocore.common import CONDUCTOR_LOGGER_NAME
 
 logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
 
-#https://stackoverflow.com/a/3809435/179412
+# https://stackoverflow.com/a/3809435/179412
 
-URL_REGEX=re.compile(r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)")
+
+USER_DIRS = {
+    "Linux": os.path.expanduser(os.path.join("~", ".conductor")),
+    "Darwin": os.path.expanduser(os.path.join("~",".conductor")),
+    "Windows": os.path.expanduser(os.path.join("~", "AppData", "Local", "Conductor")),
+}
+
+DEFAULT_USER_DIR = USER_DIRS.get(platform.system(), USER_DIRS["Linux"])
+
+URL_REGEX = re.compile(
+    r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)"
+)
 
 __config__ = None
 
+
 def config(force=False):
     """
-    DEPRECATED! Use [get()](#get) instead.
-
     Instantiate a config object if necessary.
 
-    Keyword Arguments:
+    Deprecated:
+        Use [get()](#get) instead.
 
-    * **`force`** --  Discards any existing config object and instantiate a new one -- Defaults to `False`.
+    Args:
+        force (bool): Discards any existing config object and instantiate a new one -- Defaults to `False`.
 
     Returns:
-
-    * [Config()](#Config) object containing a dictionary of configuration variables.
+        dict: A dictionary containing configuration values.
     """
 
     global __config__
@@ -42,67 +53,83 @@ def config(force=False):
         __config__ = Config()
     return __config__
 
+
 def get(force=False):
     """
     Instantiate a config object if necessary and return the dictionary.
 
-    Keyword Arguments:
-
-    * **`force`** -- Discards any existing config object and instantiate a new one -- Defaults to `False`.
+    Args:
+        force (bool): Discards any existing config object and instantiate a new one -- Defaults to `False`.
 
     Returns:
-
-    * A dictionary containing configuration.
-\
-    ???+ example
-        ``` python
-        from ciocore import config
-
-        print(config.get())
-
-        # Result:
-        # {
-        #   'thread_count': 16,
-        #   'priority': 5,
-        #   'md5_caching': True,
-        #   'log_level': 'INFO',
-        #   'url': 'https://dashboard.conductortech.com',
-        #   'auth_url': 'https://dashboard.conductortech.com',
-        #   'api_url': 'https://api.conductortech.com', 
-        #   'api_key': None
-        # }
-        ```
+        dict: A dictionary containing configuration values.
+
+    Example:
+        >>> from ciocore import config
+        >>> config.get()
+        {
+          'thread_count': 16,
+          'priority': 5,
+          'md5_caching': True,
+          'log_level': 'INFO',
+          'url': 'https://dashboard.conductortech.com',
+          'auth_url': 'https://dashboard.conductortech.com',
+          'api_url': 'https://api.conductortech.com',
+          'api_key': None,
+          'downloader_page_size': 50,
+        }
     """
     global __config__
     if force or not __config__:
         __config__ = Config()
     return __config__.config
 
+
 class Config(object):
     def __init__(self):
         """
-        Create a configuration dictionary.
+        Initialize the config object.
 
+        A config object is a dictionary containing configuration values. It is a singleton, so there is only one instance of it. It is instantiated the first time it is needed. It can be refreshed by calling get() with the `force` keyword argument set to `True`.
 
-        Raises:
+        A Config object has the following properties:
+
+        * `thread_count` The number of threads to use for downloading files. Defaults to the number of CPUs on the system times 2. It can be overridden by the `CONDUCTOR_THREAD_COUNT` environment variable.
+        * `priority` Set the priority for submissions. Defaults to 5. It can be overridden by the `CONDUCTOR_PRIORITY` environment variable.
+        * `md5_caching` Whether to cache MD5s. Defaults to `True`. It can be overridden by the `CONDUCTOR_MD5_CACHING` environment variable. Cachine MD5s significantly improves submission performance, but on rare occasions it can cause submissions to fail. If you experience this, set `md5_caching` to `False`.
+        * `log_level` The logging level. Defaults to `INFO`. It can be overridden by the `CONDUCTOR_LOG_LEVEL` environment variable.
+        * `url` The URL of the Conductor dashboard. Defaults to `https://dashboard.conductortech.com`. It can be overridden by the `CONDUCTOR_URL` environment variable.
+        * `auth_url` The URL of the Conductor dashboard. Defaults to `https://dashboard.conductortech.com`. It can be overridden by the `CONDUCTOR_AUTH_URL` environment variable. This is deprecated. Use `url` instead.
+        * `api_url` The URL of the Conductor API. Defaults to `https://api.conductortech.com`. It can be overridden by the `CONDUCTOR_API_URL` environment variable.
+        * `api_key` The API key. The API key can be acquired from the Conductor dashboard, and can be stored in an environment variable or a file. In both cases the API KEY can be a JSON object or a base64 encoded JSON object. If it is base64 encoded, it can be a string or bytes. If it is a string, it will be decoded as ASCII. If it is bytes, it will be decoded as UTF-8.
+            * Environment variable: The `CONDUCTOR_API_KEY` variable can hold the API KEY directly.
+            * File: The `CONDUCTOR_API_KEY_PATH` variable can hold the path to a file containing the API KEY.
+        * `downloader_page_size` The number of files to request from the Conductor API at a time. Defaults to 50. It can be overridden by the `CONDUCTOR_DOWNLOADER_PAGE_SIZE` environment variable.
 
-        * **`ValueError`** -- Invalid inputs, such as badly formed URLs.
+        Returns:
+            Config: A config object.
+
+        Raises:
+            ValueError -- Invalid inputs, such as badly formed URLs.
         """
+        default_downloader_page_size = 50
+
+     
 
         try:
-            default_thread_count = min(multiprocessing.cpu_count() * 2, 16)
+            default_thread_count = min( os.cpu_count() - 1, 15)
         except NotImplementedError:
-            default_thread_count = 16
+            default_thread_count = 15
 
         url = os.environ.get("CONDUCTOR_URL", "https://dashboard.conductortech.com")
 
         if not URL_REGEX.match(url):
             raise ValueError("CONDUCTOR_URL is not valid '{}'".format(url))
- 
+
         api_url = os.environ.get("CONDUCTOR_API_URL", url.replace("dashboard", "api"))
         if not URL_REGEX.match(api_url):
             raise ValueError("CONDUCTOR_API_URL is not valid '{}'".format(api_url))
- 
+
         falsy = ["false", "no", "off", "0"]
 
         log_level = os.environ.get("CONDUCTOR_LOG_LEVEL", "INFO")
@@ -110,34 +137,40 @@ class Config(object):
             log_level = "INFO"
 
         self.config = {
-            "thread_count": int(os.environ.get("CONDUCTOR_THREAD_COUNT", default_thread_count)),
+            "thread_count": int(
+                os.environ.get("CONDUCTOR_THREAD_COUNT", default_thread_count)
+            ),
+            "downloader_page_size": int(
+                os.environ.get(
+                    "CONDUCTOR_DOWNLOADER_PAGE_SIZE", default_downloader_page_size
+                )
+            ),
             "priority": int(os.environ.get("CONDUCTOR_PRIORITY", 5)),
             "md5_caching": False
             if os.environ.get("CONDUCTOR_MD5_CACHING", "True").lower() in falsy
             else True,
             "log_level": log_level,
             "url": url,
-            # Keep "auth_url" for backwwards compatibillity only. 
-            # Clients should use "url" moving forward. 
+            # Keep "auth_url" for backwwards compatibillity only.
+            # Clients should use "url" moving forward.
             # Remove "auth_url" on the next major version bump.
             "auth_url": url,
             "api_url": api_url,
             "api_key": self.get_api_key_from_variable() or self.get_api_key_from_file(),
-            "user_dir": os.environ.get('CONDUCTOR_USER_DIR', os.path.expanduser(os.path.join("~", ".conductor")))
+            "user_dir": os.environ.get('CONDUCTOR_USER_DIR', DEFAULT_USER_DIR)
         }
 
     @staticmethod
     def get_api_key_from_variable():
         """
-        Attempt to get an API key from the CONDUCTOR_API_KEY environment variable.
+        Attempt to get an API key from the `CONDUCTOR_API_KEY` environment variable.
 
         Raises:
-
-        * **`ValueError`** -- An error occurred while loading the key into JSON.
+            ValueError: An error occurred while reading or loading the key into JSON.
 
         Returns:
+            str: JSON object containing the key - base 64 decoded if necessary.
 
-        * JSON object containing the key - base 64 decoded if necessary.
         """
         api_key = os.environ.get("CONDUCTOR_API_KEY")
         if not api_key:
@@ -163,12 +196,11 @@ class Config(object):
         Attempt to get an API key from the file in the CONDUCTOR_API_KEY_PATH environment variable.
 
         Raises:
-
-        * **`ValueError`** --  An error occurred while reading or loading the key into JSON.
+            ValueError: An error occurred while reading or loading the key into JSON.
 
         Returns:
+            str: JSON object containing the key - base 64 decoded if necessary.
 
-        * JSON object containing the key - base 64 decoded if necessary.
         """
         api_key_path = os.environ.get("CONDUCTOR_API_KEY_PATH")
         if not api_key_path:

ciocore/data.py

@@ -1,17 +1,16 @@
 """
-This module is a singleton that provides the data from Conductor endpoints. Specifically, it provides projects, instance types, and software package data. It is also possible to use cached fixture data for development purposes.
+This module is a singleton that provides the data from Conductor endpoints. Specifically, it provides projects, instance types, and software package data.
 
 Since the data is stored at the module level, you can access it from anywhere in your code without the need to pass it around.
 """
 
-import json
-import os
 from ciocore.package_tree import PackageTree
 from ciocore import api_client
 from ciocore.hardware_set import HardwareSet
+import copy
+
 __data__ = {}
 __products__ = None
-__fixtures_dir__ = None
 __platforms__ = None
 
 def init(*products, **kwargs):
@@ -57,7 +56,7 @@ def data(force=False):
     Provide projects, instance types, and software package data.
 
     Keyword Args:
-        force: (bool) If `True`, then fetch fresh data -- Defaults to `False`.
+        force: (bool) If `True`, then force the system to fetch fresh data -- Defaults to `False`.
 
     Raises:
         ValueError:  Module was not initialized with [init()](/data/#ciocore.data.init).
@@ -109,36 +108,31 @@ def data(force=False):
         raise ValueError(
             'Data must be initialized before use, e.g. data.init("maya-io") or data.init().'
         )
-    products_copy = __products__.copy()
+    products_copy = copy.copy(__products__)
+
     if force:
         clear()
         init(*products_copy)
 
     if __data__ == {}:
         # PROJECTS
-        projects_json = _get_json_fixture("projects")
-        if projects_json:
-            __data__["projects"] = projects_json
-        else:
-            __data__["projects"] = sorted(api_client.request_projects())
-
+        __data__["projects"] = sorted(api_client.request_projects())
         # INST_TYPES
-        instance_types = _get_json_fixture("instance_types")
-        if not instance_types:
-            instance_types = api_client.request_instance_types()
-
-        it_platforms = set([it["operating_system"] for it in instance_types])
-        valid_platforms = it_platforms.intersection(__platforms__)
-
+        instance_types = api_client.request_instance_types()
         # SOFTWARE
-        software = _get_json_fixture("software")
-        if not software:
-            software = api_client.request_software_packages()
+        software = api_client.request_software_packages()
 
-        # This is a temporary hack to remove 3dsmax on linux from the software list, since it is a
-        # miscategorization
-        software = [sw for sw in software if not ("3dsmax" in sw["product"] and sw["platform"] == "linux")]
+        # EXTRA ENV VARS
+        extra_env_vars = []
+        try:
+            extra_env_vars = api_client.request_extra_environment()
+        except Exception:
+            pass   
+        __data__["extra_environment"] = extra_env_vars
 
+        # PLATFORMS
+        it_platforms = set([it["operating_system"] for it in instance_types])
+        valid_platforms = it_platforms.intersection(__platforms__)
         kwargs = {"platforms": valid_platforms}
 
         # If there's only one product, it's possible to initialize the software tree with a plugin.
@@ -158,14 +152,13 @@ def data(force=False):
             instance_types = [
                 it for it in instance_types if it["operating_system"] in sw_platforms
             ]
-
+        # Then adjust __platforms__ to match the instance types that are represented.
         __platforms__ = set([it["operating_system"] for it in instance_types])
 
         __data__["instance_types"] = HardwareSet(instance_types)
 
     return __data__
 
-
 def valid():
     """
     Check validity.
@@ -210,49 +203,9 @@ def products():
     """
     return __products__
 
-
-def set_fixtures_dir(path):
-    """
-    Specify a directory in which to find JSON files representing the three sets of data to provide.
-    The individual filenames are:
-
-    * `projects.json`
-    * `instance_types.json`
-    * `software.json`
-
-    These files could be used in an environment where machines can't access the internet. They are
-    also useful as a cache for developers who need to reload often as it avoids waiting for the
-    network.
-
-    In order to get the content for the fixtures files, use the following Example
-
-    Examples:
-        >>> from ciocore import api_client
-        >>> projects = api_client.request_projects()
-        >>> instance_types = api_client.request_instance_types()
-        >>> software = api_client.request_software_packages()
-
-    Write that data as JSON to the filenames listed above.
-
-    Arguments:
-        path (str): Directory in which to find the above files.
-
-    """
-
-    global __fixtures_dir__
-    __fixtures_dir__ = path or ""
-
-
-def _get_json_fixture(resource):
-    if __fixtures_dir__:
-        cache_path = os.path.join(__fixtures_dir__, "{}.json".format(resource))
-        if os.path.isfile(cache_path):
-            try:
-                with open(cache_path) as f:
-                    return json.load(f)
-            except BaseException:
-                pass
-
+def set_fixtures_dir(_):
+    """Deprecated. """
+    pass
 
 def platforms():
     """