ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl

ciocore/conductor_submit.py

@@ -6,18 +6,18 @@ import getpass
 import json
 import logging
 import os
-import re
 import threading
+import traceback
 
 from ciocore import config
-from ciocore import exceptions, file_utils, api_client, uploader
-
+from ciocore import file_utils, api_client, uploader, exceptions
 from ciocore.common import CONDUCTOR_LOGGER_NAME
- 
+
 logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
 
 FEATURE_DEV = int(os.environ.get("CIO_FEATURE_DEV", 0))
 
+
 class Submit(object):
     """Conductor Submission object."""
 
@@ -33,7 +33,7 @@ class Submit(object):
         self.enforced_md5s = args.get("enforced_md5s", {})
         self.database_filepath = args.get("database_filepath", "")
         self.api_client = api_client.ApiClient()
-        
+
         self.progress_handler = None
         self.uploader_ = None
 
@@ -70,16 +70,25 @@ class Submit(object):
                 self.payload[arg] = args[arg]
             except KeyError:
                 if default is None:
-                    logger.error("Submit: You must provide the '{}' argument.".format(arg))
+                    logger.error(
+                        "Submit: You must provide the '{}' argument.".format(arg)
+                    )
                     raise
 
+        # If no upload paths are provided, make sure the backend does not expect a daemon to be running.
+        if not self.upload_paths:
+            self.payload["local_upload"] = True
+
         # HACK: Posix -> Windows submission - must windowize output_path. Only available for
         # developers. If a customer tries to submit from Mac to Windows, then they have access to
         # Windows instances by mistake. Yes this code could get them out of a bind, but it will
         # generate support tickets when they try to use the uploader daemon for example.
-        self.ensure_windows_drive_letters = FEATURE_DEV and self.payload["instance_type"].endswith("-w")
-        self.payload["output_path"] = self._ensure_windows_drive_letter(self.payload["output_path"])
-
+        self.ensure_windows_drive_letters = FEATURE_DEV and self.payload[
+            "instance_type"
+        ].endswith("-w")
+        self.payload["output_path"] = self._ensure_windows_drive_letter(
+            self.payload["output_path"]
+        )
 
         self.payload["notify"] = {"emails": self.payload["notify"]}
 
@@ -88,25 +97,25 @@ class Submit(object):
                 logger.debug("{}:{}".format(arg, self.payload[arg]))
 
     def upload_progress_callback(self, upload_stats):
-        '''
+        """
         Call the progress handler
-        '''
-        
+        """
+
         if self.progress_handler:
-            logger.debug("Sending progress update to {}".format(self.progress_handler))        
+            logger.debug("Sending progress update to {}".format(self.progress_handler))
             self.progress_handler(upload_stats)
 
     def stop_work(self):
-        '''
+        """
         Cancel the submission process
-        '''
-        
+        """
+
         logger.debug("Submitter was requested to stop work.")
 
         if self.uploader_:
-            logger.debug("Uploader set to cancel.")            
-            self.uploader_.cancel=True
-        
+            logger.debug("Uploader set to cancel.")
+            self.uploader_.cancel = True
+
     def main(self):
         """
         Submit the job
@@ -118,39 +127,42 @@ class Submit(object):
         2. local_upload=False: md5 calcs and uploads are performed on on any machine with access to
            the filesystem on which the files reside, and by the same paths as the submission machine.
         """
-        
-        self._log_threads(message_template="{thread_count} threads before starting upload")
 
-        processed_filepaths =  file_utils.process_upload_filepaths(self.upload_paths)
-        file_map = {path: None for path in processed_filepaths}
+        self._log_threads(
+            message_template="{thread_count} threads before starting upload"
+        )
+
+        if self.upload_paths:
 
-        if self.payload["local_upload"]:
-            file_map = self._handle_local_upload(file_map)
-        
-        elif self.enforced_md5s:
-            file_map = self._enforce_md5s(file_map)
+            processed_filepaths = file_utils.process_upload_filepaths(self.upload_paths)
+            file_map = {path: None for path in processed_filepaths}
 
-        for path in file_map:
-            expanded = self._expand_stats(path, file_map[path])
+            if self.payload["local_upload"]:
+                file_map = self._handle_local_upload(file_map)
 
-  
-            self.payload["upload_files"].append(expanded)
-            self.payload["upload_size"] += expanded["st_size"]
+            elif self.enforced_md5s:
+                file_map = self._enforce_md5s(file_map)
 
+            for path in file_map:
+                expanded = self._expand_stats(path, file_map[path])
+                self.payload["upload_files"].append(expanded)
+                self.payload["upload_size"] += expanded["st_size"]
 
         self._log_threads(message_template="{thread_count} threads after upload")
-        
+
         logger.info("Sending Job...")
-        
+
         response, response_code = self.api_client.make_request(
-            uri_path="jobs/", 
-            data=json.dumps(self.payload), 
-            raise_on_error=False, 
-            use_api_key=True
+            uri_path="jobs/",
+            data=json.dumps(self.payload),
+            raise_on_error=False,
+            use_api_key=True,
         )
- 
+
         if response_code not in [201, 204]:
-            raise Exception("Job Submission failed: Error %s ...\n%s" % (response_code, response))
+            raise Exception(
+                "Job Submission failed: Error %s ...\n%s" % (response_code, response)
+            )
 
         return json.loads(response), response_code
 
@@ -160,34 +172,42 @@ class Submit(object):
 
         Returns {"path1': md5_1, path2: md5_2}
         """
-        
         cfg = config.config().config
         api_client.read_conductor_credentials(use_api_key=True)
-        
+
         # Don't use more threads than there are files
         thread_count = min(len(file_map), cfg["thread_count"])
         logger.info("Using {} threads for the uploader".format(thread_count))
-        
+
         uploader_args = {
             "location": self.payload["location"],
             "database_filepath": self.database_filepath,
             "thread_count": thread_count,
             "md5_caching": self.md5_caching,
         }
-    
+
         self.uploader_ = uploader.Uploader(uploader_args)
 
         self.uploader_.progress_callback = self.upload_progress_callback
-        
+
         self.uploader_.handle_upload_response(self.payload["project"], file_map)
-        
+
         if self.uploader_.cancel:
-            raise exceptions.UserCanceledError("Job submission was cancelled by the user")
+            raise exceptions.UserCanceledError(
+                "Job submission was cancelled by the user"
+            )
 
         if self.uploader_.error_messages:
-            # Need a better way to handle multiple exceptions
-            raise self.uploader_.error_messages[0][1]
-        
+            error_message = ""
+            for cnt, err in enumerate(self.uploader_.error_messages):
+                error_message += "Error {}:\n{}\n\n".format(
+                    cnt + 1, "".join(traceback.format_exception(*err))
+                )
+            raise Exception(
+                "\n\nCould not upload files, encountered %s errors:\n\n%s"
+                % (len(self.uploader_.error_messages), error_message)
+            )
+
         # Get the resulting dictionary of the file's and their corresponding md5 hashes
         upload_md5s = self.uploader_.return_md5s()
         for path in upload_md5s:
@@ -202,13 +222,13 @@ class Submit(object):
 
         Returns {"path1': enforced_md5_1, path2: enforced_md5_2}
         """
-        
+
         progress_title = "Processing MD5 of local files"
         file_count = len(self.enforced_md5s)
 
         for cnt, filepath in enumerate(self.enforced_md5s):
-            percentage_complete = float(cnt)/float(file_count)
-            
+            percentage_complete = float(cnt) / float(file_count)
+
             md5 = self.enforced_md5s[filepath]
             logger.debug("filepath is %s" % filepath)
             processed_filepaths = file_utils.process_upload_filepath(filepath)
@@ -219,7 +239,6 @@ class Submit(object):
 
         return file_map
 
- 
     def _expand_stats(self, file, md5):
         filestat = os.stat(file)
 
@@ -240,16 +259,15 @@ class Submit(object):
             "st_mtime": filestat.st_mtime,
             "st_ctime": filestat.st_ctime,
         }
-        
+
     def _log_threads(self, message_template):
 
         threads = list(threading.enumerate())
-        
+
         for t in threads:
             logger.debug(t)
 
-        logger.debug(message_template.format(thread_count=len(threads)))        
-        
+        logger.debug(message_template.format(thread_count=len(threads)))
 
     def _ensure_windows_drive_letter(self, filepath):
         """
@@ -261,4 +279,3 @@ class Submit(object):
             logger.debug("Windows dev hack! Setting {0} to 'X:{0}'".format(filepath))
             return "X:{}".format(filepath)
         return filepath
-

ciocore/config.py

@@ -1,49 +1,135 @@
 """
-Config is an Object containing configuration variables.
+Config is a configuration object implemented as a module-level singleton.
 
-Only contains stuff that is not suitable to specify in a submitter.
+Configuration variables can be shared by importing the module. If there are changes in environment variables or other sources, the config can be refreshed.
 """
 
 import logging
 import os
-import multiprocessing
+# import multiprocessing
 import base64
 import json
 import re
-from ciocore import loggeria
+import platform
 
 from ciocore.common import CONDUCTOR_LOGGER_NAME
 
 logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
 
-#https://stackoverflow.com/a/3809435/179412
-URL_REGEX=re.compile(r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)")
+# https://stackoverflow.com/a/3809435/179412
+
+
+USER_DIRS = {
+    "Linux": os.path.expanduser(os.path.join("~", ".conductor")),
+    "Darwin": os.path.expanduser(os.path.join("~",".conductor")),
+    "Windows": os.path.expanduser(os.path.join("~", "AppData", "Local", "Conductor")),
+}
+
+DEFAULT_USER_DIR = USER_DIRS.get(platform.system(), USER_DIRS["Linux"])
+
+URL_REGEX = re.compile(
+    r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)"
+)
 
 __config__ = None
 
+
 def config(force=False):
+    """
+    Instantiate a config object if necessary.
+
+    Deprecated:
+        Use [get()](#get) instead.
+
+    Args:
+        force (bool): Discards any existing config object and instantiate a new one -- Defaults to `False`.
+
+    Returns:
+        dict: A dictionary containing configuration values.
+    """
+
     global __config__
     if force or not __config__:
         __config__ = Config()
     return __config__
 
+
+def get(force=False):
+    """
+    Instantiate a config object if necessary and return the dictionary.
+
+    Args:
+        force (bool): Discards any existing config object and instantiate a new one -- Defaults to `False`.
+
+    Returns:
+        dict: A dictionary containing configuration values.
+
+    Example:
+        >>> from ciocore import config
+        >>> config.get()
+        {
+          'thread_count': 16,
+          'priority': 5,
+          'md5_caching': True,
+          'log_level': 'INFO',
+          'url': 'https://dashboard.conductortech.com',
+          'auth_url': 'https://dashboard.conductortech.com',
+          'api_url': 'https://api.conductortech.com',
+          'api_key': None,
+          'downloader_page_size': 50,
+        }
+    """
+    global __config__
+    if force or not __config__:
+        __config__ = Config()
+    return __config__.config
+
+
 class Config(object):
     def __init__(self):
+        """
+        Initialize the config object.
+
+        A config object is a dictionary containing configuration values. It is a singleton, so there is only one instance of it. It is instantiated the first time it is needed. It can be refreshed by calling get() with the `force` keyword argument set to `True`.
+
+        A Config object has the following properties:
+
+        * `thread_count` The number of threads to use for downloading files. Defaults to the number of CPUs on the system times 2. It can be overridden by the `CONDUCTOR_THREAD_COUNT` environment variable.
+        * `priority` Set the priority for submissions. Defaults to 5. It can be overridden by the `CONDUCTOR_PRIORITY` environment variable.
+        * `md5_caching` Whether to cache MD5s. Defaults to `True`. It can be overridden by the `CONDUCTOR_MD5_CACHING` environment variable. Cachine MD5s significantly improves submission performance, but on rare occasions it can cause submissions to fail. If you experience this, set `md5_caching` to `False`.
+        * `log_level` The logging level. Defaults to `INFO`. It can be overridden by the `CONDUCTOR_LOG_LEVEL` environment variable.
+        * `url` The URL of the Conductor dashboard. Defaults to `https://dashboard.conductortech.com`. It can be overridden by the `CONDUCTOR_URL` environment variable.
+        * `auth_url` The URL of the Conductor dashboard. Defaults to `https://dashboard.conductortech.com`. It can be overridden by the `CONDUCTOR_AUTH_URL` environment variable. This is deprecated. Use `url` instead.
+        * `api_url` The URL of the Conductor API. Defaults to `https://api.conductortech.com`. It can be overridden by the `CONDUCTOR_API_URL` environment variable.
+        * `api_key` The API key. The API key can be acquired from the Conductor dashboard, and can be stored in an environment variable or a file. In both cases the API KEY can be a JSON object or a base64 encoded JSON object. If it is base64 encoded, it can be a string or bytes. If it is a string, it will be decoded as ASCII. If it is bytes, it will be decoded as UTF-8.
+            * Environment variable: The `CONDUCTOR_API_KEY` variable can hold the API KEY directly.
+            * File: The `CONDUCTOR_API_KEY_PATH` variable can hold the path to a file containing the API KEY.
+        * `downloader_page_size` The number of files to request from the Conductor API at a time. Defaults to 50. It can be overridden by the `CONDUCTOR_DOWNLOADER_PAGE_SIZE` environment variable.
+
+        Returns:
+            Config: A config object.
+
+        Raises:
+            ValueError -- Invalid inputs, such as badly formed URLs.
+        """
+        default_downloader_page_size = 50
+
+     
 
         try:
-            default_thread_count = min(multiprocessing.cpu_count() * 2, 16)
+            default_thread_count = min( os.cpu_count() - 1, 15)
         except NotImplementedError:
-            default_thread_count = 16
+            default_thread_count = 15
 
         url = os.environ.get("CONDUCTOR_URL", "https://dashboard.conductortech.com")
 
         if not URL_REGEX.match(url):
             raise ValueError("CONDUCTOR_URL is not valid '{}'".format(url))
- 
+
         api_url = os.environ.get("CONDUCTOR_API_URL", url.replace("dashboard", "api"))
         if not URL_REGEX.match(api_url):
             raise ValueError("CONDUCTOR_API_URL is not valid '{}'".format(api_url))
- 
+
         falsy = ["false", "no", "off", "0"]
 
         log_level = os.environ.get("CONDUCTOR_LOG_LEVEL", "INFO")
@@ -51,23 +137,41 @@ class Config(object):
             log_level = "INFO"
 
         self.config = {
-            "thread_count": int(os.environ.get("CONDUCTOR_THREAD_COUNT", default_thread_count)),
+            "thread_count": int(
+                os.environ.get("CONDUCTOR_THREAD_COUNT", default_thread_count)
+            ),
+            "downloader_page_size": int(
+                os.environ.get(
+                    "CONDUCTOR_DOWNLOADER_PAGE_SIZE", default_downloader_page_size
+                )
+            ),
             "priority": int(os.environ.get("CONDUCTOR_PRIORITY", 5)),
             "md5_caching": False
             if os.environ.get("CONDUCTOR_MD5_CACHING", "True").lower() in falsy
             else True,
             "log_level": log_level,
             "url": url,
-            # Keep "auth_url" for backwwards compatibillity only. 
-            # Clients should use "url" moving forward. 
+            # Keep "auth_url" for backwwards compatibillity only.
+            # Clients should use "url" moving forward.
             # Remove "auth_url" on the next major version bump.
             "auth_url": url,
             "api_url": api_url,
             "api_key": self.get_api_key_from_variable() or self.get_api_key_from_file(),
+            "user_dir": os.environ.get('CONDUCTOR_USER_DIR', DEFAULT_USER_DIR)
         }
 
     @staticmethod
     def get_api_key_from_variable():
+        """
+        Attempt to get an API key from the `CONDUCTOR_API_KEY` environment variable.
+
+        Raises:
+            ValueError: An error occurred while reading or loading the key into JSON.
+
+        Returns:
+            str: JSON object containing the key - base 64 decoded if necessary.
+
+        """
         api_key = os.environ.get("CONDUCTOR_API_KEY")
         if not api_key:
             return
@@ -88,6 +192,16 @@ class Config(object):
 
     @staticmethod
     def get_api_key_from_file():
+        """
+        Attempt to get an API key from the file in the CONDUCTOR_API_KEY_PATH environment variable.
+
+        Raises:
+            ValueError: An error occurred while reading or loading the key into JSON.
+
+        Returns:
+            str: JSON object containing the key - base 64 decoded if necessary.
+
+        """
         api_key_path = os.environ.get("CONDUCTOR_API_KEY_PATH")
         if not api_key_path:
             return