ciocore 8.2.0b1__py2.py3-none-any.whl → 8.3.0__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-8.2.0-beta.1
+8.3.0

ciocore/conductor_submit.py

@@ -17,6 +17,7 @@ logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
 
 FEATURE_DEV = int(os.environ.get("CIO_FEATURE_DEV", 0))
 
+
 class Submit(object):
     """Conductor Submission object."""
 
@@ -32,7 +33,7 @@ class Submit(object):
         self.enforced_md5s = args.get("enforced_md5s", {})
         self.database_filepath = args.get("database_filepath", "")
         self.api_client = api_client.ApiClient()
-        
+
         self.progress_handler = None
         self.uploader_ = None
 
@@ -69,15 +70,25 @@ class Submit(object):
                 self.payload[arg] = args[arg]
             except KeyError:
                 if default is None:
-                    logger.error("Submit: You must provide the '{}' argument.".format(arg))
+                    logger.error(
+                        "Submit: You must provide the '{}' argument.".format(arg)
+                    )
                     raise
 
+        # If no upload paths are provided, make sure the backend does not expect a daemon to be running.
+        if not self.upload_paths:
+            self.payload["local_upload"] = True
+
         # HACK: Posix -> Windows submission - must windowize output_path. Only available for
         # developers. If a customer tries to submit from Mac to Windows, then they have access to
         # Windows instances by mistake. Yes this code could get them out of a bind, but it will
         # generate support tickets when they try to use the uploader daemon for example.
-        self.ensure_windows_drive_letters = FEATURE_DEV and self.payload["instance_type"].endswith("-w")
-        self.payload["output_path"] = self._ensure_windows_drive_letter(self.payload["output_path"])
+        self.ensure_windows_drive_letters = FEATURE_DEV and self.payload[
+            "instance_type"
+        ].endswith("-w")
+        self.payload["output_path"] = self._ensure_windows_drive_letter(
+            self.payload["output_path"]
+        )
 
         self.payload["notify"] = {"emails": self.payload["notify"]}
 
@@ -86,25 +97,25 @@ class Submit(object):
                 logger.debug("{}:{}".format(arg, self.payload[arg]))
 
     def upload_progress_callback(self, upload_stats):
-        '''
+        """
         Call the progress handler
-        '''
-        
+        """
+
         if self.progress_handler:
-            logger.debug("Sending progress update to {}".format(self.progress_handler))        
+            logger.debug("Sending progress update to {}".format(self.progress_handler))
             self.progress_handler(upload_stats)
 
     def stop_work(self):
-        '''
+        """
         Cancel the submission process
-        '''
-        
+        """
+
         logger.debug("Submitter was requested to stop work.")
 
         if self.uploader_:
-            logger.debug("Uploader set to cancel.")            
-            self.uploader_.cancel=True
-        
+            logger.debug("Uploader set to cancel.")
+            self.uploader_.cancel = True
+
     def main(self):
         """
         Submit the job
@@ -116,37 +127,42 @@ class Submit(object):
         2. local_upload=False: md5 calcs and uploads are performed on on any machine with access to
            the filesystem on which the files reside, and by the same paths as the submission machine.
         """
-        
-        self._log_threads(message_template="{thread_count} threads before starting upload")
 
-        processed_filepaths =  file_utils.process_upload_filepaths(self.upload_paths)
-        file_map = {path: None for path in processed_filepaths}
+        self._log_threads(
+            message_template="{thread_count} threads before starting upload"
+        )
 
-        if self.payload["local_upload"]:
-            file_map = self._handle_local_upload(file_map)
-        
-        elif self.enforced_md5s:
-            file_map = self._enforce_md5s(file_map)
+        if self.upload_paths:
 
-        for path in file_map:
-            expanded = self._expand_stats(path, file_map[path])
-            self.payload["upload_files"].append(expanded)
-            self.payload["upload_size"] += expanded["st_size"]
+            processed_filepaths = file_utils.process_upload_filepaths(self.upload_paths)
+            file_map = {path: None for path in processed_filepaths}
 
+            if self.payload["local_upload"]:
+                file_map = self._handle_local_upload(file_map)
+
+            elif self.enforced_md5s:
+                file_map = self._enforce_md5s(file_map)
+
+            for path in file_map:
+                expanded = self._expand_stats(path, file_map[path])
+                self.payload["upload_files"].append(expanded)
+                self.payload["upload_size"] += expanded["st_size"]
 
         self._log_threads(message_template="{thread_count} threads after upload")
-        
+
         logger.info("Sending Job...")
-        
+
         response, response_code = self.api_client.make_request(
-            uri_path="jobs/", 
-            data=json.dumps(self.payload), 
-            raise_on_error=False, 
-            use_api_key=True
+            uri_path="jobs/",
+            data=json.dumps(self.payload),
+            raise_on_error=False,
+            use_api_key=True,
         )
 
         if response_code not in [201, 204]:
-            raise Exception("Job Submission failed: Error %s ...\n%s" % (response_code, response))
+            raise Exception(
+                "Job Submission failed: Error %s ...\n%s" % (response_code, response)
+            )
 
         return json.loads(response), response_code
 
@@ -158,11 +174,11 @@ class Submit(object):
         """
         cfg = config.config().config
         api_client.read_conductor_credentials(use_api_key=True)
-        
+
         # Don't use more threads than there are files
         thread_count = min(len(file_map), cfg["thread_count"])
         logger.info("Using {} threads for the uploader".format(thread_count))
-        
+
         uploader_args = {
             "location": self.payload["location"],
             "database_filepath": self.database_filepath,
@@ -173,18 +189,25 @@ class Submit(object):
         self.uploader_ = uploader.Uploader(uploader_args)
 
         self.uploader_.progress_callback = self.upload_progress_callback
-        
+
         self.uploader_.handle_upload_response(self.payload["project"], file_map)
-        
+
         if self.uploader_.cancel:
-            raise exceptions.UserCanceledError("Job submission was cancelled by the user")
+            raise exceptions.UserCanceledError(
+                "Job submission was cancelled by the user"
+            )
 
         if self.uploader_.error_messages:
             error_message = ""
             for cnt, err in enumerate(self.uploader_.error_messages):
-                error_message += "Error {}:\n{}\n\n".format(cnt+1, "".join(traceback.format_exception(*err)))
-            raise Exception("\n\nCould not upload files, encountered %s errors:\n\n%s" % (len(self.uploader_.error_messages), error_message))
-                            
+                error_message += "Error {}:\n{}\n\n".format(
+                    cnt + 1, "".join(traceback.format_exception(*err))
+                )
+            raise Exception(
+                "\n\nCould not upload files, encountered %s errors:\n\n%s"
+                % (len(self.uploader_.error_messages), error_message)
+            )
+
         # Get the resulting dictionary of the file's and their corresponding md5 hashes
         upload_md5s = self.uploader_.return_md5s()
         for path in upload_md5s:
@@ -199,13 +222,13 @@ class Submit(object):
 
         Returns {"path1': enforced_md5_1, path2: enforced_md5_2}
         """
-        
+
         progress_title = "Processing MD5 of local files"
         file_count = len(self.enforced_md5s)
 
         for cnt, filepath in enumerate(self.enforced_md5s):
-            percentage_complete = float(cnt)/float(file_count)
-            
+            percentage_complete = float(cnt) / float(file_count)
+
             md5 = self.enforced_md5s[filepath]
             logger.debug("filepath is %s" % filepath)
             processed_filepaths = file_utils.process_upload_filepath(filepath)
@@ -216,7 +239,6 @@ class Submit(object):
 
         return file_map
 
-
     def _expand_stats(self, file, md5):
         filestat = os.stat(file)
 
@@ -237,16 +259,15 @@ class Submit(object):
             "st_mtime": filestat.st_mtime,
             "st_ctime": filestat.st_ctime,
         }
-        
+
     def _log_threads(self, message_template):
 
         threads = list(threading.enumerate())
-        
+
         for t in threads:
             logger.debug(t)
 
-        logger.debug(message_template.format(thread_count=len(threads)))        
-        
+        logger.debug(message_template.format(thread_count=len(threads)))
 
     def _ensure_windows_drive_letter(self, filepath):
         """

ciocore/docsite/apidoc/hardware_set/index.html

@@ -1339,14 +1339,6 @@ HardwareSet        </code>
                     <span class="k">else</span> <span class="p">[{</span><span class="s2">&quot;label&quot;</span><span class="p">:</span> <span class="s2">&quot;CPU&quot;</span><span class="p">,</span> <span class="s2">&quot;order&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span>
                 <span class="p">)</span>
             <span class="n">result</span><span class="p">[</span><span class="n">it</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="n">it</span>
-        <span class="n">result</span><span class="p">[</span><span class="s2">&quot;best_fit&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span><span class="s2">&quot;automatic_instance_type&quot;</span><span class="p">,</span> 
-                            <span class="s2">&quot;description&quot;</span><span class="p">:</span> <span class="s2">&quot;Conductor will choose.&quot;</span><span class="p">,</span>
-                            <span class="s2">&quot;categories&quot;</span><span class="p">:</span> <span class="p">[{</span><span class="s2">&quot;label&quot;</span><span class="p">:</span> <span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;order&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">}],</span>
-                            <span class="s2">&quot;gpu&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
-                            <span class="s2">&quot;cores&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
-                            <span class="s2">&quot;memory&quot;</span><span class="p">:</span> <span class="mi">0</span>
-                            <span class="p">}</span>
-
 
         <span class="k">return</span> <span class="n">result</span>