ciocore 7.0.2b4__py2.py3-none-any.whl → 8.0.0__py2.py3-none-any.whl

ciocore/uploader/_uploader.py

@@ -15,8 +15,15 @@ except ImportError:
 
 import ciocore
 from ciocore import config
-from ciocore import api_client, client_db, common, file_utils, loggeria, worker, exceptions
-from ciocore.common import CONDUCTOR_LOGGER_NAME, SIGINT_EXIT
+from ciocore import (
+    api_client,
+    client_db,
+    common,
+    file_utils,
+    loggeria,
+    worker,
+    exceptions,
+)
 
 from .upload_stats import UploadStats
 
@@ -46,19 +53,23 @@ class MD5Worker(worker.ThreadWorker):
 
         # if a submission time md5 was provided then check against it
         if submission_time_md5:
-            logger.info("Enforcing md5 match: %s for: %s", submission_time_md5, filename)
+            logger.info(
+                "Enforcing md5 match: %s for: %s", submission_time_md5, filename
+            )
             if current_md5 != submission_time_md5:
                 message = "MD5 of %s has changed since submission\n" % filename
                 message += "submitted md5: %s\n" % submission_time_md5
                 message += "current md5:   %s\n" % current_md5
-                message += "This is likely due to the file being written to after the user"
+                message += (
+                    "This is likely due to the file being written to after the user"
+                )
                 message += " submitted the job but before it got uploaded to conductor"
                 logger.error(message)
                 raise Exception(message)
         self.metric_store.set_dict("file_md5s", filename, current_md5)
         self.metric_store.set_dict("file_md5s_cache_hit", filename, cache_hit)
         size_bytes = os.path.getsize(filename)
-        
+
         return (filename, current_md5, size_bytes)
 
     def get_md5(self, filepath):
@@ -68,9 +79,9 @@ class MD5Worker(worker.ThreadWorker):
         Use the sqlite db cache to retrive this (if the cache is valid), otherwise generate the md5
         from scratch
         """
-        
+
         cache_hit = True
-        
+
         # If md5 caching is disable, then just generate the md5 from scratch
         if not self.md5_caching:
             cache_hit = False
@@ -96,7 +107,9 @@ class MD5Worker(worker.ThreadWorker):
         """
         Store the given file_info into the database
         """
-        client_db.FilesDB.add_file(file_info, db_filepath=self.database_filepath, thread_safe=True)
+        client_db.FilesDB.add_file(
+            file_info, db_filepath=self.database_filepath, thread_safe=True
+        )
 
 
 class MD5OutputWorker(worker.ThreadWorker):
@@ -127,46 +140,45 @@ class MD5OutputWorker(worker.ThreadWorker):
 
     @common.dec_catch_exception(raise_=True)
     def target(self, thread_int):
-
         while not common.SIGINT_EXIT:
-            
             job = None
 
             try:
                 logger.debug("Worker querying for job")
                 job = self.in_queue.get(block=True, timeout=self.wait_time)
                 logger.debug("Got job")
-                queue_size = self.in_queue.qsize()                       
-            
+                queue_size = self.in_queue.qsize()
+
             except:
-                
                 logger.debug("No jobs available")
-                
+
                 if self._job_counter.value >= self.task_count:
-                    
                     if self.batch:
                         self.ship_batch()
-                    
+
                     logger.debug("Worker has completed all of its tasks (%s)", job)
                     self.thread_complete_counter.decrement()
                     break
-                
+
                 elif self._job_counter.value == 0:
                     logger.debug("Worker waiting for first job")
-                
+
                 time.sleep(1)
                 continue
-                
-            logger.debug("Worker got job %S", job)
+
+            logger.debug("Worker got job %s", job)
             self._job_counter.increment()
-            logger.debug("Processing Job '%s' #%s on %s. %s tasks remaining in queue", job,
-                                                                                       self._job_counter.value, 
-                                                                                       self, 
-                                                                                       queue_size)
+            logger.debug(
+                "Processing Job '%s' #%s on %s. %s tasks remaining in queue",
+                job,
+                self._job_counter.value,
+                self,
+                queue_size,
+            )
 
             try:
                 self.check_for_poison_pill(job)
-                
+
                 # add file info to the batch list
                 self.batch.append(
                     {
@@ -175,24 +187,25 @@ class MD5OutputWorker(worker.ThreadWorker):
                         "size": job[2],
                     }
                 )
-    
+
                 # if the batch is self.batch_size, ship it
                 if len(self.batch) == self.batch_size:
                     self.ship_batch()
-    
+
                 # mark this task as done
                 self.mark_done()
-                
+
             except Exception as exception:
                 logger.exception('CAUGHT EXCEPTION on job "%s" [%s]:\n', job, self)
 
                 # if there is no error queue to dump data into, then simply raise the exception
                 if self.error_queue is None:
                     raise
-                
+
                 self.error_queue.put(sys.exc_info())
                 # exit the while loop to stop the thread
-                break                
+                break
+
 
 class HttpBatchWorker(worker.ThreadWorker):
     """
@@ -265,8 +278,10 @@ class HttpBatchWorker(worker.ThreadWorker):
             return url_list
         if response_code == 204:
             return None
-        raise Exception("%s Failed request to: %s\n%s" % (response_code, uri_path, response_str))
-    
+        raise Exception(
+            "%s Failed request to: %s\n%s" % (response_code, uri_path, response_str)
+        )
+
     def do_work(self, job, thread_int):
         logger.debug("getting upload urls for %s", job)
         result = self.make_request(job)
@@ -274,19 +289,19 @@ class HttpBatchWorker(worker.ThreadWorker):
         # Determine which files have already been uploaded by looking at the difference between
         # the file paths in job and the file paths returned by the request. Only files that need
         # to be uploaded are returned by the request
-        incoming_file_paths = set([ item['path'] for item in job] )
+        incoming_file_paths = set([item["path"] for item in job])
 
         if result:
-            
             for item_type in result.values():
                 for item in item_type:
-                    incoming_file_paths.remove(item['filePath'])
+                    incoming_file_paths.remove(item["filePath"])
 
         for path in incoming_file_paths:
             self.metric_store.increment("already_uploaded", True, path)
-        
+
         return result
 
+
 """
 This worker subscribes to a queue of list of file uploads (multipart and singlepart).
 
@@ -319,24 +334,24 @@ class FileStatWorker(worker.ThreadWorker):
                 path = singlepart_upload["filePath"]
                 file_size = singlepart_upload["fileSize"]
                 upload_url = singlepart_upload["preSignedURL"]
-                
+
                 self.metric_store.increment("bytes_to_upload", file_size, path)
                 self.metric_store.increment("num_files_to_upload")
                 logger.debug("Singlepart, adding task %s", path)
-    
+
                 self.put_job((path, file_size, upload_url, SINGLEPART))
-    
+
             # iterate through multipart
             for multipart_upload in job.get("multiPartURLs", []):
                 path = multipart_upload["filePath"]
                 file_size = multipart_upload["fileSize"]
-                
+
                 self.metric_store.increment("bytes_to_upload", file_size, path)
                 self.metric_store.increment("num_files_to_upload")
                 logger.debug("Multipart, adding task %s", path)
                 self.put_job((path, file_size, multipart_upload, MULTIPART))
 
-        # make sure we return None, so no message is automatically added to the out_queue        
+        # make sure we return None, so no message is automatically added to the out_queue
         return None
 
 
@@ -367,10 +382,9 @@ class UploadWorker(worker.ThreadWorker):
                 self.metric_store.increment("bytes_uploaded", len(data), filename)
 
     def do_work(self, job, thread_int):
-        
         if job:
             kms_key_name = None
-            
+
             try:
                 filename = job[0]
                 file_size = job[1]
@@ -380,37 +394,44 @@ class UploadWorker(worker.ThreadWorker):
             except Exception:
                 logger.error("Issue with job (%s): %s", len(job), job)
                 raise
-            
+
             if len(job) > 4:
                 kms_key_name = job[4]
-    
+
             md5 = self.metric_store.get_dict("file_md5s", filename)
-    
+
             try:
                 if upload_type == SINGLEPART:
-                    return self.do_singlepart_upload(upload, filename, file_size, md5, kms_key_name)
+                    return self.do_singlepart_upload(
+                        upload, filename, file_size, md5, kms_key_name
+                    )
                 elif upload_type == MULTIPART:
                     return self.do_multipart_upload(upload, filename, md5)
-    
-                raise Exception("upload_type is '{}' expected %s or %s" % (upload_type, SINGLEPART, MULTIPART))
-            
+
+                raise Exception(
+                    "upload_type is '%s' expected %s or %s"
+                    % (upload_type, SINGLEPART, MULTIPART)
+                )
+
             except Exception as err_msg:
-                
                 real_md5 = common.get_base64_md5(filename)
 
                 if isinstance(err_msg, requests.exceptions.HTTPError):
-                    error_message = "Upload of {} failed with a response code {} ({}) (expected '{}', got '{}')".format(filename, err_msg.response.status_code, err_msg.response.reason, md5, real_md5)
-                
+                    error_message = f"Upload of {filename} failed with a response code {err_msg.response.status_code} ({err_msg.response.reason}) (expected '{md5}', got '{real_md5}')"
                 else:
-                    error_message = "Upload of {} failed. (expected '{}', got '{}') {}".format(filename,  md5, real_md5, str(err_msg))
-                
+                    error_message = (
+                        f"Upload of {filename} failed. (expected '{md5}', got '{real_md5}') {str(err_msg)}"
+                    )
+
                 logger.error(error_message)
                 raise exceptions.UploadError(error_message)
-            
+
         return worker.EMPTY_JOB
 
     @common.DecRetry(retry_exceptions=api_client.CONNECTION_EXCEPTIONS, tries=5)
-    def do_singlepart_upload(self, upload_url, filename, file_size, md5, kms_key_name=None):
+    def do_singlepart_upload(
+        self, upload_url, filename, file_size, md5, kms_key_name=None
+    ):
         """
         Note that for GCS we don't rely on the make_request's own retry mechanism because we need to
         recreate the chunked_reader generator before retrying the request. Instead, we wrap this
@@ -450,10 +471,8 @@ class UploadWorker(worker.ThreadWorker):
 
                 return response
         else:
-            headers = {"Content-MD5": md5, 
-                       "Content-Type": "application/octet-stream"
-            }
-            
+            headers = {"Content-MD5": md5, "Content-Type": "application/octet-stream"}
+
             if kms_key_name:
                 headers["x-goog-encryption-kms-key-name"] = kms_key_name
 
@@ -543,30 +562,34 @@ class UploadWorker(worker.ThreadWorker):
             response.close()
 
             return response.headers
-        
+
     def is_complete(self):
         # Get the number of files already uploaded as they are not passed to the Upload
         # worker
         file_store = self.metric_store.get("files")
-        
+
         if isinstance(file_store, dict):
-            already_completed_uploads = len([x for x in file_store.values() if x['already_uploaded']])
+            already_completed_uploads = len(
+                [x for x in file_store.values() if x["already_uploaded"]]
+            )
             queue_size = self.out_queue.qsize()
-            logger.debug("Is complete? out_queue_size=%s, completed_uploads=%s, task_count=%s", queue_size,
-                                                                                                already_completed_uploads,
-                                                                                                self.task_count)
-        
+            logger.debug(
+                "Is complete? out_queue_size=%s, completed_uploads=%s, task_count=%s",
+                queue_size,
+                already_completed_uploads,
+                self.task_count,
+            )
+
             return (queue_size + already_completed_uploads) >= self.task_count
-        
+
         else:
             logger.debug("Is complete?: files not initialized yet")
             return False
 
 
 class Uploader(object):
-
     sleep_time = 10
-    
+
     CLIENT_NAME = "Uploader"
 
     def __init__(self, args=None):
@@ -581,24 +604,22 @@ class Uploader(object):
         self.cancel = False
         self.error_messages = []
         self.num_files_to_process = 0
-        
+
         self.report_status_thread = None
         self.monitor_status_thread = None
-        
+
     def emit_progress(self, upload_stats):
-        
-        if self.progress_callback:        
+        if self.progress_callback:
             self.progress_callback(upload_stats)
 
     def prepare_workers(self):
         logger.debug("preparing workers...")
-        
+
         if isinstance(threading.current_thread(), threading._MainThread):
             common.register_sigint_signal_handler()
         self.manager = None
 
     def create_manager(self, project=None):
-       
         job_description = [
             (
                 MD5Worker,
@@ -609,20 +630,14 @@ class Uploader(object):
                     "md5_caching": self.args["md5_caching"],
                 },
             ),
-            (   
-                MD5OutputWorker, [], {"thread_count": 1}
-            ),
+            (MD5OutputWorker, [], {"thread_count": 1}),
             (
                 HttpBatchWorker,
                 [],
                 {"thread_count": self.args["thread_count"], "project": project},
             ),
-            (   
-                FileStatWorker, [], {"thread_count": 1}
-            ),
-            (
-                UploadWorker, [], {"thread_count": self.args["thread_count"]}
-            ),
+            (FileStatWorker, [], {"thread_count": 1}),
+            (UploadWorker, [], {"thread_count": self.args["thread_count"]}),
         ]
 
         manager = worker.JobManager(job_description)
@@ -634,7 +649,6 @@ class Uploader(object):
         logger.debug("started report_status thread")
         update_interval = 15
         while True:
-
             # don't report status if we are doing a local_upload
             if not self.upload_id:
                 logger.debug("not updating status as we were not provided an upload_id")
@@ -661,7 +675,7 @@ class Uploader(object):
                     logger.error("could not report status:")
                     logger.error(traceback.print_exc())
                     logger.error(traceback.format_exc())
-                    
+
             else:
                 break
 
@@ -669,7 +683,9 @@ class Uploader(object):
 
     def create_report_status_thread(self):
         logger.debug("creating reporter thread")
-        self.report_status_thread = threading.Thread(name="ReporterThread", target=self.report_status)
+        self.report_status_thread = threading.Thread(
+            name="ReporterThread", target=self.report_status
+        )
         self.report_status_thread.daemon = True
         self.report_status_thread.start()
 
@@ -684,19 +700,27 @@ class Uploader(object):
         while True:
             if self.working:
                 try:
-                    upload_stats = UploadStats.create (self.manager.metric_store, self.num_files_to_process, self.job_start_time)
+                    upload_stats = UploadStats.create(
+                        self.manager.metric_store,
+                        self.num_files_to_process,
+                        self.job_start_time,
+                    )
                     progress_handler(upload_stats)
                 except Exception as e:
                     print(e)
                     print(traceback.format_exc())
-                    
+
             else:
                 break
             sleep()
 
     def create_monitor_status_thread(self):
         logger.debug("creating console status thread")
-        self.monitor_status_thread = threading.Thread(name="PrintStatusThread", target=self.monitor_status, args=(self.emit_progress,))
+        self.monitor_status_thread = threading.Thread(
+            name="PrintStatusThread",
+            target=self.monitor_status,
+            args=(self.emit_progress,),
+        )
 
         # make sure threads don't stop the program from exiting
         self.monitor_status_thread.daemon = True
@@ -705,11 +729,17 @@ class Uploader(object):
         self.monitor_status_thread.start()
 
     def mark_upload_finished(self, upload_id, upload_files):
-
-        data = {"upload_id": upload_id, "status": "server_pending", "upload_files": upload_files}
+        data = {
+            "upload_id": upload_id,
+            "status": "server_pending",
+            "upload_files": upload_files,
+        }
 
         self.api_client.make_request(
-            "/uploads/%s/finish" % upload_id, data=json.dumps(data), verb="POST", use_api_key=True
+            "/uploads/%s/finish" % upload_id,
+            data=json.dumps(data),
+            verb="POST",
+            use_api_key=True,
         )
         return True
 
@@ -718,11 +748,19 @@ class Uploader(object):
 
         # report error_message to the app
         self.api_client.make_request(
-            "/uploads/%s/fail" % upload_id, data=error_message, verb="POST", use_api_key=True
+            "/uploads/%s/fail" % upload_id,
+            data=error_message,
+            verb="POST",
+            use_api_key=True,
         )
 
         return True
 
+    def assets_only(self, *paths):
+        processed_filepaths = file_utils.process_upload_filepaths(paths)
+        file_map = {path: None for path in processed_filepaths}
+        self.handle_upload_response(project=None, upload_files=file_map)
+
     def handle_upload_response(self, project, upload_files, upload_id=None):
         """
         This is a really confusing method and should probably be split into to clear logic
@@ -732,7 +770,6 @@ class Uploader(object):
         only be fed uploads by the app which have valid projects attached to them.
         """
         try:
-
             logger.info("%s", "  NEXT UPLOAD  ".center(30, "#"))
             logger.info("project: %s", project)
             logger.info("upload_id is %s", upload_id)
@@ -744,7 +781,7 @@ class Uploader(object):
 
             # reset counters
             self.num_files_to_process = len(upload_files)
-            logger.debug( "Processing %s files", self.num_files_to_process)
+            logger.debug("Processing %s files", self.num_files_to_process)
             self.job_start_time = datetime.datetime.now()
             self.upload_id = upload_id
             self.job_failed = False
@@ -753,7 +790,7 @@ class Uploader(object):
             self.working = True
 
             self.prepare_workers()
-            
+
             # create worker pools
             self.manager = self.create_manager(project)
 
@@ -767,39 +804,43 @@ class Uploader(object):
                 self.manager.add_task((path, md5))
 
             logger.info("creating console status thread...")
-            self.create_monitor_status_thread()                
+            self.create_monitor_status_thread()
 
-            #wait for work to finish
+            # wait for work to finish
             while not self.manager.is_complete():
                 logger.debug("Manager is running, cancel requested?: %s", self.cancel)
-                
+
                 if self.cancel or self.manager.error or common.SIGINT_EXIT:
                     self.error_messages = self.manager.stop_work()
                     logger.debug("Manager sucesfully stopped")
                     break
-                
+
                 time.sleep(5)
-            
+
             # Shutdown the manager once all jobs are done
-            if not self.cancel and not self.manager.error:    
-                logger.debug("Waiting for Manager to join")            
+            if not self.cancel and not self.manager.error:
+                logger.debug("Waiting for Manager to join")
                 self.manager.join()
 
-            upload_stats = UploadStats.create(self.manager.metric_store, self.num_files_to_process, self.job_start_time)
+            upload_stats = UploadStats.create(
+                self.manager.metric_store,
+                self.num_files_to_process,
+                self.job_start_time,
+            )
             logger.info(upload_stats.get_formatted_text())
-            self.emit_progress(upload_stats)            
-            
+            self.emit_progress(upload_stats)
+
             logger.debug("error_message: %s", self.error_messages)
 
             # signal to the reporter to stop working
             self.working = False
             logger.info("done uploading files")
-            
+
             logger.debug("Waiting for reporter status thread to join")
             self.report_status_thread.join()
-            
+
             logger.debug("Waiting for print status thread to join")
-            self.monitor_status_thread.join()            
+            self.monitor_status_thread.join()
 
             #  Despite storing lots of data about new uploads, we will only send back the things
             #  that have changed, to keep payloads small.
@@ -807,7 +848,7 @@ class Uploader(object):
             if self.upload_id and not self.error_messages:
                 md5s = self.return_md5s()
                 for path in md5s:
-                    finished_upload_files[path] =  {"source": path, "md5": md5s[path]}
+                    finished_upload_files[path] = {"source": path, "md5": md5s[path]}
 
                 self.mark_upload_finished(self.upload_id, finished_upload_files)
 
@@ -815,13 +856,12 @@ class Uploader(object):
             self.error_messages.append(sys.exc_info())
 
     def main(self, run_one_loop=False):
-        
         def show_ouput(upload_stats):
             print(upload_stats.get_formatted_text())
             logger.info("File Progress: %s", upload_stats.file_progress)
-            
+
         self.progress_callback = show_ouput
-        
+
         logger.info("Uploader Started. Checking for uploads...")
 
         waiting_for_uploads_flag = False
@@ -833,10 +873,12 @@ class Uploader(object):
                 data["location"] = self.location
                 logger.debug("Data: %s", data)
                 resp_str, resp_code = self.api_client.make_request(
-                    "/uploads/client/next", data=json.dumps(data), verb="PUT", use_api_key=True
+                    "/uploads/client/next",
+                    data=json.dumps(data),
+                    verb="PUT",
+                    use_api_key=True,
                 )
                 if resp_code == 204:
-
                     if not waiting_for_uploads_flag:
                         sys.stdout.write("\nWaiting for jobs to upload ")
                         sys.stdout.flush()
@@ -847,19 +889,21 @@ class Uploader(object):
                     time.sleep(self.sleep_time)
                     waiting_for_uploads_flag = True
                     continue
-                
+
                 elif resp_code != 201:
-                    logger.error("received invalid response code from app %s", resp_code)
+                    logger.error(
+                        "received invalid response code from app %s", resp_code
+                    )
                     logger.error("response is %s", resp_str)
                     time.sleep(self.sleep_time)
                     continue
-                
+
                 print("")  # to make a newline after the 204 loop
 
                 try:
                     json_data = json.loads(resp_str)
                     upload = json_data.get("data", {})
-                
+
                 except ValueError:
                     logger.error("response was not valid json: %s", resp_str)
                     time.sleep(self.sleep_time)
@@ -870,28 +914,34 @@ class Uploader(object):
                 project = upload["project"]
 
                 self.handle_upload_response(project, upload_files, upload_id)
-                
+
                 logger.debug("Upload of entity %s completed.", upload_id)
-                upload_stats = UploadStats.create (self.manager.metric_store, self.num_files_to_process, self.job_start_time)
+                upload_stats = UploadStats.create(
+                    self.manager.metric_store,
+                    self.num_files_to_process,
+                    self.job_start_time,
+                )
                 show_ouput(upload_stats)
                 logger.debug(self.manager.worker_queue_status_text())
-                
+
                 error_messages = []
-                
+
                 for exception in self.error_messages:
                     error_messages.append(str(exception[1]))
-                    
+
                 if error_messages:
-                    self.mark_upload_failed(error_message="\n".join(error_messages), upload_id=upload_id)
-                
+                    self.mark_upload_failed(
+                        error_message="\n".join(error_messages), upload_id=upload_id
+                    )
+
                     log_file = loggeria.LOG_PATH
                     sys.stderr.write("Error uploading files:\n")
-                    
+
                     for err_msg in error_messages:
                         sys.stderr.write("\t%s\n", err_msg)
 
-                    sys.stderr.write("\nSee log %s for more details\n\n", log_file) 
-                
+                    sys.stderr.write("\nSee log %s for more details\n\n", log_file)
+
                 self.error_messages = []
 
                 waiting_for_uploads_flag = False
@@ -899,8 +949,8 @@ class Uploader(object):
             except KeyboardInterrupt:
                 logger.info("ctrl-c exit")
                 break
-            except Exception as errMsg:
-                logger.exception("Caught exception:\n%s", errMsg)
+            except Exception as err_msg:
+                logger.exception("Caught exception:\n%s", err_msg)
                 time.sleep(self.sleep_time)
                 continue
 
@@ -913,6 +963,7 @@ class Uploader(object):
         """
         return self.manager.metric_store.get_dict("file_md5s")
 
+
 def run_uploader(args):
     """
     Start the uploader process. This process will run indefinitely, polling
@@ -921,32 +972,34 @@ def run_uploader(args):
     # convert the Namespace object to a dictionary
     args_dict = vars(args)
     cfg = config.config().config
-    
-    api_client.ApiClient.register_client(client_name = Uploader.CLIENT_NAME, client_version=ciocore.__version__) 
+
+    api_client.ApiClient.register_client(
+        client_name=Uploader.CLIENT_NAME, client_version=ciocore.version
+    )
 
     # Set up logging
     log_level_name = args_dict.get("log_level") or cfg["log_level"]
-    
+
     loggeria.setup_conductor_logging(
         logger_level=loggeria.LEVEL_MAP.get(log_level_name),
         log_dirpath=args_dict.get("log_dir"),
         log_filename="conductor_uploader.log",
-        disable_console_logging = not args_dict['log_to_console'],
-        use_system_log=False
-    )    
+        disable_console_logging=not args_dict["log_to_console"],
+        use_system_log=False,
+    )
 
     print("Logging to %s", loggeria.LOG_PATH)
-    
+
     logger.debug("Uploader parsed_args is %s", args_dict)
-    
+
     resolved_args = resolve_args(args_dict)
     uploader = Uploader(resolved_args)
-    
+
     if args.paths:
-        processed_filepaths =  file_utils.process_upload_filepaths(args.paths[0])
+        processed_filepaths = file_utils.process_upload_filepaths(args.paths[0])
         file_map = {path: None for path in processed_filepaths}
         uploader.handle_upload_response(project=None, upload_files=file_map)
-    
+
     else:
         uploader.main()
 
@@ -972,7 +1025,7 @@ def resolve_args(args):
     Resolve all arguments, reconciling differences between command line args and config.yml args.
     See resolve_arg function.
     """
-    
+
     args["md5_caching"] = resolve_arg("md5_caching", args)
     args["database_filepath"] = resolve_arg("database_filepath", args)
     args["location"] = resolve_arg("location", args)
@@ -980,18 +1033,18 @@ def resolve_args(args):
 
     return args
 
+
 def resolve_arg(key, args):
     """
     If the key doesn't exist (or is None), grab it from the config.
     """
-    
+
     cfg = config.config().config
-    config_value = cfg.get(key) 
-    
+    config_value = cfg.get(key)
+
     value = args.get(key, config_value)
-    
+
     if value is None:
         value = config_value
-        
+
     return value
-