ciocore 8.0.0b32__py2.py3-none-any.whl → 8.0.1__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-8.0.0-beta.32
+8.0.1

ciocore/cli.py

@@ -118,9 +118,11 @@ def _set_logging(log_level, log_to_console=True, log_dir=None, log_filename=None
         log_filename=log_filename,
         console_formatter=loggeria.FORMATTER_VERBOSE,
         disable_console_logging= not log_to_console,
-        use_system_log=False,
+        use_system_log=True,
     )
 
+    print("Logging to %s" % loggeria.LOG_PATH)
+
 def _register(client):
     api_client.ApiClient.register_client(
         client_name=client.CLIENT_NAME, client_version=VERSION
@@ -131,7 +133,15 @@ def _register(client):
 @click.pass_context
 @click.option("-v", "--version", is_flag=True, help="Print the version and exit.")
 def main(ctx, version):
-    """Conductor Command-line interface."""
+    """
+    Conductor Command-line interface.
+    
+    To get help on subcommands, use the --help flag after the subcommand.
+    
+    Example:
+    conductor download --help
+    
+    """
     if not ctx.invoked_subcommand:
         if version:
             click.echo(VERSION)
@@ -158,7 +168,7 @@ def main(ctx, version):
     default=DEFAULT_CONFIG_LOG_LEVEL,
 )
 @click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
-@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=True)
+@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=False)
 @click.option(
     "-tc",
     "--thread_count",
@@ -181,15 +191,13 @@ def upload(
     Example:
     conductor upload file1 file2 file3
     """
-    logfile = log_dir and os.path.join(log_dir, "conductor_ul.log")
-    _set_logging(log_level, log_to_console=log_to_console,  log_dir=log_dir)
+
+    _set_logging(log_level, log_to_console=log_to_console,  log_dir=log_dir, log_filename="conductor_uploader.log")
 
     args_dict = {
         "database_filepath": database_filepath,
         "location": location,
         "md5_caching": md5_caching,
-        "log_level": log_level,
-        "log_dir": log_dir,
         "thread_count": thread_count,
     }
 
@@ -276,7 +284,7 @@ DEPRECATED_PATHS_HELP = "Specify a list of paths to upload."
     help=THREADS_HELP,
     default=DEFAULT_CONFIG_THREAD_COUNT,
 )
-@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=True)
+@click.option("-lcl", "--log_to_console", is_flag=True, help=LOG_TO_CONSOLE_HELP, default=False)
 @click.option("-lc", "--location", help=LOCATION_HELP)
 @click.option(
     "-p",
@@ -298,14 +306,12 @@ def uploader(
     Example:
     conductor upload file1 file2 file3
     """
-    _set_logging(log_level, log_to_console=log_to_console,  log_dir=log_dir)
+    _set_logging(log_level, log_to_console=log_to_console,  log_dir=log_dir, log_filename="conductor_uploader.log")
 
     args_dict = {
         "database_filepath": database_filepath,
         "location": location,
         "md5_caching": md5_caching,
-        "log_level": log_level,
-        "log_dir": log_dir,
         "thread_count": thread_count,
     }
 
@@ -374,11 +380,9 @@ def downloader(
         )
         ctx.exit(0)
 
-    if task_id:
-        job_id = job_id + ":" + task_id
-
     Downloader.download_jobs(
         (job_id,),
+        task_id=task_id,
         thread_count=thread_count,
         output_dir=output,
     )

ciocore/docsite/404.html

@@ -14,7 +14,7 @@
       
       
       <link rel="icon" href="/assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/api_client/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/apidoc/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/config/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/data/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/hardware_set/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/package_environment/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/apidoc/package_tree/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/cmdline/docs/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/cmdline/downloader/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/cmdline/packages/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/cmdline/uploader/index.html

@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/how-to-guides/index.html

@@ -16,7 +16,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/index.html

@@ -16,7 +16,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.png">
-      <meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.2.0-b2">
+      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.2.0-b2">
     
     
       

ciocore/docsite/stylesheets/extra.css

@@ -1,8 +1,8 @@
  
 
 :root {
-    --md-primary-fg-color: #a71ed8;
-    --md-accent-fg-color: #db4ae3;
+    --md-primary-fg-color: #3c75a4;
+    --md-accent-fg-color: #406499;
 }
 
 .md-content a {

ciocore/docsite/stylesheets/tables.css

@@ -158,9 +158,6 @@
 .opts-prop-create {
   color: #36be00
 }
-/* .opts-prop-query {
-  color: #af00db
-} */
 .opts-prop-edit {
   color: #007dd1
 }

ciocore/downloader/legacy_downloader.py

@@ -18,10 +18,10 @@ import time
 import threading
 import hashlib
 
+import ciocore
 from ciocore import config
 from ciocore import api_client, common, loggeria
 from ciocore.common import CONDUCTOR_LOGGER_NAME
-from cioseq.sequence import Sequence
 
 try:
     import Queue as queue
@@ -291,12 +291,12 @@ class Downloader(object):
         downloader.print_uptime()
 
     @classmethod
-    def download_jobs(cls, job_ids, thread_count=None, output_dir=None):
+    def download_jobs(cls, job_ids, task_id=None, thread_count=None, output_dir=None):
         """
         Run the downloader for explicit jobs, and terminate afterwards.
         """
         downloader = cls(thread_count=thread_count, output_dir=output_dir)
-        downloader.start(job_ids)
+        thread_states = downloader.start(job_ids, task_id=task_id)
         while not common.SIGINT_EXIT and (
             not downloader.pending_queue.empty() or not downloader.downloading_queue.empty()
         ):
@@ -306,7 +306,7 @@ class Downloader(object):
         downloader._print_download_history()
         downloader.print_uptime()
 
-    def start(self, job_ids=None, summary_interval=10):
+    def start(self, job_ids=None, task_id=None, summary_interval=10):
         # Create new queues
         self.start_time = time.time()
         self.pending_queue = queue.Queue()
@@ -316,7 +316,7 @@ class Downloader(object):
         # If a job id has been specified then only load the queue up with that work
         if job_ids:
             self.history_queue_max = None
-            self.get_jobs_downloads(job_ids)
+            self.get_jobs_downloads(job_ids, task_id)
 
         # otherwise create a queue thread the polls the app for wor
         else:
@@ -461,12 +461,10 @@ class Downloader(object):
             # Wait for the queue to be consumed before querying for more
             self.nap()
 
-    
     def nap(self):
         while not common.SIGINT_EXIT:
 
             time.sleep(self.naptime)
-            # Pretty sure this return should be unindented!
             return
 
     @common.dec_timer_exit(log_level=logging.DEBUG)
@@ -479,51 +477,23 @@ class Downloader(object):
         except Exception as e:
             logger.exception("Could not get next download")
 
-    def get_jobs_downloads(self, job_ids):
+    def get_jobs_downloads(self, job_ids, task_id):
         """
-        Download jobs, each with an optional task specification.
-        
-        Job ids are padded to 5 digits, and task ids are padded to 3 digits.
+        Get each job and optional comma-separated task list.
 
-        job_ids: (tuple) may take any of the following forms:
-        1. A single job id, e.g. (01234,)
-        2. It doesn't have to be padded, e.g. (1234,)
-        3. A jobs with a task specification, e.g. (01234:1-3,5,7-9)
-        4. Several of jobs:tasks, e.g. (01234, 56789:1-3,5,7-9)
-    
-        """
-        jobs = self._flatten(job_ids)
-        for job in jobs:
-            endpoint = self.endpoint_downloads_job % job["job_id"]
-            
-            for tid in job["tasks"] or [None]:
-                downloads = _get_job_download(endpoint, self.api_client, job["job_id"], tid)
+        There will only be a task list if there is just one job, due to earlier arg validation.
+
+        If there is no task list, _get_job_download is called with tid=None (i.e. the whole job)
+        """ 
+        task_ids = [t for t in task_id.split(",") if t] if task_id else [None]
+
+        for job_id in job_ids:
+            endpoint = self.endpoint_downloads_job % job_id
+            for tid in task_ids:
+                downloads = _get_job_download(endpoint, self.api_client, job_id, tid)
                 if downloads:
                     for task_download in downloads.get("downloads", []):
-                        print("putting in queue: %s" % task_download)
                         self.pending_queue.put(task_download, block=True)
-            
-    @staticmethod
-    def _flatten(job_ids):
-        """Create a list of job objects with keys: job_id and tasks. 
-        
-        see: get_jobs_downloads() function see tests/test_downloader.py for examples.
-        """
-        result = []
-        for job_id in job_ids:
-            if ":" in job_id:
-                job_id, range_spec = job_id.split(":")
-                try:
-                    seq = Sequence.create(range_spec)
-                    tasks = seq.expand("###")
-                except (ValueError, TypeError):
-                    tasks = None
-            else:
-                tasks = None
-            result.append({"job_id": job_id.zfill(5), "tasks": tasks})
-        return result
-    
-
 
     @common.dec_catch_exception(raise_=True)
     def download_target(self, pending_queue, downloading_queue, task_download_state):
@@ -620,8 +590,8 @@ class Downloader(object):
                     # account for a render's generated subdirectories
                     dirpath = os.path.dirname(local_filepath)
 
-                    # Ensure that the output_path directory exists and set open permissions
-                    logger.debug("Creating output_path directory if necessary: %s", dirpath)
+                    # Ensure that the destination directory exists and set open permissions
+                    logger.debug("Creating destination directory if necessary: %s", dirpath)
                     file_download_state.status = FileDownloadState.STATE_PREPARING_DIRECTORY
                     safe_mkdirs(dirpath)
 
@@ -1268,7 +1238,7 @@ def run_downloader(args):
         file_formatter=LOG_FORMATTER,
     )    
     
-    api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=__version__)
+    api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=ciocore.__version__)
 
     logger.debug("Downloader args: %s", args)
 
@@ -1278,6 +1248,7 @@ def run_downloader(args):
     if job_ids:
         Downloader.download_jobs(
             job_ids,
+            task_id=args.get("task_id"),
             thread_count=thread_count,
             output_dir=args.get("output"),
         )
@@ -1306,6 +1277,6 @@ def report_error(self, download_id, error_message):
         resp_str, resp_code = self.api_helper.make_request(
             "/downloads/%s/fail" % download_id, data=error_message, verb="POST", use_api_key=True
         )
-    except:
+    except e:
         pass
     return True

ciocore/file_utils.py

@@ -256,13 +256,13 @@ def get_common_dirpath(paths):
 
 
 def _is_valid_path(path_str):
-    """
+    r"""
     This is dirty/inaccurate helper function to determine whether the given "path" is considered
     valid. If so, return True.
 
     If the given path_str is any of the following characters, then it's to be considered invalid:
 
-        On linux\mac:
+        On linux/mac:
                 /
                 //
                 lettered drive (e.g. x:\)
@@ -453,7 +453,7 @@ def get_tx_path(filepath, existing_only=False):
 
 
 def strip_drive_letter(filepath):
-    """
+    r"""
     If the given filepath has a drive letter, remove it and return the rest of the path
 
         C:\cat.txt         -->    \cat.txt

ciocore/uploader/_uploader.py

@@ -640,8 +640,7 @@ class Uploader(object):
             (UploadWorker, [], {"thread_count": self.args["thread_count"]}),
         ]
 
-        manager = worker.JobManager(job_description)
-        manager.start()
+        manager = worker.JobManager(job_description)        
         return manager
 
     @common.dec_catch_exception(raise_=True)
@@ -744,7 +743,7 @@ class Uploader(object):
         return True
 
     def mark_upload_failed(self, error_message, upload_id):
-        logger.error("Upload failed: \n%s" % error_message)
+        logger.error("Upload failed: %s", error_message)
 
         # report error_message to the app
         self.api_client.make_request(
@@ -793,6 +792,7 @@ class Uploader(object):
 
             # create worker pools
             self.manager = self.create_manager(project)
+            self.manager.start()
 
             # create reporters
             logger.debug("creating report status thread...")
@@ -931,16 +931,17 @@ class Uploader(object):
 
                 if error_messages:
                     self.mark_upload_failed(
-                        error_message="\n".join(error_messages), upload_id=upload_id
+                        error_message="Uploader ERROR: {}".format("\n".join(error_messages)), 
+                                                                  upload_id=upload_id
                     )
 
                     log_file = loggeria.LOG_PATH
-                    sys.stderr.write("Error uploading files:\n")
+                    sys.stderr.write("\nError uploading files:\n")
 
                     for err_msg in error_messages:
-                        sys.stderr.write("\t%s\n", err_msg)
+                        sys.stderr.write("\t{}\n".format(err_msg))
 
-                    sys.stderr.write("\nSee log %s for more details\n\n", log_file)
+                    sys.stderr.write("\nSee log {} for more details\n\n".format(log_file))
 
                 self.error_messages = []
 

ciocore/worker.py

@@ -252,7 +252,7 @@ class ThreadWorker(object):
             return
         
         # don't to anything if we were not provided an out_queue
-        logger.debug("Attempting to put job to out_queue (%s -> %s)", job,self, self.out_queue)
+        logger.debug("Attempting to put job to out_queue (%s -> %s)", job, self.out_queue)
 
         if not self.out_queue:
             return
@@ -261,7 +261,7 @@ class ThreadWorker(object):
         if not WORKING:
             return
         
-        logger.debug("Adding job to out_queue (%s -> %s)", job,self, self.out_queue)
+        logger.debug("Adding job to out_queue (%s -> %s)", job, self.out_queue)
         
         # add item to job
         self.out_queue.put(job)
@@ -491,10 +491,15 @@ class JobManager():
         while not self.error_handler_stop:
             
             try:
-                error = self.error_queue.get(True, 2)
+                error = self.error_queue.get(True, 0.5)
                 
             except queue.Empty:
-                continue
+
+                if self.error:
+                    break
+                
+                else:
+                    continue
             
             logger.error('Got something from the error queue: %s', error)
             self.error.append(error)
@@ -503,7 +508,7 @@ class JobManager():
             except ValueError:
                 pass
             
-            self.stop_work(force=True)
+        self.stop_work(force=True)
 
     def start_error_handler(self):
         

{ciocore-8.0.0b32.dist-info → ciocore-8.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 8.0.0b32
+Version: 8.0.1
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -51,11 +51,14 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
-## Version: 8.0.0-beta.30 -- 30 Mar 2024
+## Version:8.0.1 -- 22 Apr 2024
 
+* The uploader now sends all error messages to the dashboard - not just the first error.
+* Fixed a bug where --log-to-console was not not working as expected in the command line upload subcommand.
+* Fixed some errors in DEBUG-level logging messages.
 
-Major Updates in Version 8.0.0
 
+## Version:8.0.0 -- 07 Apr 2024
 
 * The `conductor` commandline interface (CLI) has undergone a complete overhaul to improve its functionality and performance. It is now installed as a Python console script, which simplifies the installation process on all platforms and ensures that the tool is always accessible. In addition, this standardized installation allows for better integration into our Companion application and potentially customers' own proprietary tools.
 * The commandline downloader has been rewritten to improve its performance and reliability. In previous versions, the downloader would fetch all download URLs from the server before starting any downloads. This approach was inefficient and would lead to performance issues when handling jobs with large numbers of tasks. The new downloader fetches tasks in small batches, which allows the download to start almost immediately, regardless of the number of tasks. It provides several new commandline flags, such as filtering based on filenames, and better logging, giving users more control and visibility over the download process. In addition, the new downloader has been written as an event based system that can be seamlessly integrated into other pipeline tools and GUIs. Pipeline TDs need only to register handlers to any of the events that the downloader emits.

{ciocore-8.0.0b32.dist-info → ciocore-8.0.1.dist-info}/RECORD

@@ -1,16 +1,15 @@
-ciocore/VERSION,sha256=CNZsvBEqWiYC9vIgbPM4bF6nCIO2d2TSSVgiI35UuBQ,13
+ciocore/VERSION,sha256=NUqt24vsbMosc7cU07CLlLBV2gAdKgs8roFBqy0xjlA,5
 ciocore/__init__.py,sha256=aTP7LeeosQA8BZE67gDV4jgfTK5zxmwZRjiTRu_ZWj0,646
 ciocore/api_client.py,sha256=TyHXGmK4uTmKV93O_IpAvttZhbexIIZTGP-IVn_WyWM,24599
-ciocore/cli.py,sha256=BXgULpqElTHwX6nUcSFVVh99MCBSG2FjuOzRKhIa6Hk,15399
+ciocore/cli.py,sha256=jZ1lOKQiUcrMhsVmD9SVmPMFwHtgDF4SaoAf2-PBS54,15449
 ciocore/client_db.py,sha256=tTz3bl2xeDPPcYSDS3g3QgV_xYihJMx0Kj6OeN2klK0,12978
 ciocore/common.py,sha256=mBIS6KiYoQsjWe6aIFUGRRvCMl8BIN2kmLZ4J8icap8,14982
 ciocore/compat.py,sha256=5uEXPSog_jxsDMaHBswAKEtfyXT25VgU6WNGIhz9PHU,256
 ciocore/conductor_submit.py,sha256=ONE0LsA5hGavTJIOXXYx8qzl8_vBPADwhd6Ytq_0E0c,9382
 ciocore/config.py,sha256=rCL7kaFn1tYgSglN8q9Wx6SwMpoXTq0BMQGwPRVwVIg,8973
 ciocore/data.py,sha256=Ji0qUk8nJXBNakoHSqBiVx8O58SbZXyt273SHlEDn3U,7027
-ciocore/dev_inst_tagger.py,sha256=bPfoFbE7IPCpDbtbVHxW7IlMwvspdnJhPuNq-u4fJN4,3497
 ciocore/exceptions.py,sha256=4Oq-WX-qiN6kPUdBCHvvd6mtSQ0nCkDbJxWt2CNtpv8,1504
-ciocore/file_utils.py,sha256=bAlL31B4YkRgX-yT8kF8UXBFktQlsE1PvxbKqTeAeOU,17174
+ciocore/file_utils.py,sha256=swA7th9WhDEloW69YViRTKB-oeC9UmNdEGegfH1r-Gw,17176
 ciocore/hardware_set.py,sha256=FlRQiGCLRcSW7Oko_gzgVK8ZqJ_J92eT8e_AleAbS2E,17047
 ciocore/loggeria.py,sha256=2xdQRFb9NyXynU2O_pSOszJWcpoHgPwTUWJvERg7ODY,15251
 ciocore/package_environment.py,sha256=MEHV7jfs3NJIEYCIaW8JfJdBmelvPHZMmBzPlXETiRo,7808
@@ -19,22 +18,22 @@ ciocore/package_tree.py,sha256=vkORKXxQ7dO8l0_96eFwm-5AUVL0rP9bhgWYhW_v3lo,15649
 ciocore/post_install.py,sha256=zu5Ctz2ANbKD-f5G2ODLIhKkWENBi4F3UKKu50OEWrg,1000
 ciocore/retry.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ciocore/validator.py,sha256=f_K7gxz122W_i5AxVx6dKhckOygl8TnmQiVj7tyX5zw,2344
-ciocore/worker.py,sha256=h6JophGBWqLdLdvphlzkkVMt0ojLw7_1Vrz4dikPRYE,21743
+ciocore/worker.py,sha256=YnE0mGqkky9uxPMyDRmvdt1p1ER_GN39AeZPIGw6f4U,21832
 ciocore/auth/__init__.py,sha256=cdS-xZzMq41yXM5cz8sUlcYgo8CJYh8HcCCWmhbDgf0,606
 ciocore/auth/server.py,sha256=8btX9-EokUl6q55V8muDmEV2tvvbTBD0BHeWFbwkzUc,3892
-ciocore/docsite/404.html,sha256=DQCrQsaKh8M0zlCFZmjFHUtqy7_gx_i2Rpc3OoDdDI4,17207
-ciocore/docsite/index.html,sha256=p_Dq_6_8cwZZPDI5zjahfCWUkfbSj14NJtoQFAZv0WI,20945
+ciocore/docsite/404.html,sha256=zUCnDe9r5yFZh8MXcTmzDaFyYBuRHz1gvuyJS2rYiTI,17207
+ciocore/docsite/index.html,sha256=xz215H_KHWBYI1HT7IRpAZ4d1ptUiO8Y4DlOmbEPgPk,20945
 ciocore/docsite/logo.png,sha256=gArgFFWdw8w985-0TkuGIgU_pW9sziEMZdqytXb5WLo,2825
 ciocore/docsite/objects.inv,sha256=s2FKStLlVIQbfG7U2-nw-7rz4unvd1W0u00YtLBxAKo,758
 ciocore/docsite/sitemap.xml,sha256=M_V85zl0y2adRvzJAnoCxlZH_Hl7TLnIb1A-6l_xGmI,109
-ciocore/docsite/sitemap.xml.gz,sha256=TKsxI6-9nU-iyOIEJTL1yHZV8KoBweu_CZYLcbZMynM,127
-ciocore/docsite/apidoc/api_client/index.html,sha256=mHDq-zdWuilDKUluzmQNdOsImRoGEnnK5Ul2ApajgdU,170218
-ciocore/docsite/apidoc/apidoc/index.html,sha256=NQn8wjapxa7O2IQhsbE7Y-VMfMFL6Tby-L7qIF6K3m4,26171
-ciocore/docsite/apidoc/config/index.html,sha256=1GWkyClM7LJlLBpx07BqCi3xDkU-SKVPjNUEeKuwbNs,72559
-ciocore/docsite/apidoc/data/index.html,sha256=9DonjpASPYZGYRHjyaodkK3AxSlTHsvm1xdq2cqMxPI,50850
-ciocore/docsite/apidoc/hardware_set/index.html,sha256=DMlpzQCy2ypJ078tcFNrZAvG2sU7P5-ZwU4ZRCObtQg,123042
-ciocore/docsite/apidoc/package_environment/index.html,sha256=M2L6Pu00jYj5AdhHQXrNZjOKVWCXYtgL8I34zL6B0i4,69248
-ciocore/docsite/apidoc/package_tree/index.html,sha256=wKW5dsHBlcDyvCGakSNzYHX8z99F1Vhu2NyqAW6c1ko,109393
+ciocore/docsite/sitemap.xml.gz,sha256=mBXMN84utPFOOhXBKJnq47kV3Zp6NvUz5Xxy7lDOxq8,127
+ciocore/docsite/apidoc/api_client/index.html,sha256=0AdyUY-LmL5ZqUeqxIjjzmoZDbupNcImCWf7RFeO5-o,170218
+ciocore/docsite/apidoc/apidoc/index.html,sha256=KGsHBY6GBF380w338GfOIMWnwAyyLvw2i1RQFORdwlE,26171
+ciocore/docsite/apidoc/config/index.html,sha256=n3koKPRUza5-uHUt8hK8H8XyH6HG_U2CsGobdaN4KfY,72559
+ciocore/docsite/apidoc/data/index.html,sha256=QhqiXUd_3qk_jGRYB5JiqlJTem8Jg7aKa5mPPDT1978,50850
+ciocore/docsite/apidoc/hardware_set/index.html,sha256=4GEBYy5uLVlJqvdG71RdyrN7sKKuxszxipgmRi_NCbM,123042
+ciocore/docsite/apidoc/package_environment/index.html,sha256=1d4hVlBXPy4bldkBH1GsRjVd-bxdYANmB-Mg7xIAC_4,69248
+ciocore/docsite/apidoc/package_tree/index.html,sha256=7GaOHXeM_oKH8Db25f2-h5kUI1R6pcp0bXxW1COCJxs,109393
 ciocore/docsite/assets/_mkdocstrings.css,sha256=K3bqYEmxlOHQ3-M11JNbBWHCBDBLarkFRm8HuEYrAG4,341
 ciocore/docsite/assets/images/favicon.png,sha256=AjhUxD_Eslt5XuSVHIAZ494Fk__rb5GLXR8qm0elfP4,1870
 ciocore/docsite/assets/javascripts/bundle.4e31edb1.min.js,sha256=vMxCR_BtNIcbmbPV6j8Z-YDLQ9ckt4RzvGuzCTg034s,97250
@@ -77,26 +76,26 @@ ciocore/docsite/assets/stylesheets/main.83068744.min.css,sha256=gwaHRErdgDoSsYab
 ciocore/docsite/assets/stylesheets/main.83068744.min.css.map,sha256=q8dWQkgGSfCcYT2Le1J0Tmxpcqxol5kZKaokp9d8Af0,42278
 ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css,sha256=7MiWsGpIt19TLduugHr1OQv1TJ9rEsndnkc2V8D9VjM,12245
 ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css.map,sha256=T8ZT8Mw8ws4vxyx_b1QAwyC9_pnv7r0KVaUdu2ST7ug,3639
-ciocore/docsite/cmdline/docs/index.html,sha256=2SFe5X1oq58UDuYBvUXCcgSBseP5AMgfu-ldKKK1WXA,20051
-ciocore/docsite/cmdline/downloader/index.html,sha256=DAVErhWEPuEtkQr7y9D_wjylzM9j4nWIWvTi4_gMoaA,23171
-ciocore/docsite/cmdline/packages/index.html,sha256=j1YeMgdwZ6FyJGJii05AdtUWKRjyu1fOr3_bKJsLfEU,20923
-ciocore/docsite/cmdline/uploader/index.html,sha256=y0728lIAYC4b9mfpjNVfwQqXoqJrsdG49UY5u7B1p94,25123
-ciocore/docsite/how-to-guides/index.html,sha256=OFrFBTb9tuMn60Jd4oeM10Mw-d-0pyqwoKoo9g1biBM,20100
+ciocore/docsite/cmdline/docs/index.html,sha256=Vte3vd2TtRY34Ft6cKvUREeQrWEI9jgdTkQ0XzjZuss,20051
+ciocore/docsite/cmdline/downloader/index.html,sha256=917XE3waukGnIXI9qxl0zYEf3vHVIQPdY3ZybmABAqc,23171
+ciocore/docsite/cmdline/packages/index.html,sha256=Ssb2R7eSubBNR824oc3jjhJfrG3JKOPojA-uiAczgeQ,20923
+ciocore/docsite/cmdline/uploader/index.html,sha256=TtLbHNlu4ofZ4bQBqam87RelWdHnVDSL7_rw_YqBbC8,25123
+ciocore/docsite/how-to-guides/index.html,sha256=rF-DMyk4k_jGUJRF3dKtMMiOgXS8OvBqAn-zxWy3AGs,20100
 ciocore/docsite/search/search_index.json,sha256=ZMAbHLAlwOTujgA32XJTA7SHogRYh9i-ioPFOxjjJ0s,182633
-ciocore/docsite/stylesheets/extra.css,sha256=qgfcao9TEBzf6ieAWN5rPyrea7YM_YgVg5qr4NPrxoU,354
-ciocore/docsite/stylesheets/tables.css,sha256=O2PEwlKC0RfvZCRV9UERqQRvhb6jcqO84PLew0bafF0,3279
+ciocore/docsite/stylesheets/extra.css,sha256=_Cxe9Dhg1BBi6Kqaz_iZD9z9VyqxA9vtONRjP4PVic0,354
+ciocore/docsite/stylesheets/tables.css,sha256=LE_zwGRxGcdPIy-9QiVPecOzlEBSqZb_WP5vDkFE0ZM,3235
 ciocore/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ciocore/downloader/base_downloader.py,sha256=9PVmP1qy-dop4MbVsmLFvf5W6LKQLUKHPDYE6AEt71g,25784
 ciocore/downloader/download_runner_base.py,sha256=cWTWKhFX5FLG84Wh8_s7xpk3n0nmQ4fDiD2rYV6a3u0,1612
 ciocore/downloader/job_downloader.py,sha256=HAhr95RyCsj3KTp5W58Znc4JmqHY-mrM5b8vo6kRIJ0,5697
-ciocore/downloader/legacy_downloader.py,sha256=ef9hCMUTTuxYHKzpEoK5bc3gqwAEIhewkC_KpbhLVHE,52087
+ciocore/downloader/legacy_downloader.py,sha256=LIbT3BxuY56lq_UW_io9FSgK_2VIKwFVMENyF-5KllA,51158
 ciocore/downloader/log.py,sha256=WCRNx0LObx8JBO5MQucNHQGBXMgSBLGdH0VALN8dFDo,2105
 ciocore/downloader/logging_download_runner.py,sha256=iOuW8OvfAYW5MpP9zpUXh94qkWLC0GeG4PDGH7JSqJM,2865
 ciocore/downloader/perpetual_downloader.py,sha256=cD7lnBH75-c-ZVVPHZc1vSnDhgJOnGlPT85zn7IjGgA,2105
 ciocore/downloader/registry.py,sha256=_JIOuqpWkJkgJGN33nt-DCvqN9Gw3xeFhzPq4RUxIoE,2903
 ciocore/downloader/reporter.py,sha256=p1NK9k6iQ-jt7lRvZR0xFz0cGb2yo8tQcjlvYKR9SWM,4501
 ciocore/uploader/__init__.py,sha256=hxRFJf5Lo86rtRObFXSjjot8nybQd-SebSfYCbgZwow,24
-ciocore/uploader/_uploader.py,sha256=AaCwRo1rGkdMA-XACx9k4Z73R97ndJUVMemUZNV-Ark,37353
+ciocore/uploader/_uploader.py,sha256=40nzqO5DuFi4sx31VvjWxZPNkrWsWqM9jtFVxs_-o3o,37479
 ciocore/uploader/upload_stats/__init__.py,sha256=Lg1y4zq1i0cwc6Hh2K1TAQDYymLff49W-uIo1xjcvdI,5309
 ciocore/uploader/upload_stats/stats_formats.py,sha256=giNirtObU66VALWghPFSRhg3q_vw5MvESsnXhb_I3y8,2402
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -106,7 +105,7 @@ tests/package_fixtures.py,sha256=CsJnhB7oYzIxJH7b1tCOPyvnnVSCqEbSPhtCnsHL-nA,507
 tests/project_fixtures.py,sha256=iBm_th_JtAw76vlNu7Jjhh9tLH4oOaNi-MgtPzCV7yQ,138
 tests/test_api_client.py,sha256=tnYkIYvDILGJRatZM0nrtr83x8yBJstMP0Rt151U2gI,6757
 tests/test_base_downloader.py,sha256=SS7tWKv2ZZhpUDk4UCg1TkrNrpntjSewgzLl1mEubSE,3603
-tests/test_cli.py,sha256=yWWGLip8WgnDAB0vg7uhBU5eeEJieFLQ78Pn7IN-uWI,6088
+tests/test_cli.py,sha256=_WTs2SWlEgd6wtg1hmOBlFnbWVdFLvqp0KqNhy-y2e8,5532
 tests/test_common.py,sha256=tY_-SY-JmJX09UehFs9RIDqZ785AmhfTl6eVKJeIUFY,763
 tests/test_config.py,sha256=-_G682Ss3Zr1FmcMkjla1zAZprX2tQKpKc5_wD28rII,13340
 tests/test_data.py,sha256=NIBXpCjG3Os3vpc1CkiVONrebro8D_jqQyJ0N3kbucU,5433
@@ -121,8 +120,8 @@ tests/test_uploader.py,sha256=B1llTJt_fqR6e_V_Jxfw9z73QgkFlEPU87xLYGzt-TQ,2914
 tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-8.0.0b32.dist-info/METADATA,sha256=-fliSdqGREA2a5rnAhFe-yqaLDS5NJuKI_IUzlvHei4,18201
-ciocore-8.0.0b32.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-ciocore-8.0.0b32.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-8.0.0b32.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-8.0.0b32.dist-info/RECORD,,
+ciocore-8.0.1.dist-info/METADATA,sha256=798KBccoox0YMHAcd09zV7pIzVkXN_GNoBWhckmK4D4,18440
+ciocore-8.0.1.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+ciocore-8.0.1.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
+ciocore-8.0.1.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-8.0.1.dist-info/RECORD,,

tests/test_cli.py

@@ -13,8 +13,6 @@ class CliTestUploaderOptions(unittest.TestCase):
             "database_filepath": None,
             "location": None,
             "md5_caching": True,
-            "log_level": "INFO",
-            "log_dir": None,
             "thread_count": mock.ANY,
         }
         init_patcher = mock.patch("ciocore.cli.Uploader.__init__", autospec=True)
@@ -43,18 +41,6 @@ class CliTestUploaderOptions(unittest.TestCase):
         expected.update({"md5_caching": False})
         self.mock_init.assert_called_once_with(mock.ANY, expected)
 
-    def test_log_level_arg(self):
-        self.runner.invoke(cli_upload, ["--log_level", "DEBUG"])
-        expected = self.default_args
-        expected.update({"log_level": "DEBUG"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_log_dir_arg(self):
-        self.runner.invoke(cli_upload, ["--log_dir", "foo"])
-        expected = self.default_args
-        expected.update({"log_dir": "foo"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
     def test_thread_count_arg(self):
         self.runner.invoke(cli_upload, ["--thread_count", 4])
         expected = self.default_args

ciocore/dev_inst_tagger.py

@@ -1,120 +0,0 @@
-"""
-DEV only. This module is used to request instance types from the inst-tagger service. We try to isolate the cod so it can be easily removed.
-"""
-
-import os
-import requests
-from ciocore import api_client
-
-USE_INST_TAGGER = int(os.environ.get("CIO_FEATURE_USE_INST_TAGGER", 0))
-SUPABASE_URL = os.environ.get("CIO_FEATURE_INST_TAGGER_SUPABASE_URL")
-SUPABASE_KEY = os.environ.get("CIO_FEATURE_INST_TAGGER_SUPABASE_KEY")
-
-def request_supabase_data(table_name):
-    HEADERS = {
-        "Authorization": f"Bearer {SUPABASE_KEY}",
-        "apikey": SUPABASE_KEY,
-        "Content-Type": "application/json",
-    }
-
-    response = requests.get(
-        f"{SUPABASE_URL}/rest/v1/{table_name}", headers=HEADERS, timeout=5
-    )
-    if response.status_code == 200:
-        return response.json()
-    else:
-        print("Error:", response.status_code, response.text)
-        return None
-
-
-def get_cloud(cio_inst_types):
-    if not cio_inst_types:
-        return None
-    if cio_inst_types[0]["name"].startswith("cw-"):
-        return "cw"
-    elif "." in cio_inst_types[0]["name"]:
-        return "aws"
-    return "gcp"
-
-
-def normalize_sb_inst_types(sb_inst_types):
-    result = []
-    for inst_type in sb_inst_types:
-        el = inst_type["instance_type"].copy()
-        el["id"] = inst_type["id"]
-        result.append(el)
-    return result
-
-
-def to_id_dict(thelist):
-    return {d["id"]: d for d in thelist}
-
-
-def valid_inst_types():
-    sb_inst_types = request_supabase_data("instance_types")
-    sb_inst_types = normalize_sb_inst_types(sb_inst_types)
-
-    cio_inst_types = api_client.request_instance_types()
-    # print(json.dumps(cio_inst_types, indent=2))
-
-    cloud = get_cloud(cio_inst_types)
-
-    # filter out instance types that are not in the cloud
-
-    sb_inst_types = [
-        inst_type for inst_type in sb_inst_types if inst_type["cloud"] == cloud
-    ]
-    for it in sb_inst_types:
-        it["cores"] = it["total_cpu"]
-        it["memory"] = it["total_memory_gb"]
-
-        cores = it["cores"]
-        mem = it["memory_gb"]
-        cpu_desc = f"{cores} cores, {mem} GB RAM"
-        gpu_desc = ""
-        if it["gpu"]:
-            gpu_count = it["gpu"]["gpu_count"]
-            gpu_model = it["gpu"]["gpu_model"]
-            gpu_arch = it["gpu"]["gpu_architecture"]
-            gpu_mem = it["gpu"]["total_gpu_memory"]
-            gpu_desc = f" ({gpu_count}x {gpu_model} ({gpu_arch}) {gpu_mem} GB)"
-
-        desc = f"{cpu_desc}{gpu_desc}"
-        it["description"] = desc
-
-    return sb_inst_types
-
-
-def request_instance_types():
-    """
-    Request instance types from the inst-tagger service.
-
-    Returns:
-        list(dict): A list of instance types.
-    """
-    if not USE_INST_TAGGER:
-        return None
-    
-    inst_types = valid_inst_types()
-
-    tags = request_supabase_data("tags")
-    tag_ids = [tag["id"] for tag in tags]
-    tags_dict = to_id_dict(tags)
-
-    assignments = request_supabase_data("assignments")
-    assignments = [a for a in assignments if a["tag_id"] in tag_ids]
-
-
-    # assign tags to instance types
-    for inst_type in inst_types:
-        inst_type["categories"] = []
-        # find assignments relating to this instance type
-        these_assignments = [
-            a for a in assignments if a["instance_type_id"] == inst_type["id"]
-        ]
-
-        for assignment in these_assignments:
-            tag = tags_dict[assignment["tag_id"]]
-            inst_type["categories"].append({"label": tag["label"], "order": tag["order"]})
-
-    return inst_types