ciocore 6.3.1rc3__py2.py3-none-any.whl → 6.4.0__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-6.3.1-rc.3
+6.4.0

ciocore/__about__.py

@@ -2,26 +2,9 @@ import os
 
 __all__ = ["__version__"]
 
-# Get the version from the VERSION file
-# The VERSION file may be in the current directory or (in dev) one directory up
-
 try:
-    with open(
-        os.path.join(os.path.abspath(os.path.dirname(__file__)), "VERSION"),
-        "r",
-        encoding="utf-8",
-    ) as version_file:
+    with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'VERSION')) as version_file:
         __version__ = version_file.read().strip()
-
+        
 except IOError:
-    try:
-        with open(
-            os.path.join(
-                os.path.abspath(os.path.dirname(os.path.dirname(__file__))), "VERSION"
-            ),
-            "r",
-            encoding="utf-8",
-        ) as version_file:
-            __version__ = version_file.read().strip()
-    except IOError:
-        __version__ = "dev"
+    __version__ = 'dev'

ciocore/cli/__init__.py

@@ -0,0 +1,3 @@
+"""
+Commandline tools.
+"""

ciocore/cli/conductor.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+# Use absolute imports to avoid a "conductor" name clash (this module name vs conductor package).
+from __future__ import absolute_import
+
+import argparse
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from ciocore import  downloader, uploader, loggeria, config
+
+def build_parser():
+    cfg = config.config().config
+
+    # Create a parent parser. Arguments that are common across all subparsers can be added to this parser
+    parent_parser = argparse.ArgumentParser(add_help=False)
+
+    # create the main parser. Not sure why this parser is required, but got parsing tracebacks when excluding it (it gets confused about the arguments provided)
+    parser = argparse.ArgumentParser(description="description")
+    subparsers = parser.add_subparsers(title="actions")
+
+
+    #############################
+    # UPLOADER PARSER
+    #############################
+    uploader_parser_desciption = "parse uploader arguments"
+    uploader_parser_help = ("Starts the Uploader in a continous running mode, polling Conductor for "
+                            "paths to upload unless a list of paths are provided."
+                            )
+
+    uploader_parser = subparsers.add_parser("uploader", parents=[parent_parser],
+                                            help=uploader_parser_help,
+                                            description=uploader_parser_desciption,
+                                            formatter_class=argparse.RawTextHelpFormatter)
+
+    uploader_parser.add_argument("--database_filepath",
+                                 help=("The filepath to the local md5 caching database. If no filepath "
+                                       "is specified, the database will be created in a temp directory"))
+
+    uploader_parser.add_argument("--location",
+                                 help=('An optional string to indicate which location this uploader '
+                                       'executable should register as. This option is only relevant '
+                                       'for conductor accounts which submits jobs from different locations '
+                                       '(e.g. differing geographic locations or office locations that have differing file systems).'
+                                       ' Typically each location would have its own conductor uploader process running. This location '
+                                       'string allows each uploader to target specific upload jobs (files to upload) that are appropriate '
+                                       'for it. This is potentially useful as each location may have differing file systems '
+                                       'available to it (e.g. uploader1 has /filesystem1 available to it, but uploader2 only '
+                                       'has /filesystem2 available to it).  In this case uploader1 should only upload files '
+                                       'that exist on /filesystem1 and uploader2 should only upload files that exist on /filesystem2. '
+                                       'This is achieved by including a location argument (such as "location1" or "location2") '
+                                       'when submitting jobs, as well as when launching this uploader command.'))
+
+    uploader_parser.add_argument("--md5_caching",
+                                 help=("Use cached md5s. This can dramatically improve the uploading "
+                                       "times, as md5 checking can be very time consuming. Caching md5s "
+                                       "allows subsequent uploads (of the same files) to skip the "
+                                       "md5 generation process (if the files appear to not have been "
+                                       "modified since the last time they were submitted). The cache is "
+                                       "stored locally and uses a file's modification time and file size "
+                                       "to intelligently guess whether the file has changed. Set this "
+                                       "flag to False if there is concern that files may not be getting "
+                                       "re-uploaded properly"),
+                                 choices=[False, True],
+                                 type=cast_to_bool,
+                                 default=None)
+
+    uploader_parser.add_argument("--log_level",
+                                 choices=loggeria.LEVELS,
+                                 help="The logging level to display")
+
+    uploader_parser.add_argument("--log_dir",
+                                 help=("When provided, will write a log file to "
+                                       "the provided directory. This will be a "
+                                       "rotating log, creating a new log file "
+                                       "everyday, while storing the last 7 days "
+                                       "of logs"))
+
+    uploader_parser.add_argument("--thread_count",
+                                 type=int,
+                                 default=cfg["thread_count"],
+                                 help=('The number of threads that should download simultaneously'))
+    
+    uploader_parser.add_argument("--paths",
+                                 type=str,
+                                 action="append",
+                                 nargs="+",
+                                 help=('A list of explicit paths to upload. Paths with spaces and/or special characters should be encapsulated in quotes'))
+
+    uploader_parser.set_defaults(func=run_uploader)
+
+    #############################
+    # DOWNLOADER PARSER
+    #############################
+
+    downloader_parser_desciption = "parse downloader arguments"
+    downloader_parser_help = ""
+
+    downloader_parser = subparsers.add_parser("downloader", parents=[parent_parser],
+                                              help=downloader_parser_help,
+                                              description=downloader_parser_desciption,
+                                              formatter_class=argparse.RawTextHelpFormatter)
+
+    downloader_parser.add_argument("--job_id",
+                                   help=("The job id(s) to download. When specified "
+                                         "will only download those jobs and terminate "
+                                         "afterwards"),
+                                   action='append')
+
+    downloader_parser.add_argument("--task_id",
+                                   help="Manually download output for specific tasks - use a comma-separated list of tasks if you wish")
+
+    downloader_parser.add_argument("--output",
+                                   help="Override for the output directory")
+
+    downloader_parser.add_argument("--location",
+                                   help=('An optional string to indicate which location this downloader '
+                                         'executable should register as. This option is only relevant for '
+                                         'conductor accounts which submits jobs from different locations '
+                                         '(e.g. differing geographic locations or office locations that '
+                                         'have differing file systems). Typically each location would '
+                                         'have its own conductor downloader process running. This location '
+                                         'argument allows each downloader to target specific jobs (to '
+                                         'download upon job-completion) that match its appropriate location. '
+                                         'Essentially this allows the location of which a job was submitted '
+                                         'from to also be the destination in which to deliver completed '
+                                         'renders to (which would typically be the desired behavior).'))
+
+    downloader_parser.add_argument("--project",
+                                   help=('An optional string to indicate which project that this downloader executable should register as.'))
+
+    downloader_parser.add_argument("--log_level",
+                                   choices=loggeria.LEVELS,
+                                   default=cfg["log_level"],
+                                   help="The logging level to display")
+
+    downloader_parser.add_argument("--log_dir",
+                                   help=("When provided, will write a log file to "
+                                         "the provided directory. This will be a "
+                                         "rotating log, creating a new log file "
+                                         "everyday, while storing the last 7 days "
+                                         "of logs"))
+
+    downloader_parser.add_argument("--thread_count",
+                                   type=int,
+                                   default=cfg["thread_count"],
+                                   help=('The number of threads that should download simultaneously'))
+
+    downloader_parser.add_argument("--alt",
+                                   help=('Run an alternative version of the downloader'),
+                                   action='store_true')
+
+    downloader_parser.set_defaults(func=run_downloader)
+
+    return parser
+
+
+def cast_to_bool(string):
+    '''
+    Ensure that the argument provided is either "True" or "False (or "true" or
+    "false") and convert that argument to an actual bool value (True or False).
+    '''
+    string_lower = string.lower()
+    if string_lower == "true":
+        return True
+    elif string_lower == "false":
+        return False
+    raise argparse.ArgumentTypeError('Argument must be True or False')
+
+def run_uploader(args):
+    uploader.run_uploader(args)
+
+
+def run_downloader(args):
+    '''
+    Convert the argparse Namespace object to a dictionary and run the downloader
+    with the given args.
+    '''
+    # Convert Namespace args object to args dict
+    args_dict = vars(args)
+
+    if args_dict.get("task_id") and not args_dict.get("job_id"):
+        raise argparse.ArgumentTypeError('Must supply a job_id with task_id.')
+
+    # New downloader broke in python 3. It was used only for linux and in
+    # daemon mode, so for now we'll use the old downloader for everything.
+
+    return downloader.run_downloader(args_dict)
+
+
+def main():
+    parser = build_parser()
+    args = parser.parse_args()
+    # Handle calling the script without an argument, fixes argparse issue
+    # https://bugs.python.org/issue16308
+    try:
+        func = args.func
+    except AttributeError:
+        parser.error("too few arguments")
+    func(args)
+
+
+if __name__ == '__main__':
+    main()
+

ciocore/downloader.py

@@ -18,10 +18,10 @@ import time
 import threading
 import hashlib
 
+import ciocore
 from ciocore import config
 from ciocore import api_client, common, loggeria
 from ciocore.common import CONDUCTOR_LOGGER_NAME
-from cioseq.sequence import Sequence
 
 try:
     import Queue as queue
@@ -291,12 +291,12 @@ class Downloader(object):
         downloader.print_uptime()
 
     @classmethod
-    def download_jobs(cls, job_ids, thread_count=None, output_dir=None):
+    def download_jobs(cls, job_ids, task_id=None, thread_count=None, output_dir=None):
         """
         Run the downloader for explicit jobs, and terminate afterwards.
         """
         downloader = cls(thread_count=thread_count, output_dir=output_dir)
-        downloader.start(job_ids)
+        thread_states = downloader.start(job_ids, task_id=task_id)
         while not common.SIGINT_EXIT and (
             not downloader.pending_queue.empty() or not downloader.downloading_queue.empty()
         ):
@@ -306,7 +306,7 @@ class Downloader(object):
         downloader._print_download_history()
         downloader.print_uptime()
 
-    def start(self, job_ids=None, summary_interval=10):
+    def start(self, job_ids=None, task_id=None, summary_interval=10):
         # Create new queues
         self.start_time = time.time()
         self.pending_queue = queue.Queue()
@@ -316,7 +316,7 @@ class Downloader(object):
         # If a job id has been specified then only load the queue up with that work
         if job_ids:
             self.history_queue_max = None
-            self.get_jobs_downloads(job_ids)
+            self.get_jobs_downloads(job_ids, task_id)
 
         # otherwise create a queue thread the polls the app for wor
         else:
@@ -461,12 +461,10 @@ class Downloader(object):
             # Wait for the queue to be consumed before querying for more
             self.nap()
 
-    
     def nap(self):
         while not common.SIGINT_EXIT:
 
             time.sleep(self.naptime)
-            # Pretty sure this return should be unindented!
             return
 
     @common.dec_timer_exit(log_level=logging.DEBUG)
@@ -479,51 +477,24 @@ class Downloader(object):
         except Exception as e:
             logger.exception("Could not get next download")
 
-    def get_jobs_downloads(self, job_ids):
+    def get_jobs_downloads(self, job_ids, task_id):
         """
-        Download jobs, each with an optional task specification.
-        
-        Job ids are padded to 5 digits, and task ids are padded to 3 digits.
+        Get each job and optional comma-separated task list.
 
-        job_ids: (tuple) may take any of the following forms:
-        1. A single job id, e.g. (01234,)
-        2. It doesn't have to be padded, e.g. (1234,)
-        3. A jobs with a task specification, e.g. (01234:1-3,5,7-9)
-        4. Several of jobs:tasks, e.g. (01234, 56789:1-3,5,7-9)
-    
-        """
-        jobs = self._flatten(job_ids)
-        for job in jobs:
-            endpoint = self.endpoint_downloads_job % job["job_id"]
-            
-            for tid in job["tasks"] or [None]:
-                downloads = _get_job_download(endpoint, self.api_client, job["job_id"], tid)
+        There will only be a task list if there is just one job, due to earlier arg validation.
+
+        If there is no task list, _get_job_download is called with tid=None (i.e. the whole job)
+        """ 
+        task_ids = [t for t in task_id.split(",") if t] if task_id else [None]
+
+        for job_id in job_ids:
+            endpoint = self.endpoint_downloads_job % job_id
+            for tid in task_ids:
+                downloads = _get_job_download(endpoint, self.api_client, job_id, tid)
                 if downloads:
                     for task_download in downloads.get("downloads", []):
                         print("putting in queue: %s" % task_download)
                         self.pending_queue.put(task_download, block=True)
-            
-    @staticmethod
-    def _flatten(job_ids):
-        """Create a list of job objects with keys: job_id and tasks. 
-        
-        see: get_jobs_downloads() function see tests/test_downloader.py for examples.
-        """
-        result = []
-        for job_id in job_ids:
-            if ":" in job_id:
-                job_id, range_spec = job_id.split(":")
-                try:
-                    seq = Sequence.create(range_spec)
-                    tasks = seq.expand("###")
-                except (ValueError, TypeError):
-                    tasks = None
-            else:
-                tasks = None
-            result.append({"job_id": job_id.zfill(5), "tasks": tasks})
-        return result
-    
-
 
     @common.dec_catch_exception(raise_=True)
     def download_target(self, pending_queue, downloading_queue, task_download_state):
@@ -1249,6 +1220,53 @@ def _in_queue(the_queue, item_dict, key):
         if item[key] == item_dict[key]:
             return True
 
+
+def run_downloader(args):
+    """
+    Start the downloader. If a job id(s) were given, exit the downloader upon completion.
+    Otherwise, run the downloader indefinitely (daemon mode), polling the Conductor cloud app for
+    files that need to be downloaded.
+    """
+
+    # Set up logging
+    log_level_name = args.get("log_level")
+    log_level = loggeria.LEVEL_MAP.get(log_level_name)
+    log_dirpath = args.get("log_dir")
+    set_logging(log_level, log_dirpath)
+    
+    api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=ciocore.__version__)
+
+    logger.debug("Downloader args: %s", args)
+
+    job_ids = args.get("job_id")
+    thread_count = args.get("thread_count")
+
+    if job_ids:
+        Downloader.download_jobs(
+            job_ids,
+            task_id=args.get("task_id"),
+            thread_count=thread_count,
+            output_dir=args.get("output"),
+        )
+
+    else:
+        Downloader.start_daemon(
+            thread_count=thread_count, location=args.get("location"), output_dir=args.get("output")
+        )
+
+
+def set_logging(level=None, log_dirpath=None):
+    log_filepath = None
+    if log_dirpath:
+        log_filepath = os.path.join(log_dirpath, "conductor_dl_log")
+    loggeria.setup_conductor_logging(
+        logger_level=level,
+        console_formatter=LOG_FORMATTER,
+        file_formatter=LOG_FORMATTER,
+        log_filepath=log_filepath,
+    )
+
+
 def report_error(self, download_id, error_message):
     try:
         logger.error("failing upload due to: \n%s" % error_message)

ciocore/uploader/_uploader.py

@@ -728,12 +728,6 @@ class Uploader(object):
 
         return True
 
-    def assets_only(self, *paths):
-        processed_filepaths = file_utils.process_upload_filepaths(paths)
-        file_map = {path: None for path in processed_filepaths}
-        self.handle_upload_response(project=None, upload_files=file_map)
-
-
     def handle_upload_response(self, project, upload_files, upload_id=None):
         """
         This is a really confusing method and should probably be split into to clear logic
@@ -916,6 +910,7 @@ def set_logging(level=None, log_dirpath=None):
         log_filepath=log_filepath,
     )
 
+
 def run_uploader(args):
     """
     Start the uploader process. This process will run indefinitely, polling
@@ -945,6 +940,7 @@ def run_uploader(args):
     else:
         uploader.main()
 
+
 def get_file_info(filepath):
     """
     For the given filepath return the following information in a dictionary:

ciocore-6.4.0.data/scripts/conductor

@@ -0,0 +1,19 @@
+#!python
+# -*- coding: utf-8 -*-
+# WHY CHANGE
+
+# DO NOT SORT IMPORTS
+import os
+import re
+import sys
+
+CIO_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+sys.path.insert(0, CIO_DIR)
+
+print("Inserted {} into sys.path".format(CIO_DIR))
+
+from ciocore.cli.conductor import main
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

ciocore-6.4.0.data/scripts/conductor.bat

@@ -0,0 +1,13 @@
+@echo off
+
+echo Looking for Python...
+
+WHERE python.exe
+if ERRORLEVEL 1 (  
+    echo WARNING: python.exe not found.
+    echo Please find or install Python and add it to your Path variable. Then run this command again.
+    echo If you don't want to add python to your Path, then you can use the full path to a Python installation.
+    echo Example: "C:\some\tools\python\python.exe" "%~dp0conductor %*
+    exit /b
+)
+python.exe "%~dp0\conductor" %*

{ciocore-6.3.1rc3.dist-info → ciocore-6.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 6.3.1rc3
+Version: 6.4.0
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -9,23 +9,21 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Topic :: Multimedia :: Graphics :: 3D Rendering
 Description-Content-Type: text/markdown
-Requires-Dist: certifi (==2019.11.28)
-Requires-Dist: pyjwt (==1.7.1)
-Requires-Dist: requests (<=2.25.1,>=2.10.0)
-Requires-Dist: future (>=0.18.2)
-Requires-Dist: urllib3 (==1.25.11)
-Requires-Dist: cioseq (<1.0.0,>=0.1.14)
-Requires-Dist: Click
-Requires-Dist: markdown
+Requires-Dist: certifi ==2019.11.28
+Requires-Dist: pyjwt ==1.7.1
+Requires-Dist: requests <=2.25.1,>=2.10.0
+Requires-Dist: future >=0.18.2
+Requires-Dist: urllib3 ==1.25.11
 
-## Introduction
+# Conductor Core
 
-Conductor Core is a Python package that provides an API and a command-line interface for interacting with Conductor.
+Use the command-line uploader and downloader or develop your own tools using the Python API.
 
 ## Install
 
+**To install the latest version.**
 ```bash
-pip install ciocore
+pip install --upgrade ciocore
 ```
 
 Run the conductor command to confirm the package was installed.
@@ -33,13 +31,6 @@ Run the conductor command to confirm the package was installed.
 conductor --help
 ```
 
-Get help for a specific sub command
-```bash
-conductor downloader --help
-conductor uploader --help
-conductor software --help
-```
-
 Example API usage
 ```python
 from ciocore import api_client
@@ -52,20 +43,9 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
-## Version:6.3.1-rc.3 -- 24 Jul 2023
-
-  * This release has an improved commandline experience. The new subcommands, `upload` and
-    `download` have a more idiomatic feel, and better control over what gets uploaded or downloaded.
-    The old commands `uploader` and `downloader` are still available, but are deprecated.
-  * There's also now a `packages` subcommand that allows you to query the package database.
-  * The command installs with console_scripts which improves stability and avoids situations wehere
-    the Conductor command cannot be3 found. (#72)
-  * Adds a `--version` flag
-
-## Version:6.3.1-rc.2 -- 23 Jul 2023
+## Version:6.4.0 -- 12 Sep 2023
 
-* Fixes missing import.
-* Improves uploader error handling and queueing fixes.
+* Uploader now supports callbacks and is interruptable
 
 ## Version:6.3.0 -- 05 Jul 2023
 

{ciocore-6.3.1rc3.dist-info → ciocore-6.4.0.dist-info}/RECORD

@@ -1,40 +1,40 @@
-ciocore/VERSION,sha256=YPDeKbDykk0XdwIXsKbLtvYyzEWPnsY8mq21KfcUpok,10
-ciocore/__about__.py,sha256=w23ToSZc0RpHPxOe-gPr1qA_yFqOFY0WvrujXwGisTk,717
+ciocore/VERSION,sha256=Yl4u2S_TJL-hz2vF8sZWuwgSXkwNAQL0sIXjVAPmLZU,5
+ciocore/__about__.py,sha256=nTb4Xx0r9QtGROSFKgwDZ-Mr2LKjm2wVt1OkMQAkRAQ,241
 ciocore/__init__.py,sha256=zB_e7gBW2QloqbrGzcvccfpZhOhd6o9KACnScrwylm8,25
 ciocore/api_client.py,sha256=O9uvtnqgmE2IRkqlP9DoPn6CX_HAIuSLo9je5Om-800,17913
-ciocore/cli.py,sha256=ivAsgWKrws5GFJh0qFlme_P5Dqq5k6KiWB32Qg3KsVc,11313
 ciocore/client_db.py,sha256=tTz3bl2xeDPPcYSDS3g3QgV_xYihJMx0Kj6OeN2klK0,12978
 ciocore/common.py,sha256=FnggAL-IGW8VQB6kLsrKLnxWFgLpwGBhOtiZSHz130M,14725
 ciocore/compat.py,sha256=5uEXPSog_jxsDMaHBswAKEtfyXT25VgU6WNGIhz9PHU,256
 ciocore/conductor_submit.py,sha256=ONE0LsA5hGavTJIOXXYx8qzl8_vBPADwhd6Ytq_0E0c,9382
 ciocore/config.py,sha256=fcxnwaYmNAmvMXQeb5V-86Pwwy3a3NTPCA0CSLqog3s,5913
 ciocore/data.py,sha256=G6tAfSkMwy3DS_fesqkDaoMgrfd_JrvV8Fs_JFpcCCc,8950
-ciocore/downloader.py,sha256=Q5DCN3I6bvGsVoJir4XisOjlZVTQV41fgJg0H4Obagw,50636
+ciocore/downloader.py,sha256=oRmRePdIx_GdQwM39ipl9_e2c2ZkcGJRanuHOZmqpTM,51147
 ciocore/exceptions.py,sha256=4Oq-WX-qiN6kPUdBCHvvd6mtSQ0nCkDbJxWt2CNtpv8,1504
 ciocore/file_utils.py,sha256=bAlL31B4YkRgX-yT8kF8UXBFktQlsE1PvxbKqTeAeOU,17174
 ciocore/hardware_set.py,sha256=txcSLrVSNewRGxKLye-wuM8szGMVloU29ktL8WHdUtM,5401
 ciocore/loggeria.py,sha256=dKKJC8ZtRZdghqD5R5XrA6eDoy8gKacfeTA-zNzXvDE,13482
 ciocore/package_environment.py,sha256=oEbNKXRtPSPzKR-yCoKtvgzu4OCmr-zaqAcNoLAN9Uk,7238
-ciocore/package_query.py,sha256=2m5EBXfu1lmqupZrFF8f8mfkX_PgijpdMxCtFI5e5s0,5574
 ciocore/package_tree.py,sha256=kH03HVfjomj7nsaxJJtr-1KSQ_9ZSQY5msG_l9btvg8,16277
 ciocore/post_install.py,sha256=zu5Ctz2ANbKD-f5G2ODLIhKkWENBi4F3UKKu50OEWrg,1000
 ciocore/validator.py,sha256=f_K7gxz122W_i5AxVx6dKhckOygl8TnmQiVj7tyX5zw,2344
 ciocore/worker.py,sha256=_WQq2_ZhXuc5NTfYQtcWUGULU6IUZykeVW5e_4_ffZc,21728
 ciocore/auth/__init__.py,sha256=cdS-xZzMq41yXM5cz8sUlcYgo8CJYh8HcCCWmhbDgf0,606
 ciocore/auth/server.py,sha256=8btX9-EokUl6q55V8muDmEV2tvvbTBD0BHeWFbwkzUc,3892
+ciocore/cli/__init__.py,sha256=RmZKWJaMpzNyMdyYc2W3VXglaJiC8vyR2cgUlA-9Qmw,26
+ciocore/cli/conductor.py,sha256=snmlICkMgP94ZPKl6J7g299deB75QwGDWI0oCnZPfSI,10861
 ciocore/uploader/__init__.py,sha256=hxRFJf5Lo86rtRObFXSjjot8nybQd-SebSfYCbgZwow,24
-ciocore/uploader/_uploader.py,sha256=NDtoSgfUkmHCY6h0yypiPrwGH6nJwTORp_LptHCiv28,37675
+ciocore/uploader/_uploader.py,sha256=31lpXVp2VG6rzVF7nw24Uafs6Z-p9CUF_bghku386R4,37430
 ciocore/uploader/upload_stats/__init__.py,sha256=kc1YCXZOy4EBpSSRIJKe89DWgiBwTzOPMbZ7530EXWw,5354
 ciocore/uploader/upload_stats/stats_formats.py,sha256=dtjsxdbT4w6BXFay19dzijf-xLwf6TktNk75Wq3rPAw,2452
+ciocore-6.4.0.data/scripts/conductor,sha256=Nk3QsLQqbUUrtaKDp4b5mr9__4tz-xnssENpQe5vuIo,409
+ciocore-6.4.0.data/scripts/conductor.bat,sha256=T1_9ByheubBczgQZn8_LwfvMtWgE7Bt64EsEScnSXMs,447
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/instance_type_fixtures.py,sha256=gR6ordfghp34I94oM2eDg6jfIe5HAwE644GKIwspuW8,3469
 tests/package_fixtures.py,sha256=CsJnhB7oYzIxJH7b1tCOPyvnnVSCqEbSPhtCnsHL-nA,5070
 tests/test_api_client.py,sha256=dWOmAIKmX0gLaDGsT9VfAq9Hcs9HIQf2P5IMM_Bt5dE,1284
-tests/test_cli.py,sha256=Td8kUgIz9GhCG0FxbL66Kohf4tpJRhQnOFE_tZSe-8k,5970
 tests/test_common.py,sha256=lJpzRdL-7u4McXFbLuwPQQoUnuEOnCVQtZEt6e_dIYs,638
 tests/test_config.py,sha256=nSmpinX2SmDNAprIcxs9UHdB0VakJB0snXaZmAoKJSc,12863
 tests/test_data.py,sha256=YdP1kZJivQ6yb9z96UK6oMDaOfJAl4YMJqzKvlCQaes,5744
-tests/test_downloader.py,sha256=Xr6-y3SjfTAo0JM2VDGoSx-nEtRj3BY-Q4JtSNFl3ro,2090
 tests/test_hardware_set.py,sha256=TcBh63rOxf1rKXxKlCPSnHueBFlz7rNP6BcoJjgVvPs,3065
 tests/test_imports_2and3.py,sha256=ehqpRYPVY7djBcb8OT_cnh86iCJJ9wuMWnfSR9RHxmY,507
 tests/test_package_environment.py,sha256=CdiC2PDVSnbcwTb4fsDTWqGYSzs1n5ca2KMoyISckGA,5893
@@ -45,8 +45,7 @@ tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/api_client_mock.py,sha256=Wfv2JPFSZfyHftVqsqxcpXWJn136pEHx26I_esz567E,943
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-6.3.1rc3.dist-info/METADATA,sha256=q6QZYoIeKZYL7oAHY4Ztb_kUzL4Xj6hfN5-65kYlAAw,15189
-ciocore-6.3.1rc3.dist-info/WHEEL,sha256=a-zpFRIJzOq5QfuhBzbhiA1eHTzNCJn8OdRvhdNX0Rk,110
-ciocore-6.3.1rc3.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-6.3.1rc3.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-6.3.1rc3.dist-info/RECORD,,
+ciocore-6.4.0.dist-info/METADATA,sha256=ObyBNodNf9WIBCAKwIsWMLC5NAOfsCZhpLFdniY7-_0,14364
+ciocore-6.4.0.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
+ciocore-6.4.0.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-6.4.0.dist-info/RECORD,,

{ciocore-6.3.1rc3.dist-info → ciocore-6.4.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: bdist_wheel (0.41.2)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

ciocore/cli.py

@@ -1,300 +0,0 @@
-import os
-import sys
-import click
-import logging
-
-from ciocore import (
-    loggeria,
-    config,
-    api_client,
-    package_query,
-)
-
-from ciocore import __version__ as VERSION
-
-from ciocore.uploader import Uploader
-from ciocore.downloader import Downloader
-
-logger = logging.getLogger("conductor cli")
-
-LOG_LEVEL_HELP = """The logging level to display"""
-
-LOG_DIR_HELP = """
-Write a log file to the given directory. The log rotates, creating a new log file every day, while
-storing logs for the last 7 days. The log file is named conductor_ul.log
-"""
-
-THREADS_HELP = """The number of threads that should download simultaneously"""
-
-UPLOADER_DATABASE_FILEPATH_HELP = (
-    "Specify a filepath to the local md5 caching database."
-)
-LOCATION_HELP = """
-Specify a location tag to associate with uploads, downloads, and submissions. A location tag allows
-you to limit the scope of your uploads and downloads to jobs sharing the same location tag. This is
-useful while using the uploader or downloader in daemon mode.
-"""
-UPLOADER_MD5_CACHING_HELP = """
-Use cached md5s. This can dramatically improve the uploading times, as md5 checking can be very
-time consuming. Caching md5s allows subsequent uploads (of the same files) to skip the md5
-generation process (if the files appear to not have been modified since the last time they were
-submitted). The cache is stored locally and uses a file's modification time and file size to
-intelligently guess whether the file has changed. Set this flag to False if there is concern that
-files may not be getting re-uploaded properly
-"""
-
-UPLOADER_PATHS_HELP = """
-A list of paths to upload. Use quotes if paths contain spaces or special characters"""
-
-DOWNLOADER_JOB_ID_HELP = """
-The job id(s) to download. When specified will only download those jobs and terminate afterwards
-"""
-
-DOWNLOADER_DEST_HELP = """
-Override the output directory"""
-
-PACKAGES_FORMAT_OPTIONS = ["text", "markdown", "html"]
-PACKAGES_FORMAT_HELP = """
-text: The output is a simple list of software names and versions, with nesting to indicate plugin
-compatibility. Output is sent to stdout.
-
-markdown: Designed for use in other markdown documentation systems where it benefits from consistent
-styling. Output is sent to stdout and can be piped to a file.
-
-html: Opens a browser window and displays the output in a simple html page.
-"""
-
-LOG_FORMATTER = logging.Formatter(
-    "%(asctime)s  %(name)s%(levelname)9s  %(threadName)s:  %(message)s"
-)
-
-
-cfg = config.get()
-DEFAULT_CONFIG_MD5_CACHING = cfg["md5_caching"]
-DEFAULT_CONFIG_THREAD_COUNT = cfg["thread_count"]
-DEFAULT_CONFIG_LOG_LEVEL = cfg["log_level"]
-
-def _set_logging(log_level=None, log_filepath=None):
-    level = loggeria.LEVEL_MAP.get(log_level)
-    loggeria.setup_conductor_logging(
-        logger_level=level,
-        console_formatter=LOG_FORMATTER,
-        file_formatter=LOG_FORMATTER,
-        log_filepath=log_filepath,
-    )
-
-def _register(client):
-    api_client.ApiClient.register_client(
-        client_name=client.CLIENT_NAME, client_version=VERSION
-    )
-
-########################### MAIN #################################
-@click.group(invoke_without_command=True)
-@click.pass_context
-@click.option('-v', '--version', is_flag=True, help='Print the version and exit.')
-def main(ctx, version):
-    """Conductor Command-line interface."""
-    if not ctx.invoked_subcommand:
-        if version:
-            click.echo(VERSION)
-            ctx.exit()
-        click.echo(ctx.get_help())
-        ctx.exit()
-
-############################# UPLOADER #############################
-@main.command()
-@click.option("-db", "--database_filepath", help=UPLOADER_DATABASE_FILEPATH_HELP)
-@click.option( "-md5", "--md5_caching", help=UPLOADER_MD5_CACHING_HELP, type=bool, default=DEFAULT_CONFIG_MD5_CACHING,)
-@click.option("-lv", "--log_level", help=LOG_LEVEL_HELP, type=click.Choice(choices=loggeria.LEVELS, case_sensitive=False), show_choices=True, default=DEFAULT_CONFIG_LOG_LEVEL)
-@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
-@click.option("-tc", "--thread_count", type=int, help=THREADS_HELP, default=DEFAULT_CONFIG_THREAD_COUNT)
-@click.option("-lc", "--location", help=LOCATION_HELP)
-@click.argument("paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
-def upload(
-    database_filepath, location, md5_caching, log_level, log_dir, thread_count, paths
-):
-    """Upload files to Conductor.
-    
-    With no arguments, the uploader runs in daemon mode, watching for files to upload for submitted
-    jobs.
-
-    Alternatively, specify a list of paths to upload.
-    
-    Example:
-    conductor upload file1 file2 file3
-    """
-    logfile = log_dir and os.path.join(log_dir, "conductor_ul.log")
-    _set_logging(log_level, logfile)
-
-    args_dict = {
-        "database_filepath": database_filepath,
-        "location": location,
-        "md5_caching": md5_caching,
-        "log_level": log_level,
-        "log_dir": log_dir,
-        "thread_count": thread_count,
-    }
-
-    up = Uploader(args_dict)
-
-    if paths:
-        up.assets_only(*paths)
-        return
-
-    up.main()
-
-########################### DOWNLOADER #############################
-@main.command()
-@click.argument("jobids", nargs=-1)
-@click.option("-d", "--destination", help=DOWNLOADER_DEST_HELP)
-@click.option("-lv", "--log_level", help=LOG_LEVEL_HELP, type=click.Choice(choices=loggeria.LEVELS, case_sensitive=False), show_choices=True, default=DEFAULT_CONFIG_LOG_LEVEL)
-@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
-@click.option("-tc", "--thread_count", type=int, help=THREADS_HELP, default=DEFAULT_CONFIG_THREAD_COUNT)
-@click.option("-lc", "--location", help=LOCATION_HELP)
-
-def download(jobids, destination, location, log_level, log_dir, thread_count):
-    """
-    Download renders and other output files from Conductor. You can give a list of job ids to
-    download, or you can omit jobids and the downloader will run in daemon mode.
-
-    If you provide jobids, the default behavior is to download all the files from completed tasks
-    for those jobs. You can however specify an explicit set of tasks to downloade by providing a
-    task range spec after each job id. To do so, append a colon to the job id and then a compact
-    task specification. See the examples.
-
-    In daemon mode, the downloader polls for new jobs to download. You can specify a location tag to
-    limit the scope of the downloader to only download jobs that were submitted with the same
-    location tag.
-
-    Examples:
-    conductor download # daemon mode
-    conductor download 1234 1235
-    conductor download 1234:1-10
-    conductor download 1234:1-5x2,10,12-14
-    conductor download 1234:1-5 1235:5-10
-    """
-    logfile = log_dir and os.path.join(log_dir, "conductor_dl.log")
-    _set_logging(log_level, logfile)
-    _register(Downloader)
-    if jobids:
-        Downloader.download_jobs(
-            jobids,
-            thread_count=thread_count,
-            output_dir=destination,
-        )
-    else:
-        Downloader.start_daemon(
-            thread_count=thread_count, location=location, output_dir=destination
-        )
-
-
-########################### PACKAGES #############################
-@main.command()
-@click.option(
-    "-f",
-    "--fmt",
-    "--format",
-    default="text",
-    help=PACKAGES_FORMAT_HELP,
-    type=click.Choice(choices=PACKAGES_FORMAT_OPTIONS, case_sensitive=False),
-)
-def packages(fmt):
-    """List the softweare packages available on the render nodes in the cloud."""
-    package_query.pq(format=fmt)
-
-
-########################### DEPRECATIONS #############################
-
-############################# UPLOADER #############################
-DEPRECATED_PATHS_HELP="Specify a list of paths to upload."
-
-@main.command(deprecated=True)
-@click.option("-db", "--database_filepath", help=UPLOADER_DATABASE_FILEPATH_HELP)
-@click.option( "-md5", "--md5_caching", help=UPLOADER_MD5_CACHING_HELP, type=bool, default=DEFAULT_CONFIG_MD5_CACHING,)
-@click.option("-lv", "--log_level", help=LOG_LEVEL_HELP, type=click.Choice(choices=loggeria.LEVELS, case_sensitive=False), show_choices=True, default=DEFAULT_CONFIG_LOG_LEVEL)
-@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
-@click.option("-tc", "--thread_count", type=int, help=THREADS_HELP, default=DEFAULT_CONFIG_THREAD_COUNT)
-@click.option("-lc", "--location", help=LOCATION_HELP)
-@click.option("-p", "--paths", help=DEPRECATED_PATHS_HELP, multiple=True, type=click.Path(exists=True, resolve_path=True))
-
-
-
-def uploader(
-    database_filepath, location, md5_caching, log_level, log_dir, thread_count, paths
-):
-    """Upload files to Conductor.
-
-    With no arguments, the uploader runs in daemon mode, watching for files to upload for submitted
-    jobs.
-
-    Alternatively, specify a list of paths to upload with the --paths option.
-    
-    Example:
-    conductor upload file1 file2 file3
-    """
-    logfile = log_dir and os.path.join(log_dir, "conductor_ul.log")
-    _set_logging(log_level, logfile)
-
-    args_dict = {
-        "database_filepath": database_filepath,
-        "location": location,
-        "md5_caching": md5_caching,
-        "log_level": log_level,
-        "log_dir": log_dir,
-        "thread_count": thread_count,
-    }
-
-    up = Uploader(args_dict)
-
-    if paths:
-        up.assets_only(*paths)
-        return
-
-    up.main()
-
-########################### DOWNLOADER #############################
-DEPRECATED_JOBID_HELP="Download all the files from completed tasks for the given jobs."
-DEPRECATED_TASKID_HELP="Download the files from the given tasks in the specified job."
-
-@main.command(deprecated=True)
-@click.option("-o", "--output", help=DOWNLOADER_DEST_HELP)
-@click.option("-j", "--job_id", help=DEPRECATED_JOBID_HELP)
-@click.option("-t", "--task_id", help=DEPRECATED_TASKID_HELP)
-@click.option("-lv", "--log_level", help=LOG_LEVEL_HELP, type=click.Choice(choices=loggeria.LEVELS, case_sensitive=False), show_choices=True, default=DEFAULT_CONFIG_LOG_LEVEL)
-@click.option("-ld", "--log_dir", help=LOG_DIR_HELP)
-@click.option("-tc", "--thread_count", type=int, help=THREADS_HELP, default=DEFAULT_CONFIG_THREAD_COUNT)
-@click.option("-lc", "--location", help=LOCATION_HELP)
-@click.pass_context
-def downloader(ctx, job_id, task_id, output, location, log_level, log_dir, thread_count):
-    """
-    Download renders and other output files from Conductor. You can specify a job id and optional task id to
-    download, or you can omit all options and the downloader will run in daemon mode.
-    
-    In daemon mode, the downloader polls for new jobs to download. You can specify a location tag to
-    limit the scope of the downloader to only download jobs that were submitted with the same
-    location tag.
-
-    Examples:
-    conductor downloader # daemon mode
-    conductor downloader --job_id --task_id 01234 
-    conductor downloader --task_id --task_id 123
-    """
-    
-    logfile = log_dir and os.path.join(log_dir, "conductor_dl.log")
-    _set_logging(log_level, logfile)
-    _register(Downloader)
-    
-    if not job_id and not task_id:
-        Downloader.start_daemon(
-            thread_count=thread_count, location=location, output_dir=output
-        )
-        ctx.exit(0)
-    
-    if task_id:
-         job_id = job_id + ":" + task_id
-
-    Downloader.download_jobs(
-        (job_id,),
-        thread_count=thread_count,
-        output_dir=output,
-    )

ciocore/package_query.py

@@ -1,171 +0,0 @@
-"""
-Generate markdown from the software packages list.
-"""
-import os
-import sys
-import json
-from ciocore.package_tree import PackageTree
-from ciocore import api_client
-import markdown
-import io
-import tempfile
-import webbrowser
-
-PURE = """
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/pure-min.css" integrity="sha384-X38yfunGUhNzHpBaEBsWLO+A0HDYOQi8ufWDkZ0k9e0eXz/tH3II7uKZ9msv++Ls" crossorigin="anonymous">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-"""
-
-def green(rhs):
-    return "\033[92m{}\033[0m".format(rhs)
-
-
-def red(rhs):
-    return "\033[91m{}\033[0m".format(rhs)
-
-
-def blue(rhs):
-    return "\033[94m{}\033[0m".format(rhs)
-
-
-def magenta(rhs):
-    return "\033[95m{}\033[0m".format(rhs)
-
-
-def raw(rhs, stream):
-    stream.write("{}\n\n".format(rhs))
-
-
-def d(n, rhs, stream):
-    """Indent with dashes"""
-    stream.write("{} {}\n".format("-" * n, rhs))
-
-
-def hr(stream):
-    stream.write("---\n\n")
-
-
-def h(n, rhs, stream):
-    stream.write("{} {}\n\n".format("#" * n, rhs))
-
-
-def plugin_table_header(stream):
-    stream.write(
-        '|<div style="width:150px">Plugin</div> |<div style="min-width:400px">Versions</div>|\n|:------------|:-------------|\n'
-    )
-
-
-def plugin_table_row(plugin, versions, stream):
-    stream.write("|{}|{}|\n".format(plugin, versions))
-
-
-def write_markdown(hostnames, tree_data, platform, stream):
-    """
-    Write the tree of packages in Markdown.
-
-    Use this to generate docs for the Conductor mkdocs site.
-    """
-    if not hostnames:
-        return
-    h(2, "{} Software".format(platform.capitalize()), stream)
-    last_hostgroup = None
-    for hostname in hostnames:
-        display_hostname = " ".join(hostname.split()[:2])
-        hostgroup = hostname.split(" ")[0]
-        stream.write("\n")
-        if not hostgroup == last_hostgroup:
-            hr(stream)
-            h(3, hostgroup, stream)
-        h(4, display_hostname, stream)
-        last_hostgroup = hostgroup
-        plugins = tree_data.supported_plugins(hostname)
-        if plugins:
-            plugin_table_header(stream)
-            for plugin in plugins:
-                plugin_table_row(
-                    plugin["plugin"], ", ".join(plugin["versions"]), stream
-                )
-
-
-def write_text(hostnames, tree_data, platform, color_func, stream):
-    """
-    Write the tree of packages as text.
-
-    Products are indented with one dash.
-    Host packages are indented with two dashes.
-    Plugin packages are indented with three dashes.
-    """
-    if not hostnames:
-        d(0, red("There are no '{}' host packages".format(platform)), stream)
-        return
-    d(0, "{} Software".format(platform).upper(), stream)
-    last_hostgroup = None
-    for hostname in hostnames:
-        display_hostname = " ".join(hostname.split()[:2])
-        hostgroup = hostname.split(" ")[0]
-        if not hostgroup == last_hostgroup:
-            d(0, green("-" * 30), stream)
-            d(1, color_func(hostgroup), stream)
-        d(2, color_func(display_hostname), stream)
-        last_hostgroup = hostgroup
-        plugins = tree_data.supported_plugins(hostname)
-        if plugins:
-            for plugin in plugins:
-                d(
-                    3,
-                    color_func(
-                        "{} [{}]".format(
-                            plugin["plugin"], ", ".join(plugin["versions"])
-                        )
-                    ),
-                    stream,
-                )
-
-
-def pq(format="text"):
-    packages = api_client.request_software_packages()
-
-    tree_data = PackageTree(packages)
-
-    hostnames = tree_data.supported_host_names()
-    linux_hostnames = [h for h in hostnames if h.endswith("linux")]
-    windows_hostnames = [h for h in hostnames if h.endswith("windows")]
-
-    if format == "markdown":
-        stream = sys.stdout
-        raw(
-            "This page contains the complete list of software available at Conductor. If you require applications or plugins that are not in the list, please [create a support ticket](https://support.conductortech.com/hc/en-us/requests/new) and let us know.",
-            stream,
-        )
-        write_markdown(linux_hostnames, tree_data, "linux", stream)
-        write_markdown(windows_hostnames, tree_data, "windows", stream)
-    elif format == "text":
-        stream = sys.stdout
-        write_text(linux_hostnames, tree_data, "linux", magenta, stream)
-        d(0, "", stream)
-        write_text(windows_hostnames, tree_data, "windows", blue, stream)
-    elif format == "html":
-        stream = io.StringIO()
-        raw(
-            "This page contains the complete list of software available at Conductor. If you require applications or plugins that are not in the list, please [create a support ticket](https://support.conductortech.com/hc/en-us/requests/new) and let us know.",
-            stream,
-        )
-
-        write_markdown(linux_hostnames, tree_data, "linux", stream)
-        write_markdown(windows_hostnames, tree_data, "windows", stream)
-
-        html = markdown.markdown(
-            stream.getvalue(), extensions=["markdown.extensions.tables"]
-        )
-
-        html = decorate(html)
-
-        stream.close()
-        with tempfile.NamedTemporaryFile(mode="w", suffix=".html", delete=False) as f:
-            f.write(html)
-            webbrowser.open("file://" + f.name, new=2)
-
-def decorate(html):
-    html = html.replace("<table>", '<table class="pure-table pure-table-bordered">')
-    html = '<html><head>{}</head><body style="margin: 2em;">{}</body></html>'.format(PURE, html)
-    return html

ciocore-6.3.1rc3.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-conductor = ciocore.cli:main

tests/test_cli.py

@@ -1,161 +0,0 @@
-from click.testing import CliRunner
-from ciocore.cli import upload as cli_upload
-from ciocore.cli import download as cli_download
-import unittest
-import os
-from unittest import mock
-
-
-class CliTestUploaderOptions(unittest.TestCase):
-    def setUp(self):
-        self.runner = CliRunner()
-        self.default_args = {
-            "database_filepath": None,
-            "location": None,
-            "md5_caching": True,
-            "log_level": "INFO",
-            "log_dir": None,
-            "thread_count": 16,
-        }
-        init_patcher = mock.patch("ciocore.cli.Uploader.__init__", autospec=True)
-        self.mock_init = init_patcher.start()
-        self.addCleanup(init_patcher.stop)
-
-    def test_receives_full_args_dict_with_defaults_when_no_args_given(self):
-        self.runner.invoke(cli_upload, [])
-        self.mock_init.assert_called_once_with(mock.ANY, self.default_args)
-
-    def test_database_filepath_arg(self):
-        self.runner.invoke(cli_upload, ["--database_filepath", "foo"])
-        expected = self.default_args
-        expected.update({"database_filepath": "foo"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_location_arg(self):
-        self.runner.invoke(cli_upload, ["--location", "foo"])
-        expected = self.default_args
-        expected.update({"location": "foo"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_md5_caching_arg(self):
-        self.runner.invoke(cli_upload, ["--md5_caching", False])
-        expected = self.default_args
-        expected.update({"md5_caching": False})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_log_level_arg(self):
-        self.runner.invoke(cli_upload, ["--log_level", "DEBUG"])
-        expected = self.default_args
-        expected.update({"log_level": "DEBUG"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_log_dir_arg(self):
-        self.runner.invoke(cli_upload, ["--log_dir", "foo"])
-        expected = self.default_args
-        expected.update({"log_dir": "foo"})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-    def test_thread_count_arg(self):
-        self.runner.invoke(cli_upload, ["--thread_count", 4])
-        expected = self.default_args
-        expected.update({"thread_count": 4})
-        self.mock_init.assert_called_once_with(mock.ANY, expected)
-
-
-class CliTestUploaderArguments(unittest.TestCase):
-    def setUp(self):
-        self.runner = CliRunner()
-
-        uploader_patcher = mock.patch("ciocore.cli.Uploader", autospec=True)
-        self.mock_uploader = uploader_patcher.start()
-        self.mock_inst = self.mock_uploader.return_value
-        self.addCleanup(uploader_patcher.stop)
-
-    def test_path_only_branch_if_paths(self):
-        with self.runner.isolated_filesystem():
-            filenames = ["foo.txt", "bar.txt", "baz.txt", "qux.txt"]
-            filenames = [os.path.join(os.getcwd(), filename) for filename in filenames]
-            for filename in filenames:
-                with open(filename, "w") as f:
-                    f.write("hello world")
-            self.runner.invoke(cli_upload, filenames)
-            self.mock_inst.assets_only.assert_called_once_with(
-                mock.ANY, mock.ANY, mock.ANY, mock.ANY
-            )
-            self.mock_inst.main.assert_not_called()
-
-    def test_main_branch_if_no_paths(self):
-        with self.runner.isolated_filesystem():
-            self.runner.invoke(cli_upload)
-            self.mock_inst.main.assert_called_once()
-            self.mock_inst.assets_only.assert_not_called()
-
-
-class CliTestDownloader(unittest.TestCase):
-    def setUp(self):
-        self.runner = CliRunner()
-
-        dljobs_patcher = mock.patch(
-            "ciocore.cli.Downloader.download_jobs", autospec=True
-        )
-        self.mock_dljobs = dljobs_patcher.start()
-        self.addCleanup(dljobs_patcher.stop)
-
-        start_daemon_patcher = mock.patch(
-            "ciocore.cli.Downloader.start_daemon", autospec=True
-        )
-        self.mock_start_daemon = start_daemon_patcher.start()
-        self.addCleanup(start_daemon_patcher.stop)
-
-    def test_jobid_branch_if_job_id(self):
-        jid = "00000"
-        self.runner.invoke(cli_download, [jid])
-        self.mock_dljobs.assert_called_once()
-        self.mock_start_daemon.assert_not_called()
-
-    def test_job_id_only(self):
-        jid = "00000"
-        self.runner.invoke(cli_download, [jid])
-        self.mock_dljobs.assert_called_once_with(
-            (jid,), thread_count=mock.ANY, output_dir=mock.ANY
-        )
-
-    def test_several_job_ids(self):
-        jid1 = "00000"
-        jid2 = "11111"
-        self.runner.invoke(cli_download, [jid1, jid2])
-        self.mock_dljobs.assert_called_once_with(
-            (jid1,jid2),  thread_count=mock.ANY, output_dir=mock.ANY
-        )
-
-    def test_job_ids_and_others(self):
-        jid1 = "00000"
-        jid2 = "11111"
-        tc = 4
-        od = "foo"
-        self.runner.invoke(
-            cli_download,
-            ["--thread_count", tc, "--destination", od, jid1, jid2 ],
-        )
-        self.mock_dljobs.assert_called_once_with(
-            (jid1,jid2), thread_count=tc, output_dir=od
-        )
-
-    def test_daemon_branch_if_no_job_id(self):
-        self.runner.invoke(cli_download, [])
-        self.mock_start_daemon.assert_called_once()
-        self.mock_dljobs.assert_not_called()
-
-    def test_daemon_branch_args_present(self):
-        tc = 4
-        od = "foo"
-        self.runner.invoke(cli_download)
-        self.mock_start_daemon.assert_called_once_with(thread_count=mock.ANY, location=mock.ANY, output_dir=mock.ANY)
- 
-    def test_daemon_branch_args(self):
-        tc = 4
-        od = "foo"
-        loc = "bar"
-        self.runner.invoke(cli_download, ["--thread_count", tc, "--destination", od, "--location", loc])
-        self.mock_start_daemon.assert_called_once_with(thread_count=tc, location="bar", output_dir=od)
- 

tests/test_downloader.py

@@ -1,56 +0,0 @@
-
-from ciocore.downloader import Downloader
-import unittest
-from unittest import mock
-
-class TestDownloaderFlatten(unittest.TestCase):
-    
-    def test_flatten_single_job_id(self):
-        input = ("01234",)
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [{"job_id": "01234", "tasks":None}])
-
-    def test_flatten_pad_job_id(self):
-        input = ("1234",)
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [{"job_id": "01234", "tasks":None}])
-
-    def test_several_job_ids(self):
-        input = ("1234","1235","1236")
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [
-            {"job_id": "01234", "tasks":None},
-            {"job_id": "01235", "tasks":None},
-            {"job_id": "01236", "tasks":None}
-            ])
-
-    def test_job_and_tasks(self):
-        input = ("1234:1-7x2,10",)
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [{"job_id": "01234", "tasks":["001","003","005","007","010"]}])
-
-    def test_several_job_and_tasks(self):
-        input = ("1234:1-7x2,10","1235:12-15")
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [
-            {"job_id": "01234", "tasks":["001","003","005","007","010"]},
-            {"job_id": "01235", "tasks":["012","013","014","015"]}
-            ])
-
-    def test_mix_job_and_job_with_tasks(self):
-        input = ("1234","1235:12-15")
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [
-            {"job_id": "01234", "tasks":None},
-            {"job_id": "01235", "tasks":["012","013","014","015"]}
-            ])
-
-    def test_invalid_range_downloads_whole_job(self):
-        # Someone might have a bunch of stuff queued up and made a mistake and left for the night.
-        # We should download the whole job in this case so they don't have to restart the dl in the
-        # morning.
-        input = ("1234:badrange",)
-        result = Downloader._flatten(input)
-        self.assertEqual(result, [
-            {"job_id": "01234", "tasks":None}
-            ])