ciocore 6.3.2rc1__py2.py3-none-any.whl → 6.4.0__py2.py3-none-any.whl

ciocore/cli/conductor.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+# Use absolute imports to avoid a "conductor" name clash (this module name vs conductor package).
+from __future__ import absolute_import
+
+import argparse
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from ciocore import  downloader, uploader, loggeria, config
+
+def build_parser():
+    cfg = config.config().config
+
+    # Create a parent parser. Arguments that are common across all subparsers can be added to this parser
+    parent_parser = argparse.ArgumentParser(add_help=False)
+
+    # create the main parser. Not sure why this parser is required, but got parsing tracebacks when excluding it (it gets confused about the arguments provided)
+    parser = argparse.ArgumentParser(description="description")
+    subparsers = parser.add_subparsers(title="actions")
+
+
+    #############################
+    # UPLOADER PARSER
+    #############################
+    uploader_parser_desciption = "parse uploader arguments"
+    uploader_parser_help = ("Starts the Uploader in a continous running mode, polling Conductor for "
+                            "paths to upload unless a list of paths are provided."
+                            )
+
+    uploader_parser = subparsers.add_parser("uploader", parents=[parent_parser],
+                                            help=uploader_parser_help,
+                                            description=uploader_parser_desciption,
+                                            formatter_class=argparse.RawTextHelpFormatter)
+
+    uploader_parser.add_argument("--database_filepath",
+                                 help=("The filepath to the local md5 caching database. If no filepath "
+                                       "is specified, the database will be created in a temp directory"))
+
+    uploader_parser.add_argument("--location",
+                                 help=('An optional string to indicate which location this uploader '
+                                       'executable should register as. This option is only relevant '
+                                       'for conductor accounts which submits jobs from different locations '
+                                       '(e.g. differing geographic locations or office locations that have differing file systems).'
+                                       ' Typically each location would have its own conductor uploader process running. This location '
+                                       'string allows each uploader to target specific upload jobs (files to upload) that are appropriate '
+                                       'for it. This is potentially useful as each location may have differing file systems '
+                                       'available to it (e.g. uploader1 has /filesystem1 available to it, but uploader2 only '
+                                       'has /filesystem2 available to it).  In this case uploader1 should only upload files '
+                                       'that exist on /filesystem1 and uploader2 should only upload files that exist on /filesystem2. '
+                                       'This is achieved by including a location argument (such as "location1" or "location2") '
+                                       'when submitting jobs, as well as when launching this uploader command.'))
+
+    uploader_parser.add_argument("--md5_caching",
+                                 help=("Use cached md5s. This can dramatically improve the uploading "
+                                       "times, as md5 checking can be very time consuming. Caching md5s "
+                                       "allows subsequent uploads (of the same files) to skip the "
+                                       "md5 generation process (if the files appear to not have been "
+                                       "modified since the last time they were submitted). The cache is "
+                                       "stored locally and uses a file's modification time and file size "
+                                       "to intelligently guess whether the file has changed. Set this "
+                                       "flag to False if there is concern that files may not be getting "
+                                       "re-uploaded properly"),
+                                 choices=[False, True],
+                                 type=cast_to_bool,
+                                 default=None)
+
+    uploader_parser.add_argument("--log_level",
+                                 choices=loggeria.LEVELS,
+                                 help="The logging level to display")
+
+    uploader_parser.add_argument("--log_dir",
+                                 help=("When provided, will write a log file to "
+                                       "the provided directory. This will be a "
+                                       "rotating log, creating a new log file "
+                                       "everyday, while storing the last 7 days "
+                                       "of logs"))
+
+    uploader_parser.add_argument("--thread_count",
+                                 type=int,
+                                 default=cfg["thread_count"],
+                                 help=('The number of threads that should download simultaneously'))
+    
+    uploader_parser.add_argument("--paths",
+                                 type=str,
+                                 action="append",
+                                 nargs="+",
+                                 help=('A list of explicit paths to upload. Paths with spaces and/or special characters should be encapsulated in quotes'))
+
+    uploader_parser.set_defaults(func=run_uploader)
+
+    #############################
+    # DOWNLOADER PARSER
+    #############################
+
+    downloader_parser_desciption = "parse downloader arguments"
+    downloader_parser_help = ""
+
+    downloader_parser = subparsers.add_parser("downloader", parents=[parent_parser],
+                                              help=downloader_parser_help,
+                                              description=downloader_parser_desciption,
+                                              formatter_class=argparse.RawTextHelpFormatter)
+
+    downloader_parser.add_argument("--job_id",
+                                   help=("The job id(s) to download. When specified "
+                                         "will only download those jobs and terminate "
+                                         "afterwards"),
+                                   action='append')
+
+    downloader_parser.add_argument("--task_id",
+                                   help="Manually download output for specific tasks - use a comma-separated list of tasks if you wish")
+
+    downloader_parser.add_argument("--output",
+                                   help="Override for the output directory")
+
+    downloader_parser.add_argument("--location",
+                                   help=('An optional string to indicate which location this downloader '
+                                         'executable should register as. This option is only relevant for '
+                                         'conductor accounts which submits jobs from different locations '
+                                         '(e.g. differing geographic locations or office locations that '
+                                         'have differing file systems). Typically each location would '
+                                         'have its own conductor downloader process running. This location '
+                                         'argument allows each downloader to target specific jobs (to '
+                                         'download upon job-completion) that match its appropriate location. '
+                                         'Essentially this allows the location of which a job was submitted '
+                                         'from to also be the destination in which to deliver completed '
+                                         'renders to (which would typically be the desired behavior).'))
+
+    downloader_parser.add_argument("--project",
+                                   help=('An optional string to indicate which project that this downloader executable should register as.'))
+
+    downloader_parser.add_argument("--log_level",
+                                   choices=loggeria.LEVELS,
+                                   default=cfg["log_level"],
+                                   help="The logging level to display")
+
+    downloader_parser.add_argument("--log_dir",
+                                   help=("When provided, will write a log file to "
+                                         "the provided directory. This will be a "
+                                         "rotating log, creating a new log file "
+                                         "everyday, while storing the last 7 days "
+                                         "of logs"))
+
+    downloader_parser.add_argument("--thread_count",
+                                   type=int,
+                                   default=cfg["thread_count"],
+                                   help=('The number of threads that should download simultaneously'))
+
+    downloader_parser.add_argument("--alt",
+                                   help=('Run an alternative version of the downloader'),
+                                   action='store_true')
+
+    downloader_parser.set_defaults(func=run_downloader)
+
+    return parser
+
+
+def cast_to_bool(string):
+    '''
+    Ensure that the argument provided is either "True" or "False (or "true" or
+    "false") and convert that argument to an actual bool value (True or False).
+    '''
+    string_lower = string.lower()
+    if string_lower == "true":
+        return True
+    elif string_lower == "false":
+        return False
+    raise argparse.ArgumentTypeError('Argument must be True or False')
+
+def run_uploader(args):
+    uploader.run_uploader(args)
+
+
+def run_downloader(args):
+    '''
+    Convert the argparse Namespace object to a dictionary and run the downloader
+    with the given args.
+    '''
+    # Convert Namespace args object to args dict
+    args_dict = vars(args)
+
+    if args_dict.get("task_id") and not args_dict.get("job_id"):
+        raise argparse.ArgumentTypeError('Must supply a job_id with task_id.')
+
+    # New downloader broke in python 3. It was used only for linux and in
+    # daemon mode, so for now we'll use the old downloader for everything.
+
+    return downloader.run_downloader(args_dict)
+
+
+def main():
+    parser = build_parser()
+    args = parser.parse_args()
+    # Handle calling the script without an argument, fixes argparse issue
+    # https://bugs.python.org/issue16308
+    try:
+        func = args.func
+    except AttributeError:
+        parser.error("too few arguments")
+    func(args)
+
+
+if __name__ == '__main__':
+    main()
+

ciocore/config.py

@@ -1,7 +1,7 @@
 """
-Config is a configuration object implemented as a module-level singleton.
+A configuration object implemented as a module-level singleton.
 
-Configuration variables can be shared by importing the module. If there are changes in environment variables or other sources, the config can be refreshed.
+Configuration variables can be shared by importing the module. If there are changes in environment variables or other sources, the config can be refreshed. See [config()](#config)
 """
 
 import logging
@@ -24,16 +24,17 @@ __config__ = None
 
 def config(force=False):
     """
+    DEPRECATED! Use [get()](#get) instead.
+
     Instantiate a config object if necessary.
 
-    Deprecated:
-        Use [get()](#get) instead.
+    Keyword Arguments:
 
-    Args:
-        force (bool): Discards any existing config object and instantiate a new one -- Defaults to `False`.
+    * **`force`** --  Discards any existing config object and instantiate a new one -- Defaults to `False`.
 
     Returns:
-        dict: A dictionary containing configuration values.
+
+    * [Config()](#Config) object containing a dictionary of configuration variables.
     """
 
     global __config__
@@ -45,25 +46,32 @@ def get(force=False):
     """
     Instantiate a config object if necessary and return the dictionary.
 
-    Args:
-        force (bool): Discards any existing config object and instantiate a new one -- Defaults to `False`.
+    Keyword Arguments:
+
+    * **`force`** -- Discards any existing config object and instantiate a new one -- Defaults to `False`.
 
     Returns:
-        dict: A dictionary containing configuration values.
-
-    Example:
-        >>> from ciocore import config
-        >>> config.get()
-        {
-          'thread_count': 16,
-          'priority': 5,
-          'md5_caching': True,
-          'log_level': 'INFO',
-          'url': 'https://dashboard.conductortech.com',
-          'auth_url': 'https://dashboard.conductortech.com',
-          'api_url': 'https://api.conductortech.com', 
-          'api_key': None
-        }
+
+    * A dictionary containing configuration.
+\
+    ???+ example
+        ``` python
+        from ciocore import config
+
+        print(config.get())
+
+        # Result:
+        # {
+        #   'thread_count': 16,
+        #   'priority': 5,
+        #   'md5_caching': True,
+        #   'log_level': 'INFO',
+        #   'url': 'https://dashboard.conductortech.com',
+        #   'auth_url': 'https://dashboard.conductortech.com',
+        #   'api_url': 'https://api.conductortech.com', 
+        #   'api_key': None
+        # }
+        ```
     """
     global __config__
     if force or not __config__:
@@ -73,28 +81,12 @@ def get(force=False):
 class Config(object):
     def __init__(self):
         """
-        Initialize the config object.
-        
-        A config object is a dictionary containing configuration values. It is a singleton, so there is only one instance of it. It is instantiated the first time it is needed. It can be refreshed by calling get() with the `force` keyword argument set to `True`.
-        
-        A Config object has the following properties:
-
-        * `thread_count` The number of threads to use for downloading files. Defaults to the number of CPUs on the system times 2. It can be overridden by the `CONDUCTOR_THREAD_COUNT` environment variable.
-        * `priority` Set the priority for submissions. Defaults to 5. It can be overridden by the `CONDUCTOR_PRIORITY` environment variable.
-        * `md5_caching` Whether to cache MD5s. Defaults to `True`. It can be overridden by the `CONDUCTOR_MD5_CACHING` environment variable. Cachine MD5s significantly improves submission performance, but on rare occasions it can cause submissions to fail. If you experience this, set `md5_caching` to `False`.
-        * `log_level` The logging level. Defaults to `INFO`. It can be overridden by the `CONDUCTOR_LOG_LEVEL` environment variable.
-        * `url` The URL of the Conductor dashboard. Defaults to `https://dashboard.conductortech.com`. It can be overridden by the `CONDUCTOR_URL` environment variable.
-        * `auth_url` The URL of the Conductor dashboard. Defaults to `https://dashboard.conductortech.com`. It can be overridden by the `CONDUCTOR_AUTH_URL` environment variable. This is deprecated. Use `url` instead.
-        * `api_url` The URL of the Conductor API. Defaults to `https://api.conductortech.com`. It can be overridden by the `CONDUCTOR_API_URL` environment variable. 
-        * `api_key` The API key. The API key can be acquired from the Conductor dashboard, and can be stored in an environment variable or a file. In both cases the API KEY can be a JSON object or a base64 encoded JSON object. If it is base64 encoded, it can be a string or bytes. If it is a string, it will be decoded as ASCII. If it is bytes, it will be decoded as UTF-8.
-            * Environment variable: The `CONDUCTOR_API_KEY` variable can hold the API KEY directly. 
-            * File: The `CONDUCTOR_API_KEY_PATH` variable can hold the path to a file containing the API KEY.
+        Create a configuration dictionary.
 
-        Returns:
-            Config: A config object.
 
         Raises:
-            ValueError -- Invalid inputs, such as badly formed URLs.
+
+        * **`ValueError`** -- Invalid inputs, such as badly formed URLs.
         """
 
         try:
@@ -136,14 +128,15 @@ class Config(object):
     @staticmethod
     def get_api_key_from_variable():
         """
-        Attempt to get an API key from the `CONDUCTOR_API_KEY` environment variable.
+        Attempt to get an API key from the CONDUCTOR_API_KEY environment variable.
 
         Raises:
-            ValueError: An error occurred while reading or loading the key into JSON.
+
+        * **`ValueError`** -- An error occurred while loading the key into JSON.
 
         Returns:
-            str: JSON object containing the key - base 64 decoded if necessary.
-            
+
+        * JSON object containing the key - base 64 decoded if necessary.
         """
         api_key = os.environ.get("CONDUCTOR_API_KEY")
         if not api_key:
@@ -169,11 +162,12 @@ class Config(object):
         Attempt to get an API key from the file in the CONDUCTOR_API_KEY_PATH environment variable.
 
         Raises:
-            ValueError: An error occurred while reading or loading the key into JSON.
+
+        * **`ValueError`** --  An error occurred while reading or loading the key into JSON.
 
         Returns:
-            str: JSON object containing the key - base 64 decoded if necessary.
-            
+
+        * JSON object containing the key - base 64 decoded if necessary.
         """
         api_key_path = os.environ.get("CONDUCTOR_API_KEY_PATH")
         if not api_key_path: