ciocore 6.3.2rc1__py2.py3-none-any.whl → 6.4.0__py2.py3-none-any.whl

ciocore/data.py

@@ -1,20 +1,16 @@
 """
-This module is a singleton that provides the data from Conductor endpoints. Specifically, it provides projects, instance types, and software package data. It is also possible to use cached fixture data for development purposes.
+A singleton containing all the data from Conductor endpoints that's needed for writing submission
+tools. Specifiically, it provides projects, instance types, and software package data.
+
+It's also possible to use cached fixtures for development purposes.
 
-Since the data is stored at the module level, you can access it from anywhere in your code without the need to pass it around.
 """
 
 import json
 import os
-import requests
 from ciocore.package_tree import PackageTree
-from ciocore import api_client, dev_inst_tagger
+from ciocore import api_client
 from ciocore.hardware_set import HardwareSet
-
-
-
-# USE_INST_TAGGER = int(os.environ.get("CIO_FEATURE_DEV_INST_TAGGER", 0))
-
 __data__ = {}
 __products__ = None
 __fixtures_dir__ = None
@@ -22,24 +18,35 @@ __platforms__ = None
 
 def init(*products, **kwargs):
     """
-    Initialize the module and let it know what host products to provide.
+    Initialize and let the module know what host products to provide in the `software` property.
+
+    Arguments:
 
-    Args:
-        products (str): Provide a list of products for which to get software packages. If no products are given, the software data contains all products from the packages endpoint. If you provide more than one product, they must all be host level products.
+    * **`products`** -- Provide a list of products as a filter. If no products are given, the
+      software data structure is built using all products from the packages endpoint. If you provide
+      more than one product, they must all be host level products.
 
-    Keyword Args:
-        product (str): `DEPRECATED` Provide one product for which to get software packages.
+    Keyword Arguments:
 
-    Examples:
-        >>> from ciocore import data as coredata
-        >>> coredata.init()
+    * **`product`** -- (DEPRECATED) You can provide one product with the product keyword, or you can
+      provide the string `"all"`. This is ignored if any product args are present. -- Defaults to
+      `None`.
+
+    ???+ example
+        ``` python
+
+        from ciocore import data as coredata
+        coredata.init()
         # OR
-        >>> coredata.init("maya-io")
+        coredata.init("maya-io")
         # OR LEGACY
-        >>> coredata.init(product="all")
+        coredata.init(product="all")
         # OR
-        >>> coredata.init(product="maya-io")
+        coredata.init(product="maya-io")
+
+        ```
     """
+
     global __products__
     global __platforms__
     if products:
@@ -55,72 +62,76 @@ def init(*products, **kwargs):
             __products__ = [kwargs.get("product")]
     else:
         __products__ = []
-    print("SETTING PLATFORMS")
-    __platforms__ = set(kwargs.get("platforms", ["windows", "linux"]))
-    print("PLATFORMS: ", __platforms__)
-    print("DONE SETTING PLATFORMS")
 
+    __platforms__ = set(kwargs.get("platforms", ["windows", "linux"]))
+        
+         
 def data(force=False):
     """
-    Provide projects, instance types, and software package data.
+    Provides projects, instance types, and software package data.
 
-    Keyword Args:
-        force: (bool) If `True`, then fetch fresh data -- Defaults to `False`.
+    If data has been fetched already, then return it, otherwise make requests to fetch it.
+    The user may have to authentiicate.
 
-    Raises:
-        ValueError:  Module was not initialized with [init()](/data/#ciocore.data.init).
+    The set of instance types and software are each potentially pruned to match the available
+    platforms represented by the other. If the instance types come from an orchestrator that
+    provides both windows and linux machines, and the software product(s) are available on both
+    platforms, then no pruning occurs. However, if there are no windows machines, then any windows
+    software is removed from the package tree. Likewise, if a product is chosen that only runs on
+    Windows, then Linux instance types will be culled from the list of available hardware.
 
-    Returns:
-        dict: Keys are `projects`, `instance_types`, `software`.
+    Keyword Arguments:
 
-    When you access the data, if it has already been fetched, it will be returned. Otherwise,
-    requests will be made to fetch the data. You may need to authenticate in order to access the
-    data.
+    * **`force`** -- If `True`, then fetch fresh data -- Defaults to `False`.
+    
+    Raises:
 
-    The set of instance types and software can be pruned to match the available platforms
-    represented by each other. For example, if the instance types come from an orchestrator that
-    provides both Windows and Linux machines, and the software product(s) are available on both
-    platforms, no pruning occurs. However, if there are no Windows machines available, any Windows
-    software will be removed from the package tree. Similarly, if a product is chosen that only runs
-    on Windows, Linux instance types will not appearin the list of available hardware.
+    * **`ValueError`** -- Module was not initialized with the [init()](#init) method.
 
-    Here is a breakdown of each key in the dictionary:
+    Returns:
 
-    * **projects** is a list of project names for your authenticated account.
+    * A dictionary with 3 keys: `projects`, `instance_types`, `software`.
 
-    * **instance_types** is an instance of HardwareSet, providing you with access to the list of
-    available machines configurations.
+    * `projects` is a list of project names for the authenticated account.
+    * `instance_types` is an instace of HardwareSet, from which you can get the data in various forms.
+    * `software` is a [PackageTree](/developer/ciocore/package_tree) object containing either all
+      software available at conductor, or a subset according to the product specified on
+      initialization.
 
-    * **software** is a PackageTree object containing either all
-    the software available at Conductor, or a subset based on specified products.
 
+    ???+ example
+        ``` python
 
-    Examples:
-        >>> from ciocore import data as coredata
-        >>> coredata.init(product="maya-io")
+        from ciocore import data as coredata
+        coredata.init(product="maya-io")
+        coredata.data()["software"]
 
-        >>> coredata.data()["software"]
-        <ciocore.package_tree.PackageTree object at 0x10e9a4040>
+        # Result:
+        # <ciocore.package_tree.PackageTree object at 0x10e9a4040>
 
-        >>> coredata.data()["projects"][0]
-        ATestForScott
+        coredata.data()["projects"][0]
 
-        >>> coredata.data()["instance_types"]
-        <ciocore.hardware_set.HardwareSet object at 0x0000028941CD9DC0>
+        # Result:
+        # ATestForScott
+
+        coredata.data()["instance_types"][-1]["description"]
+        # Result:
+        # 160 core, 3844GB Mem
+        ```
     """
 
     global __data__
     global __products__
+    global __fixtures_dir__
     global __platforms__
 
     if __products__ is None:
         raise ValueError(
             'Data must be initialized before use, e.g. data.init("maya-io") or data.init().'
         )
-    products_copy = __products__.copy()
+
     if force:
         clear()
-        init(*products_copy)
 
     if __data__ == {}:
         # PROJECTS
@@ -132,8 +143,6 @@ def data(force=False):
 
         # INST_TYPES
         instance_types = _get_json_fixture("instance_types")
-        if not instance_types:
-            instance_types = dev_inst_tagger.request_instance_types()
         if not instance_types:
             instance_types = api_client.request_instance_types()
 
@@ -154,13 +163,14 @@ def data(force=False):
             host_products = []
             kwargs["product"] = __products__[0]
 
+        
         software_tree = PackageTree(software, *host_products, **kwargs)
 
         if software_tree:
             __data__["software"] = software_tree
             # Revisit instance types to filter out any that are not needed for any software package.
             sw_platforms = software_tree.platforms()
-
+            
             instance_types = [
                 it for it in instance_types if it["operating_system"] in sw_platforms
             ]
@@ -174,17 +184,25 @@ def data(force=False):
 
 def valid():
     """
-    Check validity.
+    Check validity of the data.
 
     Returns:
-        bool: True if `projects`, `instance_types`, and `software` are valid.
 
-    Examples:
-        >>> from ciocore import data as coredata
-        >>> coredata.valid()
-        True
-    """
+    * True if `projects`, `instance_types`, and `software` exists.
+
+    ???+ example
+        ``` python
+
+        from ciocore import data as coredata
+        coredata.valid()
+
+        # Result:
+        # True
 
+
+        ```
+    """
+    global __data__
     if not __data__.get("projects"):
         return False
     if not __data__.get("instance_types"):
@@ -196,24 +214,45 @@ def valid():
 
 def clear():
     """
-    Clear out data.
+    Clear out the data.
 
-    [valid()](/data/#ciocore.data.valid) returns False after clear().
+    [valid()](#valid) returns False after clear().
     """
     global __data__
-    global __products__
-    global __platforms__
     __data__ = {}
     __products__ = None
     __platforms__ = None
+    """
+    Let the module know what host products to provide in the `software` property.
+
+    Keyword Arguments:
+
+    * **`products`** -- Product args, such as `maya-io, cinema4d` -- Defaults to `None` which means all.
+
+    Raises:
+
+    * **`ValueError`** -- _description_.
+
+    ???+ example
+        ``` python
+
+        from ciocore import data as coredata
+        coredata.init()
+        # OR
+        coredata.init("maya-io")
+
+        ```
+    """
 
 
 def products():
     """
 
     Returns:
-        list(str): The product names. An empty list signifies all products.
+
+    * The product names. An empty array signifies all products.
     """
+    global __products__
     return __products__
 
 
@@ -222,9 +261,9 @@ def set_fixtures_dir(path):
     Specify a directory in which to find JSON files representing the three sets of data to provide.
     The individual filenames are:
 
-    * `projects.json`
-    * `instance_types.json`
-    * `software.json`
+    * projects.json
+    * instance_types.json
+    * software.json
 
     These files could be used in an environment where machines can't access the internet. They are
     also useful as a cache for developers who need to reload often as it avoids waiting for the
@@ -232,16 +271,21 @@ def set_fixtures_dir(path):
 
     In order to get the content for the fixtures files, use the following Example
 
-    Examples:
-        >>> from ciocore import api_client
-        >>> projects = api_client.request_projects()
-        >>> instance_types = api_client.request_instance_types()
-        >>> software = api_client.request_software_packages()
+    ???+ example
+        ``` python
+
+        from ciocore import api_client
 
-    Write that data as JSON to the filenames listed above.
+        projects = api_client.request_projects()
+        instance_types = api_client.request_instance_types()
+        software = api_client.request_software_packages()
+
+        # Write that data as JSON to the filenames listed above.
+        ```
 
     Arguments:
-        path (str): Directory in which to find the above files.
+
+    * **`path`** -- Directory in which to find the above files.
 
     """
 
@@ -250,6 +294,7 @@ def set_fixtures_dir(path):
 
 
 def _get_json_fixture(resource):
+    global __fixtures_dir__
     if __fixtures_dir__:
         cache_path = os.path.join(__fixtures_dir__, "{}.json".format(resource))
         if os.path.isfile(cache_path):
@@ -262,10 +307,11 @@ def _get_json_fixture(resource):
 
 def platforms():
     """
-    The set of platforms that both software and instance types are valid on.
+    The set of platforms that are found in both software and instance types.
 
     Returns:
-        set: A set containing platforms: windows and/or linux.
+
+    * A set containing platforms: windows and/or linux or neither.
     """
+    global __platforms__
     return __platforms__
-

ciocore/downloader.py

@@ -18,10 +18,10 @@ import time
 import threading
 import hashlib
 
+import ciocore
 from ciocore import config
 from ciocore import api_client, common, loggeria
 from ciocore.common import CONDUCTOR_LOGGER_NAME
-from cioseq.sequence import Sequence
 
 try:
     import Queue as queue
@@ -291,12 +291,12 @@ class Downloader(object):
         downloader.print_uptime()
 
     @classmethod
-    def download_jobs(cls, job_ids, thread_count=None, output_dir=None):
+    def download_jobs(cls, job_ids, task_id=None, thread_count=None, output_dir=None):
         """
         Run the downloader for explicit jobs, and terminate afterwards.
         """
         downloader = cls(thread_count=thread_count, output_dir=output_dir)
-        downloader.start(job_ids)
+        thread_states = downloader.start(job_ids, task_id=task_id)
         while not common.SIGINT_EXIT and (
             not downloader.pending_queue.empty() or not downloader.downloading_queue.empty()
         ):
@@ -306,7 +306,7 @@ class Downloader(object):
         downloader._print_download_history()
         downloader.print_uptime()
 
-    def start(self, job_ids=None, summary_interval=10):
+    def start(self, job_ids=None, task_id=None, summary_interval=10):
         # Create new queues
         self.start_time = time.time()
         self.pending_queue = queue.Queue()
@@ -316,7 +316,7 @@ class Downloader(object):
         # If a job id has been specified then only load the queue up with that work
         if job_ids:
             self.history_queue_max = None
-            self.get_jobs_downloads(job_ids)
+            self.get_jobs_downloads(job_ids, task_id)
 
         # otherwise create a queue thread the polls the app for wor
         else:
@@ -461,12 +461,10 @@ class Downloader(object):
             # Wait for the queue to be consumed before querying for more
             self.nap()
 
-    
     def nap(self):
         while not common.SIGINT_EXIT:
 
             time.sleep(self.naptime)
-            # Pretty sure this return should be unindented!
             return
 
     @common.dec_timer_exit(log_level=logging.DEBUG)
@@ -479,51 +477,24 @@ class Downloader(object):
         except Exception as e:
             logger.exception("Could not get next download")
 
-    def get_jobs_downloads(self, job_ids):
+    def get_jobs_downloads(self, job_ids, task_id):
         """
-        Download jobs, each with an optional task specification.
-        
-        Job ids are padded to 5 digits, and task ids are padded to 3 digits.
+        Get each job and optional comma-separated task list.
 
-        job_ids: (tuple) may take any of the following forms:
-        1. A single job id, e.g. (01234,)
-        2. It doesn't have to be padded, e.g. (1234,)
-        3. A jobs with a task specification, e.g. (01234:1-3,5,7-9)
-        4. Several of jobs:tasks, e.g. (01234, 56789:1-3,5,7-9)
-    
-        """
-        jobs = self._flatten(job_ids)
-        for job in jobs:
-            endpoint = self.endpoint_downloads_job % job["job_id"]
-            
-            for tid in job["tasks"] or [None]:
-                downloads = _get_job_download(endpoint, self.api_client, job["job_id"], tid)
+        There will only be a task list if there is just one job, due to earlier arg validation.
+
+        If there is no task list, _get_job_download is called with tid=None (i.e. the whole job)
+        """ 
+        task_ids = [t for t in task_id.split(",") if t] if task_id else [None]
+
+        for job_id in job_ids:
+            endpoint = self.endpoint_downloads_job % job_id
+            for tid in task_ids:
+                downloads = _get_job_download(endpoint, self.api_client, job_id, tid)
                 if downloads:
                     for task_download in downloads.get("downloads", []):
                         print("putting in queue: %s" % task_download)
                         self.pending_queue.put(task_download, block=True)
-            
-    @staticmethod
-    def _flatten(job_ids):
-        """Create a list of job objects with keys: job_id and tasks. 
-        
-        see: get_jobs_downloads() function see tests/test_downloader.py for examples.
-        """
-        result = []
-        for job_id in job_ids:
-            if ":" in job_id:
-                job_id, range_spec = job_id.split(":")
-                try:
-                    seq = Sequence.create(range_spec)
-                    tasks = seq.expand("###")
-                except (ValueError, TypeError):
-                    tasks = None
-            else:
-                tasks = None
-            result.append({"job_id": job_id.zfill(5), "tasks": tasks})
-        return result
-    
-
 
     @common.dec_catch_exception(raise_=True)
     def download_target(self, pending_queue, downloading_queue, task_download_state):
@@ -1249,6 +1220,53 @@ def _in_queue(the_queue, item_dict, key):
         if item[key] == item_dict[key]:
             return True
 
+
+def run_downloader(args):
+    """
+    Start the downloader. If a job id(s) were given, exit the downloader upon completion.
+    Otherwise, run the downloader indefinitely (daemon mode), polling the Conductor cloud app for
+    files that need to be downloaded.
+    """
+
+    # Set up logging
+    log_level_name = args.get("log_level")
+    log_level = loggeria.LEVEL_MAP.get(log_level_name)
+    log_dirpath = args.get("log_dir")
+    set_logging(log_level, log_dirpath)
+    
+    api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=ciocore.__version__)
+
+    logger.debug("Downloader args: %s", args)
+
+    job_ids = args.get("job_id")
+    thread_count = args.get("thread_count")
+
+    if job_ids:
+        Downloader.download_jobs(
+            job_ids,
+            task_id=args.get("task_id"),
+            thread_count=thread_count,
+            output_dir=args.get("output"),
+        )
+
+    else:
+        Downloader.start_daemon(
+            thread_count=thread_count, location=args.get("location"), output_dir=args.get("output")
+        )
+
+
+def set_logging(level=None, log_dirpath=None):
+    log_filepath = None
+    if log_dirpath:
+        log_filepath = os.path.join(log_dirpath, "conductor_dl_log")
+    loggeria.setup_conductor_logging(
+        logger_level=level,
+        console_formatter=LOG_FORMATTER,
+        file_formatter=LOG_FORMATTER,
+        log_filepath=log_filepath,
+    )
+
+
 def report_error(self, download_id, error_message):
     try:
         logger.error("failing upload due to: \n%s" % error_message)