ciocore 8.0.0b31__py2.py3-none-any.whl → 8.0.0rc1__py2.py3-none-any.whl

ciocore/VERSION

@@ -1 +1 @@
-8.0.0-beta.31
+8.0.0-rc.1

ciocore/cli.py

@@ -131,7 +131,15 @@ def _register(client):
 @click.pass_context
 @click.option("-v", "--version", is_flag=True, help="Print the version and exit.")
 def main(ctx, version):
-    """Conductor Command-line interface."""
+    """
+    Conductor Command-line interface.
+    
+    To get help on subcommands, use the --help flag after the subcommand.
+    
+    Example:
+    conductor download --help
+    
+    """
     if not ctx.invoked_subcommand:
         if version:
             click.echo(VERSION)
@@ -374,11 +382,9 @@ def downloader(
         )
         ctx.exit(0)
 
-    if task_id:
-        job_id = job_id + ":" + task_id
-
     Downloader.download_jobs(
         (job_id,),
+        task_id=task_id,
         thread_count=thread_count,
         output_dir=output,
     )

ciocore/common.py

@@ -364,7 +364,16 @@ def generate_md5(
 
         while len(file_buffer) > 0:
             hash_obj.update(file_buffer)
-            file_buffer = file_obj.read(MD5_BLOCKSIZE)
+            try:
+                file_buffer = file_obj.read(MD5_BLOCKSIZE)
+            except OSError:
+                logger.log(
+                    log_level,
+                    "Cannot read file '%s'. Is it corrupted or unavailable?",
+                    filepath,
+                )
+                raise
+
             curtime = time.time()
             bytes_processed = buffer_count * MD5_BLOCKSIZE
             percentage_processed = int((bytes_processed / float(file_size)) * 100)

ciocore/docsite/stylesheets/extra.css

@@ -1,8 +1,8 @@
  
 
 :root {
-    --md-primary-fg-color: #a71ed8;
-    --md-accent-fg-color: #db4ae3;
+    --md-primary-fg-color: #3c75a4;
+    --md-accent-fg-color: #406499;
 }
 
 .md-content a {

ciocore/docsite/stylesheets/tables.css

@@ -158,9 +158,6 @@
 .opts-prop-create {
   color: #36be00
 }
-/* .opts-prop-query {
-  color: #af00db
-} */
 .opts-prop-edit {
   color: #007dd1
 }

ciocore/downloader/legacy_downloader.py

@@ -18,10 +18,10 @@ import time
 import threading
 import hashlib
 
+import ciocore
 from ciocore import config
 from ciocore import api_client, common, loggeria
 from ciocore.common import CONDUCTOR_LOGGER_NAME
-from cioseq.sequence import Sequence
 
 try:
     import Queue as queue
@@ -291,12 +291,12 @@ class Downloader(object):
         downloader.print_uptime()
 
     @classmethod
-    def download_jobs(cls, job_ids, thread_count=None, output_dir=None):
+    def download_jobs(cls, job_ids, task_id=None, thread_count=None, output_dir=None):
         """
         Run the downloader for explicit jobs, and terminate afterwards.
         """
         downloader = cls(thread_count=thread_count, output_dir=output_dir)
-        downloader.start(job_ids)
+        thread_states = downloader.start(job_ids, task_id=task_id)
         while not common.SIGINT_EXIT and (
             not downloader.pending_queue.empty() or not downloader.downloading_queue.empty()
         ):
@@ -306,7 +306,7 @@ class Downloader(object):
         downloader._print_download_history()
         downloader.print_uptime()
 
-    def start(self, job_ids=None, summary_interval=10):
+    def start(self, job_ids=None, task_id=None, summary_interval=10):
         # Create new queues
         self.start_time = time.time()
         self.pending_queue = queue.Queue()
@@ -316,7 +316,7 @@ class Downloader(object):
         # If a job id has been specified then only load the queue up with that work
         if job_ids:
             self.history_queue_max = None
-            self.get_jobs_downloads(job_ids)
+            self.get_jobs_downloads(job_ids, task_id)
 
         # otherwise create a queue thread the polls the app for wor
         else:
@@ -461,12 +461,10 @@ class Downloader(object):
             # Wait for the queue to be consumed before querying for more
             self.nap()
 
-    
     def nap(self):
         while not common.SIGINT_EXIT:
 
             time.sleep(self.naptime)
-            # Pretty sure this return should be unindented!
             return
 
     @common.dec_timer_exit(log_level=logging.DEBUG)
@@ -479,51 +477,23 @@ class Downloader(object):
         except Exception as e:
             logger.exception("Could not get next download")
 
-    def get_jobs_downloads(self, job_ids):
+    def get_jobs_downloads(self, job_ids, task_id):
         """
-        Download jobs, each with an optional task specification.
-        
-        Job ids are padded to 5 digits, and task ids are padded to 3 digits.
+        Get each job and optional comma-separated task list.
 
-        job_ids: (tuple) may take any of the following forms:
-        1. A single job id, e.g. (01234,)
-        2. It doesn't have to be padded, e.g. (1234,)
-        3. A jobs with a task specification, e.g. (01234:1-3,5,7-9)
-        4. Several of jobs:tasks, e.g. (01234, 56789:1-3,5,7-9)
-    
-        """
-        jobs = self._flatten(job_ids)
-        for job in jobs:
-            endpoint = self.endpoint_downloads_job % job["job_id"]
-            
-            for tid in job["tasks"] or [None]:
-                downloads = _get_job_download(endpoint, self.api_client, job["job_id"], tid)
+        There will only be a task list if there is just one job, due to earlier arg validation.
+
+        If there is no task list, _get_job_download is called with tid=None (i.e. the whole job)
+        """ 
+        task_ids = [t for t in task_id.split(",") if t] if task_id else [None]
+
+        for job_id in job_ids:
+            endpoint = self.endpoint_downloads_job % job_id
+            for tid in task_ids:
+                downloads = _get_job_download(endpoint, self.api_client, job_id, tid)
                 if downloads:
                     for task_download in downloads.get("downloads", []):
-                        print("putting in queue: %s" % task_download)
                         self.pending_queue.put(task_download, block=True)
-            
-    @staticmethod
-    def _flatten(job_ids):
-        """Create a list of job objects with keys: job_id and tasks. 
-        
-        see: get_jobs_downloads() function see tests/test_downloader.py for examples.
-        """
-        result = []
-        for job_id in job_ids:
-            if ":" in job_id:
-                job_id, range_spec = job_id.split(":")
-                try:
-                    seq = Sequence.create(range_spec)
-                    tasks = seq.expand("###")
-                except (ValueError, TypeError):
-                    tasks = None
-            else:
-                tasks = None
-            result.append({"job_id": job_id.zfill(5), "tasks": tasks})
-        return result
-    
-
 
     @common.dec_catch_exception(raise_=True)
     def download_target(self, pending_queue, downloading_queue, task_download_state):
@@ -620,8 +590,8 @@ class Downloader(object):
                     # account for a render's generated subdirectories
                     dirpath = os.path.dirname(local_filepath)
 
-                    # Ensure that the output_path directory exists and set open permissions
-                    logger.debug("Creating output_path directory if necessary: %s", dirpath)
+                    # Ensure that the destination directory exists and set open permissions
+                    logger.debug("Creating destination directory if necessary: %s", dirpath)
                     file_download_state.status = FileDownloadState.STATE_PREPARING_DIRECTORY
                     safe_mkdirs(dirpath)
 
@@ -1268,7 +1238,7 @@ def run_downloader(args):
         file_formatter=LOG_FORMATTER,
     )    
     
-    api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=__version__)
+    api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=ciocore.__version__)
 
     logger.debug("Downloader args: %s", args)
 
@@ -1278,6 +1248,7 @@ def run_downloader(args):
     if job_ids:
         Downloader.download_jobs(
             job_ids,
+            task_id=args.get("task_id"),
             thread_count=thread_count,
             output_dir=args.get("output"),
         )
@@ -1306,6 +1277,6 @@ def report_error(self, download_id, error_message):
         resp_str, resp_code = self.api_helper.make_request(
             "/downloads/%s/fail" % download_id, data=error_message, verb="POST", use_api_key=True
         )
-    except:
+    except e:
         pass
     return True

ciocore/file_utils.py

@@ -256,13 +256,13 @@ def get_common_dirpath(paths):
 
 
 def _is_valid_path(path_str):
-    """
+    r"""
     This is dirty/inaccurate helper function to determine whether the given "path" is considered
     valid. If so, return True.
 
     If the given path_str is any of the following characters, then it's to be considered invalid:
 
-        On linux\mac:
+        On linux/mac:
                 /
                 //
                 lettered drive (e.g. x:\)
@@ -453,7 +453,7 @@ def get_tx_path(filepath, existing_only=False):
 
 
 def strip_drive_letter(filepath):
-    """
+    r"""
     If the given filepath has a drive letter, remove it and return the rest of the path
 
         C:\cat.txt         -->    \cat.txt

{ciocore-8.0.0b31.dist-info → ciocore-8.0.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ciocore
-Version: 8.0.0b31
+Version: 8.0.0rc1
 Summary: Core functionality for Conductor's client tools
 Home-page: https://github.com/ConductorTechnologies/ciocore
 Author: conductor
@@ -51,8 +51,7 @@ See [CONTRIBUTING](CONTRIBUTING.md)
 
 ## Changelog
 
-## Version: 8.0.0-beta.30 -- 30 Mar 2024
-
+## Version:8.0.0-rc.1 -- 05 Apr 2024
 
 Major Updates in Version 8.0.0
 

{ciocore-8.0.0b31.dist-info → ciocore-8.0.0rc1.dist-info}/RECORD

@@ -1,16 +1,15 @@
-ciocore/VERSION,sha256=8MsnQQ6bwkI9kve7IMcqGjpu77NaAnyAuJ35QhvPVE8,13
+ciocore/VERSION,sha256=oaAkSyjgoKqpHRAw-3LYleaedh24W2mf8BvHs5j0-ak,10
 ciocore/__init__.py,sha256=aTP7LeeosQA8BZE67gDV4jgfTK5zxmwZRjiTRu_ZWj0,646
 ciocore/api_client.py,sha256=TyHXGmK4uTmKV93O_IpAvttZhbexIIZTGP-IVn_WyWM,24599
-ciocore/cli.py,sha256=BXgULpqElTHwX6nUcSFVVh99MCBSG2FjuOzRKhIa6Hk,15399
+ciocore/cli.py,sha256=9OdoazUovfUHxpiAHahH7HJjJcdRLO0_QsdPXjGjlW8,15509
 ciocore/client_db.py,sha256=tTz3bl2xeDPPcYSDS3g3QgV_xYihJMx0Kj6OeN2klK0,12978
-ciocore/common.py,sha256=FnggAL-IGW8VQB6kLsrKLnxWFgLpwGBhOtiZSHz130M,14725
+ciocore/common.py,sha256=mBIS6KiYoQsjWe6aIFUGRRvCMl8BIN2kmLZ4J8icap8,14982
 ciocore/compat.py,sha256=5uEXPSog_jxsDMaHBswAKEtfyXT25VgU6WNGIhz9PHU,256
 ciocore/conductor_submit.py,sha256=ONE0LsA5hGavTJIOXXYx8qzl8_vBPADwhd6Ytq_0E0c,9382
 ciocore/config.py,sha256=rCL7kaFn1tYgSglN8q9Wx6SwMpoXTq0BMQGwPRVwVIg,8973
 ciocore/data.py,sha256=Ji0qUk8nJXBNakoHSqBiVx8O58SbZXyt273SHlEDn3U,7027
-ciocore/dev_inst_tagger.py,sha256=bPfoFbE7IPCpDbtbVHxW7IlMwvspdnJhPuNq-u4fJN4,3497
 ciocore/exceptions.py,sha256=4Oq-WX-qiN6kPUdBCHvvd6mtSQ0nCkDbJxWt2CNtpv8,1504
-ciocore/file_utils.py,sha256=bAlL31B4YkRgX-yT8kF8UXBFktQlsE1PvxbKqTeAeOU,17174
+ciocore/file_utils.py,sha256=swA7th9WhDEloW69YViRTKB-oeC9UmNdEGegfH1r-Gw,17176
 ciocore/hardware_set.py,sha256=FlRQiGCLRcSW7Oko_gzgVK8ZqJ_J92eT8e_AleAbS2E,17047
 ciocore/loggeria.py,sha256=2xdQRFb9NyXynU2O_pSOszJWcpoHgPwTUWJvERg7ODY,15251
 ciocore/package_environment.py,sha256=MEHV7jfs3NJIEYCIaW8JfJdBmelvPHZMmBzPlXETiRo,7808
@@ -27,7 +26,7 @@ ciocore/docsite/index.html,sha256=p_Dq_6_8cwZZPDI5zjahfCWUkfbSj14NJtoQFAZv0WI,20
 ciocore/docsite/logo.png,sha256=gArgFFWdw8w985-0TkuGIgU_pW9sziEMZdqytXb5WLo,2825
 ciocore/docsite/objects.inv,sha256=s2FKStLlVIQbfG7U2-nw-7rz4unvd1W0u00YtLBxAKo,758
 ciocore/docsite/sitemap.xml,sha256=M_V85zl0y2adRvzJAnoCxlZH_Hl7TLnIb1A-6l_xGmI,109
-ciocore/docsite/sitemap.xml.gz,sha256=0F5oF85vGRc0wxAJHZrvqUgip6JzhnqEmPzN_j0jzGE,127
+ciocore/docsite/sitemap.xml.gz,sha256=5eSPEBgK40Dy0VTOM8gl-guV4pLORNHnn-M_KPwVDQk,127
 ciocore/docsite/apidoc/api_client/index.html,sha256=mHDq-zdWuilDKUluzmQNdOsImRoGEnnK5Ul2ApajgdU,170218
 ciocore/docsite/apidoc/apidoc/index.html,sha256=NQn8wjapxa7O2IQhsbE7Y-VMfMFL6Tby-L7qIF6K3m4,26171
 ciocore/docsite/apidoc/config/index.html,sha256=1GWkyClM7LJlLBpx07BqCi3xDkU-SKVPjNUEeKuwbNs,72559
@@ -83,13 +82,13 @@ ciocore/docsite/cmdline/packages/index.html,sha256=j1YeMgdwZ6FyJGJii05AdtUWKRjyu
 ciocore/docsite/cmdline/uploader/index.html,sha256=y0728lIAYC4b9mfpjNVfwQqXoqJrsdG49UY5u7B1p94,25123
 ciocore/docsite/how-to-guides/index.html,sha256=OFrFBTb9tuMn60Jd4oeM10Mw-d-0pyqwoKoo9g1biBM,20100
 ciocore/docsite/search/search_index.json,sha256=ZMAbHLAlwOTujgA32XJTA7SHogRYh9i-ioPFOxjjJ0s,182633
-ciocore/docsite/stylesheets/extra.css,sha256=qgfcao9TEBzf6ieAWN5rPyrea7YM_YgVg5qr4NPrxoU,354
-ciocore/docsite/stylesheets/tables.css,sha256=O2PEwlKC0RfvZCRV9UERqQRvhb6jcqO84PLew0bafF0,3279
+ciocore/docsite/stylesheets/extra.css,sha256=_Cxe9Dhg1BBi6Kqaz_iZD9z9VyqxA9vtONRjP4PVic0,354
+ciocore/docsite/stylesheets/tables.css,sha256=LE_zwGRxGcdPIy-9QiVPecOzlEBSqZb_WP5vDkFE0ZM,3235
 ciocore/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ciocore/downloader/base_downloader.py,sha256=9PVmP1qy-dop4MbVsmLFvf5W6LKQLUKHPDYE6AEt71g,25784
 ciocore/downloader/download_runner_base.py,sha256=cWTWKhFX5FLG84Wh8_s7xpk3n0nmQ4fDiD2rYV6a3u0,1612
 ciocore/downloader/job_downloader.py,sha256=HAhr95RyCsj3KTp5W58Znc4JmqHY-mrM5b8vo6kRIJ0,5697
-ciocore/downloader/legacy_downloader.py,sha256=ef9hCMUTTuxYHKzpEoK5bc3gqwAEIhewkC_KpbhLVHE,52087
+ciocore/downloader/legacy_downloader.py,sha256=LIbT3BxuY56lq_UW_io9FSgK_2VIKwFVMENyF-5KllA,51158
 ciocore/downloader/log.py,sha256=WCRNx0LObx8JBO5MQucNHQGBXMgSBLGdH0VALN8dFDo,2105
 ciocore/downloader/logging_download_runner.py,sha256=iOuW8OvfAYW5MpP9zpUXh94qkWLC0GeG4PDGH7JSqJM,2865
 ciocore/downloader/perpetual_downloader.py,sha256=cD7lnBH75-c-ZVVPHZc1vSnDhgJOnGlPT85zn7IjGgA,2105
@@ -121,8 +120,8 @@ tests/test_uploader.py,sha256=B1llTJt_fqR6e_V_Jxfw9z73QgkFlEPU87xLYGzt-TQ,2914
 tests/test_validator.py,sha256=2fY66ayNc08PGyj2vTI-V_1yeCWJDngkj2zkUM5TTCI,1526
 tests/mocks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/mocks/glob.py,sha256=J2MH7nqi6NJOHuGdVWxhfeBd700_Ckj6cLh_8jSNkfg,215
-ciocore-8.0.0b31.dist-info/METADATA,sha256=lxGtpIynjrenOqbNpXSL4H6YypvaNWmjjV_iM1O_HKY,18201
-ciocore-8.0.0b31.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
-ciocore-8.0.0b31.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
-ciocore-8.0.0b31.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
-ciocore-8.0.0b31.dist-info/RECORD,,
+ciocore-8.0.0rc1.dist-info/METADATA,sha256=OiC555SBSXja3tKUjJCh3NWrvYpRdzUTLGHP4nSlMaY,18196
+ciocore-8.0.0rc1.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+ciocore-8.0.0rc1.dist-info/entry_points.txt,sha256=cCqcALMYbC4d8545V9w0Zysfg9MVuKWhzDQ2er4UfGE,47
+ciocore-8.0.0rc1.dist-info/top_level.txt,sha256=SvlM5JlqULzAz00JZWfiUhfjhqDzYzSWssA87zdJl0o,14
+ciocore-8.0.0rc1.dist-info/RECORD,,

{ciocore-8.0.0b31.dist-info → ciocore-8.0.0rc1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

ciocore/dev_inst_tagger.py

@@ -1,120 +0,0 @@
-"""
-DEV only. This module is used to request instance types from the inst-tagger service. We try to isolate the cod so it can be easily removed.
-"""
-
-import os
-import requests
-from ciocore import api_client
-
-USE_INST_TAGGER = int(os.environ.get("CIO_FEATURE_USE_INST_TAGGER", 0))
-SUPABASE_URL = os.environ.get("CIO_FEATURE_INST_TAGGER_SUPABASE_URL")
-SUPABASE_KEY = os.environ.get("CIO_FEATURE_INST_TAGGER_SUPABASE_KEY")
-
-def request_supabase_data(table_name):
-    HEADERS = {
-        "Authorization": f"Bearer {SUPABASE_KEY}",
-        "apikey": SUPABASE_KEY,
-        "Content-Type": "application/json",
-    }
-
-    response = requests.get(
-        f"{SUPABASE_URL}/rest/v1/{table_name}", headers=HEADERS, timeout=5
-    )
-    if response.status_code == 200:
-        return response.json()
-    else:
-        print("Error:", response.status_code, response.text)
-        return None
-
-
-def get_cloud(cio_inst_types):
-    if not cio_inst_types:
-        return None
-    if cio_inst_types[0]["name"].startswith("cw-"):
-        return "cw"
-    elif "." in cio_inst_types[0]["name"]:
-        return "aws"
-    return "gcp"
-
-
-def normalize_sb_inst_types(sb_inst_types):
-    result = []
-    for inst_type in sb_inst_types:
-        el = inst_type["instance_type"].copy()
-        el["id"] = inst_type["id"]
-        result.append(el)
-    return result
-
-
-def to_id_dict(thelist):
-    return {d["id"]: d for d in thelist}
-
-
-def valid_inst_types():
-    sb_inst_types = request_supabase_data("instance_types")
-    sb_inst_types = normalize_sb_inst_types(sb_inst_types)
-
-    cio_inst_types = api_client.request_instance_types()
-    # print(json.dumps(cio_inst_types, indent=2))
-
-    cloud = get_cloud(cio_inst_types)
-
-    # filter out instance types that are not in the cloud
-
-    sb_inst_types = [
-        inst_type for inst_type in sb_inst_types if inst_type["cloud"] == cloud
-    ]
-    for it in sb_inst_types:
-        it["cores"] = it["total_cpu"]
-        it["memory"] = it["total_memory_gb"]
-
-        cores = it["cores"]
-        mem = it["memory_gb"]
-        cpu_desc = f"{cores} cores, {mem} GB RAM"
-        gpu_desc = ""
-        if it["gpu"]:
-            gpu_count = it["gpu"]["gpu_count"]
-            gpu_model = it["gpu"]["gpu_model"]
-            gpu_arch = it["gpu"]["gpu_architecture"]
-            gpu_mem = it["gpu"]["total_gpu_memory"]
-            gpu_desc = f" ({gpu_count}x {gpu_model} ({gpu_arch}) {gpu_mem} GB)"
-
-        desc = f"{cpu_desc}{gpu_desc}"
-        it["description"] = desc
-
-    return sb_inst_types
-
-
-def request_instance_types():
-    """
-    Request instance types from the inst-tagger service.
-
-    Returns:
-        list(dict): A list of instance types.
-    """
-    if not USE_INST_TAGGER:
-        return None
-    
-    inst_types = valid_inst_types()
-
-    tags = request_supabase_data("tags")
-    tag_ids = [tag["id"] for tag in tags]
-    tags_dict = to_id_dict(tags)
-
-    assignments = request_supabase_data("assignments")
-    assignments = [a for a in assignments if a["tag_id"] in tag_ids]
-
-
-    # assign tags to instance types
-    for inst_type in inst_types:
-        inst_type["categories"] = []
-        # find assignments relating to this instance type
-        these_assignments = [
-            a for a in assignments if a["instance_type_id"] == inst_type["id"]
-        ]
-
-        for assignment in these_assignments:
-            tag = tags_dict[assignment["tag_id"]]
-            inst_type["categories"].append({"label": tag["label"], "order": tag["order"]})
-
-    return inst_types