geoseeq 0.5.6a14__tar.gz → 0.5.6a16__tar.gz

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a14
+Version: 0.5.6a16
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/geoseeq/cli/main.py

@@ -53,7 +53,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.5.6a14')  # remember to update setup
+    click.echo('0.5.6a16')  # remember to update setup
 
 
 @main.group('advanced')
@@ -101,4 +101,20 @@ def cli_config(yes, api_token, endpoint, profile, overwrite):
             click.echo('You must accept the EULA to use the GeoSeeq API.')
             return
     set_profile(api_token, endpoint=endpoint, profile=profile, overwrite=overwrite)
-    click.echo(f'Profile configured.')
+    click.echo(f'Profile configured.')
+
+
+@main.command('clear-cache')
+@yes_option
+def cli_clear_cache(yes):
+    """Clear the local cache.
+
+    ---
+    
+    Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
+    Run `geoseeq eula show` to view the EULA.
+    """
+    from geoseeq.file_system_cache import GEOSEEQ_CACHE_DIR
+    import shutil
+    if yes or click.confirm('Are you sure you want to clear the cache?'):
+        shutil.rmtree(GEOSEEQ_CACHE_DIR, ignore_errors=True)

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/geoseeq/cli/upload/upload.py

@@ -40,6 +40,7 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
 @click.option('--cores', default=1, help='Number of uploads to run in parallel', show_default=True)
 @click.option('--threads-per-upload', default=4, help='Number of threads used to upload each file', show_default=True)
 @click.option('--num-retries', default=3, help='Number of times to retry a failed upload', show_default=True)
+@click.option('--chunk-size-mb', default=-1, help='Size of chunks to upload in MB', show_default=True)
 @ignore_errors_option
 @yes_option
 @private_option
@@ -52,7 +53,7 @@ hidden_option = click.option('--hidden/--no-hidden', default=False, help='Upload
               show_default=True)
 @folder_id_arg
 @click.argument('file_paths', type=click.Path(exists=True), nargs=-1)
-def cli_upload_file(state, cores, threads_per_upload, num_retries, ignore_errors, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
+def cli_upload_file(state, cores, threads_per_upload, num_retries, chunk_size_mb, ignore_errors, yes, private, link_type, recursive, hidden, no_new_versions, geoseeq_file_name, folder_id, file_paths):
     """Upload files to GeoSeeq.
 
     This command uploads files to either a sample or project on GeoSeeq. It can be used to upload
@@ -121,7 +122,8 @@ def cli_upload_file(state, cores, threads_per_upload, num_retries, ignore_errors
         use_cache=state.use_cache,
         num_retries=num_retries,
         ignore_errors=ignore_errors,
-        session=knex.new_session(),
+        session=None, #knex.new_session(),
+        chunk_size_mb=chunk_size_mb if chunk_size_mb > 0 else None,
     )
     for geoseeq_file_name, file_path in name_pairs:
         if isfile(file_path):

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/geoseeq/result/file_upload.py

@@ -175,6 +175,7 @@ class ResultFileUpload:
         attempts = 0
         while attempts < max_retries:
             try:
+                # url = url.replace("s3.wasabisys.com", "s3.us-east-1.wasabisys.com")
                 logger.debug(f"Uploading part {num + 1} to {url}. Size: {len(file_chunk)} bytes.")
                 if session:
                     http_response = session.put(url, data=file_chunk)
@@ -192,7 +193,7 @@ class ResultFileUpload:
                     raise e
 
                 retry_time = min(8 ** attempts, 120)  # exponential backoff, max 120s
-                retry_time *= 0.8 + (random() * 0.4)  # randomize to avoid thundering herd
+                retry_time *= 0.6 + (random() * 0.8)  # randomize to avoid thundering herd
                 logger.debug(f"Retrying upload for part {num + 1} in {retry_time} seconds.")
                 time.sleep(retry_time)
             
@@ -250,7 +251,7 @@ class ResultFileUpload:
         filepath,
         file_size,
         optional_fields=None,
-        chunk_size=FIVE_MB,
+        chunk_size=None,
         max_retries=3,
         session=None,
         progress_tracker=None,
@@ -259,6 +260,11 @@ class ResultFileUpload:
     ):
         """Upload a file to S3 using the multipart upload process."""
         logger.info(f"Uploading {filepath} to S3 using multipart upload.")
+        if not chunk_size:
+            chunk_size = FIVE_MB
+            if file_size >= 10 * FIVE_MB:
+                chunk_size = 5 * FIVE_MB
+        logger.debug(f"Using chunk size of {chunk_size} bytes.")
         resumable_upload_tracker = None
         if use_cache and file_size > 10 * FIVE_MB:  # only use resumable upload tracker for larger files
             resumable_upload_tracker = ResumableUploadTracker(filepath, chunk_size)

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/geoseeq/upload_download_manager.py

@@ -22,7 +22,8 @@ def _upload_one_file(args):
     (result_file, filepath, session, progress_tracker,
      link_type, overwrite, log_level, parallel_uploads,
      use_cache, no_new_versions, threads_per_upload,
-     num_retries, ignore_errors) = args
+     num_retries, ignore_errors, chunk_size_mb) = args
+    chunk_size = chunk_size_mb * 1024 * 1024 if chunk_size_mb else None
     if parallel_uploads:
         _make_in_process_logger(log_level)
     try:
@@ -31,7 +32,7 @@ def _upload_one_file(args):
             result_file.upload_file(
                 filepath,
                 session=session, overwrite=overwrite, progress_tracker=progress_tracker,
-                threads=threads_per_upload, use_cache=use_cache,
+                threads=threads_per_upload, use_cache=use_cache, chunk_size=chunk_size,
                 no_new_versions=no_new_versions, max_retries=num_retries,
             )
         else:
@@ -57,6 +58,7 @@ class GeoSeeqUploadManager:
                  no_new_versions=False,
                  num_retries=3,
                  ignore_errors=False,
+                 chunk_size_mb=5,
                  use_cache=True):
         self.session = session
         self.n_parallel_uploads = n_parallel_uploads
@@ -70,6 +72,7 @@ class GeoSeeqUploadManager:
         self.threads_per_upload = threads_per_upload
         self.num_retries = num_retries
         self.ignore_errors = ignore_errors
+        self.chunk_size_mb = chunk_size_mb
 
     def add_result_file(self, result_file, local_path):
         self._result_files.append((result_file, local_path))
@@ -95,7 +98,8 @@ class GeoSeeqUploadManager:
                 self.session, self.progress_tracker_factory(local_path),
                 self.link_type, self.overwrite, self.log_level,
                 self.n_parallel_uploads > 1, self.use_cache, self.no_new_versions,
-                self.threads_per_upload, self.num_retries, self.ignore_errors
+                self.threads_per_upload, self.num_retries, self.ignore_errors,
+                self.chunk_size_mb,
             ) for result_file, local_path in self._result_files
         ]
         out = []

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/geoseeq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.5.6a14
+Version: 0.5.6a16
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.5.6a14 → geoseeq-0.5.6a16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "geoseeq"
-version = "0.5.6a14"
+version = "0.5.6a16"
 authors = [
   { name="David C. Danko", email="dcdanko@biotia.io" },
 ]