geoseeq 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

geoseeq/cli/main.py

@@ -54,7 +54,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.6.0')  # remember to update setup
+    click.echo('0.6.1')  # remember to update setup
 
 
 @main.group('advanced')

geoseeq/result/file_download.py

@@ -2,15 +2,22 @@
 import urllib.request
 import logging
 import requests
-from os.path import basename, getsize, join, isfile, getmtime
+import os
+from os.path import basename, getsize, join, isfile, getmtime, dirname
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 from geoseeq.utils import download_ftp
 from geoseeq.constants import FIVE_MB
+from hashlib import md5
+from .resumable_download_tracker import ResumableDownloadTracker
 
 logger = logging.getLogger("geoseeq_api")  # Same name as calling module
 
+def url_to_id(url):
+    url = url.split("?")[0]
+    return md5(url.encode()).hexdigest()[:16]
+
 
 def _download_head(url, filename, head=None, start=0, progress_tracker=None):
     headers = None
@@ -20,11 +27,43 @@ def _download_head(url, filename, head=None, start=0, progress_tracker=None):
     response.raise_for_status()
     total_size_in_bytes = int(response.headers.get('content-length', 0))
     if progress_tracker: progress_tracker.set_num_chunks(total_size_in_bytes)
-    block_size = FIVE_MB
+    if total_size_in_bytes > 10 * FIVE_MB:  # Use resumable download
+        print("Using resumable download")
+        return _download_resumable(response, filename, total_size_in_bytes, progress_tracker)
+    else:
+        block_size = FIVE_MB
+        with open(filename, 'wb') as file:
+            for data in response.iter_content(block_size):
+                if progress_tracker: progress_tracker.update(len(data))
+                file.write(data)
+        return filename
+    
+
+def _download_resumable(response, filename, total_size_in_bytes, progress_tracker=None, chunk_size=5 * FIVE_MB, part_prefix=".gs_download_{}_{}."):
+    target_id = url_to_id(response.url)
+    tracker = ResumableDownloadTracker(chunk_size, target_id, filename)
+    if not tracker.download_started: tracker.start_download(response.url)
+    n_chunks = total_size_in_bytes // chunk_size
+    for i in range(n_chunks):
+        bytes_start, bytes_end = i * chunk_size, min((i + 1) * chunk_size - 1, total_size_in_bytes - 1)
+        if tracker.part_has_been_downloaded(i):
+            logger.debug(f"Part {i} has already been downloaded.")
+        else:
+            logger.debug(f"Downloading part {i} of {n_chunks - 1}")
+            part_filename = join(dirname(filename), part_prefix.format(i, n_chunks - 1) + basename(filename))
+            _download_head(response.url, part_filename, head=bytes_end, start=bytes_start, progress_tracker=None)
+            part_info = dict(part_number=i, start=bytes_start, end=bytes_end, part_filename=part_filename)
+            tracker.add_part(part_info)
+        if progress_tracker: progress_tracker.update(bytes_end - bytes_start + 1)
+        
+    # at this point all parts have been downloaded
     with open(filename, 'wb') as file:
-        for data in response.iter_content(block_size):
-            if progress_tracker: progress_tracker.update(len(data))
-            file.write(data)
+        for i in range(n_chunks):
+            part_info = tracker.get_part_info(i)
+            part_filename = part_info["part_filename"]
+            with open(part_filename, 'rb') as part_file:
+                file.write(part_file.read())
+    tracker.cleanup()
     return filename
 
 
@@ -44,7 +83,7 @@ def guess_download_kind(url):
         return 'generic'
 
 
-def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None):
+def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None, target_uuid=None):
     """Return a local filepath to the downloaded file. Download the file."""
     if filename and isfile(filename):
         file_size = getsize(filename)
@@ -135,7 +174,7 @@ class ResultFileDownload:
         url = self.get_download_url()
         filepath = download_url(
             url, blob_type, filename,
-            head=head, progress_tracker=progress_tracker
+            head=head, progress_tracker=progress_tracker,
         )
         if cache and flag_suffix:
             # create flag file

geoseeq/result/resumable_download_tracker.py

@@ -0,0 +1,99 @@
+
+import time
+import json
+import os
+from os.path import basename, getsize, join, dirname, isfile, getctime
+from pathlib import Path
+from random import random
+import requests
+
+from geoseeq.knex import GeoseeqGeneralError
+from geoseeq.constants import FIVE_MB
+from geoseeq.utils import md5_checksum
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from .utils import *
+from geoseeq.file_system_cache import GEOSEEQ_CACHE_DIR
+from .file_chunker import FileChunker
+
+
+
+class ResumableDownloadTracker:
+
+    def __init__(self, chunk_size, download_target_id, target_local_path, tracker_file_prefix="gs_resumable_download_tracker"):
+        self.open, self.download_started = True, False
+        self.download_target_id = download_target_id
+        self.target_local_path = target_local_path
+        self.tracker_file_dir = join(GEOSEEQ_CACHE_DIR, 'download')
+        self.tracker_file = join(
+            self.tracker_file_dir,
+            tracker_file_prefix + f".{download_target_id}.{chunk_size}." + basename(target_local_path)
+        )
+        try:
+            os.makedirs(self.tracker_file_dir, exist_ok=True)
+        except Exception as e:
+            logger.warning(f'Could not create resumable download tracker directory. {e}')
+            self.open = False
+        self._loaded_parts = {}
+        self._load_parts_from_file()
+
+    def start_download(self, download_url):
+        if not self.open:
+            return
+        if self.download_started:
+            raise GeoseeqGeneralError("Download has already started.")
+        self.download_started = True
+        blob = dict(download_url=download_url,
+                    download_target_id=self.download_target_id,
+                    start_time=time.time())
+        serialized = json.dumps(blob)
+        with open(self.tracker_file, "w") as f:
+            f.write(serialized + "\n")
+        self.download_url = download_url
+        return self
+    
+    def add_part(self, part_download_info):
+        if not self.open:
+            assert False, "Cannot add part to closed ResumableDownloadTracker"
+        part_id = part_download_info["part_number"]
+        serialized = json.dumps(part_download_info)
+        with open(self.tracker_file, "a") as f:
+            f.write(serialized + "\n")
+        self._loaded_parts[part_id] = part_download_info
+
+    def _load_parts_from_file(self):
+        if not isfile(self.tracker_file):
+            return
+        with open(self.tracker_file, "r") as f:
+            header_blob = json.loads(f.readline())
+            self.download_url = header_blob["download_url"]
+            start_time = header_blob["start_time"]  # for now we don't expire resumable downloads
+            self.download_started = True
+            for line in f:
+                part_info = json.loads(line)
+                part_id = part_info["part_number"]
+                self._loaded_parts[part_id] = part_info
+
+    def part_has_been_downloaded(self, part_number):
+        if not self.open:
+            return False
+        if part_number not in self._loaded_parts:
+            return False
+        part_info = self._loaded_parts[part_number]
+        part_path = part_info["part_filename"]
+        return isfile(part_path)
+    
+    def get_part_info(self, part_number):
+        if not self.open:
+            return None
+        return self._loaded_parts.get(part_number, None)
+    
+    def cleanup(self):
+        if not self.open:
+            return
+        for part in self._loaded_parts.values():
+            part_path = part["part_filename"]
+            if isfile(part_path):
+                os.remove(part_path)
+        os.remove(self.tracker_file)
+        self.open = False
+    

geoseeq/upload_download_manager.py

@@ -194,7 +194,7 @@ class GeoSeeqDownloadManager:
         self._convert_result_files_to_urls()
         download_args = [(
             url, file_path,
-            self.progress_tracker_factory(url),
+            self.progress_tracker_factory(file_path),
             self.ignore_errors, self.head, self.log_level,
             self.n_parallel_downloads > 1
         ) for url, file_path in self._result_files]

{geoseeq-0.6.0.dist-info → geoseeq-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.6.0
+Version: 0.6.1
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.6.0.dist-info → geoseeq-0.6.1.dist-info}/RECORD

@@ -11,7 +11,7 @@ geoseeq/project.py,sha256=-9Y2ik0-BpT3iqh89v8VQBbdadhI58oaUP9oZK8oetc,13741
 geoseeq/remote_object.py,sha256=Es-JlAz8iLRmCpAzh1MOwUh2MqtbuQM-p8wHIBAqNlQ,7131
 geoseeq/sample.py,sha256=whgEVk6GnDJJLjn5uTOqFqRtVxZD3BgjTo7brAC5noU,7981
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
-geoseeq/upload_download_manager.py,sha256=DNI4nce0MCds-wGoTA3fP_msz3kGOAoJNItoUv7L0uQ,8751
+geoseeq/upload_download_manager.py,sha256=FMRqLLg77o1qFbWZc5Yc86a2pjeZrrn1rHJr1iaxKCU,8757
 geoseeq/user.py,sha256=tol8i1UGLRrbMw5jeJDnna1ikRgrCDd50Jxz0a1lSgg,690
 geoseeq/utils.py,sha256=PDRiEQIZYTcfEV9AYvloQVvfqs5JaebcFZodAa2SUW8,3577
 geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
@@ -22,7 +22,7 @@ geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
 geoseeq/cli/download.py,sha256=N_Wrg9d1kY9eJ6C1l0xc_YFjiri8gkXBo9JiuHx9xxE,17766
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=Vze6p8cNGsMQmsr5bkhglOxWPIPqxk0BM6417iKvhb4,3791
+geoseeq/cli/main.py,sha256=omW-_xyCEL-rKyELtNu84Ofi-L4hm_17udQ4X6blr4I,3791
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/project.py,sha256=V5SdXm2Hwo2lxrkpwRDedw-mAE4XnM2uwT-Gj1D90VQ,3030
@@ -66,10 +66,11 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_chunker.py,sha256=bXq1csuRtqMB5sbH-AfWo6gdPwrivv5DJPuHVj-h08w,1758
-geoseeq/result/file_download.py,sha256=gV9-C_CMPpOWYi21eagsoiri53yzRHQx351nLBUj4WM,5790
+geoseeq/result/file_download.py,sha256=2VFy_p20VxAu1ItNNM1PBcDKSp9dhRuyOhcb5UBwYEU,7805
 geoseeq/result/file_upload.py,sha256=xs1DrI-h4ZP7xN8HPBc3SFpcPAxR5HAolraP1Zu7tvE,10648
 geoseeq/result/result_file.py,sha256=1Yj9fkZhds3J-tay6eNH2-EHi00MovHGV1M80_ckHD8,8677
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
+geoseeq/result/resumable_download_tracker.py,sha256=YEzqHBBnE7L3XokTvlTAhHZ8TcDTIE_pyTQ7YadOfbU,3667
 geoseeq/result/resumable_upload_tracker.py,sha256=2aI09gYz2yw63jEXqs8lmCRKQ79TIc3YuPETvP0Jeek,3811
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
 geoseeq/vc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -84,9 +85,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.6.0.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.6.0.dist-info/METADATA,sha256=mDqowxeSFM0nNuY_354pumCtTUpztbhhRe1Dv2rqn5g,4803
-geoseeq-0.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.6.0.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.6.0.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.6.0.dist-info/RECORD,,
+geoseeq-0.6.1.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.6.1.dist-info/METADATA,sha256=aSgyentw6vNb53rZD6IlBy-ZlPSjPRGQK-dq9QnY1no,4803
+geoseeq-0.6.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.6.1.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.6.1.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.6.1.dist-info/RECORD,,