geoseeq 0.6.0__tar.gz → 0.6.1__tar.gz

{geoseeq-0.6.0 → geoseeq-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.6.0
+Version: 0.6.1
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.6.0 → geoseeq-0.6.1}/geoseeq/cli/main.py

@@ -54,7 +54,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.6.0')  # remember to update setup
+    click.echo('0.6.1')  # remember to update setup
 
 
 @main.group('advanced')

{geoseeq-0.6.0 → geoseeq-0.6.1}/geoseeq/result/file_download.py

@@ -2,15 +2,22 @@
 import urllib.request
 import logging
 import requests
-from os.path import basename, getsize, join, isfile, getmtime
+import os
+from os.path import basename, getsize, join, isfile, getmtime, dirname
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 from geoseeq.utils import download_ftp
 from geoseeq.constants import FIVE_MB
+from hashlib import md5
+from .resumable_download_tracker import ResumableDownloadTracker
 
 logger = logging.getLogger("geoseeq_api")  # Same name as calling module
 
+def url_to_id(url):
+    url = url.split("?")[0]
+    return md5(url.encode()).hexdigest()[:16]
+
 
 def _download_head(url, filename, head=None, start=0, progress_tracker=None):
     headers = None
@@ -20,11 +27,43 @@ def _download_head(url, filename, head=None, start=0, progress_tracker=None):
     response.raise_for_status()
     total_size_in_bytes = int(response.headers.get('content-length', 0))
     if progress_tracker: progress_tracker.set_num_chunks(total_size_in_bytes)
-    block_size = FIVE_MB
+    if total_size_in_bytes > 10 * FIVE_MB:  # Use resumable download
+        print("Using resumable download")
+        return _download_resumable(response, filename, total_size_in_bytes, progress_tracker)
+    else:
+        block_size = FIVE_MB
+        with open(filename, 'wb') as file:
+            for data in response.iter_content(block_size):
+                if progress_tracker: progress_tracker.update(len(data))
+                file.write(data)
+        return filename
+    
+
+def _download_resumable(response, filename, total_size_in_bytes, progress_tracker=None, chunk_size=5 * FIVE_MB, part_prefix=".gs_download_{}_{}."):
+    target_id = url_to_id(response.url)
+    tracker = ResumableDownloadTracker(chunk_size, target_id, filename)
+    if not tracker.download_started: tracker.start_download(response.url)
+    n_chunks = total_size_in_bytes // chunk_size
+    for i in range(n_chunks):
+        bytes_start, bytes_end = i * chunk_size, min((i + 1) * chunk_size - 1, total_size_in_bytes - 1)
+        if tracker.part_has_been_downloaded(i):
+            logger.debug(f"Part {i} has already been downloaded.")
+        else:
+            logger.debug(f"Downloading part {i} of {n_chunks - 1}")
+            part_filename = join(dirname(filename), part_prefix.format(i, n_chunks - 1) + basename(filename))
+            _download_head(response.url, part_filename, head=bytes_end, start=bytes_start, progress_tracker=None)
+            part_info = dict(part_number=i, start=bytes_start, end=bytes_end, part_filename=part_filename)
+            tracker.add_part(part_info)
+        if progress_tracker: progress_tracker.update(bytes_end - bytes_start + 1)
+        
+    # at this point all parts have been downloaded
     with open(filename, 'wb') as file:
-        for data in response.iter_content(block_size):
-            if progress_tracker: progress_tracker.update(len(data))
-            file.write(data)
+        for i in range(n_chunks):
+            part_info = tracker.get_part_info(i)
+            part_filename = part_info["part_filename"]
+            with open(part_filename, 'rb') as part_file:
+                file.write(part_file.read())
+    tracker.cleanup()
     return filename
 
 
@@ -44,7 +83,7 @@ def guess_download_kind(url):
         return 'generic'
 
 
-def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None):
+def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None, target_uuid=None):
     """Return a local filepath to the downloaded file. Download the file."""
     if filename and isfile(filename):
         file_size = getsize(filename)
@@ -135,7 +174,7 @@ class ResultFileDownload:
         url = self.get_download_url()
         filepath = download_url(
             url, blob_type, filename,
-            head=head, progress_tracker=progress_tracker
+            head=head, progress_tracker=progress_tracker,
         )
         if cache and flag_suffix:
             # create flag file

geoseeq-0.6.1/geoseeq/result/resumable_download_tracker.py

@@ -0,0 +1,99 @@
+
+import time
+import json
+import os
+from os.path import basename, getsize, join, dirname, isfile, getctime
+from pathlib import Path
+from random import random
+import requests
+
+from geoseeq.knex import GeoseeqGeneralError
+from geoseeq.constants import FIVE_MB
+from geoseeq.utils import md5_checksum
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from .utils import *
+from geoseeq.file_system_cache import GEOSEEQ_CACHE_DIR
+from .file_chunker import FileChunker
+
+
+
+class ResumableDownloadTracker:
+
+    def __init__(self, chunk_size, download_target_id, target_local_path, tracker_file_prefix="gs_resumable_download_tracker"):
+        self.open, self.download_started = True, False
+        self.download_target_id = download_target_id
+        self.target_local_path = target_local_path
+        self.tracker_file_dir = join(GEOSEEQ_CACHE_DIR, 'download')
+        self.tracker_file = join(
+            self.tracker_file_dir,
+            tracker_file_prefix + f".{download_target_id}.{chunk_size}." + basename(target_local_path)
+        )
+        try:
+            os.makedirs(self.tracker_file_dir, exist_ok=True)
+        except Exception as e:
+            logger.warning(f'Could not create resumable download tracker directory. {e}')
+            self.open = False
+        self._loaded_parts = {}
+        self._load_parts_from_file()
+
+    def start_download(self, download_url):
+        if not self.open:
+            return
+        if self.download_started:
+            raise GeoseeqGeneralError("Download has already started.")
+        self.download_started = True
+        blob = dict(download_url=download_url,
+                    download_target_id=self.download_target_id,
+                    start_time=time.time())
+        serialized = json.dumps(blob)
+        with open(self.tracker_file, "w") as f:
+            f.write(serialized + "\n")
+        self.download_url = download_url
+        return self
+    
+    def add_part(self, part_download_info):
+        if not self.open:
+            assert False, "Cannot add part to closed ResumableDownloadTracker"
+        part_id = part_download_info["part_number"]
+        serialized = json.dumps(part_download_info)
+        with open(self.tracker_file, "a") as f:
+            f.write(serialized + "\n")
+        self._loaded_parts[part_id] = part_download_info
+
+    def _load_parts_from_file(self):
+        if not isfile(self.tracker_file):
+            return
+        with open(self.tracker_file, "r") as f:
+            header_blob = json.loads(f.readline())
+            self.download_url = header_blob["download_url"]
+            start_time = header_blob["start_time"]  # for now we don't expire resumable downloads
+            self.download_started = True
+            for line in f:
+                part_info = json.loads(line)
+                part_id = part_info["part_number"]
+                self._loaded_parts[part_id] = part_info
+
+    def part_has_been_downloaded(self, part_number):
+        if not self.open:
+            return False
+        if part_number not in self._loaded_parts:
+            return False
+        part_info = self._loaded_parts[part_number]
+        part_path = part_info["part_filename"]
+        return isfile(part_path)
+    
+    def get_part_info(self, part_number):
+        if not self.open:
+            return None
+        return self._loaded_parts.get(part_number, None)
+    
+    def cleanup(self):
+        if not self.open:
+            return
+        for part in self._loaded_parts.values():
+            part_path = part["part_filename"]
+            if isfile(part_path):
+                os.remove(part_path)
+        os.remove(self.tracker_file)
+        self.open = False
+    

{geoseeq-0.6.0 → geoseeq-0.6.1}/geoseeq/upload_download_manager.py

@@ -194,7 +194,7 @@ class GeoSeeqDownloadManager:
         self._convert_result_files_to_urls()
         download_args = [(
             url, file_path,
-            self.progress_tracker_factory(url),
+            self.progress_tracker_factory(file_path),
             self.ignore_errors, self.head, self.log_level,
             self.n_parallel_downloads > 1
         ) for url, file_path in self._result_files]

{geoseeq-0.6.0 → geoseeq-0.6.1}/geoseeq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.6.0
+Version: 0.6.1
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.6.0 → geoseeq-0.6.1}/geoseeq.egg-info/SOURCES.txt

@@ -79,6 +79,7 @@ geoseeq/result/file_download.py
 geoseeq/result/file_upload.py
 geoseeq/result/result_file.py
 geoseeq/result/result_folder.py
+geoseeq/result/resumable_download_tracker.py
 geoseeq/result/resumable_upload_tracker.py
 geoseeq/result/utils.py
 geoseeq/vc/__init__.py

{geoseeq-0.6.0 → geoseeq-0.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "geoseeq"
-version = "0.6.0"
+version = "0.6.1"
 authors = [
   { name="David C. Danko", email="dcdanko@biotia.io" },
 ]