geoseeq 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

geoseeq/cli/main.py

@@ -54,7 +54,7 @@ def version():
     Use of this tool implies acceptance of the GeoSeeq End User License Agreement.
     Run `geoseeq eula show` to view the EULA.
     """
-    click.echo('0.6.0')  # remember to update setup
+    click.echo('0.6.2')  # remember to update setup
 
 
 @main.group('advanced')

geoseeq/file_system/filesystem_download.py

@@ -0,0 +1,434 @@
+
+import os
+import json
+from geoseeq import (
+    result_file_from_id,
+    result_folder_from_id,
+    sample_from_id,
+    project_from_id,
+)
+from geoseeq.utils import md5_checksum
+from time import time
+
+FILE_STATUS_MODIFIED_REMOTE = 'MODIFIED_REMOTE'
+FILE_STATUS_MODIFIED_LOCAL = 'MODIFIED_LOCAL'
+FILE_STATUS_NEW_LOCAL = 'NEW_LOCAL'
+FILE_STATUS_NEW_REMOTE = 'NEW_REMOTE'
+FILE_STATUS_IS_LOCAL_STUB = 'IS_LOCAL_STUB'
+
+
+def dedupe_modified_files(modified_files):
+    """Remove duplicates from a list of modified files.
+    
+    This function will remove duplicates from a list of modified files
+    based on the path to the file. The first instance of the file will be
+    kept and all others will be removed.
+    """
+    seen = set()
+    deduped = []
+    for x in modified_files:
+        if x[2] not in seen:
+            deduped.append(x)
+            seen.add(x[2])
+    return deduped
+
+
+class ResultFileOnFilesystem:
+    """
+
+    Note: unlike other filesystem classes the `path` is a file, not
+    a directory. This is because the file is downloaded directly to 
+    the path.
+    """
+
+    def __init__(self, result_file, path, kind):
+        self.result_file = result_file
+        self.path = path
+        self.kind = kind
+
+    @property
+    def info_filepath(self):
+        dirpath = os.path.dirname(self.path)
+        basename = os.path.basename(self.path)
+        return os.path.join(dirpath, f'.gs_result_file__{basename}')
+    
+    @property
+    def is_stub(self):
+        return os.path.exists(self.path) and os.path.getsize(self.path) == 0
+    
+    def file_is_ok(self, stubs_are_ok=False):
+        if self.is_stub:
+            return stubs_are_ok
+        return self.result_file.download_needs_update(self.path)
+
+    def download(self, use_stubs=False, exists_ok=False):
+        if os.path.exists(self.info_filepath):
+            if exists_ok and self.file_is_ok(stubs_are_ok=use_stubs):
+                return
+            elif not exists_ok:
+                raise ValueError('Result file already exists at path: {}'.format(self.info_filepath))
+        
+        # Download the file
+        if use_stubs:
+            open(self.path, 'w').close()
+        else:
+            self.result_file.download(self.path)
+
+        self.write_info_file()
+
+    def local_file_checksum(self):
+        if self.is_stub:
+            return "__STUB__"
+        return md5_checksum(self.path)
+    
+    def locally_modified(self):
+        raise NotImplementedError('This function is not implemented')
+
+    def status_is_ok(self, stubs_are_ok=False):
+        # check for an info file
+        if not os.path.exists(self.info_filepath):
+            return False
+        if stubs_are_ok:
+            return True
+        return not self.result_file.download_needs_update(self.path)
+    
+    def write_info_file(self):
+        result_file_info = {
+            "uuid": self.result_file.uuid,
+            "kind": self.kind,
+            "checksum": self.local_file_checksum(),
+        }
+        with open(self.info_filepath, 'w') as f:
+            json.dump(result_file_info, f)
+
+    @classmethod
+    def from_path(cls, path):
+        obj = cls(None, path, None)
+        try:
+            with open(obj.info_filepath, 'r') as f:
+                result_file_info = json.load(f)
+            obj.result_file = result_file_from_id(result_file_info['uuid'])
+            obj.kind = result_file_info['kind']
+            obj.stored_checksum = result_file_info['checksum']
+        except FileNotFoundError:
+            pass
+        return obj
+    
+    def write_info_file(self):
+        result_file_info = {
+            "uuid": self.result_file.uuid,
+            "kind": self.kind,
+            "checksum": self.local_file_checksum(),
+        }
+        with open(self.info_filepath, 'w') as f:
+            json.dump(result_file_info, f)
+    
+    def list_abnormal_objects(self):
+        """Return a list of files that have been modified.
+        
+        Since this class is a single file the list will either be empty
+        or have one element.
+
+        Note that if a file was modified locally then uploaded to the server
+        the file will be marked as modified remote.
+        """
+        if self.result_file is None:
+            return [('FILE', FILE_STATUS_NEW_LOCAL, self.path, None)]
+        if not os.path.exists(self.path):
+            return [('FILE', FILE_STATUS_NEW_REMOTE, self.path, self.result_file)]
+        if self.is_stub:
+            return [('FILE', FILE_STATUS_IS_LOCAL_STUB, self.path, self.result_file)]
+        if self.result_file and self.result_file.download_needs_update(self.path):
+            return [('FILE', FILE_STATUS_MODIFIED_REMOTE, self.path, self.result_file)]
+        if self.locally_modified():
+            return [('FILE', FILE_STATUS_MODIFIED_LOCAL, self.path, self.result_file)]
+
+        return []
+
+    
+class ResultFolderOnFilesystem:
+
+    def __init__(self, result_folder, path, kind):
+        self.result_folder = result_folder
+        self.path = path
+        self.kind = kind
+
+    @property
+    def info_filepath(self):
+        return os.path.join(self.path, '.gs_result_folder')
+
+    def download(self, use_stubs=False, exists_ok=False):
+        if os.path.exists(self.info_filepath) and not exists_ok:
+            raise ValueError('Result folder already exists at path: {}'.format(self.info_filepath))
+        
+        # Download the files in the result folder
+        for result_file in self.result_folder.get_fields():
+            result_file_local_path = os.path.join(self.path, result_file.name)
+            os.makedirs(os.path.dirname(result_file_local_path), exist_ok=True)
+            ResultFileOnFilesystem(result_file, result_file_local_path, self.kind)\
+                .download(use_stubs=use_stubs, exists_ok=exists_ok)
+        
+        # Write the result folder data
+        result_folder_info = {
+            "uuid": self.result_folder.uuid,
+            "kind": self.kind
+        }
+        with open(self.info_filepath, 'w') as f:
+            json.dump(result_folder_info, f)
+
+    def status_is_ok(self):
+        # check for an info file
+        if not os.path.exists(self.info_filepath):
+            return False
+        
+        # check that all files are downloaded
+        for result_file in self.result_folder.get_files():
+            result_file_path = os.path.join(self.path, result_file.name)
+            if not os.path.exists(result_file_path):
+                return False
+        
+        return True
+    
+    @classmethod
+    def from_path(cls, path):
+        obj = cls(None, path, None)
+        try:
+            with open(os.path.join(path, '.gs_result_folder'), 'r') as f:
+                result_folder_info = json.load(f)
+            obj.result_folder = result_folder_from_id(result_folder_info['uuid'])
+            obj.kind = result_folder_info['kind']
+        except FileNotFoundError:
+            pass
+        return obj
+    
+    def list_abnormal_objects(self):
+        """Return a list of files that have been modified.
+        
+        This function will return a list of tuples where the first element
+        is the status of the file and the second element is the path to the file.
+        """
+        modified_files = []
+        if not self.result_folder:
+            modified_files.append(('FOLDER', FILE_STATUS_NEW_LOCAL, self.path, None))
+        if not os.path.exists(self.path):
+            modified_files.append(('FOLDER', FILE_STATUS_NEW_REMOTE, self.path, self.result_folder))
+
+        # list local files
+        if os.path.exists(self.path):
+            for local_file in os.listdir(self.path):
+                if local_file.startswith('.gs_'):
+                    continue
+                local_file_path = os.path.join(self.path, local_file)
+                result_file_on_fs = ResultFileOnFilesystem.from_path(local_file_path)
+                modified_files.extend(result_file_on_fs.list_abnormal_objects())
+
+        # list remote files
+        if self.result_folder:
+            for result_file in self.result_folder.get_fields():
+                result_file_path = os.path.join(self.path, result_file.name)
+                result_file_on_fs = ResultFileOnFilesystem(result_file, result_file_path, self.kind)
+                modified_files.extend(result_file_on_fs.list_abnormal_objects())
+    
+        return dedupe_modified_files(modified_files)
+    
+
+class SampleOnFilesystem:
+
+    def __init__(self, sample, path):
+        self.sample = sample
+        self.path = path if path[-1] != '/' else path[:-1]  # remove trailing slash
+
+    @property
+    def info_filepath(self):
+        return os.path.join(self.path, '.gs_sample')
+
+    def download(self, use_stubs=False, exists_ok=False):
+        if os.path.exists(self.info_filepath) and not exists_ok:
+            raise ValueError('Sample already exists at path: {}'.format(self.info_filepath))
+        
+        # download result folders
+        for result_folder in self.sample.get_result_folders():
+            result_folder_local_path = os.path.join(self.path, result_folder.name)
+            os.makedirs(result_folder_local_path, exist_ok=True)
+            ResultFolderOnFilesystem(result_folder, result_folder_local_path, "sample")\
+                .download(use_stubs=use_stubs, exists_ok=exists_ok)
+
+        # Write the sample data
+        sample_info = {
+            "uuid": self.sample.uuid
+        }
+        with open(self.info_filepath, 'w') as f:
+            json.dump(sample_info, f)
+
+    def status_is_ok(self):
+        # check for an info file
+        if not os.path.exists(self.info_filepath):
+            return False
+        
+        # check that all result folders are downloaded
+        for result_folder in self.sample.get_result_folders():
+            result_folder_local_path = os.path.join(self.path, result_folder.name)
+            result_folder_on_fs = ResultFolderOnFilesystem.from_path(result_folder_local_path, "sample")
+            if not result_folder_on_fs.status_is_ok():
+                return False
+        
+        return True
+
+    @classmethod
+    def from_path(cls, path):
+        obj = cls(None, path)
+        try:
+            with open(os.path.join(path, '.gs_sample'), 'r') as f:
+                sample_info = json.load(f)
+            obj.sample = sample_from_id(sample_info['uuid'])
+        except FileNotFoundError:
+            pass
+        return obj
+    
+    def list_abnormal_objects(self):
+        """Return a list of files that have been modified.
+        
+        This function will return a list of tuples where the first element
+        is the status of the file and the second element is the path to the file.
+        """
+        modified_files = []
+        if not self.sample:
+            modified_files.append(('SAMPLE', FILE_STATUS_NEW_LOCAL, self.path, None))
+        if not os.path.exists(self.path):
+            modified_files.append(('SAMPLE', FILE_STATUS_NEW_REMOTE, self.path, self.sample))
+
+        # list local folders
+        if os.path.exists(self.path):
+            for local_folder in os.listdir(self.path):
+                local_folder_path = os.path.join(self.path, local_folder)
+                if not os.path.isdir(local_folder_path):
+                    continue
+                result_folder_on_fs = ResultFolderOnFilesystem.from_path(local_folder_path)
+                modified_files.extend(result_folder_on_fs.list_abnormal_objects())
+
+        # list remote folders
+        if self.sample:
+            for result_folder in self.sample.get_result_folders():
+                result_folder_path = os.path.join(self.path, result_folder.name)
+                result_folder_on_fs = ResultFolderOnFilesystem(result_folder, result_folder_path, "sample")
+                modified_files.extend(result_folder_on_fs.list_abnormal_objects())
+
+        return dedupe_modified_files(modified_files)
+
+
+class ProjectOnFilesystem:
+
+    def __init__(self, project, path):
+        self.project = project
+        self.path = path
+
+    @property
+    def info_filepath(self):
+        return os.path.join(self.path, '.gs_project')
+
+    def download(self, use_stubs=False, exists_ok=False):
+        if os.path.exists(self.info_filepath) and not exists_ok:
+            raise ValueError('Project already exists at path: {}'.format(self.info_filepath))
+        
+        # download samples
+        for sample in self.project.get_samples():
+            sample_local_path = os.path.join(self.path, "sample_results", sample.name)
+            os.makedirs(sample_local_path, exist_ok=True)
+            SampleOnFilesystem(sample, sample_local_path)\
+                .download(use_stubs=use_stubs, exists_ok=exists_ok)
+        
+        # download project result folders
+        for result_folder in self.project.get_result_folders():
+            result_folder_local_path = os.path.join(self.path, "project_results", result_folder.name)
+            os.makedirs(result_folder_local_path, exist_ok=True)
+            ResultFolderOnFilesystem(result_folder, result_folder_local_path, "project")\
+                .download(use_stubs=use_stubs, exists_ok=exists_ok)
+
+        # Write the project data
+        project_info = {
+            "uuid": self.project.uuid
+        }
+        with open(self.info_filepath, 'w') as f:
+            json.dump(project_info, f)
+
+    def status_is_ok(self):
+        # check for an info file
+        if not os.path.exists(self.info_filepath):
+            return False
+        
+        # check that all samples are downloaded
+        for sample in self.project.get_samples():
+            sample_local_path = os.path.join(self.path, "sample_results", sample.name)
+            sample_on_fs = SampleOnFilesystem.from_path(sample_local_path)
+            if not sample_on_fs.status_is_ok():
+                return False
+        
+        # check that all project result folders are downloaded
+        for result_folder in self.project.get_result_folders():
+            result_folder_local_path = os.path.join(self.path, "project_results", result_folder.name)
+            result_folder_on_fs = ResultFolderOnFilesystem.from_path(result_folder_local_path, "project")
+            if not result_folder_on_fs.status_is_ok():
+                return False
+        
+        return True
+
+    @classmethod
+    def from_path(cls, path, recursive=False):
+        try:
+            with open(os.path.join(path, '.gs_project'), 'r') as f:
+                project_info = json.load(f)
+            project = project_from_id(project_info['uuid'])
+            return cls(project, path)
+        except FileNotFoundError:
+            if not recursive:
+                raise ValueError('No project found in path or parent directories')
+            updir = os.path.dirname(os.path.abspath(path))
+            if updir == path:
+                raise ValueError('No project found in path or parent directories')
+            return cls.from_path(updir, recursive=recursive)
+        
+    def path_from_project_root(self, path):
+        if path[0] == "/":
+            return path.replace(self.path, "")[1:]
+        return path
+
+    def list_abnormal_objects(self):
+        """Return a list of files that have been modified.
+        
+        This function will return a list of tuples where the first element
+        is the status of the file and the second element is the path to the file.
+        """
+        modified_files = []
+
+        # list remote samples
+        for sample in self.project.get_samples():
+            sample_path = os.path.join(self.path, "sample_results", sample.name)
+            sample_on_fs = SampleOnFilesystem(sample, sample_path)
+            modified_files.extend(sample_on_fs.list_abnormal_objects())
+
+        # list remote project result folders
+        for result_folder in self.project.get_result_folders():
+            result_folder_path = os.path.join(self.path, "project_results", result_folder.name)
+
+            result_folder_on_fs = ResultFolderOnFilesystem(result_folder, result_folder_path, "project")
+            modified_files.extend(result_folder_on_fs.list_abnormal_objects())
+
+        # list local samples
+        for local_sample in os.listdir(os.path.join(self.path, "sample_results")):
+            local_sample_path = os.path.join(self.path, "sample_results", local_sample)
+            if not os.path.isdir(local_sample_path):
+                continue
+            sample_on_fs = SampleOnFilesystem.from_path(local_sample_path)
+            modified_files.extend(sample_on_fs.list_abnormal_objects())
+
+        # list local project result folders
+        for local_result_folder in os.listdir(os.path.join(self.path, "project_results")):
+            local_result_folder_path = os.path.join(self.path, "project_results", local_result_folder)
+            if not os.path.isdir(local_result_folder_path):
+                continue
+            result_folder_on_fs = ResultFolderOnFilesystem.from_path(local_result_folder_path)
+            modified_files.extend(result_folder_on_fs.list_abnormal_objects())
+        return dedupe_modified_files(modified_files)
+                
+
+

geoseeq/file_system/main.py

@@ -0,0 +1,122 @@
+from fuse import FUSE, Operations
+import os
+
+
+class GeoSeeqProjectFileSystem(Operations):
+    """Mount a GeoSeeq project as a filesystem.
+    
+    The project will automatically have this directory structure:
+    - <root>/project_results/<project_result_folder_name>/...
+    - <root>/sample_results/<sample_name>/...
+    - <root>/metadata/sample_metadata.csv
+    - <root>/.config/config.json
+    """
+
+    def __init__(self, root, project):
+        self.root = root
+        self.project = project
+
+    def access(self, path, mode):
+        pass
+
+    def chmod(self, path, mode):
+        pass
+
+    def chown(self, path, uid, gid):
+        pass
+
+    def getattr(self, path, fh=None):
+        pass
+
+    def readdir(self, path, fh):
+        pass
+
+    def readlink(self, path):
+        pass
+
+    def mknod(self, path, mode, dev):
+        pass
+
+    def rmdir(self, path):
+        pass
+
+    def mkdir(self, path, mode):
+        pass
+
+    def statfs(self, path):
+        pass
+
+    def unlink(self, path):
+        pass
+
+    def symlink(self, name, target):
+        pass
+
+    def rename(self, old, new):
+        pass
+
+    def link(self, target, name):
+        pass
+
+    def utimens(self, path, times=None):
+        pass
+
+    def open(self, path, flags):
+        tkns = path.split('/')
+        if tkns[0] == 'project_results':
+            result_folder_name, result_file_name = tkns[2], '/'.join(tkns[3:])
+            result_folder = self.project.get_result_folder(result_folder_name).get()
+            result_file = result_folder.get_file(result_file_name).get()
+            result_file.download(path)
+        elif tkns[0] == 'sample_results':
+            sample_name, result_folder_name, result_file_name = tkns[2], tkns[3], '/'.join(tkns[4:])
+            sample = self.project.get_sample(sample_name).get()
+            result_folder = sample.get_result_folder(result_folder_name).get()
+            result_file = result_folder.get_file(result_file_name).get()
+            result_file.download(path)
+        elif tkns[0] == 'metadata':
+            raise NotImplementedError('TODO')
+        
+        return os.open(self._full_local_path(path), flags)
+
+    def create(self, path, mode, fi=None):
+        tkns = path.split('/')
+        if tkns[0] == 'project_results':
+            result_name, file_name = tkns[2], '/'.join(tkns[3:])
+            result_folder = self.project.get_result_folder(result_name).idem()
+            result_file = result_folder.get_file(file_name).create()
+            result_file.download(path)  # nothing to download at this point
+        elif tkns[0] == 'sample_results':
+            sample_name, result_folder_name, result_file_name = tkns[2], tkns[3], '/'.join(tkns[4:])
+            sample = self.project.get_sample(sample_name).idem()
+            result_folder = sample.get_result_folder(result_folder_name).idem()
+            result_file = result_folder.get_file(result_file_name).create()
+            result_file.download(path)  # nothing to download at this point
+        elif tkns[0] == 'metadata':
+            raise NotImplementedError('TODO')
+
+    def read(self, path, length, offset, fh):
+        os.lseek(fh, offset, os.SEEK_SET)
+        return os.read(fh, length)
+
+    def write(self, path, buf, offset, fh):
+        pass
+
+    def truncate(self, path, length, fh=None):
+        pass
+
+    def flush(self, path, fh):
+        pass
+
+    def release(self, path, fh):
+        pass
+
+    def fsync(self, path, fdatasync, fh):
+        pass
+    
+    def _full_local_path(self, partial):
+        if partial.startswith("/"):
+            partial = partial[1:]
+        return os.path.join(self.root, partial)
+
+    

geoseeq/result/file_download.py

@@ -2,15 +2,22 @@
 import urllib.request
 import logging
 import requests
-from os.path import basename, getsize, join, isfile, getmtime
+import os
+from os.path import basename, getsize, join, isfile, getmtime, dirname
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 from geoseeq.utils import download_ftp
 from geoseeq.constants import FIVE_MB
+from hashlib import md5
+from .resumable_download_tracker import ResumableDownloadTracker
 
 logger = logging.getLogger("geoseeq_api")  # Same name as calling module
 
+def url_to_id(url):
+    url = url.split("?")[0]
+    return md5(url.encode()).hexdigest()[:16]
+
 
 def _download_head(url, filename, head=None, start=0, progress_tracker=None):
     headers = None
@@ -20,11 +27,43 @@ def _download_head(url, filename, head=None, start=0, progress_tracker=None):
     response.raise_for_status()
     total_size_in_bytes = int(response.headers.get('content-length', 0))
     if progress_tracker: progress_tracker.set_num_chunks(total_size_in_bytes)
-    block_size = FIVE_MB
+    if total_size_in_bytes > 10 * FIVE_MB:  # Use resumable download
+        print("Using resumable download")
+        return _download_resumable(response, filename, total_size_in_bytes, progress_tracker)
+    else:
+        block_size = FIVE_MB
+        with open(filename, 'wb') as file:
+            for data in response.iter_content(block_size):
+                if progress_tracker: progress_tracker.update(len(data))
+                file.write(data)
+        return filename
+    
+
+def _download_resumable(response, filename, total_size_in_bytes, progress_tracker=None, chunk_size=5 * FIVE_MB, part_prefix=".gs_download_{}_{}."):
+    target_id = url_to_id(response.url)
+    tracker = ResumableDownloadTracker(chunk_size, target_id, filename)
+    if not tracker.download_started: tracker.start_download(response.url)
+    n_chunks = total_size_in_bytes // chunk_size
+    for i in range(n_chunks):
+        bytes_start, bytes_end = i * chunk_size, min((i + 1) * chunk_size - 1, total_size_in_bytes - 1)
+        if tracker.part_has_been_downloaded(i):
+            logger.debug(f"Part {i} has already been downloaded.")
+        else:
+            logger.debug(f"Downloading part {i} of {n_chunks - 1}")
+            part_filename = join(dirname(filename), part_prefix.format(i, n_chunks - 1) + basename(filename))
+            _download_head(response.url, part_filename, head=bytes_end, start=bytes_start, progress_tracker=None)
+            part_info = dict(part_number=i, start=bytes_start, end=bytes_end, part_filename=part_filename)
+            tracker.add_part(part_info)
+        if progress_tracker: progress_tracker.update(bytes_end - bytes_start + 1)
+        
+    # at this point all parts have been downloaded
     with open(filename, 'wb') as file:
-        for data in response.iter_content(block_size):
-            if progress_tracker: progress_tracker.update(len(data))
-            file.write(data)
+        for i in range(n_chunks):
+            part_info = tracker.get_part_info(i)
+            part_filename = part_info["part_filename"]
+            with open(part_filename, 'rb') as part_file:
+                file.write(part_file.read())
+    tracker.cleanup()
     return filename
 
 
@@ -44,7 +83,7 @@ def guess_download_kind(url):
         return 'generic'
 
 
-def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None):
+def download_url(url, kind='guess', filename=None, head=None, progress_tracker=None, target_uuid=None):
     """Return a local filepath to the downloaded file. Download the file."""
     if filename and isfile(filename):
         file_size = getsize(filename)
@@ -135,7 +174,7 @@ class ResultFileDownload:
         url = self.get_download_url()
         filepath = download_url(
             url, blob_type, filename,
-            head=head, progress_tracker=progress_tracker
+            head=head, progress_tracker=progress_tracker,
         )
         if cache and flag_suffix:
             # create flag file

geoseeq/result/resumable_download_tracker.py

@@ -0,0 +1,99 @@
+
+import time
+import json
+import os
+from os.path import basename, getsize, join, dirname, isfile, getctime
+from pathlib import Path
+from random import random
+import requests
+
+from geoseeq.knex import GeoseeqGeneralError
+from geoseeq.constants import FIVE_MB
+from geoseeq.utils import md5_checksum
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from .utils import *
+from geoseeq.file_system_cache import GEOSEEQ_CACHE_DIR
+from .file_chunker import FileChunker
+
+
+
+class ResumableDownloadTracker:
+
+    def __init__(self, chunk_size, download_target_id, target_local_path, tracker_file_prefix="gs_resumable_download_tracker"):
+        self.open, self.download_started = True, False
+        self.download_target_id = download_target_id
+        self.target_local_path = target_local_path
+        self.tracker_file_dir = join(GEOSEEQ_CACHE_DIR, 'download')
+        self.tracker_file = join(
+            self.tracker_file_dir,
+            tracker_file_prefix + f".{download_target_id}.{chunk_size}." + basename(target_local_path)
+        )
+        try:
+            os.makedirs(self.tracker_file_dir, exist_ok=True)
+        except Exception as e:
+            logger.warning(f'Could not create resumable download tracker directory. {e}')
+            self.open = False
+        self._loaded_parts = {}
+        self._load_parts_from_file()
+
+    def start_download(self, download_url):
+        if not self.open:
+            return
+        if self.download_started:
+            raise GeoseeqGeneralError("Download has already started.")
+        self.download_started = True
+        blob = dict(download_url=download_url,
+                    download_target_id=self.download_target_id,
+                    start_time=time.time())
+        serialized = json.dumps(blob)
+        with open(self.tracker_file, "w") as f:
+            f.write(serialized + "\n")
+        self.download_url = download_url
+        return self
+    
+    def add_part(self, part_download_info):
+        if not self.open:
+            assert False, "Cannot add part to closed ResumableDownloadTracker"
+        part_id = part_download_info["part_number"]
+        serialized = json.dumps(part_download_info)
+        with open(self.tracker_file, "a") as f:
+            f.write(serialized + "\n")
+        self._loaded_parts[part_id] = part_download_info
+
+    def _load_parts_from_file(self):
+        if not isfile(self.tracker_file):
+            return
+        with open(self.tracker_file, "r") as f:
+            header_blob = json.loads(f.readline())
+            self.download_url = header_blob["download_url"]
+            start_time = header_blob["start_time"]  # for now we don't expire resumable downloads
+            self.download_started = True
+            for line in f:
+                part_info = json.loads(line)
+                part_id = part_info["part_number"]
+                self._loaded_parts[part_id] = part_info
+
+    def part_has_been_downloaded(self, part_number):
+        if not self.open:
+            return False
+        if part_number not in self._loaded_parts:
+            return False
+        part_info = self._loaded_parts[part_number]
+        part_path = part_info["part_filename"]
+        return isfile(part_path)
+    
+    def get_part_info(self, part_number):
+        if not self.open:
+            return None
+        return self._loaded_parts.get(part_number, None)
+    
+    def cleanup(self):
+        if not self.open:
+            return
+        for part in self._loaded_parts.values():
+            part_path = part["part_filename"]
+            if isfile(part_path):
+                os.remove(part_path)
+        os.remove(self.tracker_file)
+        self.open = False
+    

geoseeq/upload_download_manager.py

@@ -194,7 +194,7 @@ class GeoSeeqDownloadManager:
         self._convert_result_files_to_urls()
         download_args = [(
             url, file_path,
-            self.progress_tracker_factory(url),
+            self.progress_tracker_factory(file_path),
             self.ignore_errors, self.head, self.log_level,
             self.n_parallel_downloads > 1
         ) for url, file_path in self._result_files]

{geoseeq-0.6.0.dist-info → geoseeq-0.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geoseeq
-Version: 0.6.0
+Version: 0.6.2
 Summary: GeoSeeq command line tools and python API
 Author: David C. Danko
 Author-email: "David C. Danko" <dcdanko@biotia.io>

{geoseeq-0.6.0.dist-info → geoseeq-0.6.2.dist-info}/RECORD

@@ -11,7 +11,7 @@ geoseeq/project.py,sha256=-9Y2ik0-BpT3iqh89v8VQBbdadhI58oaUP9oZK8oetc,13741
 geoseeq/remote_object.py,sha256=Es-JlAz8iLRmCpAzh1MOwUh2MqtbuQM-p8wHIBAqNlQ,7131
 geoseeq/sample.py,sha256=whgEVk6GnDJJLjn5uTOqFqRtVxZD3BgjTo7brAC5noU,7981
 geoseeq/search.py,sha256=gawad6Cx5FxJBPlYkXWb-UKAO-UC0_yhvyU9Ca1kaNI,3388
-geoseeq/upload_download_manager.py,sha256=DNI4nce0MCds-wGoTA3fP_msz3kGOAoJNItoUv7L0uQ,8751
+geoseeq/upload_download_manager.py,sha256=FMRqLLg77o1qFbWZc5Yc86a2pjeZrrn1rHJr1iaxKCU,8757
 geoseeq/user.py,sha256=tol8i1UGLRrbMw5jeJDnna1ikRgrCDd50Jxz0a1lSgg,690
 geoseeq/utils.py,sha256=PDRiEQIZYTcfEV9AYvloQVvfqs5JaebcFZodAa2SUW8,3577
 geoseeq/work_orders.py,sha256=5uLVVfdKE8qh4gGaHkdBpXJGRTujuSg59knWCqEET4A,8071
@@ -22,7 +22,7 @@ geoseeq/cli/detail.py,sha256=q8Suu-j2k18knfSVFG-SWWGNsKM-n8y9RMA3LcIIi9Y,4132
 geoseeq/cli/download.py,sha256=N_Wrg9d1kY9eJ6C1l0xc_YFjiri8gkXBo9JiuHx9xxE,17766
 geoseeq/cli/fastq_utils.py,sha256=-bmeQLaiMBm57zWOF0R5OlWTU0_3sh1JBC1RYw2BOFM,3083
 geoseeq/cli/get_eula.py,sha256=79mbUwyiF7O1r0g6UTxG9kJGQEqKuH805E6eLkPC6Y4,997
-geoseeq/cli/main.py,sha256=Vze6p8cNGsMQmsr5bkhglOxWPIPqxk0BM6417iKvhb4,3791
+geoseeq/cli/main.py,sha256=y6OK6ryYf7TyMtufl0kGESro5Fy5Hu7_xzIc3aYDKCo,3791
 geoseeq/cli/manage.py,sha256=wGXAcVaXqE5JQEU8Jh6OlHr02nB396bpS_SFcOZdrEo,5929
 geoseeq/cli/progress_bar.py,sha256=p1Xl01nkYxSBZCB30ue2verIIi22W93m3ZAMAxipD0g,738
 geoseeq/cli/project.py,sha256=V5SdXm2Hwo2lxrkpwRDedw-mAE4XnM2uwT-Gj1D90VQ,3030
@@ -48,6 +48,9 @@ geoseeq/contrib/ncbi/api.py,sha256=WQeLoGA_-Zha-QeSO8_i7HpvXyD8UkV0qc5okm11KiA,1
 geoseeq/contrib/ncbi/bioproject.py,sha256=_oThTd_iLDOC8cLOlJKAatSr362OBYZCEV3YrqodhFg,4341
 geoseeq/contrib/ncbi/cli.py,sha256=j9zEcaZPTryK3a4xluRxigcJKDhRpRxbp3KZSx-Bfhk,2400
 geoseeq/contrib/ncbi/setup_logging.py,sha256=Tp1bY1U0f-o739aHpvVYriG2qdd1lFvCYBXZeXQgt-w,175
+geoseeq/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+geoseeq/file_system/filesystem_download.py,sha256=8bcnxjWltekmCvb5N0b1guBIjLp4-CL2VtsEok-snv4,16963
+geoseeq/file_system/main.py,sha256=4HgYGq7WhlF96JlVIf16iFBTDujlBpxImmtoh4VCzDA,3627
 geoseeq/id_constructors/__init__.py,sha256=w5E0PNQ9UuAxBeZbDI7KBnUoERd85gGz3nScz45bd2o,126
 geoseeq/id_constructors/from_blobs.py,sha256=aj7M7NRpKGs3u3xUvuFJwmJdFeIcJPmaI2_bhwbFfEs,5702
 geoseeq/id_constructors/from_ids.py,sha256=bbAJX4LnuN70v9bny6N-jAwOudb2-ztHvlMBgRuSDz0,3151
@@ -66,10 +69,11 @@ geoseeq/plotting/map/overlay.py,sha256=4VmxqOESTQra9tPr8b8OLEUhJSit9lNipabeSznEY
 geoseeq/result/__init__.py,sha256=IFHIyRV8ZzuKIfwfze1SXgcKwNMcSgMAknLHMkwjXIU,356
 geoseeq/result/bioinfo.py,sha256=QQtbyogrdro9avJSN0713sxLVnVeA24mFw3hWtKDKyw,1782
 geoseeq/result/file_chunker.py,sha256=bXq1csuRtqMB5sbH-AfWo6gdPwrivv5DJPuHVj-h08w,1758
-geoseeq/result/file_download.py,sha256=gV9-C_CMPpOWYi21eagsoiri53yzRHQx351nLBUj4WM,5790
+geoseeq/result/file_download.py,sha256=2VFy_p20VxAu1ItNNM1PBcDKSp9dhRuyOhcb5UBwYEU,7805
 geoseeq/result/file_upload.py,sha256=xs1DrI-h4ZP7xN8HPBc3SFpcPAxR5HAolraP1Zu7tvE,10648
 geoseeq/result/result_file.py,sha256=1Yj9fkZhds3J-tay6eNH2-EHi00MovHGV1M80_ckHD8,8677
 geoseeq/result/result_folder.py,sha256=6porOXPh7Tpxw3oX5yMRPYQzNCGYqszqmFJd3SwQmTc,11122
+geoseeq/result/resumable_download_tracker.py,sha256=YEzqHBBnE7L3XokTvlTAhHZ8TcDTIE_pyTQ7YadOfbU,3667
 geoseeq/result/resumable_upload_tracker.py,sha256=2aI09gYz2yw63jEXqs8lmCRKQ79TIc3YuPETvP0Jeek,3811
 geoseeq/result/utils.py,sha256=C-CxGzB3WddlnRiqFSkrY78I_m0yFgNqsTBRzGU-y8Q,2772
 geoseeq/vc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -84,9 +88,9 @@ geoseeq/vc/vc_stub.py,sha256=IQr8dI0zsWKVAeY_5ybDD6n49_3othcgfHS3P0O9tuY,3110
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_api_client.py,sha256=TS5njc5pcPP_Ycy-ljcfPVT1hQRBsFVdQ0lCqBmoesU,12810
 tests/test_plotting.py,sha256=TcTu-2ARr8sxZJ7wPQxmbs3-gHw7uRvsgrhhhg0qKik,784
-geoseeq-0.6.0.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
-geoseeq-0.6.0.dist-info/METADATA,sha256=mDqowxeSFM0nNuY_354pumCtTUpztbhhRe1Dv2rqn5g,4803
-geoseeq-0.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-geoseeq-0.6.0.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
-geoseeq-0.6.0.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
-geoseeq-0.6.0.dist-info/RECORD,,
+geoseeq-0.6.2.dist-info/LICENSE,sha256=IuhIl1XCxXLPLJT_coN1CNqQU4Khlq7x4IdW7ioOJD8,1067
+geoseeq-0.6.2.dist-info/METADATA,sha256=WSI2kZ4-2pMME7jQCQ3Hzg9dU3Gm7R6tFrKdTj1PCbg,4803
+geoseeq-0.6.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+geoseeq-0.6.2.dist-info/entry_points.txt,sha256=yF-6KDM8zXib4Al0qn49TX-qM7PUkWUIcYtsgt36rjM,45
+geoseeq-0.6.2.dist-info/top_level.txt,sha256=zZk7mmeaqAYqFJG8nq2DTgSQPbflRjJwkDIhNURPDEU,14
+geoseeq-0.6.2.dist-info/RECORD,,