cloud-files 4.23.0__tar.gz → 4.24.1__tar.gz

{cloud-files-4.23.0 → cloud-files-4.24.1}/ChangeLog

@@ -1,6 +1,21 @@
 CHANGES
 =======
 
+4.24.1
+------
+
+* fix: close unused connections for HTTP interface
+
+4.24.0
+------
+
+* fix: check for 404 and use proper key variable
+* test: modify dne test
+* docs: update authors, changelog
+* feat: add support for allow\_missing to interfaces
+* test: check to see if allow\_missing works
+* feat: add allow\_missing to transfer\_to/from
+
 4.23.0
 ------
 

{cloud-files-4.23.0 → cloud-files-4.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.23.0
+Version: 4.24.1
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud-files-4.23.0 → cloud-files-4.24.1}/automated_test.py

@@ -198,6 +198,7 @@ def test_http_read(secrets):
   cf = CloudFiles("https://storage.googleapis.com/seunglab-test/test_v0/black/", secrets=secrets)
   
   try:
+    head = cf.head('info')
     info = cf.get_json('info')
   except requests.exceptions.HTTPError:
     return
@@ -613,10 +614,11 @@ def test_s3_custom_endpoint_path():
   assert extract.path == 'world'
   assert extract.host == 'https://s3-hpcrc.rc.princeton.edu'
 
+@pytest.mark.parametrize('allow_missing', [False, True])
 @pytest.mark.parametrize('compression', (None, 'gzip', 'br', 'zstd', 'xz', 'bz2'))
 @pytest.mark.parametrize('src_protocol', ['mem', 'file', 's3'])
 @pytest.mark.parametrize('dest_protocol', ['mem', 'file', 's3'])
-def test_transfer_semantics(s3, compression, src_protocol, dest_protocol):
+def test_transfer_semantics(s3, compression, src_protocol, dest_protocol, allow_missing):
   from cloudfiles import CloudFiles, exceptions
 
   if src_protocol == "file":
@@ -650,18 +652,18 @@ def test_transfer_semantics(s3, compression, src_protocol, dest_protocol):
   cfm.delete(list(cfm))
   assert list(cfm) == []
 
-  cfm.transfer_from(f'{src_protocol}://' + path)
+  cfm.transfer_from(f'{src_protocol}://' + path, allow_missing=allow_missing)
   assert sorted(list(cfm)) == sorted([ str(i) for i in range(N) ])
   assert [ f['content'] for f in cfm[:] ] == [ content ] * N
 
   cfm.delete(list(cfm))
 
-  cff.transfer_to(cfm.cloudpath)
+  cff.transfer_to(cfm.cloudpath, allow_missing=allow_missing)
   assert sorted(list(cfm)) == sorted([ str(i) for i in range(N) ])
   assert [ f['content'] for f in cfm[:] ] == [ content ] * N  
   cfm.delete(list(cfm))
 
-  cff.transfer_to(cfm.cloudpath, reencode='br')
+  cff.transfer_to(cfm.cloudpath, reencode='br', allow_missing=allow_missing)
   assert sorted(list(cfm)) == sorted([ str(i) for i in range(N) ])
   assert [ f['content'] for f in cfm[:] ] == [ content ] * N  
 
@@ -670,6 +672,17 @@ def test_transfer_semantics(s3, compression, src_protocol, dest_protocol):
     data = [ os.path.splitext(d)[1] for d in data.keys() ] 
     assert all([ ext == '.br' for ext in data ])
 
+  if not allow_missing:
+    try:
+      cff.transfer_to(cfm.cloudpath, paths=["dne"], allow_missing=False)
+      assert False
+    except FileNotFoundError:
+      pass
+  else:
+    cff.transfer_to(cfm.cloudpath, paths=["dne"], allow_missing=True)
+    assert not cfm.exists("dne") or cfm.get("dne") == b''
+
+
   cfm.delete(list(cfm))
   cff.delete(list(cff))
 

{cloud-files-4.23.0 → cloud-files-4.24.1}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.23.0
+Version: 4.24.1
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud-files-4.24.1/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "560597e", "is_release": true}

{cloud-files-4.23.0 → cloud-files-4.24.1}/cloudfiles/cloudfiles.py

@@ -951,6 +951,7 @@ class CloudFiles:
     block_size:int = 64, 
     reencode:Optional[str] = None,
     content_type:Optional[str] = None,
+    allow_missing:bool = False,
   ) -> None:
     """
     Transfer all files from this CloudFiles storage 
@@ -992,7 +993,11 @@ class CloudFiles:
         green=self.green, num_threads=self.num_threads,
       )
 
-    return cf_dest.transfer_from(self, paths, block_size, reencode, content_type)
+    return cf_dest.transfer_from(
+      self, paths, block_size, 
+      reencode, content_type,
+      allow_missing,
+    )
 
   def transfer_from(
     self, 
@@ -1001,6 +1006,7 @@ class CloudFiles:
     block_size:int = 64, 
     reencode:Optional[str] = None,
     content_type:Optional[str] = None,
+    allow_missing:bool = False,
   ) -> None:
     """
     Transfer all files from the source CloudFiles storage 
@@ -1053,7 +1059,10 @@ class CloudFiles:
         and self.protocol == "file"
         and reencode is None
       ):
-        self.__transfer_file_to_file(cf_src, self, paths, total, pbar, block_size)
+        self.__transfer_file_to_file(
+          cf_src, self, paths, total, 
+          pbar, block_size, allow_missing
+        )
       elif (
         cf_src.protocol == "file"
         and self.protocol != "file"
@@ -1061,7 +1070,8 @@ class CloudFiles:
       ):
         self.__transfer_file_to_remote(
           cf_src, self, paths, total, 
-          pbar, block_size, content_type
+          pbar, block_size, content_type,
+          allow_missing,
         )
       elif (
         (
@@ -1076,19 +1086,22 @@ class CloudFiles:
       ):
         self.__transfer_cloud_internal(
           cf_src, self, paths, 
-          total, pbar, block_size
+          total, pbar, block_size, 
+          allow_missing,
         )
       else:
         self.__transfer_general(
           cf_src, self, paths, total, 
           pbar, block_size, 
-          reencode, content_type
+          reencode, content_type, 
+          allow_missing,
         )
 
   def __transfer_general(
     self, cf_src, cf_dest, paths, 
     total, pbar, block_size,
-    reencode, content_type
+    reencode, content_type, 
+    allow_missing
   ):
     """
     Downloads the file into RAM, transforms
@@ -1107,7 +1120,13 @@ class CloudFiles:
       if reencode is not None:
         downloaded = compression.transcode(downloaded, reencode, in_place=True)
       def renameiter():
+        nonlocal allow_missing
         for item in downloaded:
+          if item["content"] is None:
+            if allow_missing:
+              item["content"] = b""
+            else:
+              raise FileNotFoundError(f"{item['path']}")
           if (
             item["tags"] is not None
             and "dest_path" in item["tags"]
@@ -1126,7 +1145,7 @@ class CloudFiles:
 
   def __transfer_file_to_file(
     self, cf_src, cf_dest, paths, 
-    total, pbar, block_size
+    total, pbar, block_size, allow_missing
   ):
     """
     shutil.copyfile, starting in Python 3.8, uses
@@ -1148,12 +1167,21 @@ class CloudFiles:
       if dest_ext_compress != dest_ext:
         dest += dest_ext_compress
 
-      shutil.copyfile(src, dest) # avoids user space
+      try:
+        shutil.copyfile(src, dest) # avoids user space
+      except FileNotFoundError:
+        if allow_missing:
+          with open(dest, "wb") as f:
+            f.write(b'')
+        else:
+          raise
+
       pbar.update(1)
 
   def __transfer_file_to_remote(
     self, cf_src, cf_dest, paths, 
-    total, pbar, block_size, content_type
+    total, pbar, block_size, content_type,
+    allow_missing
   ):
     """
     Provide file handles instead of slurped binaries 
@@ -1174,19 +1202,29 @@ class CloudFiles:
         handle_path, encoding = FileInterface.get_encoded_file_path(
           os.path.join(srcdir, src_path)
         )
+        try: 
+          handle = open(handle_path, "rb")
+        except FileNotFoundError:
+          if allow_missing:
+            handle = b''
+          else:
+            raise
+
         to_upload.append({
           "path": dest_path,
-          "content": open(handle_path, "rb"),
+          "content": handle,
           "compress": encoding,
         })
       cf_dest.puts(to_upload, raw=True, progress=False, content_type=content_type)
       for item in to_upload:
-        item["content"].close()
+        handle = item["content"]
+        if hasattr(handle, "close"):
+          handle.close()
       pbar.update(len(block_paths))
 
   def __transfer_cloud_internal(
     self, cf_src, cf_dest, paths, 
-    total, pbar, block_size
+    total, pbar, block_size, allow_missing
   ):
     """
     For performing internal transfers in gs or s3.
@@ -1206,8 +1244,12 @@ class CloudFiles:
           dest_key = key
 
         dest_key = posixpath.join(cf_dest._path.path, dest_key)
-        conn.copy_file(src_key, cf_dest._path.bucket, dest_key)
-      return 1
+        found = conn.copy_file(src_key, cf_dest._path.bucket, dest_key)
+
+      if found == False and not allow_missing:
+        raise FileNotFoundError(src_key)
+
+      return int(found)
 
     results = schedule_jobs(
       fns=( partial(thunk_copy, path) for path in paths ),

{cloud-files-4.23.0 → cloud-files-4.24.1}/cloudfiles/interfaces.py

@@ -22,7 +22,7 @@ import fasteners
 
 from .compression import COMPRESSION_TYPES
 from .connectionpools import S3ConnectionPool, GCloudBucketPool, MemoryPool, MEMORY_DATA
-from .exceptions import MD5IntegrityError
+from .exceptions import MD5IntegrityError, CompressionError
 from .lib import mkdir, sip, md5, validate_s3_multipart_etag
 from .secrets import http_credentials, CLOUD_FILES_DIR, CLOUD_FILES_LOCK_DIR
 
@@ -82,6 +82,21 @@ retry = tenacity.retry(
   wait=tenacity.wait_random_exponential(0.5, 60.0),
 )
 
+def retry_if_not(exception_type):
+  if type(exception_type) != list:
+    exception_type = [ exception_type ]
+
+  conditions = tenacity.retry_if_not_exception_type(exception_type[0])
+  for et in exception_type[1:]:
+    conditions = conditions | tenacity.retry_if_not_exception_type(et)
+
+  return tenacity.retry(
+    retry=conditions,
+    reraise=True, 
+    stop=tenacity.stop_after_attempt(7), 
+    wait=tenacity.wait_random_exponential(0.5, 60.0),
+  ) 
+
 class StorageInterface(object):
   exists_batch_size = 1
   delete_batch_size = 1
@@ -528,7 +543,7 @@ class GoogleCloudStorageInterface(StorageInterface):
   def get_path_to_file(self, file_path):
     return posixpath.join(self._path.path, file_path)
 
-  @retry
+  @retry_if_not(CompressionError)
   def put_file(self, file_path, content, content_type,
                compress, cache_control=None, storage_class=None):
     key = self.get_path_to_file(file_path)
@@ -545,7 +560,7 @@ class GoogleCloudStorageInterface(StorageInterface):
     elif compress in ("bzip2", "bz2"):
       blob.content_encoding = "bz2"
     elif compress:
-      raise ValueError("Compression type {} not supported.".format(compress))
+      raise CompressionError("Compression type {} not supported.".format(compress))
 
     if cache_control:
       blob.cache_control = cache_control
@@ -562,11 +577,17 @@ class GoogleCloudStorageInterface(StorageInterface):
     with GCS_BUCKET_POOL_LOCK:
      pool = GC_POOL[GCloudBucketPoolParams(dest_bucket, self._request_payer)]
     dest_bucket = pool.get_connection(self._secrets, None)
-    self._bucket.copy_blob(
-      source_blob, dest_bucket, dest_key
-    )
 
-  @retry
+    try:
+      self._bucket.copy_blob(
+        source_blob, dest_bucket, dest_key
+      )
+    except google.api_core.exceptions.NotFound:
+      return False
+
+    return True
+
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def get_file(self, file_path, start=None, end=None, part_size=None):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.blob( key )
@@ -590,7 +611,7 @@ class GoogleCloudStorageInterface(StorageInterface):
 
     return (content, blob.content_encoding, hash_value, hash_type)
 
-  @retry
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def head(self, file_path):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.get_blob(key)
@@ -609,7 +630,7 @@ class GoogleCloudStorageInterface(StorageInterface):
       "Component-Count": blob.component_count,
     }
 
-  @retry
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def size(self, file_path):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.get_blob(key)
@@ -617,7 +638,7 @@ class GoogleCloudStorageInterface(StorageInterface):
       return blob.size
     return None
 
-  @retry
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def exists(self, file_path):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.blob(key)
@@ -724,9 +745,9 @@ class HttpInterface(StorageInterface):
   @retry
   def head(self, file_path):
     key = self.get_path_to_file(file_path)
-    resp = self.session.head(key)
-    resp.raise_for_status()
-    return resp.headers
+    with self.session.head(key) as resp:
+      resp.raise_for_status()
+      return resp.headers
 
   @retry
   def get_file(self, file_path, start=None, end=None, part_size=None):
@@ -741,6 +762,7 @@ class HttpInterface(StorageInterface):
       resp = self.session.get(key)
     if resp.status_code in (404, 403):
       return (None, None, None, None)
+    resp.close()
     resp.raise_for_status()
 
     # Don't check MD5 for http because the etag can come in many
@@ -758,9 +780,8 @@ class HttpInterface(StorageInterface):
   @retry
   def exists(self, file_path):
     key = self.get_path_to_file(file_path)
-    resp = self.session.get(key, stream=True)
-    resp.close()
-    return resp.ok
+    with self.session.get(key, stream=True) as resp:
+      return resp.ok
 
   def files_exist(self, file_paths):
     return {path: self.exists(path) for path in file_paths}
@@ -783,6 +804,7 @@ class HttpInterface(StorageInterface):
         params={ "prefix": prefix, "pageToken": token },
       )
       results.raise_for_status()
+      results.close()
       return results.json()
 
     token = None
@@ -907,7 +929,15 @@ class S3Interface(StorageInterface):
       'Bucket': self._path.bucket,
       'Key': key,
     }
-    dest_bucket.copy(CopySource=copy_source, Bucket=dest_bucket_name, Key=dest_key)
+    try:
+      dest_bucket.copy(CopySource=copy_source, Bucket=dest_bucket_name, Key=dest_key)
+    except botocore.exceptions.ClientError as err: 
+      if err.response['Error']['Code'] in ('NoSuchKey', '404'):
+        return False
+      else:
+        raise
+
+    return True
 
   @retry
   def get_file(self, file_path, start=None, end=None, part_size=None):

cloud-files-4.23.0/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "a123f94", "is_release": true}