cloud-files 4.23.0__py3-none-any.whl → 4.24.1__py3-none-any.whl

{cloud_files-4.23.0.dist-info → cloud_files-4.24.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.23.0
+Version: 4.24.1
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-4.23.0.dist-info → cloud_files-4.24.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
 cloudfiles/__init__.py,sha256=pLB4CcV2l3Jgv_ni1520Np1pfzFj8Cpr87vNxFT3rNI,493
 cloudfiles/buckets.py,sha256=eRAYdDfvVpNyJyK5ryDRMwgNJUeEuFBJ6doWU2JkAcA,74
-cloudfiles/cloudfiles.py,sha256=OWTQe-Lkz60E6YFrFXhqWVl1L3_3cqzRl15Lqr09nYs,43794
+cloudfiles/cloudfiles.py,sha256=YUf_-7DS8-2mCKOWT6mMKxl1glxMTEGomhpCnebtsy8,44801
 cloudfiles/compression.py,sha256=pqYdpu5vfFv-094BpfZ2pgRjVu7ESM9pAZC09P6E8bY,6150
 cloudfiles/connectionpools.py,sha256=aL8RiSjRepECfgAFmJcz80aJFKbou7hsbuEgugDKwB8,4814
 cloudfiles/exceptions.py,sha256=H2IcMlZoy2Bsn-6wCPwyLDjg66LZCyxtcf3s_p21FDw,770
 cloudfiles/gcs.py,sha256=_njJ7TpqwrHCjPHRGkBN5alCrCWKM2m9qdy5DhxMZ7U,3718
-cloudfiles/interfaces.py,sha256=j60PSGqCmeE_7o7gBUW-pi56up4fvIaX3qOlxW_fqMQ,33540
+cloudfiles/interfaces.py,sha256=LAjmO40mhg_foEjdzdP0YxUxpk9Ut8C7puortTqSGXI,34539
 cloudfiles/lib.py,sha256=fEqL5APu_WQhl2yxqQbwE7msHdu7U8pstAJw6LgoKO0,5142
 cloudfiles/paths.py,sha256=tqR9XnRdAKopTJqSM6V5xrMo1xfmBdl9b5DpBLZnoB0,9998
 cloudfiles/resumable_tools.py,sha256=pK-VcoPjQ2BjGjvlvH4dDCBf6lNsqHG-weiBgxVFbzA,5838
@@ -16,11 +16,11 @@ cloudfiles/typing.py,sha256=f3ZYkNfN9poxhGu5j-P0KCxjCCqSn9HAg5KiIPkjnCg,416
 cloudfiles_cli/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
 cloudfiles_cli/__init__.py,sha256=Wftt3R3F21QsHtWqx49ODuqT9zcSr0em7wk48kcH0WM,29
 cloudfiles_cli/cloudfiles_cli.py,sha256=eETIOK4QyztQcpA4ZRny21SobLtcrPDlzZ_JaKBmmmA,28449
-cloud_files-4.23.0.dist-info/AUTHORS,sha256=7E2vC894bbLPO_kvUuEB2LFZZbIxZn23HabxH7x0Hgo,266
-cloud_files-4.23.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
-cloud_files-4.23.0.dist-info/METADATA,sha256=1V4s9l7YchiO-D_9yqdVqSHQmK59lvWmda27YDXiGtc,26804
-cloud_files-4.23.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cloud_files-4.23.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
-cloud_files-4.23.0.dist-info/pbr.json,sha256=yW5BCBFnMvccatJ349ZLnNWUR137dFenl6ygMDm5k1c,46
-cloud_files-4.23.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
-cloud_files-4.23.0.dist-info/RECORD,,
+cloud_files-4.24.1.dist-info/AUTHORS,sha256=7E2vC894bbLPO_kvUuEB2LFZZbIxZn23HabxH7x0Hgo,266
+cloud_files-4.24.1.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
+cloud_files-4.24.1.dist-info/METADATA,sha256=neXuIXtjmY7tLeSTTNH95jTyw_rHsnrYfum-AFEboR0,26804
+cloud_files-4.24.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cloud_files-4.24.1.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
+cloud_files-4.24.1.dist-info/pbr.json,sha256=whK0oWj9vjQb1lRu9_jrdIBm1TA0onqx-o5fzWihcBo,46
+cloud_files-4.24.1.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
+cloud_files-4.24.1.dist-info/RECORD,,

cloud_files-4.24.1.dist-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "560597e", "is_release": true}

cloudfiles/cloudfiles.py

@@ -951,6 +951,7 @@ class CloudFiles:
     block_size:int = 64, 
     reencode:Optional[str] = None,
     content_type:Optional[str] = None,
+    allow_missing:bool = False,
   ) -> None:
     """
     Transfer all files from this CloudFiles storage 
@@ -992,7 +993,11 @@ class CloudFiles:
         green=self.green, num_threads=self.num_threads,
       )
 
-    return cf_dest.transfer_from(self, paths, block_size, reencode, content_type)
+    return cf_dest.transfer_from(
+      self, paths, block_size, 
+      reencode, content_type,
+      allow_missing,
+    )
 
   def transfer_from(
     self, 
@@ -1001,6 +1006,7 @@ class CloudFiles:
     block_size:int = 64, 
     reencode:Optional[str] = None,
     content_type:Optional[str] = None,
+    allow_missing:bool = False,
   ) -> None:
     """
     Transfer all files from the source CloudFiles storage 
@@ -1053,7 +1059,10 @@ class CloudFiles:
         and self.protocol == "file"
         and reencode is None
       ):
-        self.__transfer_file_to_file(cf_src, self, paths, total, pbar, block_size)
+        self.__transfer_file_to_file(
+          cf_src, self, paths, total, 
+          pbar, block_size, allow_missing
+        )
       elif (
         cf_src.protocol == "file"
         and self.protocol != "file"
@@ -1061,7 +1070,8 @@ class CloudFiles:
       ):
         self.__transfer_file_to_remote(
           cf_src, self, paths, total, 
-          pbar, block_size, content_type
+          pbar, block_size, content_type,
+          allow_missing,
         )
       elif (
         (
@@ -1076,19 +1086,22 @@ class CloudFiles:
       ):
         self.__transfer_cloud_internal(
           cf_src, self, paths, 
-          total, pbar, block_size
+          total, pbar, block_size, 
+          allow_missing,
         )
       else:
         self.__transfer_general(
           cf_src, self, paths, total, 
           pbar, block_size, 
-          reencode, content_type
+          reencode, content_type, 
+          allow_missing,
         )
 
   def __transfer_general(
     self, cf_src, cf_dest, paths, 
     total, pbar, block_size,
-    reencode, content_type
+    reencode, content_type, 
+    allow_missing
   ):
     """
     Downloads the file into RAM, transforms
@@ -1107,7 +1120,13 @@ class CloudFiles:
       if reencode is not None:
         downloaded = compression.transcode(downloaded, reencode, in_place=True)
       def renameiter():
+        nonlocal allow_missing
         for item in downloaded:
+          if item["content"] is None:
+            if allow_missing:
+              item["content"] = b""
+            else:
+              raise FileNotFoundError(f"{item['path']}")
           if (
             item["tags"] is not None
             and "dest_path" in item["tags"]
@@ -1126,7 +1145,7 @@ class CloudFiles:
 
   def __transfer_file_to_file(
     self, cf_src, cf_dest, paths, 
-    total, pbar, block_size
+    total, pbar, block_size, allow_missing
   ):
     """
     shutil.copyfile, starting in Python 3.8, uses
@@ -1148,12 +1167,21 @@ class CloudFiles:
       if dest_ext_compress != dest_ext:
         dest += dest_ext_compress
 
-      shutil.copyfile(src, dest) # avoids user space
+      try:
+        shutil.copyfile(src, dest) # avoids user space
+      except FileNotFoundError:
+        if allow_missing:
+          with open(dest, "wb") as f:
+            f.write(b'')
+        else:
+          raise
+
       pbar.update(1)
 
   def __transfer_file_to_remote(
     self, cf_src, cf_dest, paths, 
-    total, pbar, block_size, content_type
+    total, pbar, block_size, content_type,
+    allow_missing
   ):
     """
     Provide file handles instead of slurped binaries 
@@ -1174,19 +1202,29 @@ class CloudFiles:
         handle_path, encoding = FileInterface.get_encoded_file_path(
           os.path.join(srcdir, src_path)
         )
+        try: 
+          handle = open(handle_path, "rb")
+        except FileNotFoundError:
+          if allow_missing:
+            handle = b''
+          else:
+            raise
+
         to_upload.append({
           "path": dest_path,
-          "content": open(handle_path, "rb"),
+          "content": handle,
           "compress": encoding,
         })
       cf_dest.puts(to_upload, raw=True, progress=False, content_type=content_type)
       for item in to_upload:
-        item["content"].close()
+        handle = item["content"]
+        if hasattr(handle, "close"):
+          handle.close()
       pbar.update(len(block_paths))
 
   def __transfer_cloud_internal(
     self, cf_src, cf_dest, paths, 
-    total, pbar, block_size
+    total, pbar, block_size, allow_missing
   ):
     """
     For performing internal transfers in gs or s3.
@@ -1206,8 +1244,12 @@ class CloudFiles:
           dest_key = key
 
         dest_key = posixpath.join(cf_dest._path.path, dest_key)
-        conn.copy_file(src_key, cf_dest._path.bucket, dest_key)
-      return 1
+        found = conn.copy_file(src_key, cf_dest._path.bucket, dest_key)
+
+      if found == False and not allow_missing:
+        raise FileNotFoundError(src_key)
+
+      return int(found)
 
     results = schedule_jobs(
       fns=( partial(thunk_copy, path) for path in paths ),

cloudfiles/interfaces.py

@@ -22,7 +22,7 @@ import fasteners
 
 from .compression import COMPRESSION_TYPES
 from .connectionpools import S3ConnectionPool, GCloudBucketPool, MemoryPool, MEMORY_DATA
-from .exceptions import MD5IntegrityError
+from .exceptions import MD5IntegrityError, CompressionError
 from .lib import mkdir, sip, md5, validate_s3_multipart_etag
 from .secrets import http_credentials, CLOUD_FILES_DIR, CLOUD_FILES_LOCK_DIR
 
@@ -82,6 +82,21 @@ retry = tenacity.retry(
   wait=tenacity.wait_random_exponential(0.5, 60.0),
 )
 
+def retry_if_not(exception_type):
+  if type(exception_type) != list:
+    exception_type = [ exception_type ]
+
+  conditions = tenacity.retry_if_not_exception_type(exception_type[0])
+  for et in exception_type[1:]:
+    conditions = conditions | tenacity.retry_if_not_exception_type(et)
+
+  return tenacity.retry(
+    retry=conditions,
+    reraise=True, 
+    stop=tenacity.stop_after_attempt(7), 
+    wait=tenacity.wait_random_exponential(0.5, 60.0),
+  ) 
+
 class StorageInterface(object):
   exists_batch_size = 1
   delete_batch_size = 1
@@ -528,7 +543,7 @@ class GoogleCloudStorageInterface(StorageInterface):
   def get_path_to_file(self, file_path):
     return posixpath.join(self._path.path, file_path)
 
-  @retry
+  @retry_if_not(CompressionError)
   def put_file(self, file_path, content, content_type,
                compress, cache_control=None, storage_class=None):
     key = self.get_path_to_file(file_path)
@@ -545,7 +560,7 @@ class GoogleCloudStorageInterface(StorageInterface):
     elif compress in ("bzip2", "bz2"):
       blob.content_encoding = "bz2"
     elif compress:
-      raise ValueError("Compression type {} not supported.".format(compress))
+      raise CompressionError("Compression type {} not supported.".format(compress))
 
     if cache_control:
       blob.cache_control = cache_control
@@ -562,11 +577,17 @@ class GoogleCloudStorageInterface(StorageInterface):
     with GCS_BUCKET_POOL_LOCK:
      pool = GC_POOL[GCloudBucketPoolParams(dest_bucket, self._request_payer)]
     dest_bucket = pool.get_connection(self._secrets, None)
-    self._bucket.copy_blob(
-      source_blob, dest_bucket, dest_key
-    )
 
-  @retry
+    try:
+      self._bucket.copy_blob(
+        source_blob, dest_bucket, dest_key
+      )
+    except google.api_core.exceptions.NotFound:
+      return False
+
+    return True
+
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def get_file(self, file_path, start=None, end=None, part_size=None):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.blob( key )
@@ -590,7 +611,7 @@ class GoogleCloudStorageInterface(StorageInterface):
 
     return (content, blob.content_encoding, hash_value, hash_type)
 
-  @retry
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def head(self, file_path):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.get_blob(key)
@@ -609,7 +630,7 @@ class GoogleCloudStorageInterface(StorageInterface):
       "Component-Count": blob.component_count,
     }
 
-  @retry
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def size(self, file_path):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.get_blob(key)
@@ -617,7 +638,7 @@ class GoogleCloudStorageInterface(StorageInterface):
       return blob.size
     return None
 
-  @retry
+  @retry_if_not(google.cloud.exceptions.NotFound)
   def exists(self, file_path):
     key = self.get_path_to_file(file_path)
     blob = self._bucket.blob(key)
@@ -724,9 +745,9 @@ class HttpInterface(StorageInterface):
   @retry
   def head(self, file_path):
     key = self.get_path_to_file(file_path)
-    resp = self.session.head(key)
-    resp.raise_for_status()
-    return resp.headers
+    with self.session.head(key) as resp:
+      resp.raise_for_status()
+      return resp.headers
 
   @retry
   def get_file(self, file_path, start=None, end=None, part_size=None):
@@ -741,6 +762,7 @@ class HttpInterface(StorageInterface):
       resp = self.session.get(key)
     if resp.status_code in (404, 403):
       return (None, None, None, None)
+    resp.close()
     resp.raise_for_status()
 
     # Don't check MD5 for http because the etag can come in many
@@ -758,9 +780,8 @@ class HttpInterface(StorageInterface):
   @retry
   def exists(self, file_path):
     key = self.get_path_to_file(file_path)
-    resp = self.session.get(key, stream=True)
-    resp.close()
-    return resp.ok
+    with self.session.get(key, stream=True) as resp:
+      return resp.ok
 
   def files_exist(self, file_paths):
     return {path: self.exists(path) for path in file_paths}
@@ -783,6 +804,7 @@ class HttpInterface(StorageInterface):
         params={ "prefix": prefix, "pageToken": token },
       )
       results.raise_for_status()
+      results.close()
       return results.json()
 
     token = None
@@ -907,7 +929,15 @@ class S3Interface(StorageInterface):
       'Bucket': self._path.bucket,
       'Key': key,
     }
-    dest_bucket.copy(CopySource=copy_source, Bucket=dest_bucket_name, Key=dest_key)
+    try:
+      dest_bucket.copy(CopySource=copy_source, Bucket=dest_bucket_name, Key=dest_key)
+    except botocore.exceptions.ClientError as err: 
+      if err.response['Error']['Code'] in ('NoSuchKey', '404'):
+        return False
+      else:
+        raise
+
+    return True
 
   @retry
   def get_file(self, file_path, start=None, end=None, part_size=None):

cloud_files-4.23.0.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "a123f94", "is_release": true}