cloud-files 5.0.0__py3-none-any.whl → 5.0.2__py3-none-any.whl

{cloud_files-5.0.0.dist-info → cloud_files-5.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 5.0.0
+Version: 5.0.2
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-5.0.0.dist-info → cloud_files-5.0.2.dist-info}/RECORD

@@ -4,8 +4,8 @@ cloudfiles/compression.py,sha256=pqYdpu5vfFv-094BpfZ2pgRjVu7ESM9pAZC09P6E8bY,615
 cloudfiles/connectionpools.py,sha256=aL8RiSjRepECfgAFmJcz80aJFKbou7hsbuEgugDKwB8,4814
 cloudfiles/exceptions.py,sha256=N0oGQNG-St6RvnT8e5p_yC_E61q2kgAe2scwAL0F49c,843
 cloudfiles/gcs.py,sha256=unqu5KxGKaPq6N4QeHSpCDdtnK1BzPOAerTZ8FLt2_4,3820
-cloudfiles/interfaces.py,sha256=x6iRoxUWe8LFs8OAqI4DPOXu1nnJqI896h3t0IuVuNM,42024
-cloudfiles/lib.py,sha256=YOoaEkKtkXc9FdpNnC4FbZJVG1ujbyoxN07WKdUOJcs,5200
+cloudfiles/interfaces.py,sha256=rc6jblcdxfvsPAyiOrosY_kTn7idHftS026prG84x2M,43389
+cloudfiles/lib.py,sha256=9YFGMCy0mi23baqtsr-f5Gd1ewAGvD7x3UVNt772Z9E,5317
 cloudfiles/paths.py,sha256=VFgqrOqI7lTZ2EE7Ktg6I343TVXYEhn8GHTfJZZ9giE,11435
 cloudfiles/resumable_tools.py,sha256=NyuSoGh1SaP5akrHCpd9kgy2-JruEWrHW9lvJxV7jpE,6711
 cloudfiles/scheduler.py,sha256=DqDANmOpB3NdzFgJDNMMibRIkCrXQqIh2XGL8GWoc9c,3668
@@ -14,12 +14,12 @@ cloudfiles/threaded_queue.py,sha256=Nl4vfXhQ6nDLF8PZpSSBpww0M2zWtcd4DLs3W3BArBw,
 cloudfiles/typing.py,sha256=f3ZYkNfN9poxhGu5j-P0KCxjCCqSn9HAg5KiIPkjnCg,416
 cloudfiles_cli/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
 cloudfiles_cli/__init__.py,sha256=Wftt3R3F21QsHtWqx49ODuqT9zcSr0em7wk48kcH0WM,29
-cloudfiles_cli/cloudfiles_cli.py,sha256=Mwoeo0xXFaGx76f1pjU0H-qCGYoscykv_kMfWRxlnxg,34647
-cloud_files-5.0.0.dist-info/AUTHORS,sha256=BFVmobgAhaVFI5fqbuqAY5XmBQxe09ZZAsAOTy87hKQ,318
-cloud_files-5.0.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
-cloud_files-5.0.0.dist-info/METADATA,sha256=ypcRq07V2r_YR5V0NxTF5iAvOEPoPq3z7ZTBSV_pGEY,27047
-cloud_files-5.0.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cloud_files-5.0.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
-cloud_files-5.0.0.dist-info/pbr.json,sha256=9c_Y5OB-aqslSyCLAkm_Gge6KKpyRGKGU4UCZi0TOV0,46
-cloud_files-5.0.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
-cloud_files-5.0.0.dist-info/RECORD,,
+cloudfiles_cli/cloudfiles_cli.py,sha256=_2Amjp3vsdq7vQgF2utD6BJqjOPKXOTjrJyehnlojpQ,34893
+cloud_files-5.0.2.dist-info/AUTHORS,sha256=BFVmobgAhaVFI5fqbuqAY5XmBQxe09ZZAsAOTy87hKQ,318
+cloud_files-5.0.2.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
+cloud_files-5.0.2.dist-info/METADATA,sha256=ZrdyciuDQruZS6GjOll-FxtWl1sRHE2DwszlHVXVdPA,27047
+cloud_files-5.0.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cloud_files-5.0.2.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
+cloud_files-5.0.2.dist-info/pbr.json,sha256=1Ut3ABIh0b-00huoRZVkC1bPqCTSbdQ3syVtVqYnIU4,46
+cloud_files-5.0.2.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
+cloud_files-5.0.2.dist-info/RECORD,,

cloud_files-5.0.2.dist-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "b480c53", "is_release": true}

cloudfiles/interfaces.py

@@ -22,7 +22,7 @@ import fasteners
 from .compression import COMPRESSION_TYPES
 from .connectionpools import S3ConnectionPool, GCloudBucketPool, MemoryPool, MEMORY_DATA
 from .exceptions import MD5IntegrityError, CompressionError, AuthorizationError
-from .lib import mkdir, sip, md5, validate_s3_multipart_etag
+from .lib import mkdir, sip, md5, encode_crc32c_b64, validate_s3_multipart_etag
 from .secrets import (
   http_credentials,
   cave_credentials,
@@ -494,7 +494,33 @@ class MemoryInterface(StorageInterface):
     return True
 
   def head(self, file_path):
-    raise NotImplementedError()
+    path = self.get_path_to_file(file_path)
+
+    data = None
+    encoding = ''
+
+    with EXT_TEST_SEQUENCE_LOCK:
+      for ext, enc in EXT_TEST_SEQUENCE:
+        pathext = path + ext
+        if pathext in self._data:
+          data = self._data[pathext]
+          encoding = enc
+          break
+
+    return {
+      "Cache-Control": None,
+      "Content-Length": len(data),
+      "Content-Type": None,
+      "ETag": None,
+      "Last-Modified": None,
+      "Content-Md5": None,
+      "Content-Encoding": encoding,
+      "Content-Disposition": None,
+      "Content-Language": None,
+      "Storage-Class": None,
+      "Request-Charged": None,
+      "Parts-Count": None,
+    }
 
   def size(self, file_path):
     path = self.get_path_to_file(file_path)
@@ -1112,7 +1138,7 @@ class S3Interface(StorageInterface):
     elif compress in ("xz", "lzma"):
       attrs['ContentEncoding'] = 'xz'
     elif compress in ("bzip2", "bz2"):
-      attrs['ContentEncoding'] = 'bz2'
+      attrs['ContentEncoding'] = 'bzip2'
     elif compress:
       raise ValueError("Compression type {} not supported.".format(compress))
 
@@ -1136,23 +1162,39 @@ class S3Interface(StorageInterface):
 
     if multipart:
       self._conn.upload_fileobj(content, self._path.bucket, key, ExtraArgs=attrs)
+      # upload_fileobj will add 'aws-chunked' to the ContentEncoding,
+      # which after it finishes uploading is useless and messes up our
+      # software. Therefore, edit the metadata and replace it (but this incurs
+      # 2x class-A...)
+      self._conn.copy_object(
+        Bucket=self._path.bucket,
+        Key=key,
+        CopySource={'Bucket': self._path.bucket, 'Key': key},
+        MetadataDirective="REPLACE",
+        **attrs
+      )
     else:
       attrs['Bucket'] = self._path.bucket
       attrs['Body'] = content
       attrs['Key'] = key
-      attrs['ContentMD5'] = md5(content)
+      attrs["ChecksumCRC32C"] = str(encode_crc32c_b64(content))
       self._conn.put_object(**attrs)
 
   @retry
   def copy_file(self, src_path, dest_bucket_name, dest_key):
     key = self.get_path_to_file(src_path)
-    dest_bucket = self._get_bucket(dest_bucket_name)
+    s3client = self._get_bucket(dest_bucket_name)
     copy_source = {
       'Bucket': self._path.bucket,
       'Key': key,
     }
     try:
-      dest_bucket.copy(CopySource=copy_source, Bucket=dest_bucket_name, Key=dest_key)
+      s3client.copy_object(
+          CopySource=copy_source,
+          Bucket=dest_bucket_name,
+          Key=dest_key,
+          MetadataDirective='COPY'  # Ensure metadata like Content-Encoding is copied
+      )
     except botocore.exceptions.ClientError as err: 
       if err.response['Error']['Code'] in ('NoSuchKey', '404'):
         return False
@@ -1231,8 +1273,9 @@ class S3Interface(StorageInterface):
 
     mkdir(os.path.dirname(dest))
 
+    encoding = resp.get("Content-Encoding", "") or ""
     encoding = ",".join([ 
-      enc for enc in resp.get("Content-Encoding", "").split(",")
+      enc for enc in encoding.split(",")
       if enc != "aws-chunked"
     ])
     ext = FileInterface.get_extension(encoding)
@@ -1263,6 +1306,11 @@ class S3Interface(StorageInterface):
         Key=self.get_path_to_file(file_path),
         **self._additional_attrs,
       )
+
+      encoding = response.get("ContentEncoding", None)
+      if encoding == '':
+        encoding = None
+
       return {
         "Cache-Control": response.get("CacheControl", None),
         "Content-Length": response.get("ContentLength", None),
@@ -1270,7 +1318,7 @@ class S3Interface(StorageInterface):
         "ETag": response.get("ETag", None),
         "Last-Modified": response.get("LastModified", None),
         "Content-Md5": response["ResponseMetadata"]["HTTPHeaders"].get("content-md5", None),
-        "Content-Encoding": response.get("ContentEncoding", None),
+        "Content-Encoding": encoding,
         "Content-Disposition": response.get("ContentDisposition", None),
         "Content-Language": response.get("ContentLanguage", None),
         "Storage-Class": response.get("StorageClass", None),

cloudfiles/lib.py

@@ -153,6 +153,11 @@ def decode_crc32c_b64(b64digest):
   # !I means network order (big endian) and unsigned int
   return struct.unpack("!I", base64.b64decode(b64digest))[0]
 
+def encode_crc32c_b64(binary):
+  val = crc32c(binary)
+  val = val.to_bytes(4, 'big')
+  return base64.b64encode(val)
+
 def crc32c(binary):
   """
   Computes the crc32c of a binary string 

cloudfiles_cli/cloudfiles_cli.py

@@ -255,7 +255,10 @@ def _cp_single(
     xferpaths = [ x.replace("\n", "") for x in xferpaths ]
     prefix = os.path.commonprefix(xferpaths)
     xferpaths = [ x.replace(prefix, "") for x in xferpaths ]
-    srcpath = cloudpathjoin(srcpath, prefix)
+    srcpath = prefix
+    if srcpath == "":
+      print(f"cloudfiles: No common prefix found. Currently only one bucket at a time is supported for STDIN.")
+      return
   elif many:
     xferpaths = CloudFiles(
       srcpath, no_sign_request=no_sign_request
@@ -432,7 +435,10 @@ def _mv_single(
     xferpaths = [ x.replace("\n", "") for x in xferpaths ]
     prefix = os.path.commonprefix(xferpaths)
     xferpaths = [ x.replace(prefix, "") for x in xferpaths ]
-    srcpath = cloudpathjoin(srcpath, prefix)
+    srcpath = prefix
+    if srcpath == "":
+      print(f"cloudfiles: No common prefix found. Currently only one bucket at a time is supported for STDIN.")
+      return
   elif many:
     xferpaths = CloudFiles(
       srcpath, no_sign_request=no_sign_request

cloud_files-5.0.0.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "ee08b4f", "is_release": true}