cloud-files 5.5.0__py3-none-any.whl → 5.6.1__py3-none-any.whl

cloudfiles/interfaces.py

@@ -4,7 +4,7 @@ from collections import defaultdict, namedtuple
 from datetime import datetime
 from io import BytesIO
 import json
-import os.path
+import os
 import posixpath
 import re
 
@@ -467,7 +467,7 @@ class MemoryInterface(StorageInterface):
       result = result[slice(start, end)]
     return (result, encoding, None, None)
 
-  def save_file(self, src, dest, resumable):
+  def save_file(self, src, dest, resumable) -> tuple[bool,int]:
     key = self.get_path_to_file(src)
     with EXT_TEST_SEQUENCE_LOCK:
       exts = list(EXT_TEST_SEQUENCE)
@@ -489,9 +489,9 @@ class MemoryInterface(StorageInterface):
       with open(dest + true_ext, "wb") as f:
         f.write(self._data[path])
     except KeyError:
-      return False
+      return (False, 0)
 
-    return True
+    return (True, len(self._data[path]))
 
   def head(self, file_path):
     path = self.get_path_to_file(file_path)
@@ -541,13 +541,13 @@ class MemoryInterface(StorageInterface):
 
     return None
 
-  def copy_file(self, src_path, dest_bucket, dest_key):
+  def copy_file(self, src_path, dest_bucket, dest_key) -> tuple[bool,int]:
     key = self.get_path_to_file(src_path)
     with MEM_BUCKET_POOL_LOCK:
      pool = MEM_POOL[MemoryPoolParams(dest_bucket)]
     dest_bucket = pool.get_connection(None, None)
     dest_bucket[dest_key] = self._data[key]
-    return True
+    return (True, len(self._data[key]))
 
   def exists(self, file_path):
     path = self.get_path_to_file(file_path)
@@ -662,7 +662,7 @@ class GoogleCloudStorageInterface(StorageInterface):
     blob.upload_from_string(content, content_type)
 
   @retry
-  def copy_file(self, src_path, dest_bucket, dest_key):
+  def copy_file(self, src_path, dest_bucket, dest_key) -> tuple[bool,int]:
     key = self.get_path_to_file(src_path)
     source_blob = self._bucket.blob( key )
     with GCS_BUCKET_POOL_LOCK:
@@ -670,13 +670,13 @@ class GoogleCloudStorageInterface(StorageInterface):
     dest_bucket = pool.get_connection(self._secrets, None)
 
     try:
-      self._bucket.copy_blob(
+      blob = self._bucket.copy_blob(
         source_blob, dest_bucket, dest_key
       )
     except google.api_core.exceptions.NotFound:
-      return False
+      return (False, 0)
 
-    return True
+    return (True, blob.size)
 
   @retry_if_not(google.cloud.exceptions.NotFound)
   def get_file(self, file_path, start=None, end=None, part_size=None):
@@ -703,7 +703,7 @@ class GoogleCloudStorageInterface(StorageInterface):
     return (content, blob.content_encoding, hash_value, hash_type)
 
   @retry
-  def save_file(self, src, dest, resumable):
+  def save_file(self, src, dest, resumable) -> tuple[bool, int]:
     key = self.get_path_to_file(src)
     blob = self._bucket.blob(key)
     try:
@@ -714,13 +714,15 @@ class GoogleCloudStorageInterface(StorageInterface):
         checksum=None
       )
     except google.cloud.exceptions.NotFound:
-      return False
+      return (False, 0)
+
+    num_bytes = os.path.getsize(dest)
 
     ext = FileInterface.get_extension(blob.content_encoding)
     if not dest.endswith(ext):
       os.rename(dest, dest + ext)
 
-    return True
+    return (True, num_bytes)
 
   @retry_if_not(google.cloud.exceptions.NotFound)
   def head(self, file_path):
@@ -927,7 +929,7 @@ class HttpInterface(StorageInterface):
     return (resp.content, content_encoding, None, None)
 
   @retry
-  def save_file(self, src, dest, resumable):
+  def save_file(self, src, dest, resumable) -> tuple[bool, int]:
     key = self.get_path_to_file(src)
 
     headers = self.head(src)
@@ -948,20 +950,23 @@ class HttpInterface(StorageInterface):
     if resumable and os.path.exists(partname):
       downloaded_size = os.path.getsize(partname)        
 
+    streamed_bytes = 0
+
     range_headers = { "Range": f"bytes={downloaded_size}-" }
     with self.session.get(key, headers=range_headers, stream=True) as resp:
       if resp.status_code not in [200, 206]:
         resp.raise_for_status()
-        return False
+        return (False, 0)
 
       with open(partname, 'ab') as f:
         for chunk in resp.iter_content(chunk_size=int(10e6)):
           f.write(chunk)
+          streamed_bytes += len(chunk)
 
     if resumable:
       os.rename(partname, fulldest)
 
-    return True
+    return (True, streamed_bytes)
 
   @retry
   def exists(self, file_path):
@@ -1162,10 +1167,17 @@ class S3Interface(StorageInterface):
     if storage_class:
       attrs['StorageClass'] = storage_class
 
-    multipart = hasattr(content, "read") and hasattr(content, "seek")
+    multipart = False
+    is_file_handle = hasattr(content, "read") and hasattr(content, "seek")
+
+    if is_file_handle:
+      content_length = os.fstat(content.fileno()).st_size
+    else:
+      content_length = len(content)
 
-    if not multipart and len(content) > int(self.composite_upload_threshold):
-      content = BytesIO(content)
+    if not multipart and content_length > int(self.composite_upload_threshold):
+      if not is_file_handle:
+        content = BytesIO(content)
       multipart = True
 
     # gevent monkey patching has a bad interaction with s3's use
@@ -1175,19 +1187,18 @@ class S3Interface(StorageInterface):
       multipart = False
       content = content.read()
 
+    # WMS 2025-07-05: 
+    # Currently, boto3 does not properly support streaming smaller files.
+    # It uses an S3 API that requires a checksum up-front, but streaming 
+    # checksums can only be provided at the end.
+    # https://github.com/boto/boto3/issues/3738
+    # https://github.com/boto/boto3/issues/4392
+    # https://docs.aws.amazon.com/sdkref/latest/guide/feature-dataintegrity.html
+    if not multipart and is_file_handle and content_length < int(self.composite_upload_threshold):
+      content = content.read()
+
     if multipart:
       self._conn.upload_fileobj(content, self._path.bucket, key, ExtraArgs=attrs)
-      # upload_fileobj will add 'aws-chunked' to the ContentEncoding,
-      # which after it finishes uploading is useless and messes up our
-      # software. Therefore, edit the metadata and replace it (but this incurs
-      # 2x class-A...)
-      self._conn.copy_object(
-        Bucket=self._path.bucket,
-        Key=key,
-        CopySource={'Bucket': self._path.bucket, 'Key': key},
-        MetadataDirective="REPLACE",
-        **attrs
-      )
     else:
       if isinstance(content, str):
         content = content.encode('utf8')
@@ -1199,7 +1210,7 @@ class S3Interface(StorageInterface):
       self._conn.put_object(**attrs)
 
   @retry
-  def copy_file(self, src_path, dest_bucket_name, dest_key):
+  def copy_file(self, src_path, dest_bucket_name, dest_key) -> tuple[bool,int]:
     key = self.get_path_to_file(src_path)
     s3client = self._get_bucket(dest_bucket_name)
     copy_source = {
@@ -1207,7 +1218,7 @@ class S3Interface(StorageInterface):
       'Key': key,
     }
     try:
-      s3client.copy_object(
+      response = s3client.copy_object(
           CopySource=copy_source,
           Bucket=dest_bucket_name,
           Key=dest_key,
@@ -1215,11 +1226,16 @@ class S3Interface(StorageInterface):
       )
     except botocore.exceptions.ClientError as err: 
       if err.response['Error']['Code'] in ('NoSuchKey', '404'):
-        return False
+        return (False, 0)
       else:
         raise
 
-    return True
+    try:
+      num_bytes = int(response["ResponseMetadata"]["HTTPHeaders"]["content-length"])
+    except KeyError:
+      num_bytes = 0
+
+    return (True, num_bytes)
 
   @retry
   def get_file(self, file_path, start=None, end=None, part_size=None):
@@ -1280,14 +1296,14 @@ class S3Interface(StorageInterface):
         raise
 
   @retry
-  def save_file(self, src, dest, resumable):
+  def save_file(self, src, dest, resumable) -> tuple[bool,int]:
     key = self.get_path_to_file(src)
     kwargs = self._additional_attrs.copy()
 
     resp = self.head(src)
 
     if resp is None:
-      return False
+      return (False, 0)
 
     mkdir(os.path.dirname(dest))
 
@@ -1310,11 +1326,12 @@ class S3Interface(StorageInterface):
       )
     except botocore.exceptions.ClientError as err: 
       if err.response['Error']['Code'] in ('NoSuchKey', '404'):
-        return False
+        return (False, 0)
       else:
         raise
 
-    return True
+    num_bytes = os.path.getsize(dest)
+    return (True, num_bytes)
 
   @retry
   def head(self, file_path):