cloud-files 6.4.0__tar.gz → 6.4.2__tar.gz

{cloud_files-6.4.0 → cloud_files-6.4.2}/ChangeLog

@@ -1,6 +1,19 @@
 CHANGES
 =======
 
+6.4.2
+-----
+
+* fix: Dell ECS does not implement the correct checksum behavior for multipart files
+* fix: also consider crc32c
+* docs: mention comparison of byte sizes
+* fix: use same locking filename for reading and writing (#132)
+
+6.4.1
+-----
+
+* refactor: control default threads from CloudFiles not multiple globals
+
 6.4.0
 -----
 

{cloud_files-6.4.0 → cloud_files-6.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 6.4.0
+Version: 6.4.2
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-6.4.0 → cloud_files-6.4.2}/automated_test.py

@@ -146,6 +146,28 @@ def test_read_write_py_objects_mem():
 
   cf.delete(["my_set", "my_set2"])
 
+def test_reader_and_writer_share_a_lock():
+  import os, tempfile
+  import cloudfiles.interfaces as cfi
+  from cloudfiles import CloudFiles
+
+  seen = []
+  real = cfi.fasteners.InterProcessReaderWriterLock
+
+  class Recording(real):
+    def __init__(self, path, *a, **k):
+      seen.append(os.path.basename(path))
+      super().__init__(path, *a, **k)
+
+  cfi.fasteners.InterProcessReaderWriterLock = Recording
+  try:
+    cf = CloudFiles("file://" + tempfile.mkdtemp(), locking=True)
+    cf.put("chunk", b"data" * 100, compress="gzip")
+    cf.get("chunk")
+    assert seen[0] == seen[-1], f"write locked {seen[0]}, read locked {seen[-1]}"
+  finally:
+    cfi.fasteners.InterProcessReaderWriterLock = real
+
 @pytest.mark.parametrize("protocol", ('mem', 'file', 's3'))#'gs'))
 def test_get_json_order(s3, protocol):
   from cloudfiles import CloudFiles

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 6.4.0
+Version: 6.4.2
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud_files-6.4.2/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "fff3997", "is_release": true}

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloudfiles/cloudfiles.py

@@ -39,7 +39,7 @@ from .lib import (
 from .monitoring import TransmissionMonitor, IOEnum
 from .paths import ALIASES, find_common_buckets
 from .secrets import CLOUD_FILES_DIR, CLOUD_FILES_LOCK_DIR
-from .threaded_queue import ThreadedQueue, DEFAULT_THREADS
+from .threaded_queue import ThreadedQueue
 from .typing import (
   CompressType, GetPathType, PutScalarType,
   PutType, ParallelType, SecretsType

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloudfiles/connectionpools.py

@@ -116,9 +116,17 @@ class S3ConnectionPool(ConnectionPool):
     if endpoint is not None:
       additional_args['endpoint_url'] = endpoint
 
-    config = None
+    config = {}
     if no_sign_request:
-      config = Config(signature_version=UNSIGNED)
+      config['signature_version'] = UNSIGNED
+
+    if endpoint is not None:
+      config['request_checksum_calculation'] = "when_required"
+    
+    if len(config) == 0:
+      config = None
+    else:
+      config = Config(**config)
 
     return boto3.client(
       's3',

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloudfiles/interfaces.py

@@ -208,6 +208,7 @@ class FileInterface(StorageInterface):
     storage_class=None
   ):
     path = self.get_path_to_file(file_path)
+    lock_path = path
     compress_ext = self.get_extension(compress)
     _, ext = os.path.splitext(path)
     if ext != compress_ext:
@@ -236,7 +237,7 @@ class FileInterface(StorageInterface):
         with open(path, 'wb') as f:
           f.write(content)
 
-    return self.io_with_lock(do_put_file, path, exclusive=True)
+    return self.io_with_lock(do_put_file, lock_path, exclusive=True)
 
   def _try_extensions(self, file_path:str, fn:Callable, null_return:Any):
     global EXT_TEST_SEQUENCE

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloudfiles/scheduler.py

@@ -3,13 +3,11 @@ import sys
 import gevent.monkey
 from tqdm import tqdm
 
-from .threaded_queue import ThreadedQueue, DEFAULT_THREADS
+from .threaded_queue import ThreadedQueue
 from .lib import totalfn
 
-DEFAULT_THREADS = 20
-
 def schedule_threaded_jobs(
-    fns, concurrency=DEFAULT_THREADS, 
+    fns, concurrency, 
     progress=None, total=None, count_return=False
   ):
 
@@ -42,7 +40,7 @@ def schedule_threaded_jobs(
   return results
 
 def schedule_green_jobs(
-    fns, concurrency=DEFAULT_THREADS, 
+    fns, concurrency, 
     progress=None, total=None, count_return=False
   ):
   import gevent.pool
@@ -114,7 +112,7 @@ def schedule_single_threaded_jobs(
   return results 
 
 def schedule_jobs(
-    fns, concurrency=DEFAULT_THREADS, 
+    fns, concurrency, 
     progress=None, total=None, green=None,
     count_return=False
   ):

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloudfiles/threaded_queue.py

@@ -5,8 +5,6 @@ import time
 
 from tqdm import tqdm
 
-DEFAULT_THREADS = 20
-
 class ThreadedQueue(object):
   """Grant threaded task processing to any derived class."""
   def __init__(self, n_threads, queue_size=0, progress=None):

{cloud_files-6.4.0 → cloud_files-6.4.2}/cloudfiles_cli/cloudfiles_cli.py

@@ -1024,9 +1024,11 @@ def populate_md5(
 @click.option('--part-size', default=None, help="For S3, compute multipart hashes using this part size in bytes.", show_default=True)
 def verify(source, target, only_matching, verbose, md5, multipart_threshold, part_size):
   """
-  Validates checksums of two files or two directories 
+  Compares source and target directories to check
+  if filenames and sizes match. Optionally it can
+  validate if the checksums of two files or two directories
   match. These tags are usually either md5 or crc32c 
-  generated strings. These are not secure hashes so they 
+  generated strings. These are not secure hashes so they
   will only catch accidental changes to files, not 
   intentionally malicious changes.
   """
@@ -1097,16 +1099,20 @@ def verify(source, target, only_matching, verbose, md5, multipart_threshold, par
       continue
     elif not (
       (
-        (sm["ETag"] and tm["ETag"])
+        (sm.get("ETag", None) and tm.get("ETag", None))
         and (
           sm["ETag"] == tm["ETag"]
           or md5_equal(sm["ETag"], tm["ETag"])
         )
       )
       or (
-        sm["Content-Md5"] and tm["Content-Md5"]
+        (sm.get("Content-Md5", None) and tm.get("Content-Md5", None))
         and md5_equal(sm["Content-Md5"], tm["Content-Md5"])
       )
+      or (
+        (sm.get("Content-Crc32c", None) and tm.get("Content-Crc32c", None))
+        and sm["Content-Crc32c"] == tm["Content-Crc32c"]
+      )
     ):
       failed_files.append(filename)
       continue

cloud_files-6.4.0/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "f7253bf", "is_release": true}