cloud-files 4.21.0__tar.gz → 4.22.0__tar.gz

{cloud-files-4.21.0 → cloud-files-4.22.0}/.github/workflows/test-suite.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v2

{cloud-files-4.21.0 → cloud-files-4.22.0}/AUTHORS

@@ -2,4 +2,5 @@ Manuel Castro <macastro@princeton.edu>
 Nico Kemnitz <nkemnitz@princeton.edu>
 V24 <55334829+umarfarouk98@users.noreply.github.com>
 William Silversmith <william.silversmith@gmail.com>
+madiganz <madiganz@users.noreply.github.com>
 ranlu <ranlu@users.noreply.github.com>

{cloud-files-4.21.0 → cloud-files-4.22.0}/ChangeLog

@@ -1,6 +1,23 @@
 CHANGES
 =======
 
+4.22.0
+------
+
+* feat: add no\_sign\_request for s3 services
+* ci: remove py37 add py312
+* fixtest: bump moto version and fix backwards incompatible calls
+* fixtest: files present don't interfere with aws credentials
+* Add support for AWS\_SESSION\_TOKEN (#102)
+* perf: faster path extraction via caching computations
+
+4.21.1
+------
+
+* chore: update ChangeLog
+* fix: annotate other callsites with locks
+* perf: fix potential for multi-thread collision when creating pools
+
 4.21.0
 ------
 

{cloud-files-4.21.0 → cloud-files-4.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.21.0
+Version: 4.22.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith
@@ -18,10 +18,35 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.7,<4.0
 Description-Content-Type: text/markdown
-Provides-Extra: test
-Provides-Extra: numpy
 License-File: LICENSE
 License-File: AUTHORS
+Requires-Dist: boto3>=1.4.7
+Requires-Dist: brotli
+Requires-Dist: crc32c
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: click
+Requires-Dist: deflate>=0.2.0
+Requires-Dist: gevent
+Requires-Dist: google-auth>=1.10.0
+Requires-Dist: google-cloud-core>=1.1.0
+Requires-Dist: google-cloud-storage>=1.31.1
+Requires-Dist: google-crc32c>=1.0.0
+Requires-Dist: orjson
+Requires-Dist: pathos
+Requires-Dist: protobuf>=3.3.0
+Requires-Dist: requests>=2.22.0
+Requires-Dist: six>=1.14.0
+Requires-Dist: tenacity>=4.10.0
+Requires-Dist: tqdm
+Requires-Dist: urllib3>=1.26.3
+Requires-Dist: zstandard
+Requires-Dist: rsa>=4.7.2
+Requires-Dist: fasteners
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: moto>=5; extra == "test"
+Provides-Extra: numpy
+Requires-Dist: numpy; extra == "numpy"
 
 [![PyPI version](https://badge.fury.io/py/cloud-files.svg)](https://badge.fury.io/py/cloud-files) [![Test Suite](https://github.com/seung-lab/cloud-files/workflows/Test%20Suite/badge.svg)](https://github.com/seung-lab/cloud-files/actions?query=workflow%3A%22Test+Suite%22)
 

{cloud-files-4.21.0 → cloud-files-4.22.0}/automated_test.py

@@ -4,7 +4,7 @@ import re
 import shutil
 import time
 
-from moto import mock_s3
+from moto import mock_aws
 
 COMPRESSION_TYPES = [ 
   None, False, True,
@@ -36,7 +36,7 @@ def aws_credentials():
 
 @pytest.fixture(scope='function')
 def s3(aws_credentials):
-  with mock_s3():
+  with mock_aws():
     import boto3
     conn = boto3.client('s3', region_name='us-east-1')
     conn.create_bucket(Bucket="cloudfiles")
@@ -44,6 +44,16 @@ def s3(aws_credentials):
     conn.create_bucket(Bucket="cloudfiles_dest")
     yield conn
 
+def test_aws_credentials(aws_credentials):
+  from cloudfiles import secrets
+  expected = {
+    'AWS_ACCESS_KEY_ID': 'testing',
+    'AWS_SECRET_ACCESS_KEY': 'testing',
+    'AWS_SESSION_TOKEN': 'testing',
+    'AWS_DEFAULT_REGION': 'us-east-1',
+  }
+  assert secrets.aws_credentials(skip_files=True) == expected
+
 @pytest.mark.parametrize("green", (False, True))
 @pytest.mark.parametrize("num_threads", (0, 5, 20))
 @pytest.mark.parametrize("protocol", ('mem', 'file', 's3'))#'gs'))

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.21.0
+Version: 4.22.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith
@@ -18,10 +18,35 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.7,<4.0
 Description-Content-Type: text/markdown
-Provides-Extra: test
-Provides-Extra: numpy
 License-File: LICENSE
 License-File: AUTHORS
+Requires-Dist: boto3>=1.4.7
+Requires-Dist: brotli
+Requires-Dist: crc32c
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: click
+Requires-Dist: deflate>=0.2.0
+Requires-Dist: gevent
+Requires-Dist: google-auth>=1.10.0
+Requires-Dist: google-cloud-core>=1.1.0
+Requires-Dist: google-cloud-storage>=1.31.1
+Requires-Dist: google-crc32c>=1.0.0
+Requires-Dist: orjson
+Requires-Dist: pathos
+Requires-Dist: protobuf>=3.3.0
+Requires-Dist: requests>=2.22.0
+Requires-Dist: six>=1.14.0
+Requires-Dist: tenacity>=4.10.0
+Requires-Dist: tqdm
+Requires-Dist: urllib3>=1.26.3
+Requires-Dist: zstandard
+Requires-Dist: rsa>=4.7.2
+Requires-Dist: fasteners
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: moto>=5; extra == "test"
+Provides-Extra: numpy
+Requires-Dist: numpy; extra == "numpy"
 
 [![PyPI version](https://badge.fury.io/py/cloud-files.svg)](https://badge.fury.io/py/cloud-files) [![Test Suite](https://github.com/seung-lab/cloud-files/workflows/Test%20Suite/badge.svg)](https://github.com/seung-lab/cloud-files/actions?query=workflow%3A%22Test+Suite%22)
 

cloud-files-4.22.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "bf417ed", "is_release": true}

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloud_files.egg-info/requires.txt

@@ -26,4 +26,4 @@ numpy
 
 [test]
 pytest
-moto
+moto>=5

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloudfiles/cloudfiles.py

@@ -251,7 +251,8 @@ class CloudFiles:
     parallel:ParallelType = 1, request_payer:Optional[str] = None,
     locking:Optional[bool] = None,
     lock_dir:Optional[str] = None,
-    composite_upload_threshold:int = int(1e8)
+    composite_upload_threshold:int = int(1e8),
+    no_sign_request:bool = False,
   ):
     if use_https:
       cloudpath = paths.to_https_protocol(cloudpath)
@@ -265,6 +266,7 @@ class CloudFiles:
     self.request_payer = request_payer
     self.locking = locking
     self.composite_upload_threshold = composite_upload_threshold
+    self.no_sign_request = bool(no_sign_request)
 
     self._lock_dir = lock_dir
     self._path = paths.extract(cloudpath)
@@ -315,6 +317,7 @@ class CloudFiles:
       locking=self.locking,
       lock_dir=self.lock_dir,
       composite_upload_threshold=self.composite_upload_threshold,
+      no_sign_request=self.no_sign_request,
     )
 
   @property
@@ -456,6 +459,8 @@ class CloudFiles:
     num_threads = self.num_threads
     if self.protocol == "file":
       num_threads = 1
+    elif self.protocol == "mem":
+      num_threads = 0
 
     results = schedule_jobs(
       fns=( partial(download, path) for path in paths ), 

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloudfiles/connectionpools.py

@@ -7,6 +7,9 @@ import time
 from functools import partial
 
 import boto3 
+from botocore import UNSIGNED
+from botocore.config import Config
+
 from google.cloud.storage import Client
 from google.oauth2 import service_account
 import tenacity
@@ -55,13 +58,13 @@ class ConnectionPool(object):
   def _create_connection(self, secrets, endpoint):
     raise NotImplementedError
 
-  def get_connection(self, secrets=None, endpoint=None):
+  def get_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     with self._lock:
       try:        
         conn = self.pool.get(block=False)
         self.pool.task_done()
       except queue.Empty:
-        conn = self._create_connection(secrets, endpoint)
+        conn = self._create_connection(secrets, endpoint, no_sign_request)
       finally:
         self.outstanding += 1
 
@@ -103,7 +106,7 @@ class S3ConnectionPool(ConnectionPool):
     super(S3ConnectionPool, self).__init__()
 
   @retry
-  def _create_connection(self, secrets=None, endpoint=None):
+  def _create_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     if secrets is None:
       secrets = self.credentials
     if isinstance(secrets, str):
@@ -113,11 +116,17 @@ class S3ConnectionPool(ConnectionPool):
     if endpoint is not None:
       additional_args['endpoint_url'] = endpoint
 
+    config = None
+    if no_sign_request:
+      config = Config(signature_version=UNSIGNED)
+
     return boto3.client(
       's3',
       aws_access_key_id=secrets.get('AWS_ACCESS_KEY_ID', None),
       aws_secret_access_key=secrets.get('AWS_SECRET_ACCESS_KEY', None),
+      aws_session_token=secrets.get('AWS_SESSION_TOKEN', None),
       region_name=secrets.get('AWS_DEFAULT_REGION', 'us-east-1'),
+      config=config,
       **additional_args
     )
       
@@ -135,7 +144,7 @@ class GCloudBucketPool(ConnectionPool):
     super(GCloudBucketPool, self).__init__()
 
   @retry
-  def _create_connection(self, secrets=None, endpoint=None):
+  def _create_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     if secrets is None:
       secrets = self.credentials
     
@@ -162,7 +171,7 @@ class MemoryPool(ConnectionPool):
     self.data = MEMORY_DATA
     super(MemoryPool, self).__init__()
 
-  def _create_connection(self, secrets=None, endpoint=None):
+  def _create_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     if self.bucket not in self.data:
       self.data[self.bucket] = {}
     return self.data[self.bucket]

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloudfiles/interfaces.py

@@ -50,14 +50,27 @@ S3ConnectionPoolParams = namedtuple('S3ConnectionPoolParams', 'service bucket_na
 GCloudBucketPoolParams = namedtuple('GCloudBucketPoolParams', 'bucket_name request_payer')
 MemoryPoolParams = namedtuple('MemoryPoolParams', 'bucket_name')
 
+GCS_BUCKET_POOL_LOCK = threading.Lock()
+S3_BUCKET_POOL_LOCK = threading.Lock()
+MEM_BUCKET_POOL_LOCK = threading.Lock()
+
 def reset_connection_pools():
   global S3_POOL
   global GC_POOL
   global MEM_POOL
-  S3_POOL = keydefaultdict(lambda params: S3ConnectionPool(params.service, params.bucket_name))
-  GC_POOL = keydefaultdict(lambda params: GCloudBucketPool(params.bucket_name, params.request_payer))
-  MEM_POOL = keydefaultdict(lambda params: MemoryPool(params.bucket_name))
-  MEMORY_DATA.clear()
+  global GCS_BUCKET_POOL_LOCK
+  global S3_BUCKET_POOL_LOCK
+  global MEM_BUCKET_POOL_LOCK
+
+  with S3_BUCKET_POOL_LOCK:
+    S3_POOL = keydefaultdict(lambda params: S3ConnectionPool(params.service, params.bucket_name))
+
+  with GCS_BUCKET_POOL_LOCK:
+    GC_POOL = keydefaultdict(lambda params: GCloudBucketPool(params.bucket_name, params.request_payer))
+  
+  with MEM_BUCKET_POOL_LOCK:
+    MEM_POOL = keydefaultdict(lambda params: MemoryPool(params.bucket_name))
+    MEMORY_DATA.clear()
   import gc
   gc.collect()
 
@@ -340,14 +353,19 @@ class FileInterface(StorageInterface):
     filenames = list(map(stripext, filenames))
     filenames.sort()
     return iter(filenames)
-    
+  
 class MemoryInterface(StorageInterface):
   def __init__(self, path, secrets=None, request_payer=None, **kwargs):
+    global MEM_BUCKET_POOL_LOCK
+
     super(StorageInterface, self).__init__()
     if request_payer is not None:
       raise ValueError("Specifying a request payer for the MemoryInterface is not supported. request_payer must be None, got '{}'.", request_payer)
     self._path = path
-    self._data = MEM_POOL[MemoryPoolParams(path.bucket)].get_connection(secrets, None)
+
+    with MEM_BUCKET_POOL_LOCK:
+      pool = MEM_POOL[MemoryPoolParams(path.bucket)]
+    self._data = pool.get_connection(secrets, None)
 
   def get_path_to_file(self, file_path):
     return posixpath.join(
@@ -498,9 +516,13 @@ class GoogleCloudStorageInterface(StorageInterface):
   def __init__(self, path, secrets=None, request_payer=None, **kwargs):
     super(StorageInterface, self).__init__()
     global GC_POOL
+    global GCS_BUCKET_POOL_LOCK
     self._path = path
     self._request_payer = request_payer
-    self._bucket = GC_POOL[GCloudBucketPoolParams(self._path.bucket, self._request_payer)].get_connection(secrets, None)
+
+    with GCS_BUCKET_POOL_LOCK:
+      pool = GC_POOL[GCloudBucketPoolParams(self._path.bucket, self._request_payer)]
+    self._bucket = pool.get_connection(secrets, None)
     self._secrets = secrets
 
   def get_path_to_file(self, file_path):
@@ -537,7 +559,9 @@ class GoogleCloudStorageInterface(StorageInterface):
   def copy_file(self, src_path, dest_bucket, dest_key):
     key = self.get_path_to_file(src_path)
     source_blob = self._bucket.blob( key )
-    dest_bucket = GC_POOL[GCloudBucketPoolParams(dest_bucket, self._request_payer)].get_connection(self._secrets, None)
+    with GCS_BUCKET_POOL_LOCK:
+     pool = GC_POOL[GCloudBucketPoolParams(dest_bucket, self._request_payer)]
+    dest_bucket = pool.get_connection(self._secrets, None)
     self._bucket.copy_blob(
       source_blob, dest_bucket, dest_key
     )
@@ -664,7 +688,9 @@ class GoogleCloudStorageInterface(StorageInterface):
 
   def release_connection(self):
     global GC_POOL
-    GC_POOL[GCloudBucketPoolParams(self._path.bucket, self._request_payer)].release_connection(self._bucket)
+    with GCS_BUCKET_POOL_LOCK:
+      pool = GC_POOL[GCloudBucketPoolParams(self._path.bucket, self._request_payer)]
+    pool.release_connection(self._bucket)
 
 class HttpInterface(StorageInterface):
   def __init__(self, path, secrets=None, request_payer=None, **kwargs):
@@ -780,7 +806,13 @@ class S3Interface(StorageInterface):
   # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Bucket.delete_objects
   # claims batch size limit is 1000
   delete_batch_size = 1000
-  def __init__(self, path, secrets=None, request_payer=None, composite_upload_threshold=int(1e8), **kwargs):
+  def __init__(
+    self, path, secrets=None, 
+    request_payer=None, 
+    composite_upload_threshold=int(1e8), 
+    no_sign_request=False,
+    **kwargs
+  ):
     super(StorageInterface, self).__init__()
     global S3_POOL
 
@@ -797,12 +829,16 @@ class S3Interface(StorageInterface):
     self._conn = self._get_bucket(path.bucket)
 
     self.composite_upload_threshold = composite_upload_threshold
+    self.no_sign_request = no_sign_request
 
   def _get_bucket(self, bucket_name):
+    global S3_BUCKET_POOL_LOCK
     service = self._path.alias or 's3'
-    return S3_POOL[S3ConnectionPoolParams(service, bucket_name, self._request_payer)].get_connection(
-      self._secrets, self._path.host
-    )
+
+    with S3_BUCKET_POOL_LOCK:
+      pool = S3_POOL[S3ConnectionPoolParams(service, bucket_name, self._request_payer)]
+    
+    return pool.get_connection(self._secrets, self._path.host)
 
   def get_path_to_file(self, file_path):
     return posixpath.join(self._path.path, file_path)
@@ -1074,4 +1110,6 @@ class S3Interface(StorageInterface):
   def release_connection(self):
     global S3_POOL
     service = self._path.alias or 's3'
-    S3_POOL[S3ConnectionPoolParams(service, self._path.bucket, self._request_payer)].release_connection(self._conn)
+    with S3_BUCKET_POOL_LOCK:
+      pool = S3_POOL[S3ConnectionPoolParams(service, self._path.bucket, self._request_payer)]
+    pool.release_connection(self._conn)

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloudfiles/paths.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 from collections import namedtuple
 import orjson
 import os.path
@@ -295,6 +296,7 @@ def extract_format_protocol(cloudpath:str) -> tuple:
 
   return (fmt, proto, endpoint, cloudpath, alias)
 
+@lru_cache(maxsize=10, typed=False)
 def extract(cloudpath:str, windows=None) -> ExtractedPath:
   """
   Given a valid cloudpath of the form 

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloudfiles/secrets.py

@@ -96,7 +96,7 @@ def google_credentials(bucket = ''):
 
 AWS_CREDENTIALS_CACHE:CredentialCacheType = defaultdict(dict)
 aws_credentials_path = secretpath('aws-secret.json')
-def aws_credentials(bucket = '', service = 'aws'):
+def aws_credentials(bucket = '', service = 'aws', skip_files=False):
   global AWS_CREDENTIALS_CACHE
 
   if service == 's3':
@@ -115,12 +115,13 @@ def aws_credentials(bucket = '', service = 'aws'):
     paths = [ secretpath('{}-{}-secret.json'.format(bucket, service)) ] + paths
 
   aws_credentials = {}
-  aws_credentials_path = secretpath(default_file_path)
-  for aws_credentials_path in paths:
-    if os.path.exists(aws_credentials_path):
-      with open(aws_credentials_path, 'r') as f:
-        aws_credentials = json.loads(f.read())
-      break
+  if not skip_files:
+    aws_credentials_path = secretpath(default_file_path)
+    for aws_credentials_path in paths:
+      if os.path.exists(aws_credentials_path):
+        with open(aws_credentials_path, 'r') as f:
+          aws_credentials = json.loads(f.read())
+        break
   
   if not aws_credentials:
     # did not find any secret json file, will try to find it in environment variables
@@ -128,6 +129,7 @@ def aws_credentials(bucket = '', service = 'aws'):
       aws_credentials = {
         'AWS_ACCESS_KEY_ID': os.environ['AWS_ACCESS_KEY_ID'],
         'AWS_SECRET_ACCESS_KEY': os.environ['AWS_SECRET_ACCESS_KEY'],
+        'AWS_SESSION_TOKEN': os.environ['AWS_SESSION_TOKEN'],
       }
     if 'AWS_DEFAULT_REGION' in os.environ:
       aws_credentials['AWS_DEFAULT_REGION'] = os.environ['AWS_DEFAULT_REGION']

{cloud-files-4.21.0 → cloud-files-4.22.0}/cloudfiles_cli/cloudfiles_cli.py

@@ -172,11 +172,12 @@ def get_mfp(path, recursive):
 @click.option('--progress', is_flag=True, default=False, help="Show transfer progress.", show_default=True)
 @click.option('-b', '--block-size', default=128, help="Number of files to download at a time.", show_default=True)
 @click.option('--part-bytes', default=int(1e8), help="Composite upload threshold in bytes. Splits a file into pieces for some cloud services like gs and s3.", show_default=True)
+@click.option('--no-sign-request', is_flag=True, default=False, help="Use s3 in anonymous mode (don't sign requests) for the source.", show_default=True)
 @click.pass_context
 def cp(
   ctx, source, destination, 
   recursive, compression, progress, 
-  block_size, part_bytes
+  block_size, part_bytes, no_sign_request,
 ):
   """
   Copy one or more files from a source to destination.
@@ -194,9 +195,17 @@ def cp(
     return
 
   for src in source:
-    _cp_single(ctx, src, destination, recursive, compression, progress, block_size, part_bytes)
-
-def _cp_single(ctx, source, destination, recursive, compression, progress, block_size, part_bytes):
+    _cp_single(
+      ctx, src, destination, recursive, 
+      compression, progress, block_size, 
+      part_bytes, no_sign_request
+    )
+
+def _cp_single(
+  ctx, source, destination, recursive, 
+  compression, progress, block_size,
+  part_bytes, no_sign_request
+):
   use_stdin = (source == '-')
   use_stdout = (destination == '-')
 
@@ -243,7 +252,9 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
     xferpaths = [ x.replace(prefix, "") for x in xferpaths ]
     srcpath = cloudpathjoin(srcpath, prefix)
   elif many:
-    xferpaths = CloudFiles(srcpath).list(prefix=prefix, flat=flat)
+    xferpaths = CloudFiles(
+      srcpath, no_sign_request=no_sign_request
+    ).list(prefix=prefix, flat=flat)
 
   destpath = ndest
   if isinstance(xferpaths, str):
@@ -260,7 +271,7 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
 
   if not isinstance(xferpaths, str):
     if parallel == 1:
-      _cp(srcpath, destpath, compression, progress, block_size, part_bytes, xferpaths)
+      _cp(srcpath, destpath, compression, progress, block_size, part_bytes, no_sign_request, xferpaths)
       return 
 
     total = None
@@ -270,16 +281,16 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
       pass
 
     if use_stdout:
-      fn = partial(_cp_stdout, srcpath)
+      fn = partial(_cp_stdout, no_sign_request, srcpath)
     else:
-      fn = partial(_cp, srcpath, destpath, compression, False, block_size, part_bytes)
+      fn = partial(_cp, srcpath, destpath, compression, False, block_size, part_bytes, no_sign_request)
 
     with tqdm(desc="Transferring", total=total, disable=(not progress)) as pbar:
       with pathos.pools.ProcessPool(parallel) as executor:
         for _ in executor.imap(fn, sip(xferpaths, block_size)):
           pbar.update(block_size)
   else:
-    cfsrc = CloudFiles(srcpath, progress=progress)
+    cfsrc = CloudFiles(srcpath, progress=progress, no_sign_request=no_sign_request)
     if not cfsrc.exists(xferpaths):
       print(f"cloudfiles: source path not found: {cfsrc.abspath(xferpaths).replace('file://','')}")
       return
@@ -288,7 +299,11 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
       _cp_stdout(srcpath, xferpaths)
       return
 
-    cfdest = CloudFiles(destpath, progress=progress, composite_upload_threshold=part_bytes)
+    cfdest = CloudFiles(
+      destpath, 
+      progress=progress, 
+      composite_upload_threshold=part_bytes,
+    )
 
     if isdestdir:
       new_path = os.path.basename(nsrc)
@@ -300,17 +315,17 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
       "dest_path": new_path,
     }], reencode=compression)
 
-def _cp(src, dst, compression, progress, block_size, part_bytes, paths):
-  cfsrc = CloudFiles(src, progress=progress, composite_upload_threshold=part_bytes)
+def _cp(src, dst, compression, progress, block_size, part_bytes, no_sign_request, paths):
+  cfsrc = CloudFiles(src, progress=progress, composite_upload_threshold=part_bytes, no_sign_request=no_sign_request)
   cfdest = CloudFiles(dst, progress=progress, composite_upload_threshold=part_bytes)
   cfsrc.transfer_to(
     cfdest, paths=paths, 
     reencode=compression, block_size=block_size
   )
 
-def _cp_stdout(src, paths):
+def _cp_stdout(src, no_sign_request, paths):
   paths = toiter(paths)
-  cf = CloudFiles(src, progress=False)
+  cf = CloudFiles(src, progress=False, no_sign_request=no_sign_request)
   for res in cf.get(paths):
     content = res["content"].decode("utf8")
     sys.stdout.write(content)
@@ -372,7 +387,8 @@ def xferexecute(db, progress, lease_msec, block_size):
 @main.command()
 @click.argument("sources", nargs=-1)
 @click.option('-r', '--range', 'byte_range', default=None, help='Retrieve start-end bytes.')
-def cat(sources, byte_range):
+@click.option('--no-sign-request', is_flag=True, default=False, help="Use s3 in anonymous mode (don't sign requests) for the source.", show_default=True)
+def cat(sources, byte_range, no_sign_request):
   """Concatenate the contents of each input file and write to stdout."""
   if '-' in sources and len(sources) == 1:
     sources = sys.stdin.readlines()
@@ -386,7 +402,7 @@ def cat(sources, byte_range):
     byte_range[0] = int(byte_range[0] or 0)
     byte_range[1] = int(byte_range[1]) if byte_range[1] not in ("", None) else None
     src = normalize_path(sources[0])
-    cf = CloudFiles(os.path.dirname(src))
+    cf = CloudFiles(os.path.dirname(src), no_sign_request=no_sign_request)
     download = cf[os.path.basename(src), byte_range[0]:byte_range[1]]
     if download is None:
       print(f'cloudfiles: {src} does not exist')
@@ -397,7 +413,7 @@ def cat(sources, byte_range):
   for srcs in sip(sources, 10):
     srcs = [ normalize_path(src) for src in srcs ]
     order = { src: i for i, src in enumerate(srcs) }
-    files = cloudfiles.dl(srcs)
+    files = cloudfiles.dl(srcs, no_sign_request=no_sign_request)
     output = [ None for _ in range(len(srcs)) ]
     for res in files:
       if res["content"] is None:

{cloud-files-4.21.0 → cloud-files-4.22.0}/setup.cfg

@@ -31,7 +31,7 @@ packages =
 [extras]
 test = 
 	pytest
-	moto
+	moto>=5
 numpy = 
 	numpy
 

cloud-files-4.21.0/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "bee46e6", "is_release": true}