cloud-files 4.21.1__tar.gz → 4.22.0__tar.gz

{cloud-files-4.21.1 → cloud-files-4.22.0}/.github/workflows/test-suite.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v2

{cloud-files-4.21.1 → cloud-files-4.22.0}/AUTHORS

@@ -2,4 +2,5 @@ Manuel Castro <macastro@princeton.edu>
 Nico Kemnitz <nkemnitz@princeton.edu>
 V24 <55334829+umarfarouk98@users.noreply.github.com>
 William Silversmith <william.silversmith@gmail.com>
+madiganz <madiganz@users.noreply.github.com>
 ranlu <ranlu@users.noreply.github.com>

{cloud-files-4.21.1 → cloud-files-4.22.0}/ChangeLog

@@ -1,6 +1,16 @@
 CHANGES
 =======
 
+4.22.0
+------
+
+* feat: add no\_sign\_request for s3 services
+* ci: remove py37 add py312
+* fixtest: bump moto version and fix backwards incompatible calls
+* fixtest: files present don't interfere with aws credentials
+* Add support for AWS\_SESSION\_TOKEN (#102)
+* perf: faster path extraction via caching computations
+
 4.21.1
 ------
 

{cloud-files-4.21.1 → cloud-files-4.22.0}/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.21.1
+Version: 4.22.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith
 Author-email: ws9@princeton.edu
 License: License :: OSI Approved :: BSD License
-Platform: UNKNOWN
 Classifier: Intended Audience :: Developers
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: BSD License
@@ -19,10 +18,35 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.7,<4.0
 Description-Content-Type: text/markdown
-Provides-Extra: test
-Provides-Extra: numpy
 License-File: LICENSE
 License-File: AUTHORS
+Requires-Dist: boto3>=1.4.7
+Requires-Dist: brotli
+Requires-Dist: crc32c
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: click
+Requires-Dist: deflate>=0.2.0
+Requires-Dist: gevent
+Requires-Dist: google-auth>=1.10.0
+Requires-Dist: google-cloud-core>=1.1.0
+Requires-Dist: google-cloud-storage>=1.31.1
+Requires-Dist: google-crc32c>=1.0.0
+Requires-Dist: orjson
+Requires-Dist: pathos
+Requires-Dist: protobuf>=3.3.0
+Requires-Dist: requests>=2.22.0
+Requires-Dist: six>=1.14.0
+Requires-Dist: tenacity>=4.10.0
+Requires-Dist: tqdm
+Requires-Dist: urllib3>=1.26.3
+Requires-Dist: zstandard
+Requires-Dist: rsa>=4.7.2
+Requires-Dist: fasteners
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: moto>=5; extra == "test"
+Provides-Extra: numpy
+Requires-Dist: numpy; extra == "numpy"
 
 [![PyPI version](https://badge.fury.io/py/cloud-files.svg)](https://badge.fury.io/py/cloud-files) [![Test Suite](https://github.com/seung-lab/cloud-files/workflows/Test%20Suite/badge.svg)](https://github.com/seung-lab/cloud-files/actions?query=workflow%3A%22Test+Suite%22)
 
@@ -501,5 +525,3 @@ CloudFiles is derived from the [CloudVolume.Storage](https://github.com/seung-la
 
 Storage was initially created by William Silversmith and Ignacio Tartavull. It was maintained and improved by William Silversmith and includes improvements by Nico Kemnitz (extremely fast exists), Ben Falk (brotli), and Ran Lu (local file locking). Manuel Castro added the ability to chose cloud storage class. Thanks to the anonymous author from https://teppen.io/ for their s3 etag validation code.
 
-
-

{cloud-files-4.21.1 → cloud-files-4.22.0}/automated_test.py

@@ -4,7 +4,7 @@ import re
 import shutil
 import time
 
-from moto import mock_s3
+from moto import mock_aws
 
 COMPRESSION_TYPES = [ 
   None, False, True,
@@ -36,7 +36,7 @@ def aws_credentials():
 
 @pytest.fixture(scope='function')
 def s3(aws_credentials):
-  with mock_s3():
+  with mock_aws():
     import boto3
     conn = boto3.client('s3', region_name='us-east-1')
     conn.create_bucket(Bucket="cloudfiles")
@@ -44,6 +44,16 @@ def s3(aws_credentials):
     conn.create_bucket(Bucket="cloudfiles_dest")
     yield conn
 
+def test_aws_credentials(aws_credentials):
+  from cloudfiles import secrets
+  expected = {
+    'AWS_ACCESS_KEY_ID': 'testing',
+    'AWS_SECRET_ACCESS_KEY': 'testing',
+    'AWS_SESSION_TOKEN': 'testing',
+    'AWS_DEFAULT_REGION': 'us-east-1',
+  }
+  assert secrets.aws_credentials(skip_files=True) == expected
+
 @pytest.mark.parametrize("green", (False, True))
 @pytest.mark.parametrize("num_threads", (0, 5, 20))
 @pytest.mark.parametrize("protocol", ('mem', 'file', 's3'))#'gs'))

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloud_files.egg-info/PKG-INFO

@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.21.1
+Version: 4.22.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith
 Author-email: ws9@princeton.edu
 License: License :: OSI Approved :: BSD License
-Platform: UNKNOWN
 Classifier: Intended Audience :: Developers
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: BSD License
@@ -19,10 +18,35 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.7,<4.0
 Description-Content-Type: text/markdown
-Provides-Extra: test
-Provides-Extra: numpy
 License-File: LICENSE
 License-File: AUTHORS
+Requires-Dist: boto3>=1.4.7
+Requires-Dist: brotli
+Requires-Dist: crc32c
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: click
+Requires-Dist: deflate>=0.2.0
+Requires-Dist: gevent
+Requires-Dist: google-auth>=1.10.0
+Requires-Dist: google-cloud-core>=1.1.0
+Requires-Dist: google-cloud-storage>=1.31.1
+Requires-Dist: google-crc32c>=1.0.0
+Requires-Dist: orjson
+Requires-Dist: pathos
+Requires-Dist: protobuf>=3.3.0
+Requires-Dist: requests>=2.22.0
+Requires-Dist: six>=1.14.0
+Requires-Dist: tenacity>=4.10.0
+Requires-Dist: tqdm
+Requires-Dist: urllib3>=1.26.3
+Requires-Dist: zstandard
+Requires-Dist: rsa>=4.7.2
+Requires-Dist: fasteners
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: moto>=5; extra == "test"
+Provides-Extra: numpy
+Requires-Dist: numpy; extra == "numpy"
 
 [![PyPI version](https://badge.fury.io/py/cloud-files.svg)](https://badge.fury.io/py/cloud-files) [![Test Suite](https://github.com/seung-lab/cloud-files/workflows/Test%20Suite/badge.svg)](https://github.com/seung-lab/cloud-files/actions?query=workflow%3A%22Test+Suite%22)
 
@@ -501,5 +525,3 @@ CloudFiles is derived from the [CloudVolume.Storage](https://github.com/seung-la
 
 Storage was initially created by William Silversmith and Ignacio Tartavull. It was maintained and improved by William Silversmith and includes improvements by Nico Kemnitz (extremely fast exists), Ben Falk (brotli), and Ran Lu (local file locking). Manuel Castro added the ability to chose cloud storage class. Thanks to the anonymous author from https://teppen.io/ for their s3 etag validation code.
 
-
-

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloud_files.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 cloudfiles = cloudfiles_cli:main
-

cloud-files-4.22.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "bf417ed", "is_release": true}

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloud_files.egg-info/requires.txt

@@ -26,4 +26,4 @@ numpy
 
 [test]
 pytest
-moto
+moto>=5

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloudfiles/cloudfiles.py

@@ -251,7 +251,8 @@ class CloudFiles:
     parallel:ParallelType = 1, request_payer:Optional[str] = None,
     locking:Optional[bool] = None,
     lock_dir:Optional[str] = None,
-    composite_upload_threshold:int = int(1e8)
+    composite_upload_threshold:int = int(1e8),
+    no_sign_request:bool = False,
   ):
     if use_https:
       cloudpath = paths.to_https_protocol(cloudpath)
@@ -265,6 +266,7 @@ class CloudFiles:
     self.request_payer = request_payer
     self.locking = locking
     self.composite_upload_threshold = composite_upload_threshold
+    self.no_sign_request = bool(no_sign_request)
 
     self._lock_dir = lock_dir
     self._path = paths.extract(cloudpath)
@@ -315,6 +317,7 @@ class CloudFiles:
       locking=self.locking,
       lock_dir=self.lock_dir,
       composite_upload_threshold=self.composite_upload_threshold,
+      no_sign_request=self.no_sign_request,
     )
 
   @property
@@ -456,6 +459,8 @@ class CloudFiles:
     num_threads = self.num_threads
     if self.protocol == "file":
       num_threads = 1
+    elif self.protocol == "mem":
+      num_threads = 0
 
     results = schedule_jobs(
       fns=( partial(download, path) for path in paths ), 

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloudfiles/connectionpools.py

@@ -7,6 +7,9 @@ import time
 from functools import partial
 
 import boto3 
+from botocore import UNSIGNED
+from botocore.config import Config
+
 from google.cloud.storage import Client
 from google.oauth2 import service_account
 import tenacity
@@ -55,13 +58,13 @@ class ConnectionPool(object):
   def _create_connection(self, secrets, endpoint):
     raise NotImplementedError
 
-  def get_connection(self, secrets=None, endpoint=None):
+  def get_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     with self._lock:
       try:        
         conn = self.pool.get(block=False)
         self.pool.task_done()
       except queue.Empty:
-        conn = self._create_connection(secrets, endpoint)
+        conn = self._create_connection(secrets, endpoint, no_sign_request)
       finally:
         self.outstanding += 1
 
@@ -103,7 +106,7 @@ class S3ConnectionPool(ConnectionPool):
     super(S3ConnectionPool, self).__init__()
 
   @retry
-  def _create_connection(self, secrets=None, endpoint=None):
+  def _create_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     if secrets is None:
       secrets = self.credentials
     if isinstance(secrets, str):
@@ -113,11 +116,17 @@ class S3ConnectionPool(ConnectionPool):
     if endpoint is not None:
       additional_args['endpoint_url'] = endpoint
 
+    config = None
+    if no_sign_request:
+      config = Config(signature_version=UNSIGNED)
+
     return boto3.client(
       's3',
       aws_access_key_id=secrets.get('AWS_ACCESS_KEY_ID', None),
       aws_secret_access_key=secrets.get('AWS_SECRET_ACCESS_KEY', None),
+      aws_session_token=secrets.get('AWS_SESSION_TOKEN', None),
       region_name=secrets.get('AWS_DEFAULT_REGION', 'us-east-1'),
+      config=config,
       **additional_args
     )
       
@@ -135,7 +144,7 @@ class GCloudBucketPool(ConnectionPool):
     super(GCloudBucketPool, self).__init__()
 
   @retry
-  def _create_connection(self, secrets=None, endpoint=None):
+  def _create_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     if secrets is None:
       secrets = self.credentials
     
@@ -162,7 +171,7 @@ class MemoryPool(ConnectionPool):
     self.data = MEMORY_DATA
     super(MemoryPool, self).__init__()
 
-  def _create_connection(self, secrets=None, endpoint=None):
+  def _create_connection(self, secrets=None, endpoint=None, no_sign_request=False):
     if self.bucket not in self.data:
       self.data[self.bucket] = {}
     return self.data[self.bucket]

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloudfiles/interfaces.py

@@ -806,7 +806,13 @@ class S3Interface(StorageInterface):
   # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Bucket.delete_objects
   # claims batch size limit is 1000
   delete_batch_size = 1000
-  def __init__(self, path, secrets=None, request_payer=None, composite_upload_threshold=int(1e8), **kwargs):
+  def __init__(
+    self, path, secrets=None, 
+    request_payer=None, 
+    composite_upload_threshold=int(1e8), 
+    no_sign_request=False,
+    **kwargs
+  ):
     super(StorageInterface, self).__init__()
     global S3_POOL
 
@@ -823,6 +829,7 @@ class S3Interface(StorageInterface):
     self._conn = self._get_bucket(path.bucket)
 
     self.composite_upload_threshold = composite_upload_threshold
+    self.no_sign_request = no_sign_request
 
   def _get_bucket(self, bucket_name):
     global S3_BUCKET_POOL_LOCK

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloudfiles/paths.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 from collections import namedtuple
 import orjson
 import os.path
@@ -295,6 +296,7 @@ def extract_format_protocol(cloudpath:str) -> tuple:
 
   return (fmt, proto, endpoint, cloudpath, alias)
 
+@lru_cache(maxsize=10, typed=False)
 def extract(cloudpath:str, windows=None) -> ExtractedPath:
   """
   Given a valid cloudpath of the form 

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloudfiles/secrets.py

@@ -96,7 +96,7 @@ def google_credentials(bucket = ''):
 
 AWS_CREDENTIALS_CACHE:CredentialCacheType = defaultdict(dict)
 aws_credentials_path = secretpath('aws-secret.json')
-def aws_credentials(bucket = '', service = 'aws'):
+def aws_credentials(bucket = '', service = 'aws', skip_files=False):
   global AWS_CREDENTIALS_CACHE
 
   if service == 's3':
@@ -115,12 +115,13 @@ def aws_credentials(bucket = '', service = 'aws'):
     paths = [ secretpath('{}-{}-secret.json'.format(bucket, service)) ] + paths
 
   aws_credentials = {}
-  aws_credentials_path = secretpath(default_file_path)
-  for aws_credentials_path in paths:
-    if os.path.exists(aws_credentials_path):
-      with open(aws_credentials_path, 'r') as f:
-        aws_credentials = json.loads(f.read())
-      break
+  if not skip_files:
+    aws_credentials_path = secretpath(default_file_path)
+    for aws_credentials_path in paths:
+      if os.path.exists(aws_credentials_path):
+        with open(aws_credentials_path, 'r') as f:
+          aws_credentials = json.loads(f.read())
+        break
   
   if not aws_credentials:
     # did not find any secret json file, will try to find it in environment variables
@@ -128,6 +129,7 @@ def aws_credentials(bucket = '', service = 'aws'):
       aws_credentials = {
         'AWS_ACCESS_KEY_ID': os.environ['AWS_ACCESS_KEY_ID'],
         'AWS_SECRET_ACCESS_KEY': os.environ['AWS_SECRET_ACCESS_KEY'],
+        'AWS_SESSION_TOKEN': os.environ['AWS_SESSION_TOKEN'],
       }
     if 'AWS_DEFAULT_REGION' in os.environ:
       aws_credentials['AWS_DEFAULT_REGION'] = os.environ['AWS_DEFAULT_REGION']

{cloud-files-4.21.1 → cloud-files-4.22.0}/cloudfiles_cli/cloudfiles_cli.py

@@ -172,11 +172,12 @@ def get_mfp(path, recursive):
 @click.option('--progress', is_flag=True, default=False, help="Show transfer progress.", show_default=True)
 @click.option('-b', '--block-size', default=128, help="Number of files to download at a time.", show_default=True)
 @click.option('--part-bytes', default=int(1e8), help="Composite upload threshold in bytes. Splits a file into pieces for some cloud services like gs and s3.", show_default=True)
+@click.option('--no-sign-request', is_flag=True, default=False, help="Use s3 in anonymous mode (don't sign requests) for the source.", show_default=True)
 @click.pass_context
 def cp(
   ctx, source, destination, 
   recursive, compression, progress, 
-  block_size, part_bytes
+  block_size, part_bytes, no_sign_request,
 ):
   """
   Copy one or more files from a source to destination.
@@ -194,9 +195,17 @@ def cp(
     return
 
   for src in source:
-    _cp_single(ctx, src, destination, recursive, compression, progress, block_size, part_bytes)
-
-def _cp_single(ctx, source, destination, recursive, compression, progress, block_size, part_bytes):
+    _cp_single(
+      ctx, src, destination, recursive, 
+      compression, progress, block_size, 
+      part_bytes, no_sign_request
+    )
+
+def _cp_single(
+  ctx, source, destination, recursive, 
+  compression, progress, block_size,
+  part_bytes, no_sign_request
+):
   use_stdin = (source == '-')
   use_stdout = (destination == '-')
 
@@ -243,7 +252,9 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
     xferpaths = [ x.replace(prefix, "") for x in xferpaths ]
     srcpath = cloudpathjoin(srcpath, prefix)
   elif many:
-    xferpaths = CloudFiles(srcpath).list(prefix=prefix, flat=flat)
+    xferpaths = CloudFiles(
+      srcpath, no_sign_request=no_sign_request
+    ).list(prefix=prefix, flat=flat)
 
   destpath = ndest
   if isinstance(xferpaths, str):
@@ -260,7 +271,7 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
 
   if not isinstance(xferpaths, str):
     if parallel == 1:
-      _cp(srcpath, destpath, compression, progress, block_size, part_bytes, xferpaths)
+      _cp(srcpath, destpath, compression, progress, block_size, part_bytes, no_sign_request, xferpaths)
       return 
 
     total = None
@@ -270,16 +281,16 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
       pass
 
     if use_stdout:
-      fn = partial(_cp_stdout, srcpath)
+      fn = partial(_cp_stdout, no_sign_request, srcpath)
     else:
-      fn = partial(_cp, srcpath, destpath, compression, False, block_size, part_bytes)
+      fn = partial(_cp, srcpath, destpath, compression, False, block_size, part_bytes, no_sign_request)
 
     with tqdm(desc="Transferring", total=total, disable=(not progress)) as pbar:
       with pathos.pools.ProcessPool(parallel) as executor:
         for _ in executor.imap(fn, sip(xferpaths, block_size)):
           pbar.update(block_size)
   else:
-    cfsrc = CloudFiles(srcpath, progress=progress)
+    cfsrc = CloudFiles(srcpath, progress=progress, no_sign_request=no_sign_request)
     if not cfsrc.exists(xferpaths):
       print(f"cloudfiles: source path not found: {cfsrc.abspath(xferpaths).replace('file://','')}")
       return
@@ -288,7 +299,11 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
       _cp_stdout(srcpath, xferpaths)
       return
 
-    cfdest = CloudFiles(destpath, progress=progress, composite_upload_threshold=part_bytes)
+    cfdest = CloudFiles(
+      destpath, 
+      progress=progress, 
+      composite_upload_threshold=part_bytes,
+    )
 
     if isdestdir:
       new_path = os.path.basename(nsrc)
@@ -300,17 +315,17 @@ def _cp_single(ctx, source, destination, recursive, compression, progress, block
       "dest_path": new_path,
     }], reencode=compression)
 
-def _cp(src, dst, compression, progress, block_size, part_bytes, paths):
-  cfsrc = CloudFiles(src, progress=progress, composite_upload_threshold=part_bytes)
+def _cp(src, dst, compression, progress, block_size, part_bytes, no_sign_request, paths):
+  cfsrc = CloudFiles(src, progress=progress, composite_upload_threshold=part_bytes, no_sign_request=no_sign_request)
   cfdest = CloudFiles(dst, progress=progress, composite_upload_threshold=part_bytes)
   cfsrc.transfer_to(
     cfdest, paths=paths, 
     reencode=compression, block_size=block_size
   )
 
-def _cp_stdout(src, paths):
+def _cp_stdout(src, no_sign_request, paths):
   paths = toiter(paths)
-  cf = CloudFiles(src, progress=False)
+  cf = CloudFiles(src, progress=False, no_sign_request=no_sign_request)
   for res in cf.get(paths):
     content = res["content"].decode("utf8")
     sys.stdout.write(content)
@@ -372,7 +387,8 @@ def xferexecute(db, progress, lease_msec, block_size):
 @main.command()
 @click.argument("sources", nargs=-1)
 @click.option('-r', '--range', 'byte_range', default=None, help='Retrieve start-end bytes.')
-def cat(sources, byte_range):
+@click.option('--no-sign-request', is_flag=True, default=False, help="Use s3 in anonymous mode (don't sign requests) for the source.", show_default=True)
+def cat(sources, byte_range, no_sign_request):
   """Concatenate the contents of each input file and write to stdout."""
   if '-' in sources and len(sources) == 1:
     sources = sys.stdin.readlines()
@@ -386,7 +402,7 @@ def cat(sources, byte_range):
     byte_range[0] = int(byte_range[0] or 0)
     byte_range[1] = int(byte_range[1]) if byte_range[1] not in ("", None) else None
     src = normalize_path(sources[0])
-    cf = CloudFiles(os.path.dirname(src))
+    cf = CloudFiles(os.path.dirname(src), no_sign_request=no_sign_request)
     download = cf[os.path.basename(src), byte_range[0]:byte_range[1]]
     if download is None:
       print(f'cloudfiles: {src} does not exist')
@@ -397,7 +413,7 @@ def cat(sources, byte_range):
   for srcs in sip(sources, 10):
     srcs = [ normalize_path(src) for src in srcs ]
     order = { src: i for i, src in enumerate(srcs) }
-    files = cloudfiles.dl(srcs)
+    files = cloudfiles.dl(srcs, no_sign_request=no_sign_request)
     output = [ None for _ in range(len(srcs)) ]
     for res in files:
       if res["content"] is None:

{cloud-files-4.21.1 → cloud-files-4.22.0}/setup.cfg

@@ -31,7 +31,7 @@ packages =
 [extras]
 test = 
 	pytest
-	moto
+	moto>=5
 numpy = 
 	numpy
 

cloud-files-4.21.1/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "6b74d88", "is_release": true}