cloud-files 5.4.0__py3-none-any.whl → 5.5.0__py3-none-any.whl

{cloud_files-5.4.0.dist-info → cloud_files-5.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 5.4.0
+Version: 5.5.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-5.4.0.dist-info → cloud_files-5.5.0.dist-info}/RECORD

@@ -1,26 +1,26 @@
 cloudfiles/__init__.py,sha256=pLB4CcV2l3Jgv_ni1520Np1pfzFj8Cpr87vNxFT3rNI,493
-cloudfiles/cloudfiles.py,sha256=diET6pfFQKKDKF0-MsbXKa9tvs_Ik4N_vHQ7nNL5eaM,51211
+cloudfiles/cloudfiles.py,sha256=nUIW-SDyeldvmUu_SETJXS4x38Nv3Kj7TBwMwLQhjns,51196
 cloudfiles/compression.py,sha256=WXJHnoNLJ_NWyoY9ygZmFA2qMou35_9xS5dzF7-2H-M,6262
 cloudfiles/connectionpools.py,sha256=aL8RiSjRepECfgAFmJcz80aJFKbou7hsbuEgugDKwB8,4814
 cloudfiles/exceptions.py,sha256=N0oGQNG-St6RvnT8e5p_yC_E61q2kgAe2scwAL0F49c,843
 cloudfiles/gcs.py,sha256=unqu5KxGKaPq6N4QeHSpCDdtnK1BzPOAerTZ8FLt2_4,3820
-cloudfiles/interfaces.py,sha256=f_7doKCiI_8--ghsQJvh8f5HhAn5u3haSlPHlxi_jbw,43949
+cloudfiles/interfaces.py,sha256=3_ybsCKf74DPpI5Vv-BwFLLzc1cLUMfZLBtIKWkfoHQ,44478
 cloudfiles/lib.py,sha256=HHjCvjmOjA0nZWSvHGoSeYpxqd6FAG8xk8LM212LAUA,5382
 cloudfiles/paths.py,sha256=HOvtdLSIYGwlwvnZt9d_Ez3TXOe7WWd18bZNDpExUDQ,12231
 cloudfiles/resumable_tools.py,sha256=NyuSoGh1SaP5akrHCpd9kgy2-JruEWrHW9lvJxV7jpE,6711
-cloudfiles/scheduler.py,sha256=DqDANmOpB3NdzFgJDNMMibRIkCrXQqIh2XGL8GWoc9c,3668
-cloudfiles/secrets.py,sha256=791b5a8nWSBYtlleGzKeoYIR5jl-FI1bw6INRM4Wy-0,5295
+cloudfiles/scheduler.py,sha256=ioqBT5bMPCVHEHlnL-SW_wHulxGgjeThiKHlnaDOydo,3831
+cloudfiles/secrets.py,sha256=IuYKHmmvFmQTyG2Zcmbx7e8U2LIv-woG5d8qyOlyCD8,5431
 cloudfiles/test.py,sha256=jktBhzKLU448QPs5H7QVnTvpYMo4lBBWG1aE7-uzVGQ,678
 cloudfiles/threaded_queue.py,sha256=Nl4vfXhQ6nDLF8PZpSSBpww0M2zWtcd4DLs3W3BArBw,7082
 cloudfiles/typing.py,sha256=f3ZYkNfN9poxhGu5j-P0KCxjCCqSn9HAg5KiIPkjnCg,416
 cloudfiles_cli/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
 cloudfiles_cli/__init__.py,sha256=Wftt3R3F21QsHtWqx49ODuqT9zcSr0em7wk48kcH0WM,29
-cloudfiles_cli/cloudfiles_cli.py,sha256=KIK917MxYGmwiI7mXVYpL4M8iiyMew8Pd1eyWoI57yE,34866
-cloud_files-5.4.0.dist-info/AUTHORS,sha256=BFVmobgAhaVFI5fqbuqAY5XmBQxe09ZZAsAOTy87hKQ,318
-cloud_files-5.4.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
-cloud_files-5.4.0.dist-info/METADATA,sha256=5kfzAsz8p7atAOuxCL1vVGljLsbyJTKFczui7pT-mH4,26970
-cloud_files-5.4.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cloud_files-5.4.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
-cloud_files-5.4.0.dist-info/pbr.json,sha256=dzxLD7W5Cqa2-KE8tL8nKLWjTqsUGyu181SesfMhxXA,46
-cloud_files-5.4.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
-cloud_files-5.4.0.dist-info/RECORD,,
+cloudfiles_cli/cloudfiles_cli.py,sha256=L9xr_z9GCf5PcpmA21Iandim7uMviaLxTZ4Pf1B9Ui0,34865
+cloud_files-5.5.0.dist-info/AUTHORS,sha256=BFVmobgAhaVFI5fqbuqAY5XmBQxe09ZZAsAOTy87hKQ,318
+cloud_files-5.5.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
+cloud_files-5.5.0.dist-info/METADATA,sha256=m3Rlk29z4XnqeLhpKX613g7KMiT-5vGVNsWpjXNfXmU,26970
+cloud_files-5.5.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cloud_files-5.5.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
+cloud_files-5.5.0.dist-info/pbr.json,sha256=k1LwgnAaBImOMXKnlbOx2sBAWZ_Jm5AdFAhAugTuvrY,46
+cloud_files-5.5.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
+cloud_files-5.5.0.dist-info/RECORD,,

cloud_files-5.5.0.dist-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "512949d", "is_release": true}

cloudfiles/cloudfiles.py

@@ -751,8 +751,7 @@ class CloudFiles:
       res = first(self.list(prefix=prefix))
       return res is not None
     except NotImplementedError as err:
-      res = CloudFile(self.cloudpath).size() 
-      return res > 0
+      return not CloudFile(self.cloudpath).exists()
 
   def exists(
     self, paths:GetPathType, 

cloudfiles/interfaces.py

@@ -1497,8 +1497,25 @@ class CaveInterface(HttpInterface):
   is, don't worry about it.
   see: https://github.com/CAVEconnectome
   """
-  def default_headers(self):
-    cred = cave_credentials()
+  def __init__(self, path, secrets=None, **kwargs):
+    super().__init__(path, secrets=secrets, **kwargs)
+
+    secrets = kwargs.get('secrets', None)
+    if secrets is None:
+      secrets = {}
+
+    self._token = secrets.get('token', None)
+    if self._token is None:
+      server = self._path.host.replace("https://", "", 1)
+      server = server.replace("http://", "", 1)
+      self._token = cave_credentials(server)
+      if self._token is not None:
+        self._token = self._token.get('token', None)
+
+  def default_headers(self) -> dict:
+    if self._token is None:
+      return {}
+    
     return {
-      "Authorization": f"Bearer {cred['token']}",
-    }
+      "Authorization": f"Bearer {self._token}",
+    }

cloudfiles/scheduler.py

@@ -137,7 +137,12 @@ def schedule_jobs(
     or (hasattr(fns, "__len__") and len(fns) <= 1)
   ):
     return schedule_single_threaded_jobs(fns, progress, total, count_return)
-    
+  
+  if isinstance(total, int):
+    concurrency = min(concurrency, max(total, 1))
+  elif hasattr(fns, "__len__"):
+    concurrency = min(concurrency, max(len(fns), 1))
+
   if green == True or (green is None and gevent.monkey.saved):
     return schedule_green_jobs(fns, concurrency, progress, total, count_return)
 

cloudfiles/secrets.py

@@ -137,23 +137,27 @@ def aws_credentials(bucket = '', service = 'aws', skip_files=False):
   AWS_CREDENTIALS_CACHE[service][bucket] = aws_credentials
   return aws_credentials
 
-CAVE_CREDENTIALS = None
-def cave_credentials():
+CAVE_CREDENTIALS:CredentialCacheType = {}
+def cave_credentials(server = ''):
   global CAVE_CREDENTIALS
-  default_file_path = 'cave-secret.json'
-  path = secretpath(default_file_path)
 
-  if CAVE_CREDENTIALS:
-    return CAVE_CREDENTIALS
+  paths = [
+    secretpath('cave-secret.json')
+  ]
 
-  if os.path.exists(path):
-    with open(path, 'rt') as f:
-      CAVE_CREDENTIALS = json.loads(f.read())
-  else:
-    CAVE_CREDENTIALS = None
+  if server:
+    paths = [ secretpath(f'{server}-cave-secret.json') ] + paths
+
+  if server in CAVE_CREDENTIALS:
+    return CAVE_CREDENTIALS.get(server, None)
 
-  return CAVE_CREDENTIALS
+  for path in paths:
+    if os.path.exists(path):
+      with open(path, 'rt') as f:
+        CAVE_CREDENTIALS[server] = json.loads(f.read())
+      break
 
+  return CAVE_CREDENTIALS.get(server, None)
 
 HTTP_CREDENTIALS = None
 def http_credentials():

cloudfiles_cli/cloudfiles_cli.py

@@ -84,7 +84,7 @@ def license():
 @main.command()
 @click.option('--shortpath', is_flag=True, default=False, help='Don\'t print the common base path for each listed path.',show_default=True)
 @click.option('--flat', is_flag=True, default=False, help='Only produce a single level of directory hierarchy.',show_default=True)
-@click.option('-e','--expr',is_flag=True, default=False, help='Use a limited regexp language (e.g. [abc123]\{3\}) to generate prefixes.', show_default=True)
+@click.option('-e','--expr',is_flag=True, default=False, help=r'Use a limited regexp language (e.g. [abc123]{3}) to generate prefixes.', show_default=True)
 @click.option('--no-auth',is_flag=True, default=False, help='Uses the http API for read-only operations.', show_default=True)
 @click.argument("cloudpath")
 def ls(shortpath, flat, expr, cloudpath, no_auth):

cloud_files-5.4.0.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "002c3fc", "is_release": true}