cloud-files 4.25.0__py3-none-any.whl → 4.27.0__py3-none-any.whl

{cloud_files-4.25.0.dist-info → cloud_files-4.27.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.25.0
+Version: 4.27.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-4.25.0.dist-info → cloud_files-4.27.0.dist-info}/RECORD

@@ -1,26 +1,26 @@
 cloudfiles/__init__.py,sha256=pLB4CcV2l3Jgv_ni1520Np1pfzFj8Cpr87vNxFT3rNI,493
 cloudfiles/buckets.py,sha256=eRAYdDfvVpNyJyK5ryDRMwgNJUeEuFBJ6doWU2JkAcA,74
-cloudfiles/cloudfiles.py,sha256=YUf_-7DS8-2mCKOWT6mMKxl1glxMTEGomhpCnebtsy8,44801
+cloudfiles/cloudfiles.py,sha256=QlrQlU94gqNhKWyOuP0xe58UEwk2x8wtZ7n9LKiyLpM,44854
 cloudfiles/compression.py,sha256=pqYdpu5vfFv-094BpfZ2pgRjVu7ESM9pAZC09P6E8bY,6150
 cloudfiles/connectionpools.py,sha256=aL8RiSjRepECfgAFmJcz80aJFKbou7hsbuEgugDKwB8,4814
 cloudfiles/exceptions.py,sha256=H2IcMlZoy2Bsn-6wCPwyLDjg66LZCyxtcf3s_p21FDw,770
 cloudfiles/gcs.py,sha256=_njJ7TpqwrHCjPHRGkBN5alCrCWKM2m9qdy5DhxMZ7U,3718
-cloudfiles/interfaces.py,sha256=Qqhjv2GIVw3ibaTgPpiGK97i7aBDgU9O0bBuOiLI2KM,36117
+cloudfiles/interfaces.py,sha256=4ICm7tnS8HHK5IKw-HYD653V-qndprPRK8e1cxUYKgA,36782
 cloudfiles/lib.py,sha256=fEqL5APu_WQhl2yxqQbwE7msHdu7U8pstAJw6LgoKO0,5142
-cloudfiles/paths.py,sha256=tqR9XnRdAKopTJqSM6V5xrMo1xfmBdl9b5DpBLZnoB0,9998
+cloudfiles/paths.py,sha256=xadVh5Vw8wAack1cws5dzVIlYQ3r8h8lrP43umUSuT0,10547
 cloudfiles/resumable_tools.py,sha256=pK-VcoPjQ2BjGjvlvH4dDCBf6lNsqHG-weiBgxVFbzA,5838
 cloudfiles/scheduler.py,sha256=DqDANmOpB3NdzFgJDNMMibRIkCrXQqIh2XGL8GWoc9c,3668
-cloudfiles/secrets.py,sha256=3BSV2Hn8FGGn4QCs5FP3eGs4WEs5cIXRBsXuF0eIgIY,4918
+cloudfiles/secrets.py,sha256=791b5a8nWSBYtlleGzKeoYIR5jl-FI1bw6INRM4Wy-0,5295
 cloudfiles/threaded_queue.py,sha256=Nl4vfXhQ6nDLF8PZpSSBpww0M2zWtcd4DLs3W3BArBw,7082
 cloudfiles/typing.py,sha256=f3ZYkNfN9poxhGu5j-P0KCxjCCqSn9HAg5KiIPkjnCg,416
 cloudfiles_cli/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
 cloudfiles_cli/__init__.py,sha256=Wftt3R3F21QsHtWqx49ODuqT9zcSr0em7wk48kcH0WM,29
 cloudfiles_cli/cloudfiles_cli.py,sha256=eETIOK4QyztQcpA4ZRny21SobLtcrPDlzZ_JaKBmmmA,28449
-cloud_files-4.25.0.dist-info/AUTHORS,sha256=7E2vC894bbLPO_kvUuEB2LFZZbIxZn23HabxH7x0Hgo,266
-cloud_files-4.25.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
-cloud_files-4.25.0.dist-info/METADATA,sha256=wXrUnPFfMt3vMXfcxvqMpOB6fVsHRV1cEhDgFr4LdqQ,26804
-cloud_files-4.25.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cloud_files-4.25.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
-cloud_files-4.25.0.dist-info/pbr.json,sha256=v8YZnfyUynJ66Bxxf0b3fjSVxyAmHqmTI4XJ4BP25pg,46
-cloud_files-4.25.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
-cloud_files-4.25.0.dist-info/RECORD,,
+cloud_files-4.27.0.dist-info/AUTHORS,sha256=7E2vC894bbLPO_kvUuEB2LFZZbIxZn23HabxH7x0Hgo,266
+cloud_files-4.27.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
+cloud_files-4.27.0.dist-info/METADATA,sha256=7xdozjXt0yT1OKZV47M9dWeip9PUx9Wj9adUcO0qZ_M,26804
+cloud_files-4.27.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cloud_files-4.27.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
+cloud_files-4.27.0.dist-info/pbr.json,sha256=C4Xk2iNLylqeAReQ_DjwEN6rVj4PC_6x96XsvMs7138,46
+cloud_files-4.27.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
+cloud_files-4.27.0.dist-info/RECORD,,

cloud_files-4.27.0.dist-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "e4b04bf", "is_release": true}

cloudfiles/cloudfiles.py

@@ -44,7 +44,7 @@ from .scheduler import schedule_jobs
 from .interfaces import (
   FileInterface, HttpInterface, 
   S3Interface, GoogleCloudStorageInterface,
-  MemoryInterface
+  MemoryInterface, CaveInterface,
 )
 
 INTERFACES = {
@@ -54,6 +54,7 @@ INTERFACES = {
   'http': HttpInterface,
   'https': HttpInterface,
   'mem': MemoryInterface,
+  'middleauth+https': CaveInterface,
 }
 for alias in ALIASES:
   INTERFACES[alias] = S3Interface

cloudfiles/interfaces.py

@@ -24,7 +24,12 @@ from .compression import COMPRESSION_TYPES
 from .connectionpools import S3ConnectionPool, GCloudBucketPool, MemoryPool, MEMORY_DATA
 from .exceptions import MD5IntegrityError, CompressionError
 from .lib import mkdir, sip, md5, validate_s3_multipart_etag
-from .secrets import http_credentials, CLOUD_FILES_DIR, CLOUD_FILES_LOCK_DIR
+from .secrets import (
+  http_credentials,
+  cave_credentials,
+  CLOUD_FILES_DIR, 
+  CLOUD_FILES_LOCK_DIR,
+)
 
 COMPRESSION_EXTENSIONS = ('.gz', '.br', '.zstd','.bz2','.xz')
 GZIP_TYPES = (True, 'gzip', 1)
@@ -731,6 +736,9 @@ class HttpInterface(StorageInterface):
     if secrets and 'user' in secrets and 'password' in secrets:
       self.session.auth = (secrets['user'], secrets['password'])
 
+  def default_headers(self):
+    return {}
+
   def get_path_to_file(self, file_path):
     return posixpath.join(self._path.host, self._path.path, file_path)
 
@@ -749,7 +757,8 @@ class HttpInterface(StorageInterface):
   @retry
   def head(self, file_path):
     key = self.get_path_to_file(file_path)
-    with self.session.head(key) as resp:
+    headers = self.default_headers()
+    with self.session.head(key, headers=headers) as resp:
       resp.raise_for_status()
       return resp.headers
 
@@ -761,13 +770,14 @@ class HttpInterface(StorageInterface):
   def get_file(self, file_path, start=None, end=None, part_size=None):
     key = self.get_path_to_file(file_path)
 
+    headers = self.default_headers()
     if start is not None or end is not None:
       start = int(start) if start is not None else 0
       end = int(end - 1) if end is not None else ''
-      headers = { "Range": "bytes={}-{}".format(start, end) }
-      resp = self.session.get(key, headers=headers)
-    else:
-      resp = self.session.get(key)
+      headers["Range"] = f"bytes={start}-{end}"
+    
+    resp = self.session.get(key, headers=headers)
+    
     if resp.status_code in (404, 403):
       return (None, None, None, None)
     resp.close()
@@ -788,7 +798,8 @@ class HttpInterface(StorageInterface):
   @retry
   def exists(self, file_path):
     key = self.get_path_to_file(file_path)
-    with self.session.get(key, stream=True) as resp:
+    headers = self.default_headers()
+    with self.session.get(key, stream=True, headers=headers) as resp:
       return resp.ok
 
   def files_exist(self, file_paths):
@@ -805,11 +816,15 @@ class HttpInterface(StorageInterface):
     if prefix and prefix[-1] != '/':
       prefix += '/'
 
+    headers = self.default_headers()
+
     @retry
     def request(token):
+      nonlocal headers
       results = self.session.get(
         f"https://storage.googleapis.com/storage/v1/b/{bucket}/o",
         params={ "prefix": prefix, "pageToken": token },
+        headers=headers,
       )
       results.raise_for_status()
       results.close()
@@ -832,12 +847,13 @@ class HttpInterface(StorageInterface):
     baseurl = posixpath.join(self._path.host, self._path.path)
 
     directories = ['']
+    headers = self.default_headers()
 
     while directories:
       directory = directories.pop()
       url = posixpath.join(baseurl, directory)
 
-      resp = requests.get(url)
+      resp = requests.get(url, headers=headers)
       resp.raise_for_status()
 
       if 'text/html' not in resp.headers["Content-Type"]:
@@ -1200,3 +1216,16 @@ class S3Interface(StorageInterface):
     with S3_BUCKET_POOL_LOCK:
       pool = S3_POOL[S3ConnectionPoolParams(service, self._path.bucket, self._request_payer)]
     pool.release_connection(self._conn)
+
+class CaveInterface(HttpInterface):
+  """
+  CAVE is an internal system that powers proofreading 
+  systems in Seung Lab. If you have no idea what this
+  is, don't worry about it.
+  see: https://github.com/CAVEconnectome
+  """
+  def default_headers(self):
+    cred = cave_credentials()
+    return {
+      "Authorization": f"Bearer {cred['token']}",
+    }

cloudfiles/paths.py

@@ -26,10 +26,16 @@ ALIASES_FROM_FILE = None
 ALIASES = {}
 BASE_ALLOWED_PROTOCOLS = [ 
   'gs', 'file', 's3', 
-  'http', 'https', 'mem' 
+  'http', 'https', 'mem',
+  'middleauth+https', 'ngauth+https',
 ]
 ALLOWED_PROTOCOLS = list(BASE_ALLOWED_PROTOCOLS)
-ALLOWED_FORMATS = [ 'graphene', 'precomputed', 'boss', 'n5' ] 
+ALLOWED_FORMATS = [ 
+  'graphene', 'precomputed', 'boss', 
+  'n5', 'zarr', 'zarr2', 'zarr3',
+  'brainmaps', 'deepzoom', 'nggraph',
+  'render', 'vtk', 'nifti', 'dvid',
+] 
 
 def update_aliases_from_file():
   global ALIASES_FROM_FILE
@@ -64,7 +70,13 @@ def cloudpath_error(cloudpath):
 def mkregexp():
   fmt_capture = r'|'.join(ALLOWED_FORMATS)
   fmt_capture = "(?:(?P<fmt>{})://)".format(fmt_capture)
-  proto_capture = r'|'.join(ALLOWED_PROTOCOLS)
+
+  allowed_protos = [
+    p.replace('+', r'\+')
+    for p in ALLOWED_PROTOCOLS
+  ]
+
+  proto_capture = r'|'.join(allowed_protos)
   proto_capture = "(?:(?P<proto>{})://)".format(proto_capture)
   regexp = "{}?{}?".format(fmt_capture, proto_capture)
   return regexp
@@ -147,11 +159,16 @@ for alias, host in OFFICIAL_ALIASES.items():
 ## Other Path Library Functions
 
 def normalize(path):
-  proto = get_protocol(path)
+  fmt, proto, endpoint, cloudpath, alias = extract_format_protocol(
+    path, allow_defaults=False
+  )
+
   if proto is None:
     proto = "file"
-    path = toabs(path)
-    return f"file://{path}"
+    cloudpath = toabs(cloudpath)
+    fmt = f"{fmt}://" if fmt else ""
+    path = f"{fmt}{proto}://{cloudpath}"
+
   return path
 
 def dirname(path):
@@ -264,7 +281,7 @@ def pop_protocol(cloudpath):
 
   return (protocol, cloudpath)
 
-def extract_format_protocol(cloudpath:str) -> tuple:
+def extract_format_protocol(cloudpath:str, allow_defaults=True) -> tuple:
   error = UnsupportedProtocolError(cloudpath_error(cloudpath))
 
   alias, cloudpath = resolve_alias(cloudpath)
@@ -276,12 +293,18 @@ def extract_format_protocol(cloudpath:str) -> tuple:
   groups = m.groups()
   cloudpath = re.sub(CLOUDPATH_REGEXP, '', cloudpath, count=1)
 
-  fmt = m.group('fmt') or 'precomputed'
+  fmt = m.group('fmt')
+  if not fmt and allow_defaults:
+    fmt = 'precomputed'
   proto = m.group('proto')
   endpoint = None
 
-  if proto in ('http', 'https'):
-    cloudpath = proto + "://" + cloudpath
+  tmp_proto = None
+  if proto is not None:
+    tmp_proto = proto.replace("middleauth+", "").replace("ngauth+", "")
+  
+  if tmp_proto in ('http', 'https'):
+    cloudpath = tmp_proto + "://" + cloudpath
     parse = urllib.parse.urlparse(cloudpath)
     endpoint = parse.scheme + "://" + parse.netloc
     cloudpath = cloudpath.replace(endpoint, '', 1)

cloudfiles/secrets.py

@@ -137,6 +137,24 @@ def aws_credentials(bucket = '', service = 'aws', skip_files=False):
   AWS_CREDENTIALS_CACHE[service][bucket] = aws_credentials
   return aws_credentials
 
+CAVE_CREDENTIALS = None
+def cave_credentials():
+  global CAVE_CREDENTIALS
+  default_file_path = 'cave-secret.json'
+  path = secretpath(default_file_path)
+
+  if CAVE_CREDENTIALS:
+    return CAVE_CREDENTIALS
+
+  if os.path.exists(path):
+    with open(path, 'rt') as f:
+      CAVE_CREDENTIALS = json.loads(f.read())
+  else:
+    CAVE_CREDENTIALS = None
+
+  return CAVE_CREDENTIALS
+
+
 HTTP_CREDENTIALS = None
 def http_credentials():
   global HTTP_CREDENTIALS

cloud_files-4.25.0.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "7dcc163", "is_release": true}