cloud-files 4.24.2__tar.gz → 4.26.0__tar.gz

{cloud-files-4.24.2 → cloud-files-4.26.0}/ChangeLog

@@ -1,6 +1,17 @@
 CHANGES
 =======
 
+4.26.0
+------
+
+* feat: make it possible to normalize e.g. zarr2://./helloworld
+* feat: add all current neuroglancer formats to parsing list
+
+4.25.0
+------
+
+* feat: list for apache servers (#104)
+
 4.24.2
 ------
 

{cloud-files-4.24.2 → cloud-files-4.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.2
+Version: 4.26.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud-files-4.24.2 → cloud-files-4.26.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.2
+Version: 4.26.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud-files-4.26.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "3ae7c76", "is_release": true}

{cloud-files-4.24.2 → cloud-files-4.26.0}/cloudfiles/interfaces.py

@@ -714,6 +714,8 @@ class GoogleCloudStorageInterface(StorageInterface):
     pool.release_connection(self._bucket)
 
 class HttpInterface(StorageInterface):
+  adaptor = requests.adapters.HTTPAdapter()
+  
   def __init__(self, path, secrets=None, request_payer=None, **kwargs):
     super(StorageInterface, self).__init__()
     self._path = path
@@ -724,6 +726,8 @@ class HttpInterface(StorageInterface):
       secrets = http_credentials()
 
     self.session = requests.Session()
+    self.session.mount('http://', self.adaptor)
+    self.session.mount('https://', self.adaptor)
     if secrets and 'user' in secrets and 'password' in secrets:
       self.session.auth = (secrets['user'], secrets['password'])
 
@@ -822,9 +826,58 @@ class HttpInterface(StorageInterface):
       if token is None:
         break
 
+  def _list_files_apache(self, prefix, flat):
+    import lxml.html
+
+    baseurl = posixpath.join(self._path.host, self._path.path)
+
+    directories = ['']
+
+    while directories:
+      directory = directories.pop()
+      url = posixpath.join(baseurl, directory)
+
+      resp = requests.get(url)
+      resp.raise_for_status()
+
+      if 'text/html' not in resp.headers["Content-Type"]:
+        raise ValueError("Unable to parse non-HTML output from Apache servers.")
+
+      entities = lxml.html.document_fromstring(resp.content)
+      resp.close()
+
+      h1 = entities.xpath("body/h1")[0].text_content()
+      if "Index of" not in h1:
+        raise ValueError("Unable to parse non-index page.")
+
+      for li in entities.xpath("body/ul/li"):
+        txt = li.text_content().strip()
+        if txt == "Parent Directory":
+          continue
+        
+        txt = posixpath.join(directory, txt)
+        if prefix and not txt.startswith(prefix):
+          continue
+
+        if txt[-1] == '/':
+          directories.append(txt)
+          continue
+
+        yield txt
+
+      if flat:
+        break
+
   def list_files(self, prefix, flat=False):
     if self._path.host == "https://storage.googleapis.com":
       yield from self._list_files_google(prefix, flat)
+    
+    url = posixpath.join(self._path.host, self._path.path, prefix)
+    resp = requests.head(url)
+
+    server = resp.headers.get("Server", "").lower()
+    if 'apache' in server:
+      yield from self._list_files_apache(prefix, flat)
     else:
       raise NotImplementedError()
 

{cloud-files-4.24.2 → cloud-files-4.26.0}/cloudfiles/paths.py

@@ -29,7 +29,12 @@ BASE_ALLOWED_PROTOCOLS = [
   'http', 'https', 'mem' 
 ]
 ALLOWED_PROTOCOLS = list(BASE_ALLOWED_PROTOCOLS)
-ALLOWED_FORMATS = [ 'graphene', 'precomputed', 'boss', 'n5' ] 
+ALLOWED_FORMATS = [ 
+  'graphene', 'precomputed', 'boss', 
+  'n5', 'zarr', 'zarr2', 'zarr3',
+  'brainmaps', 'deepzoom', 'nggraph',
+  'render', 'vtk', 'nifti', 'dvid',
+] 
 
 def update_aliases_from_file():
   global ALIASES_FROM_FILE
@@ -147,11 +152,16 @@ for alias, host in OFFICIAL_ALIASES.items():
 ## Other Path Library Functions
 
 def normalize(path):
-  proto = get_protocol(path)
+  fmt, proto, endpoint, cloudpath, alias = extract_format_protocol(
+    path, allow_defaults=False
+  )
+
   if proto is None:
     proto = "file"
-    path = toabs(path)
-    return f"file://{path}"
+    cloudpath = toabs(cloudpath)
+    fmt = f"{fmt}://" if fmt else ""
+    path = f"{fmt}{proto}://{cloudpath}"
+
   return path
 
 def dirname(path):
@@ -264,7 +274,7 @@ def pop_protocol(cloudpath):
 
   return (protocol, cloudpath)
 
-def extract_format_protocol(cloudpath:str) -> tuple:
+def extract_format_protocol(cloudpath:str, allow_defaults=True) -> tuple:
   error = UnsupportedProtocolError(cloudpath_error(cloudpath))
 
   alias, cloudpath = resolve_alias(cloudpath)
@@ -276,7 +286,9 @@ def extract_format_protocol(cloudpath:str) -> tuple:
   groups = m.groups()
   cloudpath = re.sub(CLOUDPATH_REGEXP, '', cloudpath, count=1)
 
-  fmt = m.group('fmt') or 'precomputed'
+  fmt = m.group('fmt')
+  if not fmt and allow_defaults:
+    fmt = 'precomputed'
   proto = m.group('proto')
   endpoint = None
 

{cloud-files-4.24.2 → cloud-files-4.26.0}/setup.py

@@ -11,6 +11,9 @@ setuptools.setup(
       "cloudfiles=cloudfiles_cli:main"
     ],
   },
+  extras_require={
+    "apache": [ "lxml", ],
+  },
   pbr=True
 )
 

cloud-files-4.24.2/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "84826ee", "is_release": true}