cloud-files 4.24.2__tar.gz → 4.25.0__tar.gz

{cloud-files-4.24.2 → cloud-files-4.25.0}/ChangeLog

@@ -1,6 +1,11 @@
 CHANGES
 =======
 
+4.25.0
+------
+
+* feat: list for apache servers (#104)
+
 4.24.2
 ------
 

{cloud-files-4.24.2 → cloud-files-4.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.2
+Version: 4.25.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud-files-4.24.2 → cloud-files-4.25.0}/cloud_files.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.2
+Version: 4.25.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

cloud-files-4.25.0/cloud_files.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "7dcc163", "is_release": true}

{cloud-files-4.24.2 → cloud-files-4.25.0}/cloudfiles/interfaces.py

@@ -714,6 +714,8 @@ class GoogleCloudStorageInterface(StorageInterface):
     pool.release_connection(self._bucket)
 
 class HttpInterface(StorageInterface):
+  adaptor = requests.adapters.HTTPAdapter()
+  
   def __init__(self, path, secrets=None, request_payer=None, **kwargs):
     super(StorageInterface, self).__init__()
     self._path = path
@@ -724,6 +726,8 @@ class HttpInterface(StorageInterface):
       secrets = http_credentials()
 
     self.session = requests.Session()
+    self.session.mount('http://', self.adaptor)
+    self.session.mount('https://', self.adaptor)
     if secrets and 'user' in secrets and 'password' in secrets:
       self.session.auth = (secrets['user'], secrets['password'])
 
@@ -822,9 +826,58 @@ class HttpInterface(StorageInterface):
       if token is None:
         break
 
+  def _list_files_apache(self, prefix, flat):
+    import lxml.html
+
+    baseurl = posixpath.join(self._path.host, self._path.path)
+
+    directories = ['']
+
+    while directories:
+      directory = directories.pop()
+      url = posixpath.join(baseurl, directory)
+
+      resp = requests.get(url)
+      resp.raise_for_status()
+
+      if 'text/html' not in resp.headers["Content-Type"]:
+        raise ValueError("Unable to parse non-HTML output from Apache servers.")
+
+      entities = lxml.html.document_fromstring(resp.content)
+      resp.close()
+
+      h1 = entities.xpath("body/h1")[0].text_content()
+      if "Index of" not in h1:
+        raise ValueError("Unable to parse non-index page.")
+
+      for li in entities.xpath("body/ul/li"):
+        txt = li.text_content().strip()
+        if txt == "Parent Directory":
+          continue
+        
+        txt = posixpath.join(directory, txt)
+        if prefix and not txt.startswith(prefix):
+          continue
+
+        if txt[-1] == '/':
+          directories.append(txt)
+          continue
+
+        yield txt
+
+      if flat:
+        break
+
   def list_files(self, prefix, flat=False):
     if self._path.host == "https://storage.googleapis.com":
       yield from self._list_files_google(prefix, flat)
+    
+    url = posixpath.join(self._path.host, self._path.path, prefix)
+    resp = requests.head(url)
+
+    server = resp.headers.get("Server", "").lower()
+    if 'apache' in server:
+      yield from self._list_files_apache(prefix, flat)
     else:
       raise NotImplementedError()
 

{cloud-files-4.24.2 → cloud-files-4.25.0}/setup.py

@@ -11,6 +11,9 @@ setuptools.setup(
       "cloudfiles=cloudfiles_cli:main"
     ],
   },
+  extras_require={
+    "apache": [ "lxml", ],
+  },
   pbr=True
 )
 

cloud-files-4.24.2/cloud_files.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "84826ee", "is_release": true}