cloud-files 4.24.1__py3-none-any.whl → 4.25.0__py3-none-any.whl

{cloud_files-4.24.1.dist-info → cloud_files-4.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cloud-files
-Version: 4.24.1
+Version: 4.25.0
 Summary: Fast access to cloud storage and local FS.
 Home-page: https://github.com/seung-lab/cloud-files/
 Author: William Silversmith

{cloud_files-4.24.1.dist-info → cloud_files-4.25.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ cloudfiles/compression.py,sha256=pqYdpu5vfFv-094BpfZ2pgRjVu7ESM9pAZC09P6E8bY,615
 cloudfiles/connectionpools.py,sha256=aL8RiSjRepECfgAFmJcz80aJFKbou7hsbuEgugDKwB8,4814
 cloudfiles/exceptions.py,sha256=H2IcMlZoy2Bsn-6wCPwyLDjg66LZCyxtcf3s_p21FDw,770
 cloudfiles/gcs.py,sha256=_njJ7TpqwrHCjPHRGkBN5alCrCWKM2m9qdy5DhxMZ7U,3718
-cloudfiles/interfaces.py,sha256=LAjmO40mhg_foEjdzdP0YxUxpk9Ut8C7puortTqSGXI,34539
+cloudfiles/interfaces.py,sha256=Qqhjv2GIVw3ibaTgPpiGK97i7aBDgU9O0bBuOiLI2KM,36117
 cloudfiles/lib.py,sha256=fEqL5APu_WQhl2yxqQbwE7msHdu7U8pstAJw6LgoKO0,5142
 cloudfiles/paths.py,sha256=tqR9XnRdAKopTJqSM6V5xrMo1xfmBdl9b5DpBLZnoB0,9998
 cloudfiles/resumable_tools.py,sha256=pK-VcoPjQ2BjGjvlvH4dDCBf6lNsqHG-weiBgxVFbzA,5838
@@ -16,11 +16,11 @@ cloudfiles/typing.py,sha256=f3ZYkNfN9poxhGu5j-P0KCxjCCqSn9HAg5KiIPkjnCg,416
 cloudfiles_cli/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
 cloudfiles_cli/__init__.py,sha256=Wftt3R3F21QsHtWqx49ODuqT9zcSr0em7wk48kcH0WM,29
 cloudfiles_cli/cloudfiles_cli.py,sha256=eETIOK4QyztQcpA4ZRny21SobLtcrPDlzZ_JaKBmmmA,28449
-cloud_files-4.24.1.dist-info/AUTHORS,sha256=7E2vC894bbLPO_kvUuEB2LFZZbIxZn23HabxH7x0Hgo,266
-cloud_files-4.24.1.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
-cloud_files-4.24.1.dist-info/METADATA,sha256=neXuIXtjmY7tLeSTTNH95jTyw_rHsnrYfum-AFEboR0,26804
-cloud_files-4.24.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cloud_files-4.24.1.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
-cloud_files-4.24.1.dist-info/pbr.json,sha256=whK0oWj9vjQb1lRu9_jrdIBm1TA0onqx-o5fzWihcBo,46
-cloud_files-4.24.1.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
-cloud_files-4.24.1.dist-info/RECORD,,
+cloud_files-4.25.0.dist-info/AUTHORS,sha256=7E2vC894bbLPO_kvUuEB2LFZZbIxZn23HabxH7x0Hgo,266
+cloud_files-4.25.0.dist-info/LICENSE,sha256=Jna4xYE8CCQmaxjr5Fs-wmUBnIQJ1DGcNn9MMjbkprk,1538
+cloud_files-4.25.0.dist-info/METADATA,sha256=wXrUnPFfMt3vMXfcxvqMpOB6fVsHRV1cEhDgFr4LdqQ,26804
+cloud_files-4.25.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cloud_files-4.25.0.dist-info/entry_points.txt,sha256=xlirb1FVhn1mbcv4IoyMEGumDqKOA4VMVd3drsRQxIg,51
+cloud_files-4.25.0.dist-info/pbr.json,sha256=v8YZnfyUynJ66Bxxf0b3fjSVxyAmHqmTI4XJ4BP25pg,46
+cloud_files-4.25.0.dist-info/top_level.txt,sha256=xPyrST3okJbsmdCF5IC2gYAVxg_aD5AYVTnNo8UuoZU,26
+cloud_files-4.25.0.dist-info/RECORD,,

cloud_files-4.25.0.dist-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "7dcc163", "is_release": true}

cloudfiles/interfaces.py

@@ -714,6 +714,8 @@ class GoogleCloudStorageInterface(StorageInterface):
     pool.release_connection(self._bucket)
 
 class HttpInterface(StorageInterface):
+  adaptor = requests.adapters.HTTPAdapter()
+  
   def __init__(self, path, secrets=None, request_payer=None, **kwargs):
     super(StorageInterface, self).__init__()
     self._path = path
@@ -724,6 +726,8 @@ class HttpInterface(StorageInterface):
       secrets = http_credentials()
 
     self.session = requests.Session()
+    self.session.mount('http://', self.adaptor)
+    self.session.mount('https://', self.adaptor)
     if secrets and 'user' in secrets and 'password' in secrets:
       self.session.auth = (secrets['user'], secrets['password'])
 
@@ -749,6 +753,10 @@ class HttpInterface(StorageInterface):
       resp.raise_for_status()
       return resp.headers
 
+  def size(self, file_path):
+    headers = self.head(file_path)
+    return int(headers["Content-Length"])
+
   @retry
   def get_file(self, file_path, start=None, end=None, part_size=None):
     key = self.get_path_to_file(file_path)
@@ -818,9 +826,58 @@ class HttpInterface(StorageInterface):
       if token is None:
         break
 
+  def _list_files_apache(self, prefix, flat):
+    import lxml.html
+
+    baseurl = posixpath.join(self._path.host, self._path.path)
+
+    directories = ['']
+
+    while directories:
+      directory = directories.pop()
+      url = posixpath.join(baseurl, directory)
+
+      resp = requests.get(url)
+      resp.raise_for_status()
+
+      if 'text/html' not in resp.headers["Content-Type"]:
+        raise ValueError("Unable to parse non-HTML output from Apache servers.")
+
+      entities = lxml.html.document_fromstring(resp.content)
+      resp.close()
+
+      h1 = entities.xpath("body/h1")[0].text_content()
+      if "Index of" not in h1:
+        raise ValueError("Unable to parse non-index page.")
+
+      for li in entities.xpath("body/ul/li"):
+        txt = li.text_content().strip()
+        if txt == "Parent Directory":
+          continue
+        
+        txt = posixpath.join(directory, txt)
+        if prefix and not txt.startswith(prefix):
+          continue
+
+        if txt[-1] == '/':
+          directories.append(txt)
+          continue
+
+        yield txt
+
+      if flat:
+        break
+
   def list_files(self, prefix, flat=False):
     if self._path.host == "https://storage.googleapis.com":
       yield from self._list_files_google(prefix, flat)
+    
+    url = posixpath.join(self._path.host, self._path.path, prefix)
+    resp = requests.head(url)
+
+    server = resp.headers.get("Server", "").lower()
+    if 'apache' in server:
+      yield from self._list_files_apache(prefix, flat)
     else:
       raise NotImplementedError()
 

cloud_files-4.24.1.dist-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "560597e", "is_release": true}