fsspec 2024.6.1__py3-none-any.whl → 2024.9.0__py3-none-any.whl

fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.6.1'
-__version_tuple__ = version_tuple = (2024, 6, 1)
+__version__ = version = '2024.9.0'
+__version_tuple__ = version_tuple = (2024, 9, 0)

fsspec/asyn.py

@@ -1072,7 +1072,7 @@ class AbstractAsyncStreamedFile(AbstractBufferedFile):
             self.offset = 0
             try:
                 await self._initiate_upload()
-            except:  # noqa: E722
+            except:
                 self.closed = True
                 raise
 

fsspec/core.py

@@ -639,7 +639,7 @@ def get_fs_token_paths(
     if isinstance(urlpath, (list, tuple, set)):
         if not urlpath:
             raise ValueError("empty urlpath sequence")
-        urlpath0 = stringify_path(list(urlpath)[0])
+        urlpath0 = stringify_path(next(iter(urlpath)))
     else:
         urlpath0 = stringify_path(urlpath)
     storage_options = storage_options or {}

fsspec/fuse.py

@@ -31,8 +31,8 @@ class FUSEr(Operations):
         path = "".join([self.root, path.lstrip("/")]).rstrip("/")
         try:
             info = self.fs.info(path)
-        except FileNotFoundError:
-            raise FuseOSError(ENOENT)
+        except FileNotFoundError as exc:
+            raise FuseOSError(ENOENT) from exc
 
         data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)}
         perm = info.get("mode", 0o777)
@@ -119,8 +119,8 @@ class FUSEr(Operations):
         fn = "".join([self.root, path.lstrip("/")])
         try:
             self.fs.rm(fn, False)
-        except (OSError, FileNotFoundError):
-            raise FuseOSError(EIO)
+        except (OSError, FileNotFoundError) as exc:
+            raise FuseOSError(EIO) from exc
 
     def release(self, path, fh):
         try:

fsspec/gui.py

@@ -93,8 +93,10 @@ class SigSlot:
         """Display in a notebook or a server"""
         try:
             return self.panel._repr_mimebundle_(*args, **kwargs)
-        except (ValueError, AttributeError):
-            raise NotImplementedError("Panel does not seem to be set up properly")
+        except (ValueError, AttributeError) as exc:
+            raise NotImplementedError(
+                "Panel does not seem to be set up properly"
+            ) from exc
 
     def connect(self, signal, slot):
         """Associate call back with given event

fsspec/implementations/arrow.py

@@ -128,7 +128,7 @@ class ArrowFSWrapper(AbstractFileSystem):
                 with self.open(tmp_fname, "wb") as rstream:
                     shutil.copyfileobj(lstream, rstream)
                 self.fs.move(tmp_fname, path2)
-            except BaseException:  # noqa
+            except BaseException:
                 with suppress(FileNotFoundError):
                     self.fs.delete_file(tmp_fname)
                 raise

fsspec/implementations/dbfs.py

@@ -77,9 +77,9 @@ class DatabricksFileSystem(AbstractFileSystem):
                 )
             except DatabricksException as e:
                 if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                    raise FileNotFoundError(e.message)
+                    raise FileNotFoundError(e.message) from e
 
-                raise e
+                raise
             files = r["files"]
             out = [
                 {
@@ -123,9 +123,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             self._send_to_api(method="post", endpoint="mkdirs", json={"path": path})
         except DatabricksException as e:
             if e.error_code == "RESOURCE_ALREADY_EXISTS":
-                raise FileExistsError(e.message)
+                raise FileExistsError(e.message) from e
 
-            raise e
+            raise
         self.invalidate_cache(self._parent(path))
 
     def mkdir(self, path, create_parents=True, **kwargs):
@@ -169,9 +169,9 @@ class DatabricksFileSystem(AbstractFileSystem):
                 self.rm(path=path, recursive=recursive)
             elif e.error_code == "IO_ERROR":
                 # Using the same exception as the os module would use here
-                raise OSError(e.message)
+                raise OSError(e.message) from e
 
-            raise e
+            raise
         self.invalidate_cache(self._parent(path))
 
     def mv(
@@ -212,11 +212,11 @@ class DatabricksFileSystem(AbstractFileSystem):
             )
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
             elif e.error_code == "RESOURCE_ALREADY_EXISTS":
-                raise FileExistsError(e.message)
+                raise FileExistsError(e.message) from e
 
-            raise e
+            raise
         self.invalidate_cache(self._parent(source_path))
         self.invalidate_cache(self._parent(destination_path))
 
@@ -264,9 +264,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             try:
                 exception_json = e.response.json()
             except Exception:
-                raise e
+                raise e from None
 
-            raise DatabricksException(**exception_json)
+            raise DatabricksException(**exception_json) from e
 
         return r.json()
 
@@ -297,9 +297,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             return r["handle"]
         except DatabricksException as e:
             if e.error_code == "RESOURCE_ALREADY_EXISTS":
-                raise FileExistsError(e.message)
+                raise FileExistsError(e.message) from e
 
-            raise e
+            raise
 
     def _close_handle(self, handle):
         """
@@ -314,9 +314,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             self._send_to_api(method="post", endpoint="close", json={"handle": handle})
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
 
-            raise e
+            raise
 
     def _add_data(self, handle, data):
         """
@@ -342,11 +342,11 @@ class DatabricksFileSystem(AbstractFileSystem):
             )
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
             elif e.error_code == "MAX_BLOCK_SIZE_EXCEEDED":
-                raise ValueError(e.message)
+                raise ValueError(e.message) from e
 
-            raise e
+            raise
 
     def _get_data(self, path, start, end):
         """
@@ -372,11 +372,11 @@ class DatabricksFileSystem(AbstractFileSystem):
             return base64.b64decode(r["data"])
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
             elif e.error_code in ["INVALID_PARAMETER_VALUE", "MAX_READ_SIZE_EXCEEDED"]:
-                raise ValueError(e.message)
+                raise ValueError(e.message) from e
 
-            raise e
+            raise
 
     def invalidate_cache(self, path=None):
         if path is None:

fsspec/implementations/dirfs.py

@@ -64,9 +64,15 @@ class DirFileSystem(AsyncFileSystem):
         if isinstance(path, str):
             if not self.path:
                 return path
-            if path == self.path:
+            # We need to account for S3FileSystem returning paths that do not
+            # start with a '/'
+            if path == self.path or (
+                self.path.startswith(self.fs.sep) and path == self.path[1:]
+            ):
                 return ""
             prefix = self.path + self.fs.sep
+            if self.path.startswith(self.fs.sep) and not path.startswith(self.fs.sep):
+                prefix = prefix[1:]
             assert path.startswith(prefix)
             return path[len(prefix) :]
         return [self._relpath(_path) for _path in path]

fsspec/implementations/ftp.py

@@ -2,7 +2,7 @@ import os
 import sys
 import uuid
 import warnings
-from ftplib import FTP, Error, error_perm
+from ftplib import FTP, FTP_TLS, Error, error_perm
 from typing import Any
 
 from ..spec import AbstractBufferedFile, AbstractFileSystem
@@ -27,6 +27,7 @@ class FTPFileSystem(AbstractFileSystem):
         tempdir=None,
         timeout=30,
         encoding="utf-8",
+        tls=False,
         **kwargs,
     ):
         """
@@ -56,28 +57,37 @@ class FTPFileSystem(AbstractFileSystem):
             Timeout of the ftp connection in seconds
         encoding: str
             Encoding to use for directories and filenames in FTP connection
+        tls: bool
+            Use FTP-TLS, by default False
         """
         super().__init__(**kwargs)
         self.host = host
         self.port = port
         self.tempdir = tempdir or "/tmp"
-        self.cred = username, password, acct
+        self.cred = username or "", password or "", acct or ""
         self.timeout = timeout
         self.encoding = encoding
         if block_size is not None:
             self.blocksize = block_size
         else:
             self.blocksize = 2**16
+        self.tls = tls
         self._connect()
+        if self.tls:
+            self.ftp.prot_p()
 
     def _connect(self):
+        if self.tls:
+            ftp_cls = FTP_TLS
+        else:
+            ftp_cls = FTP
         if sys.version_info >= (3, 9):
-            self.ftp = FTP(timeout=self.timeout, encoding=self.encoding)
+            self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
         elif self.encoding:
             warnings.warn("`encoding` not supported for python<3.9, ignoring")
-            self.ftp = FTP(timeout=self.timeout)
+            self.ftp = ftp_cls(timeout=self.timeout)
         else:
-            self.ftp = FTP(timeout=self.timeout)
+            self.ftp = ftp_cls(timeout=self.timeout)
         self.ftp.connect(self.host, self.port)
         self.ftp.login(*self.cred)
 
@@ -107,9 +117,9 @@ class FTPFileSystem(AbstractFileSystem):
                 except error_perm:
                     out = _mlsd2(self.ftp, path)  # Not platform independent
                 for fn, details in out:
-                    if path == "/":
-                        path = ""  # just for forming the names, below
-                    details["name"] = "/".join([path, fn.lstrip("/")])
+                    details["name"] = "/".join(
+                        ["" if path == "/" else path, fn.lstrip("/")]
+                    )
                     if details["type"] == "file":
                         details["size"] = int(details["size"])
                     else:
@@ -122,8 +132,8 @@ class FTPFileSystem(AbstractFileSystem):
                     info = self.info(path)
                     if info["type"] == "file":
                         out = [(path, info)]
-                except (Error, IndexError):
-                    raise FileNotFoundError(path)
+                except (Error, IndexError) as exc:
+                    raise FileNotFoundError(path) from exc
         files = self.dircache.get(path, out)
         if not detail:
             return sorted([fn for fn, details in files])
@@ -137,9 +147,9 @@ class FTPFileSystem(AbstractFileSystem):
             return {"name": "/", "size": 0, "type": "directory"}
         files = self.ls(self._parent(path).lstrip("/"), True)
         try:
-            out = [f for f in files if f["name"] == path][0]
-        except IndexError:
-            raise FileNotFoundError(path)
+            out = next(f for f in files if f["name"] == path)
+        except StopIteration as exc:
+            raise FileNotFoundError(path) from exc
         return out
 
     def get_file(self, rpath, lpath, **kwargs):

fsspec/implementations/http.py

@@ -254,7 +254,7 @@ class HTTPFileSystem(AsyncFileSystem):
             if isfilelike(lpath):
                 outfile = lpath
             else:
-                outfile = open(lpath, "wb")  # noqa: ASYNC101
+                outfile = open(lpath, "wb")  # noqa: ASYNC101, ASYNC230
 
             try:
                 chunk = True
@@ -282,7 +282,7 @@ class HTTPFileSystem(AsyncFileSystem):
                 context = nullcontext(lpath)
                 use_seek = False  # might not support seeking
             else:
-                context = open(lpath, "rb")  # noqa: ASYNC101
+                context = open(lpath, "rb")  # noqa: ASYNC101, ASYNC230
                 use_seek = True
 
             with context as f:
@@ -805,7 +805,7 @@ async def get_range(session, url, start, end, file=None, **kwargs):
     async with r:
         out = await r.read()
     if file:
-        with open(file, "r+b") as f:  # noqa: ASYNC101
+        with open(file, "r+b") as f:  # noqa: ASYNC101, ASYNC230
             f.seek(start)
             f.write(out)
     else:

fsspec/implementations/local.py

@@ -79,6 +79,14 @@ class LocalFileSystem(AbstractFileSystem):
                 t = "file"
             else:
                 t = "other"
+
+            size = out.st_size
+            if link:
+                try:
+                    out2 = path.stat(follow_symlinks=True)
+                    size = out2.st_size
+                except OSError:
+                    size = 0
             path = self._strip_protocol(path.path)
         else:
             # str or path-like
@@ -87,6 +95,7 @@ class LocalFileSystem(AbstractFileSystem):
             link = stat.S_ISLNK(out.st_mode)
             if link:
                 out = os.stat(path, follow_symlinks=True)
+            size = out.st_size
             if stat.S_ISDIR(out.st_mode):
                 t = "directory"
             elif stat.S_ISREG(out.st_mode):
@@ -95,20 +104,15 @@ class LocalFileSystem(AbstractFileSystem):
                 t = "other"
         result = {
             "name": path,
-            "size": out.st_size,
+            "size": size,
             "type": t,
             "created": out.st_ctime,
             "islink": link,
         }
         for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]:
             result[field] = getattr(out, f"st_{field}")
-        if result["islink"]:
+        if link:
             result["destination"] = os.readlink(path)
-            try:
-                out2 = os.stat(path, follow_symlinks=True)
-                result["size"] = out2.st_size
-            except OSError:
-                result["size"] = 0
         return result
 
     def lexists(self, path, **kwargs):

fsspec/implementations/memory.py

@@ -224,8 +224,8 @@ class MemoryFileSystem(AbstractFileSystem):
         path = self._strip_protocol(path)
         try:
             return bytes(self.store[path].getbuffer()[start:end])
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as e:
+            raise FileNotFoundError(path) from e
 
     def _rm(self, path):
         path = self._strip_protocol(path)
@@ -238,15 +238,15 @@ class MemoryFileSystem(AbstractFileSystem):
         path = self._strip_protocol(path)
         try:
             return self.store[path].modified
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as e:
+            raise FileNotFoundError(path) from e
 
     def created(self, path):
         path = self._strip_protocol(path)
         try:
             return self.store[path].created
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as e:
+            raise FileNotFoundError(path) from e
 
     def rm(self, path, recursive=False, maxdepth=None):
         if isinstance(path, str):

fsspec/implementations/reference.py

@@ -5,6 +5,7 @@ import itertools
 import logging
 import math
 import os
+from itertools import chain
 from functools import lru_cache
 from typing import TYPE_CHECKING
 
@@ -16,10 +17,10 @@ except ImportError:
     if not TYPE_CHECKING:
         import json
 
-from ..asyn import AsyncFileSystem
-from ..callbacks import DEFAULT_CALLBACK
-from ..core import filesystem, open, split_protocol
-from ..utils import isfilelike, merge_offset_ranges, other_paths
+from fsspec.asyn import AsyncFileSystem
+from fsspec.callbacks import DEFAULT_CALLBACK
+from fsspec.core import filesystem, open, split_protocol
+from fsspec.utils import isfilelike, merge_offset_ranges, other_paths
 
 logger = logging.getLogger("fsspec.reference")
 
@@ -35,7 +36,7 @@ class ReferenceNotReachable(RuntimeError):
 
 
 def _first(d):
-    return list(d.values())[0]
+    return next(iter(d.values()))
 
 
 def _prot_in_references(path, references):
@@ -131,7 +132,6 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         self.out_root = out_root or self.root
         self.cat_thresh = categorical_threshold
         self.cache_size = cache_size
-        self.dirs = None
         self.url = self.root + "/{field}/refs.{record}.parq"
         # TODO: derive fs from `root`
         self.fs = fsspec.filesystem("file") if fs is None else fs
@@ -159,7 +159,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
             path = self.url.format(field=field, record=record)
             data = io.BytesIO(self.fs.cat_file(path))
             df = self.pd.read_parquet(data, engine="fastparquet")
-            refs = {c: df[c].values for c in df.columns}
+            refs = {c: df[c].to_numpy() for c in df.columns}
             return refs
 
         self.open_refs = open_refs
@@ -195,32 +195,36 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
-    def listdir(self, basename=True):
+    @lru_cache()
+    def listdir(self):
         """List top-level directories"""
-        # cache me?
-        if self.dirs is None:
-            dirs = [p.split("/", 1)[0] for p in self.zmetadata]
-            self.dirs = {p for p in dirs if p and not p.startswith(".")}
-        listing = self.dirs
-        if basename:
-            listing = [os.path.basename(path) for path in listing]
-        return listing
+        dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))
+        return set(dirs)
 
     def ls(self, path="", detail=True):
         """Shortcut file listings"""
-        if not path:
-            dirnames = self.listdir()
-            others = set(
-                [".zmetadata"]
-                + [name for name in self.zmetadata if "/" not in name]
-                + [name for name in self._items if "/" not in name]
-            )
+        path = path.rstrip("/")
+        pathdash = path + "/" if path else ""
+        dirnames = self.listdir()
+        dirs = [
+            d
+            for d in dirnames
+            if d.startswith(pathdash) and "/" not in d.lstrip(pathdash)
+        ]
+        if dirs:
+            others = {
+                f
+                for f in chain(
+                    [".zmetadata"],
+                    (name for name in self.zmetadata),
+                    (name for name in self._items),
+                )
+                if f.startswith(pathdash) and "/" not in f.lstrip(pathdash)
+            }
             if detail is False:
-                others.update(dirnames)
+                others.update(dirs)
                 return sorted(others)
-            dirinfo = [
-                {"name": name, "type": "directory", "size": 0} for name in dirnames
-            ]
+            dirinfo = [{"name": name, "type": "directory", "size": 0} for name in dirs]
             fileinfo = [
                 {
                     "name": name,
@@ -234,10 +238,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
                 for name in others
             ]
             return sorted(dirinfo + fileinfo, key=lambda s: s["name"])
-        parts = path.split("/", 1)
-        if len(parts) > 1:
-            raise FileNotFoundError("Cannot list within directories right now")
-        field = parts[0]
+        field = path
         others = set(
             [name for name in self.zmetadata if name.startswith(f"{path}/")]
             + [name for name in self._items if name.startswith(f"{path}/")]
@@ -291,8 +292,8 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         # Chunk keys can be loaded from row group and cached in LRU cache
         try:
             refs = self.open_refs(field, record)
-        except (ValueError, TypeError, FileNotFoundError):
-            raise KeyError(key)
+        except (ValueError, TypeError, FileNotFoundError) as exc:
+            raise KeyError(key) from exc
         columns = ["path", "offset", "size", "raw"]
         selection = [refs[c][ri] if c in refs else None for c in columns]
         raw = selection[-1]
@@ -501,6 +502,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
             if k != ".zmetadata" and ".z" in k:
                 self.zmetadata[k] = json.loads(self._items.pop(k))
         met = {"metadata": self.zmetadata, "record_size": self.record_size}
+        self._items.clear()
         self._items[".zmetadata"] = json.dumps(met).encode()
         self.fs.pipe(
             "/".join([base_url or self.out_root, ".zmetadata"]),
@@ -732,8 +734,8 @@ class ReferenceFileSystem(AsyncFileSystem):
         logger.debug(f"cat: {path}")
         try:
             part = self.references[path]
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as exc:
+            raise FileNotFoundError(path) from exc
         if isinstance(part, str):
             part = part.encode()
         if isinstance(part, bytes):
@@ -995,9 +997,11 @@ class ReferenceFileSystem(AsyncFileSystem):
         out = {}
         for gen in gens:
             dimension = {
-                k: v
-                if isinstance(v, list)
-                else range(v.get("start", 0), v["stop"], v.get("step", 1))
+                k: (
+                    v
+                    if isinstance(v, list)
+                    else range(v.get("start", 0), v["stop"], v.get("step", 1))
+                )
                 for k, v in gen["dimensions"].items()
             }
             products = (
@@ -1084,7 +1088,7 @@ class ReferenceFileSystem(AsyncFileSystem):
         if self.dircache:
             return path in self.dircache
         elif isinstance(self.references, LazyReferenceMapper):
-            return path in self.references.listdir("")
+            return path in self.references.listdir()
         else:
             # this may be faster than building dircache for single calls, but
             # by looping will be slow for many calls; could cache it?

fsspec/implementations/smb.py

@@ -4,10 +4,12 @@ Windows Samba network shares by using package smbprotocol
 """
 
 import datetime
+import re
 import uuid
 from stat import S_ISDIR, S_ISLNK
 
 import smbclient
+import smbprotocol.exceptions
 
 from .. import AbstractFileSystem
 from ..utils import infer_storage_options
@@ -67,7 +69,9 @@ class SMBFileSystem(AbstractFileSystem):
         timeout=60,
         encrypt=None,
         share_access=None,
-        register_session_retries=5,
+        register_session_retries=4,
+        register_session_retry_wait=1,
+        register_session_retry_factor=10,
         auto_mkdir=False,
         **kwargs,
     ):
@@ -103,6 +107,19 @@ class SMBFileSystem(AbstractFileSystem):
             - 'r': Allow other handles to be opened with read access.
             - 'w': Allow other handles to be opened with write access.
             - 'd': Allow other handles to be opened with delete access.
+        register_session_retries: int
+            Number of retries to register a session with the server. Retries are not performed
+            for authentication errors, as they are considered as invalid credentials and not network
+            issues. If set to negative value, no register attempts will be performed.
+        register_session_retry_wait: int
+            Time in seconds to wait between each retry. Number must be non-negative.
+        register_session_retry_factor: int
+            Base factor for the wait time between each retry. The wait time
+            is calculated using exponential function. For factor=1 all wait times
+            will be equal to `register_session_retry_wait`. For any number of retries,
+            the last wait time will be equal to `register_session_retry_wait` and for retries>1
+            the first wait time will be equal to `register_session_retry_wait / factor`.
+            Number must be equal to or greater than 1. Optimal factor is 10.
         auto_mkdir: bool
             Whether, when opening a file, the directory containing it should
             be created (if it doesn't already exist). This is assumed by pyarrow
@@ -118,6 +135,17 @@ class SMBFileSystem(AbstractFileSystem):
         self.temppath = kwargs.pop("temppath", "")
         self.share_access = share_access
         self.register_session_retries = register_session_retries
+        if register_session_retry_wait < 0:
+            raise ValueError(
+                "register_session_retry_wait must be a non-negative integer"
+            )
+        self.register_session_retry_wait = register_session_retry_wait
+        if register_session_retry_factor < 1:
+            raise ValueError(
+                "register_session_retry_factor must be a positive "
+                "integer equal to or greater than 1"
+            )
+        self.register_session_retry_factor = register_session_retry_factor
         self.auto_mkdir = auto_mkdir
         self._connect()
 
@@ -128,7 +156,26 @@ class SMBFileSystem(AbstractFileSystem):
     def _connect(self):
         import time
 
-        for _ in range(self.register_session_retries):
+        if self.register_session_retries <= -1:
+            return
+
+        retried_errors = []
+
+        wait_time = self.register_session_retry_wait
+        n_waits = (
+            self.register_session_retries - 1
+        )  # -1 = No wait time after the last retry
+        factor = self.register_session_retry_factor
+
+        # Generate wait times for each retry attempt.
+        # Wait times are calculated using exponential function. For factor=1 all wait times
+        # will be equal to `wait`. For any number of retries the last wait time will be
+        # equal to `wait` and for retries>2 the first wait time will be equal to `wait / factor`.
+        wait_times = iter(
+            factor ** (n / n_waits - 1) * wait_time for n in range(0, n_waits + 1)
+        )
+
+        for attempt in range(self.register_session_retries + 1):
             try:
                 smbclient.register_session(
                     self.host,
@@ -138,9 +185,35 @@ class SMBFileSystem(AbstractFileSystem):
                     encrypt=self.encrypt,
                     connection_timeout=self.timeout,
                 )
-                break
-            except Exception:
-                time.sleep(0.1)
+                return
+            except (
+                smbprotocol.exceptions.SMBAuthenticationError,
+                smbprotocol.exceptions.LogonFailure,
+            ):
+                # These exceptions should not be repeated, as they clearly indicate
+                # that the credentials are invalid and not a network issue.
+                raise
+            except ValueError as exc:
+                if re.findall(r"\[Errno -\d+]", str(exc)):
+                    # This exception is raised by the smbprotocol.transport:Tcp.connect
+                    # and originates from socket.gaierror (OSError). These exceptions might
+                    # be raised due to network instability. We will retry to connect.
+                    retried_errors.append(exc)
+                else:
+                    # All another ValueError exceptions should be raised, as they are not
+                    # related to network issues.
+                    raise
+            except Exception as exc:
+                # Save the exception and retry to connect. This except might be dropped
+                # in the future, once all exceptions suited for retry are identified.
+                retried_errors.append(exc)
+
+            if attempt < self.register_session_retries:
+                time.sleep(next(wait_times))
+
+        # Raise last exception to inform user about the connection issues.
+        # Note: Should we use ExceptionGroup to raise all exceptions?
+        raise retried_errors[-1]
 
     @classmethod
     def _strip_protocol(cls, path):

fsspec/implementations/webhdfs.py

@@ -102,7 +102,7 @@ class WebHDFS(AbstractFileSystem):
         if self._cached:
             return
         super().__init__(**kwargs)
-        self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1"  # noqa
+        self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1"
         self.kerb = kerberos
         self.kerb_kwargs = kerb_kwargs or {}
         self.pars = {}
@@ -393,7 +393,7 @@ class WebHDFS(AbstractFileSystem):
                 with self.open(tmp_fname, "wb") as rstream:
                     shutil.copyfileobj(lstream, rstream)
                 self.mv(tmp_fname, rpath)
-            except BaseException:  # noqa
+            except BaseException:
                 with suppress(FileNotFoundError):
                     self.rm(tmp_fname)
                 raise

fsspec/implementations/zip.py

@@ -132,3 +132,45 @@ class ZipFileSystem(AbstractArchiveFileSystem):
             out.size = info["size"]
             out.name = info["name"]
         return out
+
+    def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
+
+        # Remove the leading slash, as the zip file paths are always
+        # given without a leading slash
+        path = path.lstrip("/")
+        path_parts = list(filter(lambda s: bool(s), path.split("/")))
+
+        def _matching_starts(file_path):
+            file_parts = filter(lambda s: bool(s), file_path.split("/"))
+            return all(a == b for a, b in zip(path_parts, file_parts))
+
+        self._get_dirs()
+
+        result = {}
+        # To match posix find, if an exact file name is given, we should
+        # return only that file
+        if path in self.dir_cache and self.dir_cache[path]["type"] == "file":
+            result[path] = self.dir_cache[path]
+            return result if detail else [path]
+
+        for file_path, file_info in self.dir_cache.items():
+            if not (path == "" or _matching_starts(file_path)):
+                continue
+
+            if file_info["type"] == "directory":
+                if withdirs:
+                    if file_path not in result:
+                        result[file_path.strip("/")] = file_info
+                continue
+
+            if file_path not in result:
+                result[file_path] = file_info if detail else None
+
+        if maxdepth:
+            path_depth = path.count("/")
+            result = {
+                k: v for k, v in result.items() if k.count("/") - path_depth < maxdepth
+            }
+        return result if detail else sorted(result)

fsspec/mapping.py

@@ -153,10 +153,10 @@ class FSMap(MutableMapping):
         k = self._key_to_str(key)
         try:
             result = self.fs.cat(k)
-        except self.missing_exceptions:
+        except self.missing_exceptions as exc:
             if default is not None:
                 return default
-            raise KeyError(key)
+            raise KeyError(key) from exc
         return result
 
     def pop(self, key, default=None):
@@ -184,8 +184,8 @@ class FSMap(MutableMapping):
         """Remove key"""
         try:
             self.fs.rm(self._key_to_str(key))
-        except:  # noqa: E722
-            raise KeyError
+        except Exception as exc:
+            raise KeyError from exc
 
     def __contains__(self, key):
         """Does key exist in mapping?"""

fsspec/spec.py

@@ -1892,7 +1892,7 @@ class AbstractBufferedFile(io.IOBase):
             self.offset = 0
             try:
                 self._initiate_upload()
-            except:  # noqa: E722
+            except:
                 self.closed = True
                 raise
 

fsspec/tests/abstract/__init__.py

@@ -4,9 +4,9 @@ from hashlib import md5
 import pytest
 
 from fsspec.implementations.local import LocalFileSystem
-from fsspec.tests.abstract.copy import AbstractCopyTests  # noqa
-from fsspec.tests.abstract.get import AbstractGetTests  # noqa
-from fsspec.tests.abstract.put import AbstractPutTests  # noqa
+from fsspec.tests.abstract.copy import AbstractCopyTests  # noqa: F401
+from fsspec.tests.abstract.get import AbstractGetTests  # noqa: F401
+from fsspec.tests.abstract.put import AbstractPutTests  # noqa: F401
 
 
 class BaseAbstractFixtures:

fsspec/utils.py

@@ -427,10 +427,7 @@ def is_exception(obj: Any) -> bool:
 
 
 def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
-    for attr in ["read", "close", "tell"]:
-        if not hasattr(f, attr):
-            return False
-    return True
+    return all(hasattr(f, attr) for attr in ["read", "close", "tell"])
 
 
 def get_protocol(url: str) -> str:

{fsspec-2024.6.1.dist-info → fsspec-2024.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fsspec
-Version: 2024.6.1
+Version: 2024.9.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/

{fsspec-2024.6.1.dist-info → fsspec-2024.9.0.dist-info}/RECORD

@@ -1,55 +1,55 @@
 fsspec/__init__.py,sha256=l9MJaNNV2d4wKpCtMvXDr55n92DkdrAayGy3F9ICjzk,1998
-fsspec/_version.py,sha256=bs5gFL9Mlwh1IvRh5hZawz_SleC88gaDg6jodp6omsQ,417
+fsspec/_version.py,sha256=1O0P6wbwqtkyltAT0n4UISDOJbtOp9uOwvifb7ASk-U,417
 fsspec/archive.py,sha256=S__DzfZj-urAN3tp2W6jJ6YDiXG1fAl7FjvWUN73qIE,2386
-fsspec/asyn.py,sha256=AOd2SXH2YPCaQL5jA6IegYevdMFkAnGD7Seh9DC2gSE,36404
+fsspec/asyn.py,sha256=MTe85f2Rmvwg-uhZbckpU_GemYYYSZ3AAj8Et9CCgmk,36390
 fsspec/caching.py,sha256=x6IEdxtR3cMDjy40sNHyawR2SLtNSahVuP5i_TImdso,31600
 fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210
 fsspec/compression.py,sha256=jCSUMJu-zSNyrusnHT0wKXgOd1tTJR6vM126i5SR5Zc,4865
 fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
 fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
-fsspec/core.py,sha256=Iln37fNZqjjk5vaDGU_0WWuwOxN1iVsQ6sDmCmuEvrs,23681
+fsspec/core.py,sha256=299qCp0H3w3e6zbiK5YOm3pJjxuPr4IYcK6Yg1Zgcos,23684
 fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
 fsspec/exceptions.py,sha256=pauSLDMxzTJMOjvX1WEUK0cMyFkrFxpWJsyFywav7A8,331
-fsspec/fuse.py,sha256=66amOa6wdIbS0DMhhfAPUoOB37HPorfXD1izV0prmTY,10145
+fsspec/fuse.py,sha256=Q-3NOOyLqBfYa4Db5E19z_ZY36zzYHtIs1mOUasItBQ,10177
 fsspec/generic.py,sha256=AFbo-mHBt5QJV1Aplg5CJuUiiJ4bNQhcKRuwkZJdWac,13761
-fsspec/gui.py,sha256=k46F11VGBLlrliPj3XbxHKlVGByWoX67Ofmu9ijaPBQ,13929
+fsspec/gui.py,sha256=xBnHL2-r0LVwhDAtnHoPpXts7jd4Z32peawCJiI-7lI,13975
 fsspec/json.py,sha256=65sQ0Y7mTj33u_Y4IId5up4abQ3bAel4E4QzbKMiQSg,3826
-fsspec/mapping.py,sha256=hSsiRo-dgAOj6oHf67bF3i11U4xREglXToHGUX4GhRY,8261
+fsspec/mapping.py,sha256=CtD_GEmyYgXefQHndkxu7Zb_kbTS3mlFP2zIwlAoQTY,8289
 fsspec/parquet.py,sha256=ONG29Enesp0ToCH2bQ7zkpimnVIsZ2S4xCLj35-fY78,19455
 fsspec/registry.py,sha256=HVC-4HWDZnA6rycJwAu8F8ZXzON_85MTQVIyS6LOHxo,11320
-fsspec/spec.py,sha256=6rb-C3hTZLLtMGx2HDp37N_sZKs5RtYdcj8XOlHFi_c,69586
+fsspec/spec.py,sha256=A48RUDL50AwSOGB1VT114GC1TY93SGlr3fkBO1Yp0Fk,69572
 fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398
-fsspec/utils.py,sha256=8czEIoX4GpcC42WLGoy3t_EMeZjJE8e5rTpOT_nEPo0,22987
+fsspec/utils.py,sha256=dVaokocjhMOnO3B1KmKlgxYqojQJyzb3mgIfaAaz8Pk,22941
 fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fsspec/implementations/arrow.py,sha256=Y4F_IwWXuJI1mRO_c0_PI5o-Wp58RLmoiH_s-x88w4M,8631
+fsspec/implementations/arrow.py,sha256=721Dikne_lV_0tlgk9jyKmHL6W-5MT0h2LKGvOYQTPI,8623
 fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421
 fsspec/implementations/cache_metadata.py,sha256=pcOJYcBQY5OaC7Yhw0F3wjg08QLYApGmoISCrbs59ks,8511
 fsspec/implementations/cached.py,sha256=t5atYATgjuABm-mUyReqjGqVyyP1XBSuROX92aMecxY,32826
 fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
 fsspec/implementations/data.py,sha256=LDLczxRh8h7x39Zjrd-GgzdQHr78yYxDlrv2C9Uxb5E,1658
-fsspec/implementations/dbfs.py,sha256=cix9OYUveuSOx5UO5uRUwNUkYqjzyY0fkKnca1kTgZ0,15014
-fsspec/implementations/dirfs.py,sha256=0H6k67e2lZgq3U4K64ao6894L4134COUCekc3PCwTq8,11488
-fsspec/implementations/ftp.py,sha256=rp6cTog8xqjDPlKdSLKcsyP7K593_ByMabxGbNSEpTo,11655
+fsspec/implementations/dbfs.py,sha256=a0eNjLxyfFK7pbEa52U8K-PhNHukzdGVx1eLcVniaXY,15092
+fsspec/implementations/dirfs.py,sha256=VPSJhy2wFZodY5BB-o1tkWGYecvU2EmbgubmX3lOwuw,11815
+fsspec/implementations/ftp.py,sha256=VpJWnQscdEKRu4fzkCtuf3jD9A74mBaerS2ijUwZ-_I,11936
 fsspec/implementations/git.py,sha256=vKGI-Vd5q4H2RrvhebkPc9NwlfkZ980OUGhebeCw-M0,4034
 fsspec/implementations/github.py,sha256=eAn1kJ7VeWR6gVoVRLBYclF_rQDXSJU-xzMXpvPQWqs,8002
-fsspec/implementations/http.py,sha256=ymjMQTXW6-akqqEoEKpjf416JAzP9N4VhnWiNYbRklk,29665
+fsspec/implementations/http.py,sha256=BjDJ72IoUCe_EHKc44J__sy8VHCOdc98gVEq27sWUk8,29695
 fsspec/implementations/jupyter.py,sha256=B2uj7OEm7yIk-vRSsO37_ND0t0EBvn4B-Su43ibN4Pg,3811
 fsspec/implementations/libarchive.py,sha256=5_I2DiLXwQ1JC8x-K7jXu-tBwhO9dj7tFLnb0bTnVMQ,7102
-fsspec/implementations/local.py,sha256=qc68w69-I7zqVO8njv_s-THVImwICOqxyt-_2EK1VLg,15042
-fsspec/implementations/memory.py,sha256=-BpOVwaWyW2rDvxWIIcrZTNFAhvuG66VWeIM6vLwhkc,10134
-fsspec/implementations/reference.py,sha256=iDisTIZ8kIWG_FNSGaDf88RClywAwoF8yMgoVcxM4cY,44308
+fsspec/implementations/local.py,sha256=DNBZhF9LYYTPR4PKedeWuk32Tztc9jlgXtGRFGX7nv4,15103
+fsspec/implementations/memory.py,sha256=-0AedWR-jBaw2zamEuL4ku73lJQwRdp-Muia0u1j6pU,10170
+fsspec/implementations/reference.py,sha256=43lG6cq9GP0JfMv_n_CyPznRcpEaWAaxG-rgeZt9BV4,44375
 fsspec/implementations/sftp.py,sha256=fMY9XZcmpjszQ2tCqO_TPaJesaeD_Dv7ptYzgUPGoO0,5631
-fsspec/implementations/smb.py,sha256=RcqCvVBPD3U0I0Rc31ns6HRhqKVDugjPQMDPVpvZSNg,11408
+fsspec/implementations/smb.py,sha256=5fhu8h06nOLBPh2c48aT7WBRqh9cEcbIwtyu06wTjec,15236
 fsspec/implementations/tar.py,sha256=dam78Tp_CozybNqCY2JYgGBS3Uc9FuJUAT9oB0lolOs,4111
-fsspec/implementations/webhdfs.py,sha256=Wm7zr0iX3SZx5LtWfJIo-5rkIaoEoWq_Ev87NWbUgug,16721
-fsspec/implementations/zip.py,sha256=vc1fNz-yO8uWQ9bQUqBFYpTcgsfZQq9vDwwg4Aufs9Y,4417
-fsspec/tests/abstract/__init__.py,sha256=i1wcFixV6QhOwdoB24c8oXjzobISNqiKVz9kl2DvAY8,10028
+fsspec/implementations/webhdfs.py,sha256=aet-AOfMoK91C3jNu5xBxK0Mu2iaAWiL9Xfu12KyjQI,16705
+fsspec/implementations/zip.py,sha256=XoRukvrnJWngLbE8Exp2XCVf3SgSPmOqdeCqQ3NpSr0,6047
+fsspec/tests/abstract/__init__.py,sha256=o3rQBCeTTTdji0OxKdTvBvwL7q78sEIh5J5-Q-If6z0,10046
 fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973
 fsspec/tests/abstract/copy.py,sha256=gU5-d97U3RSde35Vp4RxPY4rWwL744HiSrJ8IBOp9-8,19967
 fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755
 fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1982
 fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201
-fsspec-2024.6.1.dist-info/METADATA,sha256=ijt16ZAzPN9P0_1AU4zcKdiM18pkIyf1Gkr-IFXrlLw,11749
-fsspec-2024.6.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-fsspec-2024.6.1.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
-fsspec-2024.6.1.dist-info/RECORD,,
+fsspec-2024.9.0.dist-info/METADATA,sha256=Jgjl4t19VdvyQ81LzLYeXagQpe2Om5QJDQ4nDfKdGbc,11749
+fsspec-2024.9.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+fsspec-2024.9.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
+fsspec-2024.9.0.dist-info/RECORD,,