fsspec 2023.6.0__py3-none-any.whl → 2023.9.1__py3-none-any.whl

fsspec/implementations/reference.py

@@ -150,7 +150,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         """List top-level directories"""
         if self.dirs is None:
             dirs = [p.split("/", 1)[0] for p in self.zmetadata]
-            self.dirs = set(sorted(p for p in dirs if p and not p.startswith(".")))
+            self.dirs = {p for p in dirs if p and not p.startswith(".")}
         listing = self.dirs
         if basename:
             listing = [os.path.basename(path) for path in listing]
@@ -381,17 +381,17 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
                 raws[j] = kerchunk.df._proc_raw(data)
         # TODO: only save needed columns
         df = pd.DataFrame(
-            dict(
-                path=paths,
-                offset=offsets,
-                size=sizes,
-                raw=raws,
-            ),
+            {
+                "path": paths,
+                "offset": offsets,
+                "size": sizes,
+                "raw": raws,
+            },
             copy=False,
         )
         if df.path.count() / (df.path.nunique() or 1) > self.cat_thresh:
             df["path"] = df["path"].astype("category")
-        object_encoding = dict(raw="bytes", path="utf8")
+        object_encoding = {"raw": "bytes", "path": "utf8"}
         has_nulls = ["path", "raw"]
 
         self.fs.mkdirs(f"{base_url or self.out_root}/{field}", exist_ok=True)

fsspec/implementations/sftp.py

@@ -110,8 +110,12 @@ class SFTPFileSystem(AbstractFileSystem):
             "type": t,
             "uid": stat.st_uid,
             "gid": stat.st_gid,
-            "time": datetime.datetime.utcfromtimestamp(stat.st_atime),
-            "mtime": datetime.datetime.utcfromtimestamp(stat.st_mtime),
+            "time": datetime.datetime.fromtimestamp(
+                stat.st_atime, tz=datetime.timezone.utc
+            ),
+            "mtime": datetime.datetime.fromtimestamp(
+                stat.st_mtime, tz=datetime.timezone.utc
+            ),
         }
         if parent_path:
             out["name"] = "/".join([parent_path.rstrip("/"), stat.filename])

fsspec/implementations/smb.py

@@ -81,7 +81,7 @@ class SMBFileSystem(AbstractFileSystem):
         ----------
         host: str
             The remote server name/ip to connect to
-        port: int
+        port: int or None
             Port to connect with. Usually 445, sometimes 139.
         username: str or None
             Username to connect with. Required if Kerberos auth is not being used.
@@ -114,12 +114,16 @@ class SMBFileSystem(AbstractFileSystem):
         self.share_access = share_access
         self._connect()
 
+    @property
+    def _port(self):
+        return 445 if self.port is None else self.port
+
     def _connect(self):
         smbclient.register_session(
             self.host,
             username=self.username,
             password=self.password,
-            port=445 if self.port is None else self.port,
+            port=self._port,
             encrypt=self.encrypt,
             connection_timeout=self.timeout,
         )
@@ -139,23 +143,23 @@ class SMBFileSystem(AbstractFileSystem):
     def mkdir(self, path, create_parents=True, **kwargs):
         wpath = _as_unc_path(self.host, path)
         if create_parents:
-            smbclient.makedirs(wpath, exist_ok=False, **kwargs)
+            smbclient.makedirs(wpath, exist_ok=False, port=self._port, **kwargs)
         else:
-            smbclient.mkdir(wpath, **kwargs)
+            smbclient.mkdir(wpath, port=self._port, **kwargs)
 
     def makedirs(self, path, exist_ok=False):
         if _share_has_path(path):
             wpath = _as_unc_path(self.host, path)
-            smbclient.makedirs(wpath, exist_ok=exist_ok)
+            smbclient.makedirs(wpath, exist_ok=exist_ok, port=self._port)
 
     def rmdir(self, path):
         if _share_has_path(path):
             wpath = _as_unc_path(self.host, path)
-            smbclient.rmdir(wpath)
+            smbclient.rmdir(wpath, port=self._port)
 
     def info(self, path, **kwargs):
         wpath = _as_unc_path(self.host, path)
-        stats = smbclient.stat(wpath, **kwargs)
+        stats = smbclient.stat(wpath, port=self._port, **kwargs)
         if S_ISDIR(stats.st_mode):
             stype = "directory"
         elif S_ISLNK(stats.st_mode):
@@ -176,18 +180,18 @@ class SMBFileSystem(AbstractFileSystem):
     def created(self, path):
         """Return the created timestamp of a file as a datetime.datetime"""
         wpath = _as_unc_path(self.host, path)
-        stats = smbclient.stat(wpath)
-        return datetime.datetime.utcfromtimestamp(stats.st_ctime)
+        stats = smbclient.stat(wpath, port=self._port)
+        return datetime.datetime.fromtimestamp(stats.st_ctime, tz=datetime.timezone.utc)
 
     def modified(self, path):
         """Return the modified timestamp of a file as a datetime.datetime"""
         wpath = _as_unc_path(self.host, path)
-        stats = smbclient.stat(wpath)
-        return datetime.datetime.utcfromtimestamp(stats.st_mtime)
+        stats = smbclient.stat(wpath, port=self._port)
+        return datetime.datetime.fromtimestamp(stats.st_mtime, tz=datetime.timezone.utc)
 
     def ls(self, path, detail=True, **kwargs):
         unc = _as_unc_path(self.host, path)
-        listed = smbclient.listdir(unc, **kwargs)
+        listed = smbclient.listdir(unc, port=self._port, **kwargs)
         dirs = ["/".join([path.rstrip("/"), p]) for p in listed]
         if detail:
             dirs = [self.info(d) for d in dirs]
@@ -217,30 +221,37 @@ class SMBFileSystem(AbstractFileSystem):
         share_access = kwargs.pop("share_access", self.share_access)
         if "w" in mode and autocommit is False:
             temp = _as_temp_path(self.host, path, self.temppath)
-            return SMBFileOpener(wpath, temp, mode, block_size=bls, **kwargs)
+            return SMBFileOpener(
+                wpath, temp, mode, port=self._port, block_size=bls, **kwargs
+            )
         return smbclient.open_file(
-            wpath, mode, buffering=bls, share_access=share_access, **kwargs
+            wpath,
+            mode,
+            buffering=bls,
+            share_access=share_access,
+            port=self._port,
+            **kwargs,
         )
 
     def copy(self, path1, path2, **kwargs):
         """Copy within two locations in the same filesystem"""
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
-        smbclient.copyfile(wpath1, wpath2, **kwargs)
+        smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs)
 
     def _rm(self, path):
         if _share_has_path(path):
             wpath = _as_unc_path(self.host, path)
-            stats = smbclient.stat(wpath)
+            stats = smbclient.stat(wpath, port=self._port)
             if S_ISDIR(stats.st_mode):
-                smbclient.rmdir(wpath)
+                smbclient.rmdir(wpath, port=self._port)
             else:
-                smbclient.remove(wpath)
+                smbclient.remove(wpath, port=self._port)
 
     def mv(self, path1, path2, **kwargs):
         wpath1 = _as_unc_path(self.host, path1)
         wpath2 = _as_unc_path(self.host, path2)
-        smbclient.rename(wpath1, wpath2, **kwargs)
+        smbclient.rename(wpath1, wpath2, port=self._port, **kwargs)
 
 
 def _as_unc_path(host, path):
@@ -266,7 +277,7 @@ def _share_has_path(path):
 class SMBFileOpener(object):
     """writes to remote temporary file, move on commit"""
 
-    def __init__(self, path, temp, mode, block_size=-1, **kwargs):
+    def __init__(self, path, temp, mode, port=445, block_size=-1, **kwargs):
         self.path = path
         self.temp = temp
         self.mode = mode
@@ -274,22 +285,27 @@ class SMBFileOpener(object):
         self.kwargs = kwargs
         self.smbfile = None
         self._incontext = False
+        self.port = port
         self._open()
 
     def _open(self):
         if self.smbfile is None or self.smbfile.closed:
             self.smbfile = smbclient.open_file(
-                self.temp, self.mode, buffering=self.block_size, **self.kwargs
+                self.temp,
+                self.mode,
+                port=self.port,
+                buffering=self.block_size,
+                **self.kwargs,
             )
 
     def commit(self):
         """Move temp file to definitive on success."""
         # TODO: use transaction support in SMB protocol
-        smbclient.replace(self.temp, self.path)
+        smbclient.replace(self.temp, self.path, port=self.port)
 
     def discard(self):
         """Remove the temp file on failure."""
-        smbclient.remove(self.temp)
+        smbclient.remove(self.temp, port=self.port)
 
     def __fspath__(self):
         return self.path

fsspec/mapping.py

@@ -2,6 +2,7 @@ import array
 import posixpath
 import warnings
 from collections.abc import MutableMapping
+from functools import cached_property
 
 from .core import url_to_fs
 
@@ -59,6 +60,13 @@ class FSMap(MutableMapping):
             self.fs.touch(root + "/a")
             self.fs.rm(root + "/a")
 
+    @cached_property
+    def dirfs(self):
+        """dirfs instance that can be used with the same keys as the mapper"""
+        from .implementations.dirfs import DirFileSystem
+
+        return DirFileSystem(path=self._root_key_to_str, fs=self.fs)
+
     def clear(self):
         """Remove all keys below root - empties out mapping"""
         try:

fsspec/registry.py

@@ -119,6 +119,10 @@ known_implementations = {
         "class": "ocifs.OCIFileSystem",
         "err": "Install ocifs to access OCI Object Storage",
     },
+    "ocilake": {
+        "class": "ocifs.OCIFileSystem",
+        "err": "Install ocifs to access OCI Data Lake",
+    },
     "asynclocal": {
         "class": "morefs.asyn_local.AsyncLocalFileSystem",
         "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem",
@@ -200,6 +204,10 @@ known_implementations = {
         "class": "boxfs.BoxFileSystem",
         "err": "Please install boxfs to access BoxFileSystem",
     },
+    "lakefs": {
+        "class": "lakefs_spec.LakeFSFileSystem",
+        "err": "Please install lakefs-spec to access LakeFSFileSystem",
+    },
 }
 
 
@@ -233,6 +241,14 @@ def get_filesystem_class(protocol):
     return cls
 
 
+s3_msg = """Your installed version of s3fs is very old and known to cause
+severe performance issues, see also https://github.com/dask/dask/issues/10276
+
+To fix, you should specify a lower version bound on s3fs, or
+update the current installation.
+"""
+
+
 def _import_class(cls, minv=None):
     """Take a string FQP and return the imported class or identifier
 
@@ -240,13 +256,19 @@ def _import_class(cls, minv=None):
     """
     if ":" in cls:
         mod, name = cls.rsplit(":", 1)
+        s3 = mod == "s3fs"
         mod = importlib.import_module(mod)
+        if s3 and mod.__version__.split(".") < ["0", "5"]:
+            warnings.warn(s3_msg)
         for part in name.split("."):
             mod = getattr(mod, part)
         return mod
     else:
         mod, name = cls.rsplit(".", 1)
+        s3 = mod == "s3fs"
         mod = importlib.import_module(mod)
+        if s3 and mod.__version__.split(".") < ["0", "5"]:
+            warnings.warn(s3_msg)
         return getattr(mod, name)