fsspec 2024.6.1__tar.gz → 2024.9.0__tar.gz

{fsspec-2024.6.1 → fsspec-2024.9.0}/.github/workflows/main.yaml

@@ -25,19 +25,15 @@ jobs:
           fetch-depth: 0
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-py38.yml
-          create-args: >-
-            python=${{ matrix.PY }}
+          python-version: ${{ matrix.PY }}
 
       - name: Run Tests
         shell: bash -l {0}
         run: |
-          pip install s3fs
-          pip uninstall s3fs
           pip install -e .[test_full]
-          pip install s3fs --no-deps
           pytest -v
 
   win:
@@ -54,17 +50,14 @@ jobs:
           fetch-depth: 0
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-win.yml
 
       - name: Run Tests
         shell: bash -l {0}
         run: |
-          pip install s3fs
-          pip uninstall s3fs
           pip install -e .[test]
-          pip install s3fs --no-deps
           pytest -v
 
   lint:
@@ -84,7 +77,7 @@ jobs:
 #        uses: actions/checkout@v4
 #
 #      - name: Setup conda
-#        uses: mamba-org/setup-micromamba@v1
+#        uses: conda-incubator/setup-miniconda@v3
 #        with:
 #          environment-file: ci/environment-typecheck.yml
 #
@@ -104,7 +97,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-downstream.yml
 
@@ -145,7 +138,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup conda
-        uses: mamba-org/setup-micromamba@v1
+        uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: ci/environment-friends.yml
 

{fsspec-2024.6.1 → fsspec-2024.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fsspec
-Version: 2024.6.1
+Version: 2024.9.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/

{fsspec-2024.6.1 → fsspec-2024.9.0}/ci/environment-downstream.yml

@@ -2,6 +2,6 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.11
   - pip:
       - git+https://github.com/dask/dask

{fsspec-2024.6.1 → fsspec-2024.9.0}/ci/environment-friends.yml

@@ -2,7 +2,7 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.12
   - pytest
   - pytest-asyncio !=0.22.0
   - pytest-benchmark

{fsspec-2024.6.1 → fsspec-2024.9.0}/ci/environment-py38.yml

@@ -5,6 +5,7 @@ dependencies:
   - pip
   - git <2.45.0
   - py
+  - s3fs
   - pip:
     - hadoop-test-cluster
     - smbprotocol

{fsspec-2024.6.1 → fsspec-2024.9.0}/ci/environment-win.yml

@@ -2,4 +2,6 @@ name: test_env
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.11
+  - s3fs
+  - pytest

{fsspec-2024.6.1 → fsspec-2024.9.0}/docs/source/api.rst

@@ -226,6 +226,7 @@ Other Known Implementations
 - `s3fs`_ for Amazon S3 and other compatible stores
 - `wandbfs`_ to access Wandb run data (experimental)
 - `webdav4`_ for WebDAV
+- `xrootd`_ for xrootd, with protocol "root://"
 
 .. _abfs: https://github.com/dask/adlfs
 .. _adl: https://github.com/dask/adlfs
@@ -244,6 +245,7 @@ Other Known Implementations
 .. _s3fs: https://s3fs.readthedocs.io/en/latest/
 .. _wandbfs: https://github.com/jkulhanek/wandbfs
 .. _webdav4: https://github.com/skshetry/webdav4
+.. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
 
 .. _readbuffering:
 

{fsspec-2024.6.1 → fsspec-2024.9.0}/docs/source/changelog.rst

@@ -1,6 +1,28 @@
 Changelog
 =========
 
+2024.9.0
+--------
+
+Enhancements
+
+- fewer stat calls in localFS (#1659)
+- faster find in ZIP (#1664)
+
+Fixes
+
+- paths without "/" in dirFS (#1638)
+- paths with "/" in FTS (#1643, 1644)
+- ls in parquet-based nested reference sets, and append (#1645, 1657)
+- exception handling for SMB (#1650)
+
+
+Other
+
+- style (#1640, 1641, 1660)
+- docs: xrootd (#1646)
+- CI back on miniconda (#1658)
+
 2024.6.1
 --------
 

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.6.1'
-__version_tuple__ = version_tuple = (2024, 6, 1)
+__version__ = version = '2024.9.0'
+__version_tuple__ = version_tuple = (2024, 9, 0)

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/asyn.py

@@ -1072,7 +1072,7 @@ class AbstractAsyncStreamedFile(AbstractBufferedFile):
             self.offset = 0
             try:
                 await self._initiate_upload()
-            except:  # noqa: E722
+            except:
                 self.closed = True
                 raise
 

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/core.py

@@ -639,7 +639,7 @@ def get_fs_token_paths(
     if isinstance(urlpath, (list, tuple, set)):
         if not urlpath:
             raise ValueError("empty urlpath sequence")
-        urlpath0 = stringify_path(list(urlpath)[0])
+        urlpath0 = stringify_path(next(iter(urlpath)))
     else:
         urlpath0 = stringify_path(urlpath)
     storage_options = storage_options or {}

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/fuse.py

@@ -31,8 +31,8 @@ class FUSEr(Operations):
         path = "".join([self.root, path.lstrip("/")]).rstrip("/")
         try:
             info = self.fs.info(path)
-        except FileNotFoundError:
-            raise FuseOSError(ENOENT)
+        except FileNotFoundError as exc:
+            raise FuseOSError(ENOENT) from exc
 
         data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)}
         perm = info.get("mode", 0o777)
@@ -119,8 +119,8 @@ class FUSEr(Operations):
         fn = "".join([self.root, path.lstrip("/")])
         try:
             self.fs.rm(fn, False)
-        except (OSError, FileNotFoundError):
-            raise FuseOSError(EIO)
+        except (OSError, FileNotFoundError) as exc:
+            raise FuseOSError(EIO) from exc
 
     def release(self, path, fh):
         try:

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/gui.py

@@ -93,8 +93,10 @@ class SigSlot:
         """Display in a notebook or a server"""
         try:
             return self.panel._repr_mimebundle_(*args, **kwargs)
-        except (ValueError, AttributeError):
-            raise NotImplementedError("Panel does not seem to be set up properly")
+        except (ValueError, AttributeError) as exc:
+            raise NotImplementedError(
+                "Panel does not seem to be set up properly"
+            ) from exc
 
     def connect(self, signal, slot):
         """Associate call back with given event

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/arrow.py

@@ -128,7 +128,7 @@ class ArrowFSWrapper(AbstractFileSystem):
                 with self.open(tmp_fname, "wb") as rstream:
                     shutil.copyfileobj(lstream, rstream)
                 self.fs.move(tmp_fname, path2)
-            except BaseException:  # noqa
+            except BaseException:
                 with suppress(FileNotFoundError):
                     self.fs.delete_file(tmp_fname)
                 raise

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/dbfs.py

@@ -77,9 +77,9 @@ class DatabricksFileSystem(AbstractFileSystem):
                 )
             except DatabricksException as e:
                 if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                    raise FileNotFoundError(e.message)
+                    raise FileNotFoundError(e.message) from e
 
-                raise e
+                raise
             files = r["files"]
             out = [
                 {
@@ -123,9 +123,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             self._send_to_api(method="post", endpoint="mkdirs", json={"path": path})
         except DatabricksException as e:
             if e.error_code == "RESOURCE_ALREADY_EXISTS":
-                raise FileExistsError(e.message)
+                raise FileExistsError(e.message) from e
 
-            raise e
+            raise
         self.invalidate_cache(self._parent(path))
 
     def mkdir(self, path, create_parents=True, **kwargs):
@@ -169,9 +169,9 @@ class DatabricksFileSystem(AbstractFileSystem):
                 self.rm(path=path, recursive=recursive)
             elif e.error_code == "IO_ERROR":
                 # Using the same exception as the os module would use here
-                raise OSError(e.message)
+                raise OSError(e.message) from e
 
-            raise e
+            raise
         self.invalidate_cache(self._parent(path))
 
     def mv(
@@ -212,11 +212,11 @@ class DatabricksFileSystem(AbstractFileSystem):
             )
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
             elif e.error_code == "RESOURCE_ALREADY_EXISTS":
-                raise FileExistsError(e.message)
+                raise FileExistsError(e.message) from e
 
-            raise e
+            raise
         self.invalidate_cache(self._parent(source_path))
         self.invalidate_cache(self._parent(destination_path))
 
@@ -264,9 +264,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             try:
                 exception_json = e.response.json()
             except Exception:
-                raise e
+                raise e from None
 
-            raise DatabricksException(**exception_json)
+            raise DatabricksException(**exception_json) from e
 
         return r.json()
 
@@ -297,9 +297,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             return r["handle"]
         except DatabricksException as e:
             if e.error_code == "RESOURCE_ALREADY_EXISTS":
-                raise FileExistsError(e.message)
+                raise FileExistsError(e.message) from e
 
-            raise e
+            raise
 
     def _close_handle(self, handle):
         """
@@ -314,9 +314,9 @@ class DatabricksFileSystem(AbstractFileSystem):
             self._send_to_api(method="post", endpoint="close", json={"handle": handle})
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
 
-            raise e
+            raise
 
     def _add_data(self, handle, data):
         """
@@ -342,11 +342,11 @@ class DatabricksFileSystem(AbstractFileSystem):
             )
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
             elif e.error_code == "MAX_BLOCK_SIZE_EXCEEDED":
-                raise ValueError(e.message)
+                raise ValueError(e.message) from e
 
-            raise e
+            raise
 
     def _get_data(self, path, start, end):
         """
@@ -372,11 +372,11 @@ class DatabricksFileSystem(AbstractFileSystem):
             return base64.b64decode(r["data"])
         except DatabricksException as e:
             if e.error_code == "RESOURCE_DOES_NOT_EXIST":
-                raise FileNotFoundError(e.message)
+                raise FileNotFoundError(e.message) from e
             elif e.error_code in ["INVALID_PARAMETER_VALUE", "MAX_READ_SIZE_EXCEEDED"]:
-                raise ValueError(e.message)
+                raise ValueError(e.message) from e
 
-            raise e
+            raise
 
     def invalidate_cache(self, path=None):
         if path is None:

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/dirfs.py

@@ -64,9 +64,15 @@ class DirFileSystem(AsyncFileSystem):
         if isinstance(path, str):
             if not self.path:
                 return path
-            if path == self.path:
+            # We need to account for S3FileSystem returning paths that do not
+            # start with a '/'
+            if path == self.path or (
+                self.path.startswith(self.fs.sep) and path == self.path[1:]
+            ):
                 return ""
             prefix = self.path + self.fs.sep
+            if self.path.startswith(self.fs.sep) and not path.startswith(self.fs.sep):
+                prefix = prefix[1:]
             assert path.startswith(prefix)
             return path[len(prefix) :]
         return [self._relpath(_path) for _path in path]

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/ftp.py

@@ -2,7 +2,7 @@ import os
 import sys
 import uuid
 import warnings
-from ftplib import FTP, Error, error_perm
+from ftplib import FTP, FTP_TLS, Error, error_perm
 from typing import Any
 
 from ..spec import AbstractBufferedFile, AbstractFileSystem
@@ -27,6 +27,7 @@ class FTPFileSystem(AbstractFileSystem):
         tempdir=None,
         timeout=30,
         encoding="utf-8",
+        tls=False,
         **kwargs,
     ):
         """
@@ -56,28 +57,37 @@ class FTPFileSystem(AbstractFileSystem):
             Timeout of the ftp connection in seconds
         encoding: str
             Encoding to use for directories and filenames in FTP connection
+        tls: bool
+            Use FTP-TLS, by default False
         """
         super().__init__(**kwargs)
         self.host = host
         self.port = port
         self.tempdir = tempdir or "/tmp"
-        self.cred = username, password, acct
+        self.cred = username or "", password or "", acct or ""
         self.timeout = timeout
         self.encoding = encoding
         if block_size is not None:
             self.blocksize = block_size
         else:
             self.blocksize = 2**16
+        self.tls = tls
         self._connect()
+        if self.tls:
+            self.ftp.prot_p()
 
     def _connect(self):
+        if self.tls:
+            ftp_cls = FTP_TLS
+        else:
+            ftp_cls = FTP
         if sys.version_info >= (3, 9):
-            self.ftp = FTP(timeout=self.timeout, encoding=self.encoding)
+            self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
         elif self.encoding:
             warnings.warn("`encoding` not supported for python<3.9, ignoring")
-            self.ftp = FTP(timeout=self.timeout)
+            self.ftp = ftp_cls(timeout=self.timeout)
         else:
-            self.ftp = FTP(timeout=self.timeout)
+            self.ftp = ftp_cls(timeout=self.timeout)
         self.ftp.connect(self.host, self.port)
         self.ftp.login(*self.cred)
 
@@ -107,9 +117,9 @@ class FTPFileSystem(AbstractFileSystem):
                 except error_perm:
                     out = _mlsd2(self.ftp, path)  # Not platform independent
                 for fn, details in out:
-                    if path == "/":
-                        path = ""  # just for forming the names, below
-                    details["name"] = "/".join([path, fn.lstrip("/")])
+                    details["name"] = "/".join(
+                        ["" if path == "/" else path, fn.lstrip("/")]
+                    )
                     if details["type"] == "file":
                         details["size"] = int(details["size"])
                     else:
@@ -122,8 +132,8 @@ class FTPFileSystem(AbstractFileSystem):
                     info = self.info(path)
                     if info["type"] == "file":
                         out = [(path, info)]
-                except (Error, IndexError):
-                    raise FileNotFoundError(path)
+                except (Error, IndexError) as exc:
+                    raise FileNotFoundError(path) from exc
         files = self.dircache.get(path, out)
         if not detail:
             return sorted([fn for fn, details in files])
@@ -137,9 +147,9 @@ class FTPFileSystem(AbstractFileSystem):
             return {"name": "/", "size": 0, "type": "directory"}
         files = self.ls(self._parent(path).lstrip("/"), True)
         try:
-            out = [f for f in files if f["name"] == path][0]
-        except IndexError:
-            raise FileNotFoundError(path)
+            out = next(f for f in files if f["name"] == path)
+        except StopIteration as exc:
+            raise FileNotFoundError(path) from exc
         return out
 
     def get_file(self, rpath, lpath, **kwargs):

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/http.py

@@ -254,7 +254,7 @@ class HTTPFileSystem(AsyncFileSystem):
             if isfilelike(lpath):
                 outfile = lpath
             else:
-                outfile = open(lpath, "wb")  # noqa: ASYNC101
+                outfile = open(lpath, "wb")  # noqa: ASYNC101, ASYNC230
 
             try:
                 chunk = True
@@ -282,7 +282,7 @@ class HTTPFileSystem(AsyncFileSystem):
                 context = nullcontext(lpath)
                 use_seek = False  # might not support seeking
             else:
-                context = open(lpath, "rb")  # noqa: ASYNC101
+                context = open(lpath, "rb")  # noqa: ASYNC101, ASYNC230
                 use_seek = True
 
             with context as f:
@@ -805,7 +805,7 @@ async def get_range(session, url, start, end, file=None, **kwargs):
     async with r:
         out = await r.read()
     if file:
-        with open(file, "r+b") as f:  # noqa: ASYNC101
+        with open(file, "r+b") as f:  # noqa: ASYNC101, ASYNC230
             f.seek(start)
             f.write(out)
     else:

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/local.py

@@ -79,6 +79,14 @@ class LocalFileSystem(AbstractFileSystem):
                 t = "file"
             else:
                 t = "other"
+
+            size = out.st_size
+            if link:
+                try:
+                    out2 = path.stat(follow_symlinks=True)
+                    size = out2.st_size
+                except OSError:
+                    size = 0
             path = self._strip_protocol(path.path)
         else:
             # str or path-like
@@ -87,6 +95,7 @@ class LocalFileSystem(AbstractFileSystem):
             link = stat.S_ISLNK(out.st_mode)
             if link:
                 out = os.stat(path, follow_symlinks=True)
+            size = out.st_size
             if stat.S_ISDIR(out.st_mode):
                 t = "directory"
             elif stat.S_ISREG(out.st_mode):
@@ -95,20 +104,15 @@ class LocalFileSystem(AbstractFileSystem):
                 t = "other"
         result = {
             "name": path,
-            "size": out.st_size,
+            "size": size,
             "type": t,
             "created": out.st_ctime,
             "islink": link,
         }
         for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]:
             result[field] = getattr(out, f"st_{field}")
-        if result["islink"]:
+        if link:
             result["destination"] = os.readlink(path)
-            try:
-                out2 = os.stat(path, follow_symlinks=True)
-                result["size"] = out2.st_size
-            except OSError:
-                result["size"] = 0
         return result
 
     def lexists(self, path, **kwargs):

{fsspec-2024.6.1 → fsspec-2024.9.0}/fsspec/implementations/memory.py

@@ -224,8 +224,8 @@ class MemoryFileSystem(AbstractFileSystem):
         path = self._strip_protocol(path)
         try:
             return bytes(self.store[path].getbuffer()[start:end])
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as e:
+            raise FileNotFoundError(path) from e
 
     def _rm(self, path):
         path = self._strip_protocol(path)
@@ -238,15 +238,15 @@ class MemoryFileSystem(AbstractFileSystem):
         path = self._strip_protocol(path)
         try:
             return self.store[path].modified
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as e:
+            raise FileNotFoundError(path) from e
 
     def created(self, path):
         path = self._strip_protocol(path)
         try:
             return self.store[path].created
-        except KeyError:
-            raise FileNotFoundError(path)
+        except KeyError as e:
+            raise FileNotFoundError(path) from e
 
     def rm(self, path, recursive=False, maxdepth=None):
         if isinstance(path, str):