fsspec 2025.3.2__tar.gz → 2025.5.0__tar.gz

{fsspec-2025.3.2 → fsspec-2025.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.3.2
+Version: 2025.5.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/

{fsspec-2025.3.2 → fsspec-2025.5.0}/docs/source/api.rst

@@ -221,7 +221,7 @@ documentation carefully before using any particular package.
 - `dropbox`_ for access to dropbox shares, with protocol "dropbox://"
 - `dvc`_ to access DVC/Git repository as a filesystem
 - `fsspec-encrypted`_ for transparent encryption on top of other fsspec filesystems.
-- `gcsfs`_ for Google Cloud Storage, with protocol "gcs://"
+- `gcsfs`_ for Google Cloud Storage, with protocol "gs://" or "gcs://"
 - `gdrive`_ to access Google Drive and shares (experimental)
 - `git`_ to access Git repositories
 - `huggingface_hub`_ to access the Hugging Face Hub filesystem, with protocol "hf://"
@@ -237,6 +237,7 @@ documentation carefully before using any particular package.
 - `p9fs`_ for 9P (Plan 9 Filesystem Protocol) servers
 - `PyAthena`_ for S3 access to Amazon Athena, with protocol "s3://" or "s3a://"
 - `PyDrive2`_ for Google Drive access
+- `fsspec-proxy`_ for "pyscript:" URLs via a proxy server
 - `s3fs`_ for Amazon S3 and other compatible stores, with protocol "s3://"
 - `sshfs`_ for access to SSH servers, with protocol "ssh://" or "sftp://"
 - `swiftspec`_ for OpenStack SWIFT, with protocol "swift://"
@@ -254,6 +255,7 @@ documentation carefully before using any particular package.
 .. _dropbox: https://github.com/fsspec/dropboxdrivefs
 .. _dvc: https://github.com/iterative/dvc
 .. _fsspec-encrypted: https://github.com/thevgergroup/fsspec-encrypted
+.. _fsspec-proxy: https://github.com/fsspec/fsspec-proxy
 .. _gcsfs: https://gcsfs.readthedocs.io/en/latest/
 .. _gdrive: https://github.com/fsspec/gdrivefs
 .. _git: https://github.com/iterative/scmrepo

{fsspec-2025.3.2 → fsspec-2025.5.0}/docs/source/changelog.rst

@@ -1,6 +1,31 @@
 Changelog
 =========
 
+2025.5.0
+--------
+
+Enhancements
+
+- add rm in github: (##1839)
+- cachingFS performance, don't ls every time (#1833)
+- pyscript: protocol and optional entrypoint (#1828)
+
+Fixes
+
+- improve cp in generic (#1835)
+- fix ls in dbfs (#1834)
+- fix parquet tests for pyarrow 20 (#1831)
+- fix crosslink for local tempfile (#1829)
+- keep permissions in local transaction (#1826)
+- signature compatibility in local cp() (#1820)
+
+Other
+
+- add "gs" at mention of gcsfs in docs (#1840)
+- lint style (#1837)
+- glob docstring to reflect reality (#1825)
+
+
 2025.3.2
 --------
 

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/__init__.py

@@ -1,5 +1,3 @@
-from importlib.metadata import entry_points
-
 from . import caching
 from ._version import __version__  # noqa: F401
 from .callbacks import Callback
@@ -38,6 +36,10 @@ __all__ = [
 
 
 def process_entries():
+    try:
+        from importlib.metadata import entry_points
+    except ImportError:
+        return
     if entry_points is not None:
         try:
             eps = entry_points()

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2025.3.2'
-__version_tuple__ = version_tuple = (2025, 3, 2)
+__version__ = version = '2025.5.0'
+__version_tuple__ = version_tuple = (2025, 5, 0)

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/generic.py

@@ -16,15 +16,13 @@ logger = logging.getLogger("fsspec.generic")
 
 
 def set_generic_fs(protocol, **storage_options):
+    """Populate the dict used for method=="generic" lookups"""
     _generic_fs[protocol] = filesystem(protocol, **storage_options)
 
 
-default_method = "default"
-
-
-def _resolve_fs(url, method=None, protocol=None, storage_options=None):
+def _resolve_fs(url, method, protocol=None, storage_options=None):
     """Pick instance of backend FS"""
-    method = method or default_method
+    url = url[0] if isinstance(url, (list, tuple)) else url
     protocol = protocol or split_protocol(url)[0]
     storage_options = storage_options or {}
     if method == "default":
@@ -159,7 +157,7 @@ class GenericFileSystem(AsyncFileSystem):
 
     protocol = "generic"  # there is no real reason to ever use a protocol with this FS
 
-    def __init__(self, default_method="default", **kwargs):
+    def __init__(self, default_method="default", storage_options=None, **kwargs):
         """
 
         Parameters
@@ -171,22 +169,25 @@ class GenericFileSystem(AsyncFileSystem):
               configured via the config system
             - "generic": takes instances from the `_generic_fs` dict in this module,
               which you must populate before use. Keys are by protocol
+            - "options": expects storage_options, a dict mapping protocol to
+              kwargs to use when constructing the filesystem
             - "current": takes the most recently instantiated version of each FS
         """
         self.method = default_method
+        self.st_opts = storage_options
         super().__init__(**kwargs)
 
     def _parent(self, path):
-        fs = _resolve_fs(path, self.method)
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
         return fs.unstrip_protocol(fs._parent(path))
 
     def _strip_protocol(self, path):
         # normalization only
-        fs = _resolve_fs(path, self.method)
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
         return fs.unstrip_protocol(fs._strip_protocol(path))
 
     async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs):
-        fs = _resolve_fs(path, self.method)
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
         if fs.async_impl:
             out = await fs._find(
                 path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs
@@ -251,7 +252,7 @@ class GenericFileSystem(AsyncFileSystem):
         value,
         **kwargs,
     ):
-        fs = _resolve_fs(path, self.method)
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
         if fs.async_impl:
             return await fs._pipe_file(path, value, **kwargs)
         else:
@@ -269,7 +270,7 @@ class GenericFileSystem(AsyncFileSystem):
 
     async def _makedirs(self, path, exist_ok=False):
         logger.debug("Make dir %s", path)
-        fs = _resolve_fs(path, self.method)
+        fs = _resolve_fs(path, self.method, storage_options=self.st_opts)
         if fs.async_impl:
             await fs._makedirs(path, exist_ok=exist_ok)
         else:
@@ -288,6 +289,7 @@ class GenericFileSystem(AsyncFileSystem):
         url2,
         blocksize=2**20,
         callback=DEFAULT_CALLBACK,
+        tempdir: Optional[str] = None,
         **kwargs,
     ):
         fs = _resolve_fs(url, self.method)
@@ -295,35 +297,10 @@ class GenericFileSystem(AsyncFileSystem):
         if fs is fs2:
             # pure remote
             if fs.async_impl:
-                return await fs._cp_file(url, url2, **kwargs)
+                return await fs._copy(url, url2, **kwargs)
             else:
-                return fs.cp_file(url, url2, **kwargs)
-        kw = {"blocksize": 0, "cache_type": "none"}
-        try:
-            f1 = (
-                await fs.open_async(url, "rb")
-                if hasattr(fs, "open_async")
-                else fs.open(url, "rb", **kw)
-            )
-            callback.set_size(await maybe_await(f1.size))
-            f2 = (
-                await fs2.open_async(url2, "wb")
-                if hasattr(fs2, "open_async")
-                else fs2.open(url2, "wb", **kw)
-            )
-            while f1.size is None or f2.tell() < f1.size:
-                data = await maybe_await(f1.read(blocksize))
-                if f1.size is None and not data:
-                    break
-                await maybe_await(f2.write(data))
-                callback.absolute_update(f2.tell())
-        finally:
-            try:
-                await maybe_await(f2.close())
-                await maybe_await(f1.close())
-            except NameError:
-                # fail while opening f1 or f2
-                pass
+                return fs.copy(url, url2, **kwargs)
+        await copy_file_op(fs, [url], fs2, [url2], tempdir, 1, on_error="raise")
 
     async def _make_many_dirs(self, urls, exist_ok=True):
         fs = _resolve_fs(urls[0], self.method)
@@ -347,17 +324,22 @@ class GenericFileSystem(AsyncFileSystem):
         tempdir: Optional[str] = None,
         **kwargs,
     ):
+        # TODO: special case for one FS being local, which can use get/put
+        # TODO: special case for one being memFS, which can use cat/pipe
         if recursive:
-            raise NotImplementedError
-        fs = _resolve_fs(path1[0], self.method)
-        fs2 = _resolve_fs(path2[0], self.method)
-        # not expanding paths atm., assume call is from rsync()
+            raise NotImplementedError("Please use fsspec.generic.rsync")
+        path1 = [path1] if isinstance(path1, str) else path1
+        path2 = [path2] if isinstance(path2, str) else path2
+
+        fs = _resolve_fs(path1, self.method)
+        fs2 = _resolve_fs(path2, self.method)
+
         if fs is fs2:
-            # pure remote
             if fs.async_impl:
                 return await fs._copy(path1, path2, **kwargs)
             else:
                 return fs.copy(path1, path2, **kwargs)
+
         await copy_file_op(
             fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error
         )
@@ -377,31 +359,33 @@ async def copy_file_op(
                 fs2,
                 u2,
                 os.path.join(tempdir, uuid.uuid4().hex),
-                on_error=on_error,
             )
             for u1, u2 in zip(url1, url2)
         ]
-        await _run_coros_in_chunks(coros, batch_size=batch_size)
+        out = await _run_coros_in_chunks(
+            coros, batch_size=batch_size, return_exceptions=True
+        )
     finally:
         shutil.rmtree(tempdir)
+    if on_error == "return":
+        return out
+    elif on_error == "raise":
+        for o in out:
+            if isinstance(o, Exception):
+                raise o
 
 
 async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"):
-    ex = () if on_error == "raise" else Exception
-    logger.debug("Copy %s -> %s", url1, url2)
-    try:
-        if fs1.async_impl:
-            await fs1._get_file(url1, local)
-        else:
-            fs1.get_file(url1, local)
-        if fs2.async_impl:
-            await fs2._put_file(local, url2)
-        else:
-            fs2.put_file(local, url2)
-        os.unlink(local)
-        logger.debug("Copy %s -> %s; done", url1, url2)
-    except ex as e:
-        logger.debug("ignoring cp exception for %s: %s", url1, e)
+    if fs1.async_impl:
+        await fs1._get_file(url1, local)
+    else:
+        fs1.get_file(url1, local)
+    if fs2.async_impl:
+        await fs2._put_file(local, url2)
+    else:
+        fs2.put_file(local, url2)
+    os.unlink(local)
+    logger.debug("Copy %s -> %s; done", url1, url2)
 
 
 async def maybe_await(cor):

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/implementations/asyn_wrapper.py

@@ -2,6 +2,7 @@ import asyncio
 import functools
 import inspect
 
+import fsspec
 from fsspec.asyn import AsyncFileSystem, running_async
 
 
@@ -42,14 +43,24 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         The synchronous filesystem instance to wrap.
     """
 
-    protocol = "async_wrapper"
+    protocol = "asyncwrapper", "async_wrapper"
     cachable = False
 
-    def __init__(self, fs, *args, asynchronous=None, **kwargs):
+    def __init__(
+        self,
+        fs=None,
+        asynchronous=None,
+        target_protocol=None,
+        target_options=None,
+        **kwargs,
+    ):
         if asynchronous is None:
             asynchronous = running_async()
-        super().__init__(*args, asynchronous=asynchronous, **kwargs)
-        self.sync_fs = fs
+        super().__init__(asynchronous=asynchronous, **kwargs)
+        if fs is not None:
+            self.sync_fs = fs
+        else:
+            self.sync_fs = fsspec.filesystem(target_protocol, **target_options)
         self.protocol = self.sync_fs.protocol
         self._wrap_all_sync_methods()
 

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/implementations/cached.py

@@ -335,6 +335,11 @@ class CachingFileSystem(AbstractFileSystem):
             self._metadata.update_file(path, detail)
             logger.debug("Creating local sparse file for %s", path)
 
+        # explicitly submitting the size to the open call will avoid extra
+        # operations when opening. This is particularly relevant
+        # for any file that is read over a network, e.g. S3.
+        size = detail.get("size", None)
+
         # call target filesystems open
         self._mkcache()
         f = self.fs._open(
@@ -344,8 +349,15 @@ class CachingFileSystem(AbstractFileSystem):
             autocommit=autocommit,
             cache_options=cache_options,
             cache_type="none",
+            size=size,
             **kwargs,
         )
+
+        # set size if not already set
+        if size is None:
+            detail["size"] = f.size
+            self._metadata.update_file(path, detail)
+
         if self.compression:
             comp = (
                 infer_compression(path)

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/implementations/dbfs.py

@@ -14,12 +14,13 @@ class DatabricksException(Exception):
     Helper class for exceptions raised in this module.
     """
 
-    def __init__(self, error_code, message):
+    def __init__(self, error_code, message, details=None):
         """Create a new DatabricksException"""
         super().__init__(message)
 
         self.error_code = error_code
         self.message = message
+        self.details = details
 
 
 class DatabricksFileSystem(AbstractFileSystem):
@@ -80,7 +81,7 @@ class DatabricksFileSystem(AbstractFileSystem):
                     raise FileNotFoundError(e.message) from e
 
                 raise
-            files = r["files"]
+            files = r.get("files", [])
             out = [
                 {
                     "name": o["path"],

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/implementations/github.py

@@ -1,4 +1,5 @@
 import base64
+import re
 
 import requests
 
@@ -265,3 +266,70 @@ class GithubFileSystem(AbstractFileSystem):
             cache_options=cache_options,
             **kwargs,
         )
+
+    def rm(self, path, recursive=False, maxdepth=None, message=None):
+        path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth)
+        for p in reversed(path):
+            self.rm_file(p, message=message)
+
+    def rm_file(self, path, message=None, **kwargs):
+        """
+        Remove a file from a specified branch using a given commit message.
+
+        Since Github DELETE operation requires a branch name, and we can't reliably
+        determine whether the provided SHA refers to a branch, tag, or commit, we
+        assume it's a branch. If it's not, the user will encounter an error when
+        attempting to retrieve the file SHA or delete the file.
+
+        Parameters
+        ----------
+        path: str
+            The file's location relative to the repository root.
+        message: str, optional
+            The commit message for the deletion.
+        """
+
+        if not self.username:
+            raise ValueError("Authentication required")
+
+        path = self._strip_protocol(path)
+
+        # Attempt to get SHA from cache or Github API
+        sha = self._get_sha_from_cache(path)
+        if not sha:
+            url = self.content_url.format(
+                org=self.org, repo=self.repo, path=path.lstrip("/"), sha=self.root
+            )
+            r = requests.get(url, timeout=self.timeout, **self.kw)
+            if r.status_code == 404:
+                raise FileNotFoundError(path)
+            r.raise_for_status()
+            sha = r.json()["sha"]
+
+        # Delete the file
+        delete_url = self.content_url.format(
+            org=self.org, repo=self.repo, path=path, sha=self.root
+        )
+        branch = self.root
+        data = {
+            "message": message or f"Delete {path}",
+            "sha": sha,
+            **({"branch": branch} if branch else {}),
+        }
+
+        r = requests.delete(delete_url, json=data, timeout=self.timeout, **self.kw)
+        error_message = r.json().get("message", "")
+        if re.search(r"Branch .+ not found", error_message):
+            error = "Remove only works when the filesystem is initialised from a branch or default (None)"
+            raise ValueError(error)
+        r.raise_for_status()
+
+        self.invalidate_cache(path)
+
+    def _get_sha_from_cache(self, path):
+        for entries in self.dircache.values():
+            for entry in entries:
+                entry_path = entry.get("name")
+                if entry_path and entry_path == path and "sha" in entry:
+                    return entry["sha"]
+        return None

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/implementations/local.py

@@ -6,6 +6,7 @@ import os.path as osp
 import shutil
 import stat
 import tempfile
+from functools import lru_cache
 
 from fsspec import AbstractFileSystem
 from fsspec.compression import compr
@@ -154,7 +155,11 @@ class LocalFileSystem(AbstractFileSystem):
     def put_file(self, path1, path2, callback=None, **kwargs):
         return self.cp_file(path1, path2, **kwargs)
 
-    def mv(self, path1, path2, **kwargs):
+    def mv(self, path1, path2, recursive: bool = True, **kwargs):
+        """Move files/directories
+        For the specific case of local, all ops on directories are recursive and
+        the recursive= kwarg is ignored.
+        """
         path1 = self._strip_protocol(path1)
         path2 = self._strip_protocol(path2)
         shutil.move(path1, path2)
@@ -350,6 +355,19 @@ def trailing_sep(path):
     return path.endswith(os.sep) or (os.altsep is not None and path.endswith(os.altsep))
 
 
+@lru_cache(maxsize=1)
+def get_umask(mask: int = 0o666) -> int:
+    """Get the current umask.
+
+    Follows https://stackoverflow.com/a/44130549 to get the umask.
+    Temporarily sets the umask to the given value, and then resets it to the
+    original value.
+    """
+    value = os.umask(mask)
+    os.umask(value)
+    return value
+
+
 class LocalFileOpener(io.IOBase):
     def __init__(
         self, path, mode, autocommit=True, fs=None, compression=None, **kwargs
@@ -412,7 +430,22 @@ class LocalFileOpener(io.IOBase):
     def commit(self):
         if self.autocommit:
             raise RuntimeError("Can only commit if not already set to autocommit")
-        shutil.move(self.temp, self.path)
+        try:
+            shutil.move(self.temp, self.path)
+        except PermissionError as e:
+            # shutil.move raises PermissionError if os.rename
+            # and the default copy2 fallback with shutil.copystats fail.
+            # The file should be there nonetheless, but without copied permissions.
+            # If it doesn't exist, there was no permission to create the file.
+            if not os.path.exists(self.path):
+                raise e
+        else:
+            # If PermissionError is not raised, permissions can be set.
+            try:
+                mask = 0o666
+                os.chmod(self.path, mask & ~get_umask(mask))
+            except RuntimeError:
+                pass
 
     def discard(self):
         if self.autocommit:

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/registry.py

@@ -72,13 +72,13 @@ known_implementations = {
         "class": "fsspec.implementations.arrow.HadoopFileSystem",
         "err": "pyarrow and local java libraries required for HDFS",
     },
-    "async_wrapper": {
-        "class": "fsspec.asyn_wrapper.AsyncWrapperFileSystem",
-    },
     "asynclocal": {
         "class": "morefs.asyn_local.AsyncLocalFileSystem",
         "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem",
     },
+    "asyncwrapper": {
+        "class": "fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper",
+    },
     "az": {
         "class": "adlfs.AzureBlobFileSystem",
         "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage",
@@ -180,6 +180,10 @@ known_implementations = {
         "class": "ossfs.OSSFileSystem",
         "err": "Install ossfs to access Alibaba Object Storage System",
     },
+    "pyscript": {
+        "class": "pyscript_fsspec_client.client.PyscriptFileSystem",
+        "err": "Install requests (cpython) or run in pyscript",
+    },
     "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"},
     "root": {
         "class": "fsspec_xrootd.XRootDFileSystem",

{fsspec-2025.3.2 → fsspec-2025.5.0}/fsspec/spec.py

@@ -548,17 +548,45 @@ class AbstractFileSystem(metaclass=_Cached):
             return sizes
 
     def glob(self, path, maxdepth=None, **kwargs):
-        """
-        Find files by glob-matching.
+        """Find files by glob-matching.
 
-        If the path ends with '/', only folders are returned.
+        Pattern matching capabilities for finding files that match the given pattern.
 
-        We support ``"**"``,
-        ``"?"`` and ``"[..]"``. We do not support ^ for pattern negation.
+        Parameters
+        ----------
+        path: str
+            The glob pattern to match against
+        maxdepth: int or None
+            Maximum depth for '**' patterns. Applied on the first '**' found.
+            Must be at least 1 if provided.
+        **kwargs:
+            Additional arguments passed to ``find`` (e.g., detail=True)
 
-        The `maxdepth` option is applied on the first `**` found in the path.
+        Returns
+        -------
+        List of matched paths, or dict of paths and their info if detail=True
 
-        kwargs are passed to ``ls``.
+        Notes
+        -----
+        Supported patterns:
+        - '*': Matches any sequence of characters within a single directory level
+        - '**': Matches any number of directory levels (must be an entire path component)
+        - '?': Matches exactly one character
+        - '[abc]': Matches any character in the set
+        - '[a-z]': Matches any character in the range
+        - '[!abc]': Matches any character NOT in the set
+
+        Special behaviors:
+        - If the path ends with '/', only folders are returned
+        - Consecutive '*' characters are compressed into a single '*'
+        - Empty brackets '[]' never match anything
+        - Negated empty brackets '[!]' match any single character
+        - Special characters in character classes are escaped properly
+
+        Limitations:
+        - '**' must be a complete path component (e.g., 'a/**/b', not 'a**b')
+        - No brace expansion ('{a,b}.txt')
+        - No extended glob patterns ('+(pattern)', '!(pattern)')
         """
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")

{fsspec-2025.3.2 → fsspec-2025.5.0}/pyproject.toml

@@ -183,7 +183,7 @@ select = [
     "SIM",
     "SLOT",
     "SIM101",
-    "TCH",
+    "TC",
     "UP",
 ]
 ignore = [