fsspec 2025.7.0__py3-none-any.whl → 2025.10.0__py3-none-any.whl

fsspec/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '2025.7.0'
-__version_tuple__ = version_tuple = (2025, 7, 0)
+__version__ = version = '2025.10.0'
+__version_tuple__ = version_tuple = (2025, 10, 0)
+
+__commit_id__ = commit_id = None

fsspec/conftest.py

@@ -3,11 +3,12 @@ import shutil
 import subprocess
 import sys
 import time
+from collections import deque
+from collections.abc import Generator, Sequence
 
 import pytest
 
 import fsspec
-from fsspec.implementations.cached import CachingFileSystem
 
 
 @pytest.fixture()
@@ -27,16 +28,85 @@ def m():
         m.pseudo_dirs.append("")
 
 
-@pytest.fixture
+class InstanceCacheInspector:
+    """
+    Helper class to inspect instance caches of filesystem classes in tests.
+    """
+
+    def clear(self) -> None:
+        """
+        Clear instance caches of all currently imported filesystem classes.
+        """
+        classes = deque([fsspec.spec.AbstractFileSystem])
+        while classes:
+            cls = classes.popleft()
+            cls.clear_instance_cache()
+            classes.extend(cls.__subclasses__())
+
+    def gather_counts(self, *, omit_zero: bool = True) -> dict[str, int]:
+        """
+        Gather counts of filesystem instances in the instance caches
+        of all currently imported filesystem classes.
+
+        Parameters
+        ----------
+        omit_zero:
+            Whether to omit instance types with no cached instances.
+        """
+        out: dict[str, int] = {}
+        classes = deque([fsspec.spec.AbstractFileSystem])
+        while classes:
+            cls = classes.popleft()
+            count = len(cls._cache)  # there is no public interface for the cache
+            # note: skip intermediate AbstractFileSystem subclasses
+            #   if they proxy the protocol attribute via a property.
+            if isinstance(cls.protocol, (Sequence, str)):
+                key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0]
+                if count or not omit_zero:
+                    out[key] = count
+            classes.extend(cls.__subclasses__())
+        return out
+
+
+@pytest.fixture(scope="function", autouse=True)
+def instance_caches() -> Generator[InstanceCacheInspector, None, None]:
+    """
+    Fixture to ensure empty filesystem instance caches before and after a test.
+
+    Used by default for all tests.
+    Clears caches of all imported filesystem classes.
+    Can be used to write test assertions about instance caches.
+
+    Usage:
+
+        def test_something(instance_caches):
+            # Test code here
+            fsspec.open("file://abc")
+            fsspec.open("memory://foo/bar")
+
+            # Test assertion
+            assert instance_caches.gather_counts() == {"file": 1, "memory": 1}
+
+    Returns
+    -------
+    instance_caches: An instance cache inspector for clearing and inspecting caches.
+    """
+    ic = InstanceCacheInspector()
+
+    ic.clear()
+    try:
+        yield ic
+    finally:
+        ic.clear()
+
+
+@pytest.fixture(scope="function")
 def ftp_writable(tmpdir):
     """
     Fixture providing a writable FTP filesystem.
     """
     pytest.importorskip("pyftpdlib")
-    from fsspec.implementations.ftp import FTPFileSystem
 
-    FTPFileSystem.clear_instance_cache()  # remove lingering connections
-    CachingFileSystem.clear_instance_cache()
     d = str(tmpdir)
     with open(os.path.join(d, "out"), "wb") as f:
         f.write(b"hello" * 10000)

fsspec/core.py

@@ -330,7 +330,7 @@ def open_files(
 
 def _un_chain(path, kwargs):
     # Avoid a circular import
-    from fsspec.implementations.cached import CachingFileSystem
+    from fsspec.implementations.chained import ChainedFileSystem
 
     if "::" in path:
         x = re.compile(".*[^a-z]+.*")  # test for non protocol-like single word
@@ -358,7 +358,7 @@ def _un_chain(path, kwargs):
             **kws,
         )
         bit = cls._strip_protocol(bit)
-        if "target_protocol" not in kw and issubclass(cls, CachingFileSystem):
+        if "target_protocol" not in kw and issubclass(cls, ChainedFileSystem):
             bit = previous_bit
         out.append((bit, protocol, kw))
         previous_bit = bit

fsspec/generic.py

@@ -118,6 +118,8 @@ def rsync(
         if otherfile in otherfiles:
             if update_cond == "always":
                 allfiles[k] = otherfile
+            elif update_cond == "never":
+                allfiles.pop(k)
             elif update_cond == "different":
                 inf1 = source_field(v) if callable(source_field) else v[source_field]
                 v2 = otherfiles[otherfile]

fsspec/implementations/arrow.py

@@ -75,10 +75,13 @@ class ArrowFSWrapper(AbstractFileSystem):
         path = self._strip_protocol(path)
         from pyarrow.fs import FileSelector
 
-        entries = [
-            self._make_entry(entry)
-            for entry in self.fs.get_file_info(FileSelector(path))
-        ]
+        try:
+            entries = [
+                self._make_entry(entry)
+                for entry in self.fs.get_file_info(FileSelector(path))
+            ]
+        except (FileNotFoundError, NotADirectoryError):
+            entries = [self.info(path, **kwargs)]
         if detail:
             return entries
         else:

fsspec/implementations/asyn_wrapper.py

@@ -6,7 +6,7 @@ import fsspec
 from fsspec.asyn import AsyncFileSystem, running_async
 
 
-def async_wrapper(func, obj=None):
+def async_wrapper(func, obj=None, semaphore=None):
     """
     Wraps a synchronous function to make it awaitable.
 
@@ -16,6 +16,8 @@ def async_wrapper(func, obj=None):
         The synchronous function to wrap.
     obj : object, optional
         The instance to bind the function to, if applicable.
+    semaphore : asyncio.Semaphore, optional
+        A semaphore to limit concurrent calls.
 
     Returns
     -------
@@ -25,6 +27,9 @@ def async_wrapper(func, obj=None):
 
     @functools.wraps(func)
     async def wrapper(*args, **kwargs):
+        if semaphore:
+            async with semaphore:
+                return await asyncio.to_thread(func, *args, **kwargs)
         return await asyncio.to_thread(func, *args, **kwargs)
 
     return wrapper
@@ -52,6 +57,8 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         asynchronous=None,
         target_protocol=None,
         target_options=None,
+        semaphore=None,
+        max_concurrent_tasks=None,
         **kwargs,
     ):
         if asynchronous is None:
@@ -62,6 +69,7 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         else:
             self.sync_fs = fsspec.filesystem(target_protocol, **target_options)
         self.protocol = self.sync_fs.protocol
+        self.semaphore = semaphore
         self._wrap_all_sync_methods()
 
     @property
@@ -83,7 +91,7 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
 
             method = getattr(self.sync_fs, method_name)
             if callable(method) and not inspect.iscoroutinefunction(method):
-                async_method = async_wrapper(method, obj=self)
+                async_method = async_wrapper(method, obj=self, semaphore=self.semaphore)
                 setattr(self, f"_{method_name}", async_method)
 
     @classmethod

fsspec/implementations/cached.py

@@ -9,13 +9,14 @@ import weakref
 from shutil import rmtree
 from typing import TYPE_CHECKING, Any, Callable, ClassVar
 
-from fsspec import AbstractFileSystem, filesystem
+from fsspec import filesystem
 from fsspec.callbacks import DEFAULT_CALLBACK
 from fsspec.compression import compr
 from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
 from fsspec.implementations.cache_mapper import create_cache_mapper
 from fsspec.implementations.cache_metadata import CacheMetadata
+from fsspec.implementations.chained import ChainedFileSystem
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.spec import AbstractBufferedFile
 from fsspec.transaction import Transaction
@@ -39,7 +40,7 @@ class WriteCachedTransaction(Transaction):
         self.fs = None  # break cycle
 
 
-class CachingFileSystem(AbstractFileSystem):
+class CachingFileSystem(ChainedFileSystem):
     """Locally caching filesystem, layer over any other FS
 
     This class implements chunk-wise local storage of remote files, for quick
@@ -60,6 +61,7 @@ class CachingFileSystem(AbstractFileSystem):
     """
 
     protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached")
+    _strip_tokenize_options = ("fo",)
 
     def __init__(
         self,
@@ -478,7 +480,7 @@ class CachingFileSystem(AbstractFileSystem):
         if item in ["transaction"]:
             # property
             return type(self).transaction.__get__(self)
-        if item in ["_cache", "transaction_type"]:
+        if item in {"_cache", "transaction_type", "protocol"}:
             # class attributes
             return getattr(type(self), item)
         if item == "__class__":
@@ -886,6 +888,7 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         rpaths = [p for l, p in zip(lpaths, paths) if l is False]
         lpaths = [l for l, p in zip(lpaths, paths) if l is False]
         self.fs.get(rpaths, lpaths)
+        paths = [self._check_file(p) for p in paths]
         return LocalFileSystem().cat_ranges(
             paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
         )
@@ -983,7 +986,9 @@ class LocalTempFile:
         os.remove(self.fn)
 
     def commit(self):
-        self.fs.put(self.fn, self.path, **self.kwargs)
+        # calling put() with list arguments avoids path expansion and additional operations
+        # like isdir()
+        self.fs.put([self.fn], [self.path], **self.kwargs)
         # we do not delete the local copy, it's still in the cache.
 
     @property

fsspec/implementations/chained.py

@@ -0,0 +1,23 @@
+from typing import ClassVar
+
+from fsspec import AbstractFileSystem
+
+__all__ = ("ChainedFileSystem",)
+
+
+class ChainedFileSystem(AbstractFileSystem):
+    """Chained filesystem base class.
+
+    A chained filesystem is designed to be layered over another FS.
+    This is useful to implement things like caching.
+
+    This base class does very little on its own, but is used as a marker
+    that the class is designed for chaining.
+
+    Right now this is only used in `url_to_fs` to provide the path argument
+    (`fo`) to the chained filesystem from the underlying filesystem.
+
+    Additional functionality may be added in the future.
+    """
+
+    protocol: ClassVar[str] = "chained"

fsspec/implementations/dbfs.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
+
 import base64
 import urllib
 
 import requests
-import requests.exceptions
 from requests.adapters import HTTPAdapter, Retry
+from typing_extensions import override
 
 from fsspec import AbstractFileSystem
 from fsspec.spec import AbstractBufferedFile
@@ -57,6 +59,24 @@ class DatabricksFileSystem(AbstractFileSystem):
 
         super().__init__(**kwargs)
 
+    @override
+    def _ls_from_cache(self, path) -> list[dict[str, str | int]] | None:
+        """Check cache for listing
+
+        Returns listing, if found (may be empty list for a directory that
+        exists but contains nothing), None if not in cache.
+        """
+        self.dircache.pop(path.rstrip("/"), None)
+
+        parent = self._parent(path)
+        if parent in self.dircache:
+            for entry in self.dircache[parent]:
+                if entry["name"] == path.rstrip("/"):
+                    if entry["type"] != "directory":
+                        return [entry]
+                    return []
+            raise FileNotFoundError(path)
+
     def ls(self, path, detail=True, **kwargs):
         """
         List the contents of the given path.
@@ -70,7 +90,15 @@ class DatabricksFileSystem(AbstractFileSystem):
             but also additional information on file sizes
             and types.
         """
-        out = self._ls_from_cache(path)
+        try:
+            out = self._ls_from_cache(path)
+        except FileNotFoundError:
+            # This happens if the `path`'s parent was cached, but `path` is not
+            # there. This suggests that `path` is new since the parent was
+            # cached. Attempt to invalidate parent's cache before continuing.
+            self.dircache.pop(self._parent(path), None)
+            out = None
+
         if not out:
             try:
                 r = self._send_to_api(
@@ -460,7 +488,7 @@ class DatabricksFile(AbstractBufferedFile):
         return return_buffer
 
     def _to_sized_blocks(self, length, start=0):
-        """Helper function to split a range from 0 to total_length into bloksizes"""
+        """Helper function to split a range from 0 to total_length into blocksizes"""
         end = start + length
         for data_chunk in range(start, end, self.blocksize):
             data_start = data_chunk

fsspec/implementations/gist.py

@@ -14,21 +14,21 @@ class GistFileSystem(AbstractFileSystem):
 
     Parameters
     ----------
-    gist_id : str
+    gist_id: str
         The ID of the gist you want to access (the long hex value from the URL).
-    filenames : list[str] (optional)
+    filenames: list[str] (optional)
         If provided, only make a file system representing these files, and do not fetch
         the list of all files for this gist.
-    sha : str (optional)
+    sha: str (optional)
         If provided, fetch a particular revision of the gist. If omitted,
         the latest revision is used.
-    username : str (optional)
-        GitHub username for authentication (required if token is given).
-    token : str (optional)
-        GitHub personal access token (required if username is given).
-    timeout : (float, float) or float, optional
+    username: str (optional)
+        GitHub username for authentication.
+    token: str (optional)
+        GitHub personal access token (required if username is given), or.
+    timeout: (float, float) or float, optional
         Connect and read timeouts for requests (default 60s each).
-    kwargs : dict
+    kwargs: dict
         Stored on `self.request_kw` and passed to `requests.get` when fetching Gist
         metadata or reading ("opening") a file.
     """
@@ -51,10 +51,8 @@ class GistFileSystem(AbstractFileSystem):
         self.gist_id = gist_id
         self.filenames = filenames
         self.sha = sha  # revision of the gist (optional)
-        if (username is None) ^ (token is None):
-            # Both or neither must be set
-            if username or token:
-                raise ValueError("Auth requires both username and token, or neither.")
+        if username is not None and token is None:
+            raise ValueError("User auth requires a token")
         self.username = username
         self.token = token
         self.request_kw = kwargs
@@ -67,9 +65,18 @@ class GistFileSystem(AbstractFileSystem):
     @property
     def kw(self):
         """Auth parameters passed to 'requests' if we have username/token."""
-        if self.username is not None and self.token is not None:
-            return {"auth": (self.username, self.token), **self.request_kw}
-        return self.request_kw
+        kw = {
+            "headers": {
+                "Accept": "application/vnd.github+json",
+                "X-GitHub-Api-Version": "2022-11-28",
+            }
+        }
+        kw.update(self.request_kw)
+        if self.username and self.token:
+            kw["auth"] = (self.username, self.token)
+        elif self.token:
+            kw["headers"]["Authorization"] = f"Bearer {self.token}"
+        return kw
 
     def _fetch_gist_metadata(self):
         """
@@ -229,4 +236,6 @@ class GistFileSystem(AbstractFileSystem):
                     pass  # skip
                 else:
                     out[p] = e
+        if len(paths) == 1 and paths[0] == path:
+            return out[path]
         return out

fsspec/implementations/http.py

@@ -43,6 +43,7 @@ class HTTPFileSystem(AsyncFileSystem):
     HTML href tags will be used.
     """
 
+    protocol = ("http", "https")
     sep = "/"
 
     def __init__(
@@ -873,7 +874,7 @@ async def _file_info(url, session, size_policy="head", **kwargs):
 
         info["url"] = str(r.url)
 
-        for checksum_field in ["ETag", "Content-MD5", "Digest"]:
+        for checksum_field in ["ETag", "Content-MD5", "Digest", "Last-Modified"]:
             if r.headers.get(checksum_field):
                 info[checksum_field] = r.headers[checksum_field]
 

fsspec/implementations/jupyter.py

@@ -42,7 +42,7 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
         path = self._strip_protocol(path)
         r = self.session.get(f"{self.url}/{path}")
         if r.status_code == 404:
-            return FileNotFoundError(path)
+            raise FileNotFoundError(path)
         r.raise_for_status()
         out = r.json()
 
@@ -63,7 +63,7 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
         path = self._strip_protocol(path)
         r = self.session.get(f"{self.url}/{path}")
         if r.status_code == 404:
-            return FileNotFoundError(path)
+            raise FileNotFoundError(path)
         r.raise_for_status()
         out = r.json()
         if out["format"] == "text":
@@ -98,6 +98,11 @@ class JupyterFileSystem(fsspec.AbstractFileSystem):
         }
         self.session.put(f"{self.url}/{path}", json=json)
 
+    def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
+        if path1 == path2:
+            return
+        self.session.patch(f"{self.url}/{path1}", json={"path": path2})
+
     def _rm(self, path):
         path = self._strip_protocol(path)
         self.session.delete(f"{self.url}/{path}")

fsspec/implementations/memory.py

@@ -187,10 +187,10 @@ class MemoryFileSystem(AbstractFileSystem):
             parent = self._parent(parent)
             if self.isfile(parent):
                 raise FileExistsError(parent)
-        if mode in ["rb", "ab", "r+b"]:
+        if mode in ["rb", "ab", "r+b", "a+b"]:
             if path in self.store:
                 f = self.store[path]
-                if mode == "ab":
+                if "a" in mode:
                     # position at the end of file
                     f.seek(0, 2)
                 else:
@@ -199,8 +199,8 @@ class MemoryFileSystem(AbstractFileSystem):
                 return f
             else:
                 raise FileNotFoundError(path)
-        elif mode in {"wb", "xb"}:
-            if mode == "xb" and self.exists(path):
+        elif mode in {"wb", "w+b", "xb", "x+b"}:
+            if "x" in mode and self.exists(path):
                 raise FileExistsError
             m = MemoryFile(self, path, kwargs.get("data"))
             if not self._intrans:

fsspec/implementations/reference.py

@@ -22,7 +22,11 @@ from fsspec.asyn import AsyncFileSystem
 from fsspec.callbacks import DEFAULT_CALLBACK
 from fsspec.core import filesystem, open, split_protocol
 from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
-from fsspec.utils import isfilelike, merge_offset_ranges, other_paths
+from fsspec.utils import (
+    isfilelike,
+    merge_offset_ranges,
+    other_paths,
+)
 
 logger = logging.getLogger("fsspec.reference")
 
@@ -698,13 +702,9 @@ class ReferenceFileSystem(AsyncFileSystem):
                 **(ref_storage_args or target_options or {}), protocol=target_protocol
             )
             ref_fs, fo2 = fsspec.core.url_to_fs(fo, **dic)
-            if ref_fs.isfile(fo2):
-                # text JSON
-                with fsspec.open(fo, "rb", **dic) as f:
-                    logger.info("Read reference from URL %s", fo)
-                    text = json.load(f)
-                self._process_references(text, template_overrides)
-            else:
+            if ".json" not in fo2 and (
+                fo.endswith(("parq", "parquet", "/")) or ref_fs.isdir(fo2)
+            ):
                 # Lazy parquet refs
                 logger.info("Open lazy reference dict from URL %s", fo)
                 self.references = LazyReferenceMapper(
@@ -712,6 +712,12 @@ class ReferenceFileSystem(AsyncFileSystem):
                     fs=ref_fs,
                     cache_size=cache_size,
                 )
+            else:
+                # text JSON
+                with fsspec.open(fo, "rb", **dic) as f:
+                    logger.info("Read reference from URL %s", fo)
+                    text = json.load(f)
+                self._process_references(text, template_overrides)
         else:
             # dictionaries
             self._process_references(fo, template_overrides)

fsspec/implementations/sftp.py

@@ -66,6 +66,7 @@ class SFTPFileSystem(AbstractFileSystem):
         return out
 
     def mkdir(self, path, create_parents=True, mode=511):
+        path = self._strip_protocol(path)
         logger.debug("Creating folder %s", path)
         if self.exists(path):
             raise FileExistsError(f"File exists: {path}")
@@ -89,10 +90,12 @@ class SFTPFileSystem(AbstractFileSystem):
                     self.ftp.mkdir(new_path, mode)
 
     def rmdir(self, path):
+        path = self._strip_protocol(path)
         logger.debug("Removing folder %s", path)
         self.ftp.rmdir(path)
 
     def info(self, path):
+        path = self._strip_protocol(path)
         stat = self._decode_stat(self.ftp.stat(path))
         stat["name"] = path
         return stat
@@ -123,6 +126,7 @@ class SFTPFileSystem(AbstractFileSystem):
         return out
 
     def ls(self, path, detail=False):
+        path = self._strip_protocol(path)
         logger.debug("Listing folder %s", path)
         stats = [self._decode_stat(stat, path) for stat in self.ftp.listdir_iter(path)]
         if detail:
@@ -132,6 +136,7 @@ class SFTPFileSystem(AbstractFileSystem):
             return sorted(paths)
 
     def put(self, lpath, rpath, callback=None, **kwargs):
+        rpath = self._strip_protocol(rpath)
         logger.debug("Put file %s into %s", lpath, rpath)
         self.ftp.put(lpath, rpath)
 
@@ -168,6 +173,8 @@ class SFTPFileSystem(AbstractFileSystem):
             self.ftp.remove(path)
 
     def mv(self, old, new):
+        new = self._strip_protocol(new)
+        old = self._strip_protocol(old)
         logger.debug("Renaming %s into %s", old, new)
         self.ftp.posix_rename(old, new)
 

fsspec/implementations/webhdfs.py

@@ -268,7 +268,7 @@ class WebHDFS(AbstractFileSystem):
         info["name"] = path
         return self._process_info(info)
 
-    def ls(self, path, detail=False):
+    def ls(self, path, detail=False, **kwargs):
         out = self._call("LISTSTATUS", path=path)
         infos = out.json()["FileStatuses"]["FileStatus"]
         for info in infos:

fsspec/spec.py

@@ -67,6 +67,9 @@ class _Cached(type):
         extra_tokens = tuple(
             getattr(cls, attr, None) for attr in cls._extra_tokenize_attributes
         )
+        strip_tokenize_options = {
+            k: kwargs.pop(k) for k in cls._strip_tokenize_options if k in kwargs
+        }
         token = tokenize(
             cls, cls._pid, threading.get_ident(), *args, *extra_tokens, **kwargs
         )
@@ -78,7 +81,7 @@ class _Cached(type):
             cls._latest = token
             return cls._cache[token]
         else:
-            obj = super().__call__(*args, **kwargs)
+            obj = super().__call__(*args, **kwargs, **strip_tokenize_options)
             # Setting _fs_token here causes some static linters to complain.
             obj._fs_token_ = token
             obj.storage_args = args
@@ -115,6 +118,8 @@ class AbstractFileSystem(metaclass=_Cached):
 
     #: Extra *class attributes* that should be considered when hashing.
     _extra_tokenize_attributes = ()
+    #: *storage options* that should not be considered when hashing.
+    _strip_tokenize_options = ()
 
     # Set by _Cached metaclass
     storage_args: tuple[Any, ...]
@@ -892,7 +897,7 @@ class AbstractFileSystem(metaclass=_Cached):
         dict of {path: contents} if there are multiple paths
         or the path has been otherwise expanded
         """
-        paths = self.expand_path(path, recursive=recursive)
+        paths = self.expand_path(path, recursive=recursive, **kwargs)
         if (
             len(paths) > 1
             or isinstance(path, list)
@@ -972,7 +977,9 @@ class AbstractFileSystem(metaclass=_Cached):
             )
 
             source_is_str = isinstance(rpath, str)
-            rpaths = self.expand_path(rpath, recursive=recursive, maxdepth=maxdepth)
+            rpaths = self.expand_path(
+                rpath, recursive=recursive, maxdepth=maxdepth, **kwargs
+            )
             if source_is_str and (not recursive or maxdepth is not None):
                 # Non-recursive glob does not copy directories
                 rpaths = [p for p in rpaths if not (trailing_sep(p) or self.isdir(p))]
@@ -1060,7 +1067,9 @@ class AbstractFileSystem(metaclass=_Cached):
             if source_is_str:
                 lpath = make_path_posix(lpath)
             fs = LocalFileSystem()
-            lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth)
+            lpaths = fs.expand_path(
+                lpath, recursive=recursive, maxdepth=maxdepth, **kwargs
+            )
             if source_is_str and (not recursive or maxdepth is not None):
                 # Non-recursive glob does not copy directories
                 lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))]
@@ -1131,7 +1140,9 @@ class AbstractFileSystem(metaclass=_Cached):
             from .implementations.local import trailing_sep
 
             source_is_str = isinstance(path1, str)
-            paths1 = self.expand_path(path1, recursive=recursive, maxdepth=maxdepth)
+            paths1 = self.expand_path(
+                path1, recursive=recursive, maxdepth=maxdepth, **kwargs
+            )
             if source_is_str and (not recursive or maxdepth is not None):
                 # Non-recursive glob does not copy directories
                 paths1 = [p for p in paths1 if not (trailing_sep(p) or self.isdir(p))]
@@ -1172,7 +1183,7 @@ class AbstractFileSystem(metaclass=_Cached):
             raise ValueError("maxdepth must be at least 1")
 
         if isinstance(path, (str, os.PathLike)):
-            out = self.expand_path([path], recursive, maxdepth)
+            out = self.expand_path([path], recursive, maxdepth, **kwargs)
         else:
             out = set()
             path = [self._strip_protocol(p) for p in path]

fsspec/utils.py

@@ -438,6 +438,14 @@ def get_protocol(url: str) -> str:
     return "file"
 
 
+def get_file_extension(url: str) -> str:
+    url = stringify_path(url)
+    ext_parts = url.rsplit(".", 1)
+    if len(ext_parts) > 1:
+        return ext_parts[-1]
+    return ""
+
+
 def can_be_local(path: str) -> bool:
     """Can the given URL be used with open_local?"""
     from fsspec import get_filesystem_class

{fsspec-2025.7.0.dist-info → fsspec-2025.10.0.dist-info}/METADATA

@@ -1,45 +1,16 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.7.0
+Version: 2025.10.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
 Project-URL: Homepage, https://github.com/fsspec/filesystem_spec
 Maintainer-email: Martin Durant <mdurant@anaconda.com>
-License: BSD 3-Clause License
-        
-        Copyright (c) 2018, Martin Durant
-        All rights reserved.
-        
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        
-        * Redistributions of source code must retain the above copyright notice, this
-          list of conditions and the following disclaimer.
-        
-        * Redistributions in binary form must reproduce the above copyright notice,
-          this list of conditions and the following disclaimer in the documentation
-          and/or other materials provided with the distribution.
-        
-        * Neither the name of the copyright holder nor the names of its
-          contributors may be used to endorse or promote products derived from
-          this software without specific prior written permission.
-        
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+License-Expression: BSD-3-Clause
 License-File: LICENSE
 Keywords: file
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10

{fsspec-2025.7.0.dist-info → fsspec-2025.10.0.dist-info}/RECORD

@@ -1,50 +1,51 @@
 fsspec/__init__.py,sha256=L7qwNBU1iMNQd8Of87HYSNFT9gWlNMSESaJC8fY0AaQ,2053
-fsspec/_version.py,sha256=BxtkhBSbP2A9Z9pLCoNSk_l6NzaY9SDK9dmjDgIXO54,517
+fsspec/_version.py,sha256=fXgQLiXV0scw4LTidVAhOWJj_BwnxWigALeToXadaR0,712
 fsspec/archive.py,sha256=vM6t_lgV6lBWbBYwpm3S4ofBQFQxUPr5KkDQrrQcQro,2411
 fsspec/asyn.py,sha256=mE55tO_MmGcxD14cUuaiS3veAqo0h6ZqANfnUuCN3sk,36365
 fsspec/caching.py,sha256=86uSgPa5E55b28XEhuC-dMcKAxJtZZnpQqnHTwaF3hI,34294
 fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210
 fsspec/compression.py,sha256=gBK2MV_oTFVW2XDq8bZVbYQKYrl6JDUou6_-kyvmxuk,5086
 fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
-fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
-fsspec/core.py,sha256=1tLctwr7sF1VO3djc_UkjhJ8IAEy0TUMH_bb07Sw17E,23828
+fsspec/conftest.py,sha256=uWfm_Qs5alPRxOhRpDfQ0-1jqSJ54pni4y96IxOREXM,3446
+fsspec/core.py,sha256=ETQrATK6ZSkuIoy5-40N_NWUfMGx1KVSl5XGuJsaoYI,23829
 fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
 fsspec/exceptions.py,sha256=pauSLDMxzTJMOjvX1WEUK0cMyFkrFxpWJsyFywav7A8,331
 fsspec/fuse.py,sha256=Q-3NOOyLqBfYa4Db5E19z_ZY36zzYHtIs1mOUasItBQ,10177
-fsspec/generic.py,sha256=K-b03ifKidHUo99r8nz2pB6oGyf88RtTKahCuBF9ZVU,13409
+fsspec/generic.py,sha256=9QHQYMNb-8w8-eYuIqShcTjO_LeHXFoQTyt8J5oEq5Q,13482
 fsspec/gui.py,sha256=CQ7QsrTpaDlWSLNOpwNoJc7khOcYXIZxmrAJN9bHWQU,14002
 fsspec/json.py,sha256=3BfNSQ96MB4Xao_ocjheINeqZM2ev7oljUzR5XmNXrE,3814
 fsspec/mapping.py,sha256=m2ndB_gtRBXYmNJg0Ie1-BVR75TFleHmIQBzC-yWhjU,8343
 fsspec/parquet.py,sha256=6ibAmG527L5JNFS0VO8BDNlxHdA3bVYqdByeiFgpUVM,19448
 fsspec/registry.py,sha256=epoYryFFzDWjbkQJfh6xkF3nEu8RTiOzV3-voi8Pshs,12048
-fsspec/spec.py,sha256=7cOUe5PC5Uyf56HtGBUHEoym8ktPj-BI8G4HR8Xd_C8,77298
+fsspec/spec.py,sha256=Ym-Ust6LRjHgbhrmvNqwOBZxoVnaw3g3xHXMZGHx_xg,77692
 fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398
-fsspec/utils.py,sha256=HC8RFbb7KpEDedsYxExvWvsTObEuUcuuWxd0B_MyGpo,22995
+fsspec/utils.py,sha256=12npx0j-z0qpG9ko3laZ0us4KhjMe-2epmyocJB2ih8,23173
 fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fsspec/implementations/arrow.py,sha256=721Dikne_lV_0tlgk9jyKmHL6W-5MT0h2LKGvOYQTPI,8623
-fsspec/implementations/asyn_wrapper.py,sha256=435NV_LyrRic3WvSxMWq7B8QGV_Ovzi-vYd2W1_1YtM,3326
+fsspec/implementations/arrow.py,sha256=CVVyjNt9B_pRa5Ac1XIlhVLBs7vA5kCUZjsLPELb4d4,8758
+fsspec/implementations/asyn_wrapper.py,sha256=fox9yjsEu7NCgzdAZJYfNALtUnFkIc_QmeKzaSllZho,3679
 fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421
 fsspec/implementations/cache_metadata.py,sha256=rddh5-0SXIeyWCPpBpOFcaAyWoPyeYmFfeubEWt-nRM,8536
-fsspec/implementations/cached.py,sha256=59lyWvbzvX_yYC9cVASrktOdjmK6w-e7dNtNBJHaONQ,35103
+fsspec/implementations/cached.py,sha256=d3IE33J5QA4QU_e43fPLF-dpbGszv6JM9mcFmHBI99o,35365
+fsspec/implementations/chained.py,sha256=iGivpNaHUFjB_ea0-HAPhcmm6CL8qnDf270PSj7JwuE,680
 fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
 fsspec/implementations/data.py,sha256=LDLczxRh8h7x39Zjrd-GgzdQHr78yYxDlrv2C9Uxb5E,1658
-fsspec/implementations/dbfs.py,sha256=2Bp-0m9SqlaroDa0KbXxb5BobCyBJ7_5YQBISf3fxbQ,15145
+fsspec/implementations/dbfs.py,sha256=1cvvC6KBWOb8pBVpc01xavVbEPXO1xsgZvPD7H73M9k,16217
 fsspec/implementations/dirfs.py,sha256=f1sGnQ9Vf0xTxrXo4jDeBy4Qfq3RTqAEemqBSeb0hwY,12108
 fsspec/implementations/ftp.py,sha256=bzL_TgH77nMMtTMewRGkbq4iObSHGu7YoMRCXBH4nrc,11639
-fsspec/implementations/gist.py,sha256=Ost985hmFr50KsA-QD0shY3hP4KX5qJ9rb5C-X4ehK8,8341
+fsspec/implementations/gist.py,sha256=Y6jTDrE-wuTwvpPyAQDuuOMBGxlajafKWoB1_yX6jdY,8528
 fsspec/implementations/git.py,sha256=qBDWMz5LNllPqVjr5jf_1FuNha4P5lyQI3IlhYg-wUE,3731
 fsspec/implementations/github.py,sha256=aCsZL8UvXZgdkcB1RUs3DdLeNrjLKcFsFYeQFDWbBFo,11653
-fsspec/implementations/http.py,sha256=3LhYuRU3yw3v3tN8Oqz6EbJRl3ab2Sg_zsGOIv0E2gE,30418
+fsspec/implementations/http.py,sha256=IxwekrxehP_l02yMiSHE7Xl1trCO5FzKTycH_iuHQG0,30468
 fsspec/implementations/http_sync.py,sha256=UydDqSdUBdhiJ1KufzV8rKGrTftFR4QmNV0safILb8g,30133
-fsspec/implementations/jupyter.py,sha256=B2uj7OEm7yIk-vRSsO37_ND0t0EBvn4B-Su43ibN4Pg,3811
+fsspec/implementations/jupyter.py,sha256=q1PlQ66AAswGFyr8MFKWyobaV2YekMWRtqENBDQtD28,4002
 fsspec/implementations/libarchive.py,sha256=5_I2DiLXwQ1JC8x-K7jXu-tBwhO9dj7tFLnb0bTnVMQ,7102
 fsspec/implementations/local.py,sha256=DQeK7jRGv4_mJAweLKALO5WzIIkjXxZ_jRvwQ_xadSA,16936
-fsspec/implementations/memory.py,sha256=Kc6TZSbZ4tdi-6cE5ttEPIgMyq9aAt6cDdVLFRTJvf8,10488
-fsspec/implementations/reference.py,sha256=npYj49AmR8rmON9t_BLpfEXqhgsardUeynamqyraOXo,48704
-fsspec/implementations/sftp.py,sha256=fMY9XZcmpjszQ2tCqO_TPaJesaeD_Dv7ptYzgUPGoO0,5631
+fsspec/implementations/memory.py,sha256=TDdLtSPWXxZKrrVGwmc3uS3oK_2mlcVTk2BiqR8IeII,10507
+fsspec/implementations/reference.py,sha256=3hr_CusIR1wBGo20MsKGoWCEnZJ626_QlHhRYobVYo0,48816
+fsspec/implementations/sftp.py,sha256=L9pZOa6eLUWfJNtxkxeG2YI96SQwrM5Hj6ocyUZXUbg,5923
 fsspec/implementations/smb.py,sha256=5fhu8h06nOLBPh2c48aT7WBRqh9cEcbIwtyu06wTjec,15236
 fsspec/implementations/tar.py,sha256=dam78Tp_CozybNqCY2JYgGBS3Uc9FuJUAT9oB0lolOs,4111
-fsspec/implementations/webhdfs.py,sha256=G9wGywj7BkZk4Mu9zXu6HaDlEqX4F8Gw1i4k46CP_-o,16769
+fsspec/implementations/webhdfs.py,sha256=PUgZM9HbVPAeW4u4B-rWl8wTcKKpPhwZO7xcINDmTNQ,16779
 fsspec/implementations/zip.py,sha256=9LBMHPft2OutJl2Ft-r9u_z3GptLkc2n91ur2A3bCbg,6072
 fsspec/tests/abstract/__init__.py,sha256=4xUJrv7gDgc85xAOz1p-V_K1hrsdMWTSa0rviALlJk8,10181
 fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973
@@ -54,7 +55,7 @@ fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1
 fsspec/tests/abstract/open.py,sha256=Fi2PBPYLbRqysF8cFm0rwnB41kMdQVYjq8cGyDXp3BU,329
 fsspec/tests/abstract/pipe.py,sha256=LFzIrLCB5GLXf9rzFKJmE8AdG7LQ_h4bJo70r8FLPqM,402
 fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201
-fsspec-2025.7.0.dist-info/METADATA,sha256=fPEhTbN6wi6KOz3IkiJQcOTYvQXVEjzTv9gzyX-KRHI,12161
-fsspec-2025.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fsspec-2025.7.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
-fsspec-2025.7.0.dist-info/RECORD,,
+fsspec-2025.10.0.dist-info/METADATA,sha256=M950PL-JM4aP_1zCRiebQ-lOaUqy9_4kra0-dqA8tCI,10398
+fsspec-2025.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fsspec-2025.10.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
+fsspec-2025.10.0.dist-info/RECORD,,