fsspec 2025.9.0__py3-none-any.whl → 2025.12.0__py3-none-any.whl

fsspec/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2025.9.0'
-__version_tuple__ = version_tuple = (2025, 9, 0)
+__version__ = version = '2025.12.0'
+__version_tuple__ = version_tuple = (2025, 12, 0)
 
 __commit_id__ = commit_id = None

fsspec/asyn.py

@@ -328,6 +328,11 @@ class AsyncFileSystem(AbstractFileSystem):
         return self._loop
 
     async def _rm_file(self, path, **kwargs):
+        if (
+            inspect.iscoroutinefunction(self._rm)
+            and type(self)._rm is not AsyncFileSystem._rm
+        ):
+            return await self._rm(path, recursive=False, batch_size=1, **kwargs)
         raise NotImplementedError
 
     async def _rm(self, path, recursive=False, batch_size=None, **kwargs):
@@ -776,6 +781,7 @@ class AsyncFileSystem(AbstractFileSystem):
         min_idx = min(idx_star, idx_qmark, idx_brace)
 
         detail = kwargs.pop("detail", False)
+        withdirs = kwargs.pop("withdirs", True)
 
         if not has_magic(path):
             if await self._exists(path, **kwargs):
@@ -805,7 +811,7 @@ class AsyncFileSystem(AbstractFileSystem):
                 depth = None
 
         allpaths = await self._find(
-            root, maxdepth=depth, withdirs=True, detail=True, **kwargs
+            root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs
         )
 
         pattern = glob_translate(path + ("/" if ends_with_sep else ""))

fsspec/caching.py

@@ -6,20 +6,12 @@ import logging
 import math
 import os
 import threading
-import warnings
 from collections import OrderedDict
+from collections.abc import Callable
 from concurrent.futures import Future, ThreadPoolExecutor
 from itertools import groupby
 from operator import itemgetter
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    ClassVar,
-    Generic,
-    NamedTuple,
-    TypeVar,
-)
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, NamedTuple, TypeVar
 
 if TYPE_CHECKING:
     import mmap
@@ -629,7 +621,7 @@ class KnownPartsOfAFile(BaseCache):
         fetcher: Fetcher,
         size: int,
         data: dict[tuple[int, int], bytes] | None = None,
-        strict: bool = True,
+        strict: bool = False,
         **_: Any,
     ):
         super().__init__(blocksize, fetcher, size)
@@ -653,50 +645,65 @@ class KnownPartsOfAFile(BaseCache):
         else:
             self.data = {}
 
+    @property
+    def size(self):
+        return sum(_[1] - _[0] for _ in self.data)
+
+    @size.setter
+    def size(self, value):
+        pass
+
+    @property
+    def nblocks(self):
+        return len(self.data)
+
+    @nblocks.setter
+    def nblocks(self, value):
+        pass
+
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:
             start = 0
         if stop is None:
             stop = self.size
+        self.total_requested_bytes += stop - start
 
         out = b""
-        for (loc0, loc1), data in self.data.items():
-            # If self.strict=False, use zero-padded data
-            # for reads beyond the end of a "known" buffer
+        started = False
+        loc_old = 0
+        for loc0, loc1 in sorted(self.data):
+            if (loc0 <= start < loc1) and (loc0 <= stop <= loc1):
+                # entirely within the block
+                off = start - loc0
+                self.hit_count += 1
+                return self.data[(loc0, loc1)][off : off + stop - start]
+            if stop <= loc0:
+                break
+            if started and loc0 > loc_old:
+                # a gap where we need data
+                self.miss_count += 1
+                if self.strict:
+                    raise ValueError
+                out += b"\x00" * (loc0 - loc_old)
             if loc0 <= start < loc1:
+                # found the start
+                self.hit_count += 1
                 off = start - loc0
-                out = data[off : off + stop - start]
-                if not self.strict or loc0 <= stop <= loc1:
-                    # The request is within a known range, or
-                    # it begins within a known range, and we
-                    # are allowed to pad reads beyond the
-                    # buffer with zero
-                    out += b"\x00" * (stop - start - len(out))
-                    self.hit_count += 1
-                    return out
-                else:
-                    # The request ends outside a known range,
-                    # and we are being "strict" about reads
-                    # beyond the buffer
-                    start = loc1
-                    break
-
-        # We only get here if there is a request outside the
-        # known parts of the file. In an ideal world, this
-        # should never happen
-        if self.fetcher is None:
-            # We cannot fetch the data, so raise an error
-            raise ValueError(f"Read is outside the known file parts: {(start, stop)}. ")
-        # We can fetch the data, but should warn the user
-        # that this may be slow
-        warnings.warn(
-            f"Read is outside the known file parts: {(start, stop)}. "
-            f"IO/caching performance may be poor!"
-        )
-        logger.debug(f"KnownPartsOfAFile cache fetching {start}-{stop}")
-        self.total_requested_bytes += stop - start
+                out = self.data[(loc0, loc1)][off : off + stop - start]
+                started = True
+            elif start < loc0 and stop > loc1:
+                # the whole block
+                self.hit_count += 1
+                out += self.data[(loc0, loc1)]
+            elif loc0 <= stop <= loc1:
+                # end block
+                self.hit_count += 1
+                return out + self.data[(loc0, loc1)][: stop - loc0]
+            loc_old = loc1
         self.miss_count += 1
-        return out + super()._fetch(start, stop)
+        if started and not self.strict:
+            return out + b"\x00" * (stop - loc_old)
+        raise ValueError
 
 
 class UpdatableLRU(Generic[P, T]):

fsspec/conftest.py

@@ -3,11 +3,12 @@ import shutil
 import subprocess
 import sys
 import time
+from collections import deque
+from collections.abc import Generator, Sequence
 
 import pytest
 
 import fsspec
-from fsspec.implementations.cached import CachingFileSystem
 
 
 @pytest.fixture()
@@ -27,16 +28,85 @@ def m():
         m.pseudo_dirs.append("")
 
 
-@pytest.fixture
+class InstanceCacheInspector:
+    """
+    Helper class to inspect instance caches of filesystem classes in tests.
+    """
+
+    def clear(self) -> None:
+        """
+        Clear instance caches of all currently imported filesystem classes.
+        """
+        classes = deque([fsspec.spec.AbstractFileSystem])
+        while classes:
+            cls = classes.popleft()
+            cls.clear_instance_cache()
+            classes.extend(cls.__subclasses__())
+
+    def gather_counts(self, *, omit_zero: bool = True) -> dict[str, int]:
+        """
+        Gather counts of filesystem instances in the instance caches
+        of all currently imported filesystem classes.
+
+        Parameters
+        ----------
+        omit_zero:
+            Whether to omit instance types with no cached instances.
+        """
+        out: dict[str, int] = {}
+        classes = deque([fsspec.spec.AbstractFileSystem])
+        while classes:
+            cls = classes.popleft()
+            count = len(cls._cache)  # there is no public interface for the cache
+            # note: skip intermediate AbstractFileSystem subclasses
+            #   if they proxy the protocol attribute via a property.
+            if isinstance(cls.protocol, (Sequence, str)):
+                key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0]
+                if count or not omit_zero:
+                    out[key] = count
+            classes.extend(cls.__subclasses__())
+        return out
+
+
+@pytest.fixture(scope="function", autouse=True)
+def instance_caches() -> Generator[InstanceCacheInspector, None, None]:
+    """
+    Fixture to ensure empty filesystem instance caches before and after a test.
+
+    Used by default for all tests.
+    Clears caches of all imported filesystem classes.
+    Can be used to write test assertions about instance caches.
+
+    Usage:
+
+        def test_something(instance_caches):
+            # Test code here
+            fsspec.open("file://abc")
+            fsspec.open("memory://foo/bar")
+
+            # Test assertion
+            assert instance_caches.gather_counts() == {"file": 1, "memory": 1}
+
+    Returns
+    -------
+    instance_caches: An instance cache inspector for clearing and inspecting caches.
+    """
+    ic = InstanceCacheInspector()
+
+    ic.clear()
+    try:
+        yield ic
+    finally:
+        ic.clear()
+
+
+@pytest.fixture(scope="function")
 def ftp_writable(tmpdir):
     """
     Fixture providing a writable FTP filesystem.
     """
     pytest.importorskip("pyftpdlib")
-    from fsspec.implementations.ftp import FTPFileSystem
 
-    FTPFileSystem.clear_instance_cache()  # remove lingering connections
-    CachingFileSystem.clear_instance_cache()
     d = str(tmpdir)
     with open(os.path.join(d, "out"), "wb") as f:
         f.write(b"hello" * 10000)

fsspec/core.py

@@ -18,7 +18,7 @@ from fsspec.caching import (  # noqa: F401
 )
 from fsspec.compression import compr
 from fsspec.config import conf
-from fsspec.registry import filesystem, get_filesystem_class
+from fsspec.registry import available_protocols, filesystem, get_filesystem_class
 from fsspec.utils import (
     _unstrip_protocol,
     build_name_function,
@@ -330,38 +330,55 @@ def open_files(
 
 def _un_chain(path, kwargs):
     # Avoid a circular import
-    from fsspec.implementations.cached import CachingFileSystem
+    from fsspec.implementations.chained import ChainedFileSystem
 
     if "::" in path:
         x = re.compile(".*[^a-z]+.*")  # test for non protocol-like single word
+        known_protocols = set(available_protocols())
         bits = []
+
+        # split on '::', then ensure each bit has a protocol
         for p in path.split("::"):
-            if "://" in p or x.match(p):
+            if p in known_protocols:
+                bits.append(p + "://")
+            elif "://" in p or x.match(p):
                 bits.append(p)
             else:
                 bits.append(p + "://")
     else:
         bits = [path]
+
     # [[url, protocol, kwargs], ...]
     out = []
     previous_bit = None
     kwargs = kwargs.copy()
+
     for bit in reversed(bits):
         protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file"
         cls = get_filesystem_class(protocol)
         extra_kwargs = cls._get_kwargs_from_urls(bit)
         kws = kwargs.pop(protocol, {})
+
         if bit is bits[0]:
             kws.update(kwargs)
+
         kw = dict(
             **{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]},
             **kws,
         )
         bit = cls._strip_protocol(bit)
-        if "target_protocol" not in kw and issubclass(cls, CachingFileSystem):
+
+        if (
+            "target_protocol" not in kw
+            and issubclass(cls, ChainedFileSystem)
+            and not bit
+        ):
+            # replace bit if we are chaining and no path given
             bit = previous_bit
+
         out.append((bit, protocol, kw))
         previous_bit = bit
+
     out.reverse()
     return out
 

fsspec/generic.py

@@ -118,6 +118,8 @@ def rsync(
         if otherfile in otherfiles:
             if update_cond == "always":
                 allfiles[k] = otherfile
+            elif update_cond == "never":
+                allfiles.pop(k)
             elif update_cond == "different":
                 inf1 = source_field(v) if callable(source_field) else v[source_field]
                 v2 = otherfiles[otherfile]

fsspec/implementations/arrow.py

@@ -75,10 +75,13 @@ class ArrowFSWrapper(AbstractFileSystem):
         path = self._strip_protocol(path)
         from pyarrow.fs import FileSelector
 
-        entries = [
-            self._make_entry(entry)
-            for entry in self.fs.get_file_info(FileSelector(path))
-        ]
+        try:
+            entries = [
+                self._make_entry(entry)
+                for entry in self.fs.get_file_info(FileSelector(path))
+            ]
+        except (FileNotFoundError, NotADirectoryError):
+            entries = [self.info(path, **kwargs)]
         if detail:
             return entries
         else:
@@ -202,11 +205,11 @@ class ArrowFSWrapper(AbstractFileSystem):
         return self.fs.get_file_info(path).mtime
 
     def cat_file(self, path, start=None, end=None, **kwargs):
-        kwargs["seekable"] = start not in [None, 0]
+        kwargs.setdefault("seekable", start not in [None, 0])
         return super().cat_file(path, start=None, end=None, **kwargs)
 
     def get_file(self, rpath, lpath, **kwargs):
-        kwargs["seekable"] = False
+        kwargs.setdefault("seekable", False)
         super().get_file(rpath, lpath, **kwargs)
 
 
@@ -220,7 +223,6 @@ class ArrowFSWrapper(AbstractFileSystem):
         "readable",
         "writable",
         "close",
-        "size",
         "seekable",
     ],
 )
@@ -238,6 +240,10 @@ class ArrowFile(io.IOBase):
     def __enter__(self):
         return self
 
+    @property
+    def size(self):
+        return self.stream.size()
+
     def __exit__(self, *args):
         return self.close()
 

fsspec/implementations/asyn_wrapper.py

@@ -5,6 +5,8 @@ import inspect
 import fsspec
 from fsspec.asyn import AsyncFileSystem, running_async
 
+from .chained import ChainedFileSystem
+
 
 def async_wrapper(func, obj=None, semaphore=None):
     """
@@ -35,7 +37,7 @@ def async_wrapper(func, obj=None, semaphore=None):
     return wrapper
 
 
-class AsyncFileSystemWrapper(AsyncFileSystem):
+class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem):
     """
     A wrapper class to convert a synchronous filesystem into an asynchronous one.
 

fsspec/implementations/cache_metadata.py

@@ -15,9 +15,7 @@ except ImportError:
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
-    from typing import Any, Literal
-
-    from typing_extensions import TypeAlias
+    from typing import Any, Literal, TypeAlias
 
     from .cached import CachingFileSystem
 

fsspec/implementations/cached.py

@@ -6,16 +6,18 @@ import os
 import tempfile
 import time
 import weakref
+from collections.abc import Callable
 from shutil import rmtree
-from typing import TYPE_CHECKING, Any, Callable, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
 
-from fsspec import AbstractFileSystem, filesystem
+from fsspec import filesystem
 from fsspec.callbacks import DEFAULT_CALLBACK
 from fsspec.compression import compr
 from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
 from fsspec.implementations.cache_mapper import create_cache_mapper
 from fsspec.implementations.cache_metadata import CacheMetadata
+from fsspec.implementations.chained import ChainedFileSystem
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.spec import AbstractBufferedFile
 from fsspec.transaction import Transaction
@@ -39,7 +41,7 @@ class WriteCachedTransaction(Transaction):
         self.fs = None  # break cycle
 
 
-class CachingFileSystem(AbstractFileSystem):
+class CachingFileSystem(ChainedFileSystem):
     """Locally caching filesystem, layer over any other FS
 
     This class implements chunk-wise local storage of remote files, for quick
@@ -60,6 +62,7 @@ class CachingFileSystem(AbstractFileSystem):
     """
 
     protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached")
+    _strip_tokenize_options = ("fo",)
 
     def __init__(
         self,
@@ -984,7 +987,9 @@ class LocalTempFile:
         os.remove(self.fn)
 
     def commit(self):
-        self.fs.put(self.fn, self.path, **self.kwargs)
+        # calling put() with list arguments avoids path expansion and additional operations
+        # like isdir()
+        self.fs.put([self.fn], [self.path], **self.kwargs)
         # we do not delete the local copy, it's still in the cache.
 
     @property

fsspec/implementations/chained.py

@@ -0,0 +1,23 @@
+from typing import ClassVar
+
+from fsspec import AbstractFileSystem
+
+__all__ = ("ChainedFileSystem",)
+
+
+class ChainedFileSystem(AbstractFileSystem):
+    """Chained filesystem base class.
+
+    A chained filesystem is designed to be layered over another FS.
+    This is useful to implement things like caching.
+
+    This base class does very little on its own, but is used as a marker
+    that the class is designed for chaining.
+
+    Right now this is only used in `url_to_fs` to provide the path argument
+    (`fo`) to the chained filesystem from the underlying filesystem.
+
+    Additional functionality may be added in the future.
+    """
+
+    protocol: ClassVar[str] = "chained"

fsspec/implementations/data.py

@@ -1,6 +1,5 @@
 import base64
 import io
-from typing import Optional
 from urllib.parse import unquote
 
 from fsspec import AbstractFileSystem
@@ -50,7 +49,7 @@ class DataFileSystem(AbstractFileSystem):
         return io.BytesIO(self.cat_file(path))
 
     @staticmethod
-    def encode(data: bytes, mime: Optional[str] = None):
+    def encode(data: bytes, mime: str | None = None):
         """Format the given data into data-URL syntax
 
         This version always base64 encodes, even when the data is ascii/url-safe.

fsspec/implementations/dirfs.py

@@ -1,8 +1,9 @@
 from .. import filesystem
 from ..asyn import AsyncFileSystem
+from .chained import ChainedFileSystem
 
 
-class DirFileSystem(AsyncFileSystem):
+class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
     """Directory prefix filesystem
 
     The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with

fsspec/implementations/gist.py

@@ -14,21 +14,21 @@ class GistFileSystem(AbstractFileSystem):
 
     Parameters
     ----------
-    gist_id : str
+    gist_id: str
         The ID of the gist you want to access (the long hex value from the URL).
-    filenames : list[str] (optional)
+    filenames: list[str] (optional)
         If provided, only make a file system representing these files, and do not fetch
         the list of all files for this gist.
-    sha : str (optional)
+    sha: str (optional)
         If provided, fetch a particular revision of the gist. If omitted,
         the latest revision is used.
-    username : str (optional)
-        GitHub username for authentication (required if token is given).
-    token : str (optional)
-        GitHub personal access token (required if username is given).
-    timeout : (float, float) or float, optional
+    username: str (optional)
+        GitHub username for authentication.
+    token: str (optional)
+        GitHub personal access token (required if username is given), or.
+    timeout: (float, float) or float, optional
         Connect and read timeouts for requests (default 60s each).
-    kwargs : dict
+    kwargs: dict
         Stored on `self.request_kw` and passed to `requests.get` when fetching Gist
         metadata or reading ("opening") a file.
     """
@@ -51,10 +51,8 @@ class GistFileSystem(AbstractFileSystem):
         self.gist_id = gist_id
         self.filenames = filenames
         self.sha = sha  # revision of the gist (optional)
-        if (username is None) ^ (token is None):
-            # Both or neither must be set
-            if username or token:
-                raise ValueError("Auth requires both username and token, or neither.")
+        if username is not None and token is None:
+            raise ValueError("User auth requires a token")
         self.username = username
         self.token = token
         self.request_kw = kwargs
@@ -67,9 +65,18 @@ class GistFileSystem(AbstractFileSystem):
     @property
     def kw(self):
         """Auth parameters passed to 'requests' if we have username/token."""
-        if self.username is not None and self.token is not None:
-            return {"auth": (self.username, self.token), **self.request_kw}
-        return self.request_kw
+        kw = {
+            "headers": {
+                "Accept": "application/vnd.github+json",
+                "X-GitHub-Api-Version": "2022-11-28",
+            }
+        }
+        kw.update(self.request_kw)
+        if self.username and self.token:
+            kw["auth"] = (self.username, self.token)
+        elif self.token:
+            kw["headers"]["Authorization"] = f"Bearer {self.token}"
+        return kw
 
     def _fetch_gist_metadata(self):
         """
@@ -229,4 +236,6 @@ class GistFileSystem(AbstractFileSystem):
                     pass  # skip
                 else:
                     out[p] = e
+        if len(paths) == 1 and paths[0] == path:
+            return out[path]
         return out

fsspec/implementations/http.py

@@ -43,6 +43,7 @@ class HTTPFileSystem(AsyncFileSystem):
     HTML href tags will be used.
     """
 
+    protocol = ("http", "https")
     sep = "/"
 
     def __init__(
@@ -326,7 +327,7 @@ class HTTPFileSystem(AsyncFileSystem):
         async with meth(self.encode_url(rpath), data=gen_chunks(), **kw) as resp:
             self._raise_not_found_for_status(resp, rpath)
 
-    async def _exists(self, path, **kwargs):
+    async def _exists(self, path, strict=False, **kwargs):
         kw = self.kwargs.copy()
         kw.update(kwargs)
         try:
@@ -334,8 +335,14 @@ class HTTPFileSystem(AsyncFileSystem):
             session = await self.set_session()
             r = await session.get(self.encode_url(path), **kw)
             async with r:
+                if strict:
+                    self._raise_not_found_for_status(r, path)
                 return r.status < 400
+        except FileNotFoundError:
+            return False
         except aiohttp.ClientError:
+            if strict:
+                raise
             return False
 
     async def _isfile(self, path, **kwargs):

fsspec/implementations/http_sync.py

@@ -463,14 +463,20 @@ class HTTPFileSystem(AbstractFileSystem):
                 end -= 1  # bytes range is inclusive
         return f"bytes={start}-{end}"
 
-    def exists(self, path, **kwargs):
+    def exists(self, path, strict=False, **kwargs):
         kw = self.kwargs.copy()
         kw.update(kwargs)
         try:
             logger.debug(path)
             r = self.session.get(self.encode_url(path), **kw)
+            if strict:
+                self._raise_not_found_for_status(r, path)
             return r.status_code < 400
+        except FileNotFoundError:
+            return False
         except Exception:
+            if strict:
+                raise
             return False
 
     def isfile(self, path, **kwargs):