fsspec 2025.9.0__tar.gz → 2025.10.0__tar.gz

{fsspec-2025.9.0 → fsspec-2025.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.9.0
+Version: 2025.10.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/

{fsspec-2025.9.0 → fsspec-2025.10.0}/docs/source/api.rst

@@ -7,17 +7,17 @@ User Functions
 --------------
 
 .. autosummary::
-   fsspec.available_compressions
-   fsspec.available_protocols
-   fsspec.filesystem
-   fsspec.fuse.run
-   fsspec.generic.rsync
-   fsspec.get_filesystem_class
-   fsspec.get_mapper
-   fsspec.gui.FileSelector
-   fsspec.open
-   fsspec.open_files
-   fsspec.open_local
+   available_compressions
+   available_protocols
+   filesystem
+   fuse.run
+   generic.rsync
+   get_filesystem_class
+   get_mapper
+   gui.FileSelector
+   open
+   open_files
+   open_local
 
 .. autofunction:: fsspec.available_compressions
 .. autofunction:: fsspec.available_protocols
@@ -36,24 +36,24 @@ Base Classes
 ------------
 
 .. autosummary::
-   fsspec.archive.AbstractArchiveFileSystem
-   fsspec.asyn.AsyncFileSystem
-   fsspec.callbacks.Callback
-   fsspec.callbacks.DotPrinterCallback
-   fsspec.callbacks.NoOpCallback
-   fsspec.callbacks.TqdmCallback
-   fsspec.core.BaseCache
-   fsspec.core.OpenFile
-   fsspec.core.OpenFiles
-   fsspec.core.get_fs_token_paths
-   fsspec.core.url_to_fs
-   fsspec.dircache.DirCache
-   fsspec.FSMap
-   fsspec.generic.GenericFileSystem
-   fsspec.registry.register_implementation
-   fsspec.spec.AbstractBufferedFile
-   fsspec.spec.AbstractFileSystem
-   fsspec.spec.Transaction
+   archive.AbstractArchiveFileSystem
+   asyn.AsyncFileSystem
+   callbacks.Callback
+   callbacks.DotPrinterCallback
+   callbacks.NoOpCallback
+   callbacks.TqdmCallback
+   core.BaseCache
+   core.OpenFile
+   core.OpenFiles
+   core.get_fs_token_paths
+   core.url_to_fs
+   dircache.DirCache
+   FSMap
+   generic.GenericFileSystem
+   registry.register_implementation
+   spec.AbstractBufferedFile
+   spec.AbstractFileSystem
+   spec.Transaction
 
 .. autoclass:: fsspec.archive.AbstractArchiveFileSystem
    :members:
@@ -107,31 +107,31 @@ Built-in Implementations
 ------------------------
 
 .. autosummary::
-   fsspec.implementations.arrow.ArrowFSWrapper
-   fsspec.implementations.arrow.HadoopFileSystem
-   fsspec.implementations.cached.CachingFileSystem
-   fsspec.implementations.cached.SimpleCacheFileSystem
-   fsspec.implementations.cached.WholeFileCacheFileSystem
-   fsspec.implementations.dask.DaskWorkerFileSystem
-   fsspec.implementations.data.DataFileSystem
-   fsspec.implementations.dbfs.DatabricksFileSystem
-   fsspec.implementations.dirfs.DirFileSystem
-   fsspec.implementations.ftp.FTPFileSystem
-   fsspec.implementations.gist.GistFileSystem
-   fsspec.implementations.git.GitFileSystem
-   fsspec.implementations.github.GithubFileSystem
-   fsspec.implementations.http.HTTPFileSystem
-   fsspec.implementations.jupyter.JupyterFileSystem
-   fsspec.implementations.libarchive.LibArchiveFileSystem
-   fsspec.implementations.local.LocalFileSystem
-   fsspec.implementations.memory.MemoryFileSystem
-   fsspec.implementations.reference.ReferenceFileSystem
-   fsspec.implementations.reference.LazyReferenceMapper
-   fsspec.implementations.sftp.SFTPFileSystem
-   fsspec.implementations.smb.SMBFileSystem
-   fsspec.implementations.tar.TarFileSystem
-   fsspec.implementations.webhdfs.WebHDFS
-   fsspec.implementations.zip.ZipFileSystem
+   implementations.arrow.ArrowFSWrapper
+   implementations.arrow.HadoopFileSystem
+   implementations.cached.CachingFileSystem
+   implementations.cached.SimpleCacheFileSystem
+   implementations.cached.WholeFileCacheFileSystem
+   implementations.dask.DaskWorkerFileSystem
+   implementations.data.DataFileSystem
+   implementations.dbfs.DatabricksFileSystem
+   implementations.dirfs.DirFileSystem
+   implementations.ftp.FTPFileSystem
+   implementations.gist.GistFileSystem
+   implementations.git.GitFileSystem
+   implementations.github.GithubFileSystem
+   implementations.http.HTTPFileSystem
+   implementations.jupyter.JupyterFileSystem
+   implementations.libarchive.LibArchiveFileSystem
+   implementations.local.LocalFileSystem
+   implementations.memory.MemoryFileSystem
+   implementations.reference.ReferenceFileSystem
+   implementations.reference.LazyReferenceMapper
+   implementations.sftp.SFTPFileSystem
+   implementations.smb.SMBFileSystem
+   implementations.tar.TarFileSystem
+   implementations.webhdfs.WebHDFS
+   implementations.zip.ZipFileSystem
 
 .. autoclass:: fsspec.implementations.arrow.ArrowFSWrapper
    :members: __init__
@@ -296,12 +296,12 @@ Read Buffering
 --------------
 
 .. autosummary::
-   fsspec.caching.BlockCache
-   fsspec.caching.BytesCache
-   fsspec.caching.MMapCache
-   fsspec.caching.ReadAheadCache
-   fsspec.caching.FirstChunkCache
-   fsspec.caching.BackgroundBlockCache
+   caching.BlockCache
+   caching.BytesCache
+   caching.MMapCache
+   caching.ReadAheadCache
+   caching.FirstChunkCache
+   caching.BackgroundBlockCache
 
 .. autoclass:: fsspec.caching.BlockCache
    :members:
@@ -326,7 +326,7 @@ Utilities
 
 .. autosummary::
 
-   fsspec.utils.read_block
+   utils.read_block
 
 .. autofunction:: fsspec.utils.read_block
 

{fsspec-2025.9.0 → fsspec-2025.10.0}/docs/source/changelog.rst

@@ -1,6 +1,33 @@
 Changelog
 =========
 
+2025.10.0
+---------
+
+Enhancements
+
+- specify storage_options not use use for tokenising; test instance cache (#1933)
+- base class "chained" for FSs that pass through operations (#1929)
+
+Fixes
+
+- strip protocol for sftp (#1940)
+- use one-step mv in jupyterfs (#1937)
+- raise errors in jupyterfs, not return (#1936)
+- add protocol attribute to http(#1935)
+- allow ls() on a single file for arrowFS (#1931)
+- support kwargs in webhdfs ls() (#1928)
+- better discrimination for parquet in referenceFS (#1923)
+- accept all mode= types in memoryFS (#1922)
+- pass path list to put() in localtempfile for efficiency (#1920)
+- fix docs errors (#1917)
+
+Other
+
+- add CoC (#1924)
+- restrict github/gist tests because of rate limiting (#1918)
+
+
 2025.9.0
 --------
 

fsspec-2025.10.0/docs/source/code-of-conduct.rst

@@ -0,0 +1,126 @@
+Code of Conduct
+===============
+
+All participants in the fsspec community are expected to adhere to a Code of Conduct.
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, treating everyone as unique humans deserving of
+respect.
+
+Examples of unacceptable behaviour by participants include:
+
+- The use of sexualized language or imagery
+- Personal attacks
+- Trolling or insulting/derogatory comments
+- Public or private harassment
+- Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+- Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviours that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves
+to fairly and consistently applying these principles to every aspect of
+managing this project. Project maintainers who do not follow or enforce
+the Code of Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public
+spaces when an individual is representing the project or its community.
+
+If you feel the code of conduct has been violated, please report the
+incident to the fsspec core team.
+
+Reporting
+---------
+
+If you believe someone is violating theCode of Conduct we ask that you report it
+to the  Project by emailing community@anaconda.com. All reports will be kept
+confidential. In some cases we may determine that a public statement will need
+to be made. If that's the case, the identities of all victims and reporters
+will remain confidential unless those individuals instruct us otherwise.
+If you believe anyone is in physical danger, please notify appropriate law
+enforcement first.
+
+In your report please include:
+
+- Your contact info
+- Names (real, nicknames, or pseudonyms) of any individuals involved.
+  If there were other witnesses besides you, please try to include them as well.
+- When and where the incident occurred. Please be as specific as possible.
+- Your account of what occurred. If there is a publicly available record
+  please include a link.
+- Any extra context you believe existed for the incident.
+- If you believe this incident is ongoing.
+- If you believe any member of the core team has a conflict of interest
+  in adjudicating the incident.
+- What, if any, corrective response you believe would be appropriate.
+- Any other information you believe we should have.
+
+Core team members are obligated to maintain confidentiality with regard
+to the reporter and details of an incident.
+
+What happens next?
+~~~~~~~~~~~~~~~~~~
+
+You will receive an email acknowledging receipt of your complaint.
+The core team will immediately meet to review the incident and determine:
+
+- What happened.
+- Whether this event constitutes a code of conduct violation.
+- Who the bad actor was.
+- Whether this is an ongoing situation, or if there is a threat to anyone's
+  physical safety.
+- If this is determined to be an ongoing incident or a threat to physical safety,
+  the working groups' immediate priority will be to protect everyone involved.
+
+If a member of the core team is one of the named parties, they will not be
+included in any discussions, and will not be provided with any confidential
+details from the reporter.
+
+If anyone on the core team believes they have a conflict of interest in
+adjudicating on a reported issue, they will inform the other core team
+members, and exempt themselves from any discussion about the issue.
+Following this declaration, they will not be provided with any confidential
+details from the reporter.
+
+Once the working group has a complete account of the events they will make a
+decision as to how to response. Responses may include:
+
+- Nothing (if we determine no violation occurred).
+- A private reprimand from the working group to the individual(s) involved.
+- A public reprimand.
+- An imposed vacation
+- A permanent or temporary ban from some or all spaces (GitHub repositories, etc.)
+- A request for a public or private apology.
+
+We'll respond within one week to the person who filed the report with either a
+resolution or an explanation of why the situation is not yet resolved.
+
+Once we've determined our final action, we'll contact the original reporter
+to let them know what action (if any) we'll be taking. We'll take into account
+feedback from the reporter on the appropriateness of our response, but we
+don't guarantee we'll act on it.
+
+Acknowledgement
+---------------
+
+This CoC is modified from the one by `BeeWare`_, which in turn refers to
+the `Contributor Covenant`_ and the `Django`_ project.
+
+.. _BeeWare: https://beeware.org/community/behavior/code-of-conduct/
+.. _Contributor Covenant: https://www.contributor-covenant.org/version/1/3/0/code-of-conduct/
+.. _Django: https://www.djangoproject.com/conduct/reporting/
+
+.. raw:: html
+
+    <script data-goatcounter="https://projspec.goatcounter.com/count"
+        async src="//gc.zgo.at/count.js"></script>

{fsspec-2025.9.0 → fsspec-2025.10.0}/docs/source/features.rst

@@ -241,7 +241,9 @@ reads the same zip-file, but extracts the CSV files and stores them locally in t
 **For developers**: this "chaining" methods works by formatting the arguments passed to ``open_*``
 into ``target_protocol`` (a simple string) and ``target_options`` (a dict) and also optionally
 ``fo`` (target path, if a specific file is required). In order for an implementation to chain
-successfully like this, it must look for exactly those named arguments.
+successfully like this, it must look for exactly those named arguments. Implementations that
+require access to the target path of their nested targets should inherit from ``ChainedFileSystem``,
+which will trigger pass-through of the nested path automatically.
 
 Caching Files Locally
 ---------------------

{fsspec-2025.9.0 → fsspec-2025.10.0}/docs/source/index.rst

@@ -119,6 +119,11 @@ The current list of known implementations can be found as follows
    async.rst
    api.rst
    changelog.rst
+   code-of-conduct.rst
+
+
+These docs pages collect anonymous tracking data using goatcounter, and the
+dashboard is available to the public: https://fsspec.goatcounter.com/ .
 
 .. raw:: html
 

{fsspec-2025.9.0 → fsspec-2025.10.0}/fsspec/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2025.9.0'
-__version_tuple__ = version_tuple = (2025, 9, 0)
+__version__ = version = '2025.10.0'
+__version_tuple__ = version_tuple = (2025, 10, 0)
 
 __commit_id__ = commit_id = None

fsspec-2025.10.0/fsspec/conftest.py

@@ -0,0 +1,125 @@
+import os
+import shutil
+import subprocess
+import sys
+import time
+from collections import deque
+from collections.abc import Generator, Sequence
+
+import pytest
+
+import fsspec
+
+
+@pytest.fixture()
+def m():
+    """
+    Fixture providing a memory filesystem.
+    """
+    m = fsspec.filesystem("memory")
+    m.store.clear()
+    m.pseudo_dirs.clear()
+    m.pseudo_dirs.append("")
+    try:
+        yield m
+    finally:
+        m.store.clear()
+        m.pseudo_dirs.clear()
+        m.pseudo_dirs.append("")
+
+
+class InstanceCacheInspector:
+    """
+    Helper class to inspect instance caches of filesystem classes in tests.
+    """
+
+    def clear(self) -> None:
+        """
+        Clear instance caches of all currently imported filesystem classes.
+        """
+        classes = deque([fsspec.spec.AbstractFileSystem])
+        while classes:
+            cls = classes.popleft()
+            cls.clear_instance_cache()
+            classes.extend(cls.__subclasses__())
+
+    def gather_counts(self, *, omit_zero: bool = True) -> dict[str, int]:
+        """
+        Gather counts of filesystem instances in the instance caches
+        of all currently imported filesystem classes.
+
+        Parameters
+        ----------
+        omit_zero:
+            Whether to omit instance types with no cached instances.
+        """
+        out: dict[str, int] = {}
+        classes = deque([fsspec.spec.AbstractFileSystem])
+        while classes:
+            cls = classes.popleft()
+            count = len(cls._cache)  # there is no public interface for the cache
+            # note: skip intermediate AbstractFileSystem subclasses
+            #   if they proxy the protocol attribute via a property.
+            if isinstance(cls.protocol, (Sequence, str)):
+                key = cls.protocol if isinstance(cls.protocol, str) else cls.protocol[0]
+                if count or not omit_zero:
+                    out[key] = count
+            classes.extend(cls.__subclasses__())
+        return out
+
+
+@pytest.fixture(scope="function", autouse=True)
+def instance_caches() -> Generator[InstanceCacheInspector, None, None]:
+    """
+    Fixture to ensure empty filesystem instance caches before and after a test.
+
+    Used by default for all tests.
+    Clears caches of all imported filesystem classes.
+    Can be used to write test assertions about instance caches.
+
+    Usage:
+
+        def test_something(instance_caches):
+            # Test code here
+            fsspec.open("file://abc")
+            fsspec.open("memory://foo/bar")
+
+            # Test assertion
+            assert instance_caches.gather_counts() == {"file": 1, "memory": 1}
+
+    Returns
+    -------
+    instance_caches: An instance cache inspector for clearing and inspecting caches.
+    """
+    ic = InstanceCacheInspector()
+
+    ic.clear()
+    try:
+        yield ic
+    finally:
+        ic.clear()
+
+
+@pytest.fixture(scope="function")
+def ftp_writable(tmpdir):
+    """
+    Fixture providing a writable FTP filesystem.
+    """
+    pytest.importorskip("pyftpdlib")
+
+    d = str(tmpdir)
+    with open(os.path.join(d, "out"), "wb") as f:
+        f.write(b"hello" * 10000)
+    P = subprocess.Popen(
+        [sys.executable, "-m", "pyftpdlib", "-d", d, "-u", "user", "-P", "pass", "-w"]
+    )
+    try:
+        time.sleep(1)
+        yield "localhost", 2121, "user", "pass"
+    finally:
+        P.terminate()
+        P.wait()
+        try:
+            shutil.rmtree(tmpdir)
+        except Exception:
+            pass

{fsspec-2025.9.0 → fsspec-2025.10.0}/fsspec/core.py

@@ -330,7 +330,7 @@ def open_files(
 
 def _un_chain(path, kwargs):
     # Avoid a circular import
-    from fsspec.implementations.cached import CachingFileSystem
+    from fsspec.implementations.chained import ChainedFileSystem
 
     if "::" in path:
         x = re.compile(".*[^a-z]+.*")  # test for non protocol-like single word
@@ -358,7 +358,7 @@ def _un_chain(path, kwargs):
             **kws,
         )
         bit = cls._strip_protocol(bit)
-        if "target_protocol" not in kw and issubclass(cls, CachingFileSystem):
+        if "target_protocol" not in kw and issubclass(cls, ChainedFileSystem):
             bit = previous_bit
         out.append((bit, protocol, kw))
         previous_bit = bit

{fsspec-2025.9.0 → fsspec-2025.10.0}/fsspec/generic.py

@@ -118,6 +118,8 @@ def rsync(
         if otherfile in otherfiles:
             if update_cond == "always":
                 allfiles[k] = otherfile
+            elif update_cond == "never":
+                allfiles.pop(k)
             elif update_cond == "different":
                 inf1 = source_field(v) if callable(source_field) else v[source_field]
                 v2 = otherfiles[otherfile]

{fsspec-2025.9.0 → fsspec-2025.10.0}/fsspec/implementations/arrow.py

@@ -75,10 +75,13 @@ class ArrowFSWrapper(AbstractFileSystem):
         path = self._strip_protocol(path)
         from pyarrow.fs import FileSelector
 
-        entries = [
-            self._make_entry(entry)
-            for entry in self.fs.get_file_info(FileSelector(path))
-        ]
+        try:
+            entries = [
+                self._make_entry(entry)
+                for entry in self.fs.get_file_info(FileSelector(path))
+            ]
+        except (FileNotFoundError, NotADirectoryError):
+            entries = [self.info(path, **kwargs)]
         if detail:
             return entries
         else:

{fsspec-2025.9.0 → fsspec-2025.10.0}/fsspec/implementations/cached.py

@@ -9,13 +9,14 @@ import weakref
 from shutil import rmtree
 from typing import TYPE_CHECKING, Any, Callable, ClassVar
 
-from fsspec import AbstractFileSystem, filesystem
+from fsspec import filesystem
 from fsspec.callbacks import DEFAULT_CALLBACK
 from fsspec.compression import compr
 from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
 from fsspec.implementations.cache_mapper import create_cache_mapper
 from fsspec.implementations.cache_metadata import CacheMetadata
+from fsspec.implementations.chained import ChainedFileSystem
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.spec import AbstractBufferedFile
 from fsspec.transaction import Transaction
@@ -39,7 +40,7 @@ class WriteCachedTransaction(Transaction):
         self.fs = None  # break cycle
 
 
-class CachingFileSystem(AbstractFileSystem):
+class CachingFileSystem(ChainedFileSystem):
     """Locally caching filesystem, layer over any other FS
 
     This class implements chunk-wise local storage of remote files, for quick
@@ -60,6 +61,7 @@ class CachingFileSystem(AbstractFileSystem):
     """
 
     protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached")
+    _strip_tokenize_options = ("fo",)
 
     def __init__(
         self,
@@ -984,7 +986,9 @@ class LocalTempFile:
         os.remove(self.fn)
 
     def commit(self):
-        self.fs.put(self.fn, self.path, **self.kwargs)
+        # calling put() with list arguments avoids path expansion and additional operations
+        # like isdir()
+        self.fs.put([self.fn], [self.path], **self.kwargs)
         # we do not delete the local copy, it's still in the cache.
 
     @property

fsspec-2025.10.0/fsspec/implementations/chained.py

@@ -0,0 +1,23 @@
+from typing import ClassVar
+
+from fsspec import AbstractFileSystem
+
+__all__ = ("ChainedFileSystem",)
+
+
+class ChainedFileSystem(AbstractFileSystem):
+    """Chained filesystem base class.
+
+    A chained filesystem is designed to be layered over another FS.
+    This is useful to implement things like caching.
+
+    This base class does very little on its own, but is used as a marker
+    that the class is designed for chaining.
+
+    Right now this is only used in `url_to_fs` to provide the path argument
+    (`fo`) to the chained filesystem from the underlying filesystem.
+
+    Additional functionality may be added in the future.
+    """
+
+    protocol: ClassVar[str] = "chained"

{fsspec-2025.9.0 → fsspec-2025.10.0}/fsspec/implementations/gist.py

@@ -14,21 +14,21 @@ class GistFileSystem(AbstractFileSystem):
 
     Parameters
     ----------
-    gist_id : str
+    gist_id: str
         The ID of the gist you want to access (the long hex value from the URL).
-    filenames : list[str] (optional)
+    filenames: list[str] (optional)
         If provided, only make a file system representing these files, and do not fetch
         the list of all files for this gist.
-    sha : str (optional)
+    sha: str (optional)
         If provided, fetch a particular revision of the gist. If omitted,
         the latest revision is used.
-    username : str (optional)
-        GitHub username for authentication (required if token is given).
-    token : str (optional)
-        GitHub personal access token (required if username is given).
-    timeout : (float, float) or float, optional
+    username: str (optional)
+        GitHub username for authentication.
+    token: str (optional)
+        GitHub personal access token (required if username is given), or.
+    timeout: (float, float) or float, optional
         Connect and read timeouts for requests (default 60s each).
-    kwargs : dict
+    kwargs: dict
         Stored on `self.request_kw` and passed to `requests.get` when fetching Gist
         metadata or reading ("opening") a file.
     """
@@ -51,10 +51,8 @@ class GistFileSystem(AbstractFileSystem):
         self.gist_id = gist_id
         self.filenames = filenames
         self.sha = sha  # revision of the gist (optional)
-        if (username is None) ^ (token is None):
-            # Both or neither must be set
-            if username or token:
-                raise ValueError("Auth requires both username and token, or neither.")
+        if username is not None and token is None:
+            raise ValueError("User auth requires a token")
         self.username = username
         self.token = token
         self.request_kw = kwargs
@@ -67,9 +65,18 @@ class GistFileSystem(AbstractFileSystem):
     @property
     def kw(self):
         """Auth parameters passed to 'requests' if we have username/token."""
-        if self.username is not None and self.token is not None:
-            return {"auth": (self.username, self.token), **self.request_kw}
-        return self.request_kw
+        kw = {
+            "headers": {
+                "Accept": "application/vnd.github+json",
+                "X-GitHub-Api-Version": "2022-11-28",
+            }
+        }
+        kw.update(self.request_kw)
+        if self.username and self.token:
+            kw["auth"] = (self.username, self.token)
+        elif self.token:
+            kw["headers"]["Authorization"] = f"Bearer {self.token}"
+        return kw
 
     def _fetch_gist_metadata(self):
         """
@@ -229,4 +236,6 @@ class GistFileSystem(AbstractFileSystem):
                     pass  # skip
                 else:
                     out[p] = e
+        if len(paths) == 1 and paths[0] == path:
+            return out[path]
         return out