fsspec 2025.5.1__tar.gz → 2025.9.0__tar.gz

{fsspec-2025.5.1 → fsspec-2025.9.0}/.github/workflows/main.yaml

@@ -32,7 +32,7 @@ jobs:
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v3
         with:
-          environment-file: ci/environment-py38.yml
+          environment-file: ci/environment-linux.yml
           python-version: ${{ matrix.PY }}
 
       - name: Run Tests
@@ -43,7 +43,7 @@ jobs:
 
   win:
     name: pytest-win
-    runs-on: windows-2019
+    runs-on: windows-2022
 
     env:
       CIRUN: true
@@ -75,22 +75,6 @@ jobs:
           python-version: "3.11"
       - uses: pre-commit/action@main
 
-#  typecheck:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v4
-#
-#      - name: Setup conda
-#        uses: conda-incubator/setup-miniconda@v3
-#        with:
-#          environment-file: ci/environment-typecheck.yml
-#
-#      - name: mypy
-#        shell: bash -l {0}
-#        run: |
-#          mypy fsspec
-#
   downstream:
     name: downstream
     runs-on: ubuntu-24.04

{fsspec-2025.5.1 → fsspec-2025.9.0}/.pre-commit-config.yaml

@@ -13,11 +13,10 @@ repos:
       - id: check-json
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.9.2
+    rev: v0.12.2
     hooks:
       # Run the linter.
-      - id: ruff
+      - id: ruff-check
         args: [ --fix, "--show-fixes"]
       - id: ruff-format
         types_or: [python]

{fsspec-2025.5.1 → fsspec-2025.9.0}/PKG-INFO

@@ -1,45 +1,16 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.5.1
+Version: 2025.9.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
 Project-URL: Homepage, https://github.com/fsspec/filesystem_spec
 Maintainer-email: Martin Durant <mdurant@anaconda.com>
-License: BSD 3-Clause License
-        
-        Copyright (c) 2018, Martin Durant
-        All rights reserved.
-        
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        
-        * Redistributions of source code must retain the above copyright notice, this
-          list of conditions and the following disclaimer.
-        
-        * Redistributions in binary form must reproduce the above copyright notice,
-          this list of conditions and the following disclaimer in the documentation
-          and/or other materials provided with the distribution.
-        
-        * Neither the name of the copyright holder nor the names of its
-          contributors may be used to endorse or promote products derived from
-          this software without specific prior written permission.
-        
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+License-Expression: BSD-3-Clause
 License-File: LICENSE
 Keywords: file
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
@@ -58,7 +29,7 @@ Requires-Dist: dask; extra == 'dask'
 Requires-Dist: distributed; extra == 'dask'
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
-Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
 Provides-Extra: doc
 Requires-Dist: numpydoc; extra == 'doc'
 Requires-Dist: sphinx; extra == 'doc'
@@ -172,7 +143,7 @@ Requires-Dist: smbprotocol; extra == 'test-full'
 Requires-Dist: tqdm; extra == 'test-full'
 Requires-Dist: urllib3; extra == 'test-full'
 Requires-Dist: zarr; extra == 'test-full'
-Requires-Dist: zstandard; extra == 'test-full'
+Requires-Dist: zstandard; (python_version < '3.14') and extra == 'test-full'
 Provides-Extra: tqdm
 Requires-Dist: tqdm; extra == 'tqdm'
 Description-Content-Type: text/markdown
@@ -275,3 +246,11 @@ filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
 before you commit, reformatting any changed files. You can format without
 committing via ``pre-commit run`` or skip these checks with ``git commit
 --no-verify``.
+
+## Support
+
+Work on this repository is supported in part by:
+
+"Anaconda, Inc. - Advancing AI through open source."
+
+<a href="https://anaconda.com/"><img src="https://camo.githubusercontent.com/b8555ef2222598ed37ce38ac86955febbd25de7619931bb7dd3c58432181d3b6/68747470733a2f2f626565776172652e6f72672f636f6d6d756e6974792f6d656d626572732f616e61636f6e64612f616e61636f6e64612d6c617267652e706e67" alt="anaconda logo" width="40%"/></a>

{fsspec-2025.5.1 → fsspec-2025.9.0}/README.md

@@ -96,3 +96,11 @@ filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
 before you commit, reformatting any changed files. You can format without
 committing via ``pre-commit run`` or skip these checks with ``git commit
 --no-verify``.
+
+## Support
+
+Work on this repository is supported in part by:
+
+"Anaconda, Inc. - Advancing AI through open source."
+
+<a href="https://anaconda.com/"><img src="https://camo.githubusercontent.com/b8555ef2222598ed37ce38ac86955febbd25de7619931bb7dd3c58432181d3b6/68747470733a2f2f626565776172652e6f72672f636f6d6d756e6974792f6d656d626572732f616e61636f6e64612f616e61636f6e64612d6c617267652e706e67" alt="anaconda logo" width="40%"/></a>

{fsspec-2025.5.1 → fsspec-2025.9.0}/docs/source/api.rst

@@ -235,6 +235,7 @@ documentation carefully before using any particular package.
 - `irods`_ for access to iRODS servers, with protocol "irods://"
 - `lakefs`_ for lakeFS data lakes, with protocol "lakefs://"
 - `morefs`_ for `OverlayFileSystem`, `DictFileSystem`, and others
+- `obstore`_: zero-dependency access to Amazon S3, Google Cloud Storage, and Azure Blob Storage using the underlying Rust `object_store`_ library, with protocols "s3://", "gs://", and "abfs://".
 - `ocifs`_ for access to Oracle Cloud Object Storage, with protocol "oci://"
 - `ocilake`_ for OCI Data Lake storage
 - `ossfs`_ for Alibaba Cloud (Aliyun) Object Storage System (OSS)
@@ -250,6 +251,7 @@ documentation carefully before using any particular package.
 - `wandbfsspec`_ to access Weights & Biases (experimental)
 - `webdav4`_ for WebDAV, with protocol "webdav://" or "dav://"
 - `xrootd`_ for xrootd, with protocol "root://"
+- `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
 
 .. _abfs: https://github.com/dask/adlfs
 .. _adl: https://github.com/dask/adlfs
@@ -270,6 +272,8 @@ documentation carefully before using any particular package.
 .. _irods: https://github.com/xwcl/irods_fsspec
 .. _lakefs: https://github.com/aai-institute/lakefs-spec
 .. _morefs: https://github.com/iterative/morefs
+.. _object_store: https://docs.rs/object_store/latest/object_store/
+.. _obstore: https://developmentseed.org/obstore/latest/
 .. _ocifs: https://ocifs.readthedocs.io/en/latest/
 .. _ocilake: https://github.com/oracle/ocifs
 .. _ossfs: https://github.com/fsspec/ossfs
@@ -284,6 +288,7 @@ documentation carefully before using any particular package.
 .. _wandbfsspec: https://github.com/alvarobartt/wandbfsspec
 .. _webdav4: https://github.com/skshetry/webdav4
 .. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
+.. _msgraphfs: https://github.com/acsone/msgraphfs
 
 .. _readbuffering:
 

{fsspec-2025.5.1 → fsspec-2025.9.0}/docs/source/changelog.rst

@@ -1,6 +1,53 @@
 Changelog
 =========
 
+2025.9.0
+--------
+
+Enhancements
+
+- include Last_modified info from HTTP headers to info (#1909)
+- add optional semaphore to async-wrapper (#1908)
+
+Fixes
+
+- ensure cachingFSs show correct protocol (#1897)
+- fix simplecache cat_ranges (#1892)
+
+Other
+
+- Style (#1894)
+
+
+2025.7.0
+--------
+
+Enhancements
+
+- only download HTML mime for http listing (#1889)
+- add tos:// to registry (#1878)
+
+Fixes
+
+- use st_birthtime in localFS, if available (#1883)
+- allow cat_* in simplecache (#1881)
+- remove deprecated asyncio use (#1862)
+- create event loop if it doesn't exist (#1857)
+
+Other
+
+- remove references to py38 (#1888)
+- ruff updates (#1887, 1864)
+- github rate limits in CI (#1879, 1877)
+- acknowledge Anaconda support (#1876)
+- add obstore to known implementations (#1875)
+- add Microsoft storage to known implementations (#1853)
+- use builtins zstd for py3.14 (#1874)
+- gdrivefs -> gdrive_fsspec (#1858)
+- windows version in CI (#1855)
+- error message typo (#1854)
+
+
 2025.5.1
 --------
 

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '2025.5.1'
-__version_tuple__ = version_tuple = (2025, 5, 1)
+__version__ = version = '2025.9.0'
+__version_tuple__ = version_tuple = (2025, 9, 0)
+
+__commit_id__ = commit_id = None

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/asyn.py

@@ -7,9 +7,9 @@ import numbers
 import os
 import re
 import threading
-from contextlib import contextmanager
+from collections.abc import Iterable
 from glob import has_magic
-from typing import TYPE_CHECKING, Iterable
+from typing import TYPE_CHECKING
 
 from .callbacks import DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
@@ -120,18 +120,6 @@ def sync_wrapper(func, obj=None):
     return wrapper
 
 
-@contextmanager
-def _selector_policy():
-    original_policy = asyncio.get_event_loop_policy()
-    try:
-        if os.name == "nt" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
-            asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-
-        yield
-    finally:
-        asyncio.set_event_loop_policy(original_policy)
-
-
 def get_loop():
     """Create or return the default fsspec IO loop
 
@@ -142,8 +130,7 @@ def get_loop():
             # repeat the check just in case the loop got filled between the
             # previous two calls from another thread
             if loop[0] is None:
-                with _selector_policy():
-                    loop[0] = asyncio.new_event_loop()
+                loop[0] = asyncio.new_event_loop()
                 th = threading.Thread(target=loop[0].run_forever, name="fsspecIO")
                 th.daemon = True
                 th.start()

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/caching.py

@@ -7,6 +7,7 @@ import math
 import os
 import threading
 import warnings
+from collections import OrderedDict
 from concurrent.futures import Future, ThreadPoolExecutor
 from itertools import groupby
 from operator import itemgetter
@@ -17,8 +18,6 @@ from typing import (
     ClassVar,
     Generic,
     NamedTuple,
-    Optional,
-    OrderedDict,
     TypeVar,
 )
 
@@ -629,7 +628,7 @@ class KnownPartsOfAFile(BaseCache):
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: Optional[dict[tuple[int, int], bytes]] = None,
+        data: dict[tuple[int, int], bytes] | None = None,
         strict: bool = True,
         **_: Any,
     ):

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/compression.py

@@ -155,19 +155,26 @@ except ImportError:
     pass
 
 try:
-    import zstandard as zstd
+    # zstd in the standard library for python >= 3.14
+    from compression.zstd import ZstdFile
 
-    def zstandard_file(infile, mode="rb"):
-        if "r" in mode:
-            cctx = zstd.ZstdDecompressor()
-            return cctx.stream_reader(infile)
-        else:
-            cctx = zstd.ZstdCompressor(level=10)
-            return cctx.stream_writer(infile)
+    register_compression("zstd", ZstdFile, "zst")
 
-    register_compression("zstd", zstandard_file, "zst")
 except ImportError:
-    pass
+    try:
+        import zstandard as zstd
+
+        def zstandard_file(infile, mode="rb"):
+            if "r" in mode:
+                cctx = zstd.ZstdDecompressor()
+                return cctx.stream_reader(infile)
+            else:
+                cctx = zstd.ZstdCompressor(level=10)
+                return cctx.stream_writer(infile)
+
+        register_compression("zstd", zstandard_file, "zst")
+    except ImportError:
+        pass
 
 
 def available_compressions():

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/generic.py

@@ -5,7 +5,6 @@ import logging
 import os
 import shutil
 import uuid
-from typing import Optional
 
 from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper
 from .callbacks import DEFAULT_CALLBACK
@@ -289,7 +288,7 @@ class GenericFileSystem(AsyncFileSystem):
         url2,
         blocksize=2**20,
         callback=DEFAULT_CALLBACK,
-        tempdir: Optional[str] = None,
+        tempdir: str | None = None,
         **kwargs,
     ):
         fs = _resolve_fs(url, self.method)
@@ -319,9 +318,9 @@ class GenericFileSystem(AsyncFileSystem):
         path2: list[str],
         recursive: bool = False,
         on_error: str = "ignore",
-        maxdepth: Optional[int] = None,
-        batch_size: Optional[int] = None,
-        tempdir: Optional[str] = None,
+        maxdepth: int | None = None,
+        batch_size: int | None = None,
+        tempdir: str | None = None,
         **kwargs,
     ):
         # TODO: special case for one FS being local, which can use get/put

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/gui.py

@@ -3,7 +3,8 @@ import contextlib
 import logging
 import os
 import re
-from typing import ClassVar, Sequence
+from collections.abc import Sequence
+from typing import ClassVar
 
 import panel as pn
 

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/asyn_wrapper.py

@@ -6,7 +6,7 @@ import fsspec
 from fsspec.asyn import AsyncFileSystem, running_async
 
 
-def async_wrapper(func, obj=None):
+def async_wrapper(func, obj=None, semaphore=None):
     """
     Wraps a synchronous function to make it awaitable.
 
@@ -16,6 +16,8 @@ def async_wrapper(func, obj=None):
         The synchronous function to wrap.
     obj : object, optional
         The instance to bind the function to, if applicable.
+    semaphore : asyncio.Semaphore, optional
+        A semaphore to limit concurrent calls.
 
     Returns
     -------
@@ -25,6 +27,9 @@ def async_wrapper(func, obj=None):
 
     @functools.wraps(func)
     async def wrapper(*args, **kwargs):
+        if semaphore:
+            async with semaphore:
+                return await asyncio.to_thread(func, *args, **kwargs)
         return await asyncio.to_thread(func, *args, **kwargs)
 
     return wrapper
@@ -52,6 +57,8 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         asynchronous=None,
         target_protocol=None,
         target_options=None,
+        semaphore=None,
+        max_concurrent_tasks=None,
         **kwargs,
     ):
         if asynchronous is None:
@@ -62,6 +69,7 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         else:
             self.sync_fs = fsspec.filesystem(target_protocol, **target_options)
         self.protocol = self.sync_fs.protocol
+        self.semaphore = semaphore
         self._wrap_all_sync_methods()
 
     @property
@@ -82,8 +90,8 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
                 continue
 
             method = getattr(self.sync_fs, method_name)
-            if callable(method) and not asyncio.iscoroutinefunction(method):
-                async_method = async_wrapper(method, obj=self)
+            if callable(method) and not inspect.iscoroutinefunction(method):
+                async_method = async_wrapper(method, obj=self, semaphore=self.semaphore)
                 setattr(self, f"_{method_name}", async_method)
 
     @classmethod

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/cache_metadata.py

@@ -14,13 +14,14 @@ except ImportError:
         import json
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Iterator, Literal
+    from collections.abc import Iterator
+    from typing import Any, Literal
 
     from typing_extensions import TypeAlias
 
     from .cached import CachingFileSystem
 
-    Detail: TypeAlias = Dict[str, Any]
+    Detail: TypeAlias = dict[str, Any]
 
 
 class CacheMetadata:

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/cached.py

@@ -16,6 +16,7 @@ from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
 from fsspec.implementations.cache_mapper import create_cache_mapper
 from fsspec.implementations.cache_metadata import CacheMetadata
+from fsspec.implementations.local import LocalFileSystem
 from fsspec.spec import AbstractBufferedFile
 from fsspec.transaction import Transaction
 from fsspec.utils import infer_compression
@@ -338,7 +339,7 @@ class CachingFileSystem(AbstractFileSystem):
         # explicitly submitting the size to the open call will avoid extra
         # operations when opening. This is particularly relevant
         # for any file that is read over a network, e.g. S3.
-        size = detail.get("size", None)
+        size = detail.get("size")
 
         # call target filesystems open
         self._mkcache()
@@ -433,7 +434,9 @@ class CachingFileSystem(AbstractFileSystem):
             "open",
             "cat",
             "cat_file",
+            "_cat_file",
             "cat_ranges",
+            "_cat_ranges",
             "get",
             "read_block",
             "tail",
@@ -475,7 +478,7 @@ class CachingFileSystem(AbstractFileSystem):
         if item in ["transaction"]:
             # property
             return type(self).transaction.__get__(self)
-        if item in ["_cache", "transaction_type"]:
+        if item in {"_cache", "transaction_type", "protocol"}:
             # class attributes
             return getattr(type(self), item)
         if item == "__class__":
@@ -835,14 +838,56 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         else:
             raise ValueError("path must be str or dict")
 
+    async def _cat_file(self, path, start=None, end=None, **kwargs):
+        logger.debug("async cat_file %s", path)
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+        fn = self._check_file(path)
+
+        if not fn:
+            fn = os.path.join(self.storage[-1], sha)
+            await self.fs._get_file(path, fn, **kwargs)
+
+        with open(fn, "rb") as f:  # noqa ASYNC230
+            if start:
+                f.seek(start)
+            size = -1 if end is None else end - f.tell()
+            return f.read(size)
+
+    async def _cat_ranges(
+        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
+    ):
+        logger.debug("async cat ranges %s", paths)
+        lpaths = []
+        rset = set()
+        download = []
+        rpaths = []
+        for p in paths:
+            fn = self._check_file(p)
+            if fn is None and p not in rset:
+                sha = self._mapper(p)
+                fn = os.path.join(self.storage[-1], sha)
+                download.append(fn)
+                rset.add(p)
+                rpaths.append(p)
+            lpaths.append(fn)
+        if download:
+            await self.fs._get(rpaths, download, on_error=on_error)
+
+        return LocalFileSystem().cat_ranges(
+            lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
+        )
+
     def cat_ranges(
         self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
     ):
+        logger.debug("cat ranges %s", paths)
         lpaths = [self._check_file(p) for p in paths]
         rpaths = [p for l, p in zip(lpaths, paths) if l is False]
         lpaths = [l for l, p in zip(lpaths, paths) if l is False]
         self.fs.get(rpaths, lpaths)
-        return super().cat_ranges(
+        paths = [self._check_file(p) for p in paths]
+        return LocalFileSystem().cat_ranges(
             paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
         )
 
@@ -940,7 +985,7 @@ class LocalTempFile:
 
     def commit(self):
         self.fs.put(self.fn, self.path, **self.kwargs)
-        # we do not delete local copy - it's still in the cache
+        # we do not delete the local copy, it's still in the cache.
 
     @property
     def name(self):

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/dbfs.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
+
 import base64
 import urllib
 
 import requests
-import requests.exceptions
 from requests.adapters import HTTPAdapter, Retry
+from typing_extensions import override
 
 from fsspec import AbstractFileSystem
 from fsspec.spec import AbstractBufferedFile
@@ -57,6 +59,24 @@ class DatabricksFileSystem(AbstractFileSystem):
 
         super().__init__(**kwargs)
 
+    @override
+    def _ls_from_cache(self, path) -> list[dict[str, str | int]] | None:
+        """Check cache for listing
+
+        Returns listing, if found (may be empty list for a directory that
+        exists but contains nothing), None if not in cache.
+        """
+        self.dircache.pop(path.rstrip("/"), None)
+
+        parent = self._parent(path)
+        if parent in self.dircache:
+            for entry in self.dircache[parent]:
+                if entry["name"] == path.rstrip("/"):
+                    if entry["type"] != "directory":
+                        return [entry]
+                    return []
+            raise FileNotFoundError(path)
+
     def ls(self, path, detail=True, **kwargs):
         """
         List the contents of the given path.
@@ -70,7 +90,15 @@ class DatabricksFileSystem(AbstractFileSystem):
             but also additional information on file sizes
             and types.
         """
-        out = self._ls_from_cache(path)
+        try:
+            out = self._ls_from_cache(path)
+        except FileNotFoundError:
+            # This happens if the `path`'s parent was cached, but `path` is not
+            # there. This suggests that `path` is new since the parent was
+            # cached. Attempt to invalidate parent's cache before continuing.
+            self.dircache.pop(self._parent(path), None)
+            out = None
+
         if not out:
             try:
                 r = self._send_to_api(
@@ -460,7 +488,7 @@ class DatabricksFile(AbstractBufferedFile):
         return return_buffer
 
     def _to_sized_blocks(self, length, start=0):
-        """Helper function to split a range from 0 to total_length into bloksizes"""
+        """Helper function to split a range from 0 to total_length into blocksizes"""
         end = start + length
         for data_chunk in range(start, end, self.blocksize):
             data_start = data_chunk

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/ftp.py

@@ -1,7 +1,5 @@
 import os
-import sys
 import uuid
-import warnings
 from ftplib import FTP, FTP_TLS, Error, error_perm
 from typing import Any
 
@@ -81,13 +79,7 @@ class FTPFileSystem(AbstractFileSystem):
             ftp_cls = FTP_TLS
         else:
             ftp_cls = FTP
-        if sys.version_info >= (3, 9):
-            self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
-        elif self.encoding:
-            warnings.warn("`encoding` not supported for python<3.9, ignoring")
-            self.ftp = ftp_cls(timeout=self.timeout)
-        else:
-            self.ftp = ftp_cls(timeout=self.timeout)
+        self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
         self.ftp.connect(self.host, self.port)
         self.ftp.login(*self.cred)
 

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/git.py

@@ -62,8 +62,7 @@ class GitFileSystem(AbstractFileSystem):
 
     @staticmethod
     def _get_kwargs_from_urls(path):
-        if path.startswith("git://"):
-            path = path[6:]
+        path = path.removeprefix("git://")
         out = {}
         if ":" in path:
             out["path"], path = path.split(":", 1)

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/http.py

@@ -80,12 +80,12 @@ class HTTPFileSystem(AsyncFileSystem):
             https://docs.aiohttp.org/en/stable/client_reference.html
             For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}``
         get_client: Callable[..., aiohttp.ClientSession]
-            A callable which takes keyword arguments and constructs
-            an aiohttp.ClientSession. It's state will be managed by
+            A callable, which takes keyword arguments and constructs
+            an aiohttp.ClientSession. Its state will be managed by
             the HTTPFileSystem class.
         storage_options: key-value
             Any other parameters passed on to requests
-        cache_type, cache_options: defaults used in open
+        cache_type, cache_options: defaults used in open()
         """
         super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options)
         self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE
@@ -158,14 +158,24 @@ class HTTPFileSystem(AsyncFileSystem):
         session = await self.set_session()
         async with session.get(self.encode_url(url), **self.kwargs) as r:
             self._raise_not_found_for_status(r, url)
-            try:
-                text = await r.text()
-                if self.simple_links:
-                    links = ex2.findall(text) + [u[2] for u in ex.findall(text)]
-                else:
-                    links = [u[2] for u in ex.findall(text)]
-            except UnicodeDecodeError:
-                links = []  # binary, not HTML
+
+            if "Content-Type" in r.headers:
+                mimetype = r.headers["Content-Type"].partition(";")[0]
+            else:
+                mimetype = None
+
+            if mimetype in ("text/html", None):
+                try:
+                    text = await r.text(errors="ignore")
+                    if self.simple_links:
+                        links = ex2.findall(text) + [u[2] for u in ex.findall(text)]
+                    else:
+                        links = [u[2] for u in ex.findall(text)]
+                except UnicodeDecodeError:
+                    links = []  # binary, not HTML
+            else:
+                links = []
+
         out = set()
         parts = urlparse(url)
         for l in links:
@@ -254,7 +264,7 @@ class HTTPFileSystem(AsyncFileSystem):
             if isfilelike(lpath):
                 outfile = lpath
             else:
-                outfile = open(lpath, "wb")  # noqa: ASYNC101, ASYNC230
+                outfile = open(lpath, "wb")  # noqa: ASYNC230
 
             try:
                 chunk = True
@@ -286,7 +296,7 @@ class HTTPFileSystem(AsyncFileSystem):
                 context = nullcontext(lpath)
                 use_seek = False  # might not support seeking
             else:
-                context = open(lpath, "rb")  # noqa: ASYNC101, ASYNC230
+                context = open(lpath, "rb")  # noqa: ASYNC230
                 use_seek = True
 
             with context as f:
@@ -812,7 +822,7 @@ async def get_range(session, url, start, end, file=None, **kwargs):
     async with r:
         out = await r.read()
     if file:
-        with open(file, "r+b") as f:  # noqa: ASYNC101, ASYNC230
+        with open(file, "r+b") as f:  # noqa: ASYNC230
             f.seek(start)
             f.write(out)
     else:
@@ -863,7 +873,7 @@ async def _file_info(url, session, size_policy="head", **kwargs):
 
         info["url"] = str(r.url)
 
-        for checksum_field in ["ETag", "Content-MD5", "Digest"]:
+        for checksum_field in ["ETag", "Content-MD5", "Digest", "Last-Modified"]:
             if r.headers.get(checksum_field):
                 info[checksum_field] = r.headers[checksum_field]
 

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/local.py

@@ -109,11 +109,15 @@ class LocalFileSystem(AbstractFileSystem):
                 t = "file"
             else:
                 t = "other"
+
+        # Check for the 'st_birthtime' attribute, which is not always present; fallback to st_ctime
+        created_time = getattr(out, "st_birthtime", out.st_ctime)
+
         result = {
             "name": path,
             "size": size,
             "type": t,
-            "created": out.st_ctime,
+            "created": created_time,
             "islink": link,
         }
         for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]:

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/memory.py

@@ -34,8 +34,7 @@ class MemoryFileSystem(AbstractFileSystem):
             else:
                 path = stringify_path(path)
 
-        if path.startswith("memory://"):
-            path = path[len("memory://") :]
+        path = path.removeprefix("memory://")
         if "::" in path or "://" in path:
             return path.rstrip("/")
         path = path.lstrip("/").rstrip("/")

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/implementations/reference.py

@@ -768,7 +768,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                 self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous)
             elif self.asynchronous ^ f.asynchronous:
                 raise ValueError(
-                    "Reference-FS's target filesystem must have same value"
+                    "Reference-FS's target filesystem must have same value "
                     "of asynchronous"
                 )
 

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/json.py

@@ -1,16 +1,12 @@
 import json
+from collections.abc import Mapping, Sequence
 from contextlib import suppress
 from pathlib import PurePath
 from typing import (
     Any,
     Callable,
     ClassVar,
-    Dict,
-    List,
-    Mapping,
     Optional,
-    Sequence,
-    Tuple,
 )
 
 from .registry import _import_class, get_filesystem_class
@@ -49,12 +45,12 @@ class FilesystemJSONDecoder(json.JSONDecoder):
     def __init__(
         self,
         *,
-        object_hook: Optional[Callable[[Dict[str, Any]], Any]] = None,
+        object_hook: Optional[Callable[[dict[str, Any]], Any]] = None,
         parse_float: Optional[Callable[[str], Any]] = None,
         parse_int: Optional[Callable[[str], Any]] = None,
         parse_constant: Optional[Callable[[str], Any]] = None,
         strict: bool = True,
-        object_pairs_hook: Optional[Callable[[List[Tuple[str, Any]]], Any]] = None,
+        object_pairs_hook: Optional[Callable[[list[tuple[str, Any]]], Any]] = None,
     ) -> None:
         self.original_object_hook = object_hook
 
@@ -68,7 +64,7 @@ class FilesystemJSONDecoder(json.JSONDecoder):
         )
 
     @classmethod
-    def try_resolve_path_cls(cls, dct: Dict[str, Any]):
+    def try_resolve_path_cls(cls, dct: dict[str, Any]):
         with suppress(Exception):
             fqp = dct["cls"]
 
@@ -80,7 +76,7 @@ class FilesystemJSONDecoder(json.JSONDecoder):
         return None
 
     @classmethod
-    def try_resolve_fs_cls(cls, dct: Dict[str, Any]):
+    def try_resolve_fs_cls(cls, dct: dict[str, Any]):
         with suppress(Exception):
             if "cls" in dct:
                 try:
@@ -95,7 +91,7 @@ class FilesystemJSONDecoder(json.JSONDecoder):
 
         return None
 
-    def custom_object_hook(self, dct: Dict[str, Any]):
+    def custom_object_hook(self, dct: dict[str, Any]):
         if "cls" in dct:
             if (obj_cls := self.try_resolve_fs_cls(dct)) is not None:
                 return AbstractFileSystem.from_dict(dct)

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/registry.py

@@ -118,8 +118,8 @@ known_implementations = {
         "err": "Please install gcsfs to access Google Storage",
     },
     "gdrive": {
-        "class": "gdrivefs.GoogleDriveFileSystem",
-        "err": "Please install gdrivefs for access to Google Drive",
+        "class": "gdrive_fsspec.GoogleDriveFileSystem",
+        "err": "Please install gdrive_fs for access to Google Drive",
     },
     "generic": {"class": "fsspec.generic.GenericFileSystem"},
     "gist": {
@@ -213,6 +213,10 @@ known_implementations = {
         "err": 'SFTPFileSystem requires "paramiko" to be installed',
     },
     "tar": {"class": "fsspec.implementations.tar.TarFileSystem"},
+    "tos": {
+        "class": "tosfs.TosFileSystem",
+        "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage",
+    },
     "tosfs": {
         "class": "tosfs.TosFileSystem",
         "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage",

{fsspec-2025.5.1 → fsspec-2025.9.0}/fsspec/utils.py

@@ -7,6 +7,7 @@ import os
 import re
 import sys
 import tempfile
+from collections.abc import Iterable, Iterator, Sequence
 from functools import partial
 from hashlib import md5
 from importlib.metadata import version
@@ -15,9 +16,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
-    Iterable,
-    Iterator,
-    Sequence,
     TypeVar,
 )
 from urllib.parse import urlsplit

{fsspec-2025.5.1 → fsspec-2025.9.0}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling", "hatch-vcs"]
+requires = ["hatchling>=1.27.0", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [project]
@@ -7,14 +7,14 @@ name = "fsspec"
 dynamic = ["version"]
 description = "File-system specification"
 readme = "README.md"
-license = { file = "LICENSE" }
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
 requires-python = ">=3.9"
 maintainers = [{ name = "Martin Durant", email = "mdurant@anaconda.com" }]
 keywords = ["file"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
-    "License :: OSI Approved :: BSD License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
@@ -28,7 +28,7 @@ abfs = ["adlfs"]
 adl = ["adlfs"]
 arrow = ["pyarrow >= 1"]
 dask = ["dask", "distributed"]
-dev = ["ruff", "pre-commit"]
+dev = ["ruff >= 0.5", "pre-commit"]
 doc = ["sphinx", "numpydoc", "sphinx-design", "sphinx-rtd-theme", "yarl"]
 dropbox = ["dropbox", "dropboxdrivefs", "requests"]
 entrypoints = []
@@ -117,7 +117,7 @@ test_full = [
     'tqdm',
     'urllib3',
     'zarr',
-    'zstandard',
+    'zstandard; python_version < "3.14"',
 ]
 test_downstream = [
     "dask[dataframe,test]",
@@ -144,7 +144,6 @@ version-file = "fsspec/_version.py"
 exclude = ["**/tests/*", "!**/tests/abstract/"]
 
 [tool.ruff]
-target-version = "py38"
 exclude = [".tox", "build", "docs/source/conf.py", "fsspec/_version"]
 line-length = 88
 
@@ -209,6 +208,7 @@ ignore = [
     # Fix these codes later
     "G004",
     "PERF203",
+    "PLC0415",
     "UP007",
     "UP011",
     "UP015",
@@ -219,6 +219,7 @@ ignore = [
     "SIM114",
     "SIM115",
     "SIM117",
+    "TC003",
     # https://github.com/astral-sh/ruff/issues/7871
     "UP038",
     # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules

fsspec-2025.5.1/ci/environment-typecheck.yml

@@ -1,15 +0,0 @@
-name: test_env
-channels:
-  - conda-forge
-dependencies:
-  - mypy=1.4.1
-  - pyarrow
-  - python=3.8
-  - pip
-  - pip:
-    - types-paramiko
-    - types-requests
-    - types-tqdm
-    - types-paramiko
-    - types-PyYAML
-    - types-ujson