fsspec 2025.5.0__tar.gz → 2025.7.0__tar.gz

{fsspec-2025.5.0 → fsspec-2025.7.0}/.github/workflows/main.yaml

@@ -32,7 +32,7 @@ jobs:
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v3
         with:
-          environment-file: ci/environment-py38.yml
+          environment-file: ci/environment-linux.yml
           python-version: ${{ matrix.PY }}
 
       - name: Run Tests
@@ -43,7 +43,7 @@ jobs:
 
   win:
     name: pytest-win
-    runs-on: windows-2019
+    runs-on: windows-2022
 
     env:
       CIRUN: true
@@ -75,22 +75,6 @@ jobs:
           python-version: "3.11"
       - uses: pre-commit/action@main
 
-#  typecheck:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v4
-#
-#      - name: Setup conda
-#        uses: conda-incubator/setup-miniconda@v3
-#        with:
-#          environment-file: ci/environment-typecheck.yml
-#
-#      - name: mypy
-#        shell: bash -l {0}
-#        run: |
-#          mypy fsspec
-#
   downstream:
     name: downstream
     runs-on: ubuntu-24.04

{fsspec-2025.5.0 → fsspec-2025.7.0}/.pre-commit-config.yaml

@@ -13,11 +13,10 @@ repos:
       - id: check-json
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.9.2
+    rev: v0.12.2
     hooks:
       # Run the linter.
-      - id: ruff
+      - id: ruff-check
         args: [ --fix, "--show-fixes"]
       - id: ruff-format
         types_or: [python]

{fsspec-2025.5.0 → fsspec-2025.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.5.0
+Version: 2025.7.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -58,7 +58,7 @@ Requires-Dist: dask; extra == 'dask'
 Requires-Dist: distributed; extra == 'dask'
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
-Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
 Provides-Extra: doc
 Requires-Dist: numpydoc; extra == 'doc'
 Requires-Dist: sphinx; extra == 'doc'
@@ -172,7 +172,7 @@ Requires-Dist: smbprotocol; extra == 'test-full'
 Requires-Dist: tqdm; extra == 'test-full'
 Requires-Dist: urllib3; extra == 'test-full'
 Requires-Dist: zarr; extra == 'test-full'
-Requires-Dist: zstandard; extra == 'test-full'
+Requires-Dist: zstandard; (python_version < '3.14') and extra == 'test-full'
 Provides-Extra: tqdm
 Requires-Dist: tqdm; extra == 'tqdm'
 Description-Content-Type: text/markdown
@@ -275,3 +275,11 @@ filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
 before you commit, reformatting any changed files. You can format without
 committing via ``pre-commit run`` or skip these checks with ``git commit
 --no-verify``.
+
+## Support
+
+Work on this repository is supported in part by:
+
+"Anaconda, Inc. - Advancing AI through open source."
+
+<a href="https://anaconda.com/"><img src="https://camo.githubusercontent.com/b8555ef2222598ed37ce38ac86955febbd25de7619931bb7dd3c58432181d3b6/68747470733a2f2f626565776172652e6f72672f636f6d6d756e6974792f6d656d626572732f616e61636f6e64612f616e61636f6e64612d6c617267652e706e67" alt="anaconda logo" width="40%"/></a>

{fsspec-2025.5.0 → fsspec-2025.7.0}/README.md

@@ -96,3 +96,11 @@ filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
 before you commit, reformatting any changed files. You can format without
 committing via ``pre-commit run`` or skip these checks with ``git commit
 --no-verify``.
+
+## Support
+
+Work on this repository is supported in part by:
+
+"Anaconda, Inc. - Advancing AI through open source."
+
+<a href="https://anaconda.com/"><img src="https://camo.githubusercontent.com/b8555ef2222598ed37ce38ac86955febbd25de7619931bb7dd3c58432181d3b6/68747470733a2f2f626565776172652e6f72672f636f6d6d756e6974792f6d656d626572732f616e61636f6e64612f616e61636f6e64612d6c617267652e706e67" alt="anaconda logo" width="40%"/></a>

{fsspec-2025.5.0 → fsspec-2025.7.0}/docs/source/api.rst

@@ -117,6 +117,7 @@ Built-in Implementations
    fsspec.implementations.dbfs.DatabricksFileSystem
    fsspec.implementations.dirfs.DirFileSystem
    fsspec.implementations.ftp.FTPFileSystem
+   fsspec.implementations.gist.GistFileSystem
    fsspec.implementations.git.GitFileSystem
    fsspec.implementations.github.GithubFileSystem
    fsspec.implementations.http.HTTPFileSystem
@@ -162,6 +163,9 @@ Built-in Implementations
 .. autoclass:: fsspec.implementations.ftp.FTPFileSystem
    :members: __init__
 
+.. autoclass:: fsspec.implementations.gist.GistFileSystem
+   :members: __init__
+
 .. autoclass:: fsspec.implementations.git.GitFileSystem
    :members: __init__
 
@@ -231,6 +235,7 @@ documentation carefully before using any particular package.
 - `irods`_ for access to iRODS servers, with protocol "irods://"
 - `lakefs`_ for lakeFS data lakes, with protocol "lakefs://"
 - `morefs`_ for `OverlayFileSystem`, `DictFileSystem`, and others
+- `obstore`_: zero-dependency access to Amazon S3, Google Cloud Storage, and Azure Blob Storage using the underlying Rust `object_store`_ library, with protocols "s3://", "gs://", and "abfs://".
 - `ocifs`_ for access to Oracle Cloud Object Storage, with protocol "oci://"
 - `ocilake`_ for OCI Data Lake storage
 - `ossfs`_ for Alibaba Cloud (Aliyun) Object Storage System (OSS)
@@ -246,6 +251,7 @@ documentation carefully before using any particular package.
 - `wandbfsspec`_ to access Weights & Biases (experimental)
 - `webdav4`_ for WebDAV, with protocol "webdav://" or "dav://"
 - `xrootd`_ for xrootd, with protocol "root://"
+- `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
 
 .. _abfs: https://github.com/dask/adlfs
 .. _adl: https://github.com/dask/adlfs
@@ -266,6 +272,8 @@ documentation carefully before using any particular package.
 .. _irods: https://github.com/xwcl/irods_fsspec
 .. _lakefs: https://github.com/aai-institute/lakefs-spec
 .. _morefs: https://github.com/iterative/morefs
+.. _object_store: https://docs.rs/object_store/latest/object_store/
+.. _obstore: https://developmentseed.org/obstore/latest/
 .. _ocifs: https://ocifs.readthedocs.io/en/latest/
 .. _ocilake: https://github.com/oracle/ocifs
 .. _ossfs: https://github.com/fsspec/ossfs
@@ -280,6 +288,7 @@ documentation carefully before using any particular package.
 .. _wandbfsspec: https://github.com/alvarobartt/wandbfsspec
 .. _webdav4: https://github.com/skshetry/webdav4
 .. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
+.. _msgraphfs: https://github.com/acsone/msgraphfs
 
 .. _readbuffering:
 

{fsspec-2025.5.0 → fsspec-2025.7.0}/docs/source/changelog.rst

@@ -1,6 +1,46 @@
 Changelog
 =========
 
+2025.7.0
+--------
+
+Enhancements
+
+- only download HTML mime for http listing (#1889)
+- add tos:// to registry (#1878)
+
+Fixes
+
+- use st_birthtime in localFS, if available (#1883)
+- allow cat_* in simplecache (#1881)
+- remove deprecated asyncio use (#1862)
+- create event loop if it doesn't exist (#1857)
+
+Other
+
+- remove references to py38 (#1888)
+- ruff updates (#1887, 1864)
+- github rate limits in CI (#1879, 1877)
+- acknowledge Anaconda support (#1876)
+- add obstore to known implementations (#1875)
+- add Microsoft storage to known implementations (#1853)
+- use builtins zstd for py3.14 (#1874)
+- gdrivefs -> gdrive_fsspec (#1858)
+- windows version in CI (#1855)
+- error message typo (#1854)
+
+
+2025.5.1
+--------
+
+Enhancements
+
+- file system for GitHub gists (#1791)
+
+Other
+
+- doc fixes (#1847, 1848)
+
 2025.5.0
 --------
 

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2025.5.0'
-__version_tuple__ = version_tuple = (2025, 5, 0)
+__version__ = version = '2025.7.0'
+__version_tuple__ = version_tuple = (2025, 7, 0)

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/asyn.py

@@ -7,9 +7,9 @@ import numbers
 import os
 import re
 import threading
-from contextlib import contextmanager
+from collections.abc import Iterable
 from glob import has_magic
-from typing import TYPE_CHECKING, Iterable
+from typing import TYPE_CHECKING
 
 from .callbacks import DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
@@ -120,18 +120,6 @@ def sync_wrapper(func, obj=None):
     return wrapper
 
 
-@contextmanager
-def _selector_policy():
-    original_policy = asyncio.get_event_loop_policy()
-    try:
-        if os.name == "nt" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
-            asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-
-        yield
-    finally:
-        asyncio.set_event_loop_policy(original_policy)
-
-
 def get_loop():
     """Create or return the default fsspec IO loop
 
@@ -142,8 +130,7 @@ def get_loop():
             # repeat the check just in case the loop got filled between the
             # previous two calls from another thread
             if loop[0] is None:
-                with _selector_policy():
-                    loop[0] = asyncio.new_event_loop()
+                loop[0] = asyncio.new_event_loop()
                 th = threading.Thread(target=loop[0].run_forever, name="fsspecIO")
                 th.daemon = True
                 th.start()

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/caching.py

@@ -7,6 +7,7 @@ import math
 import os
 import threading
 import warnings
+from collections import OrderedDict
 from concurrent.futures import Future, ThreadPoolExecutor
 from itertools import groupby
 from operator import itemgetter
@@ -17,8 +18,6 @@ from typing import (
     ClassVar,
     Generic,
     NamedTuple,
-    Optional,
-    OrderedDict,
     TypeVar,
 )
 
@@ -629,7 +628,7 @@ class KnownPartsOfAFile(BaseCache):
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: Optional[dict[tuple[int, int], bytes]] = None,
+        data: dict[tuple[int, int], bytes] | None = None,
         strict: bool = True,
         **_: Any,
     ):

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/compression.py

@@ -155,19 +155,26 @@ except ImportError:
     pass
 
 try:
-    import zstandard as zstd
+    # zstd in the standard library for python >= 3.14
+    from compression.zstd import ZstdFile
 
-    def zstandard_file(infile, mode="rb"):
-        if "r" in mode:
-            cctx = zstd.ZstdDecompressor()
-            return cctx.stream_reader(infile)
-        else:
-            cctx = zstd.ZstdCompressor(level=10)
-            return cctx.stream_writer(infile)
+    register_compression("zstd", ZstdFile, "zst")
 
-    register_compression("zstd", zstandard_file, "zst")
 except ImportError:
-    pass
+    try:
+        import zstandard as zstd
+
+        def zstandard_file(infile, mode="rb"):
+            if "r" in mode:
+                cctx = zstd.ZstdDecompressor()
+                return cctx.stream_reader(infile)
+            else:
+                cctx = zstd.ZstdCompressor(level=10)
+                return cctx.stream_writer(infile)
+
+        register_compression("zstd", zstandard_file, "zst")
+    except ImportError:
+        pass
 
 
 def available_compressions():

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/generic.py

@@ -5,7 +5,6 @@ import logging
 import os
 import shutil
 import uuid
-from typing import Optional
 
 from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper
 from .callbacks import DEFAULT_CALLBACK
@@ -289,7 +288,7 @@ class GenericFileSystem(AsyncFileSystem):
         url2,
         blocksize=2**20,
         callback=DEFAULT_CALLBACK,
-        tempdir: Optional[str] = None,
+        tempdir: str | None = None,
         **kwargs,
     ):
         fs = _resolve_fs(url, self.method)
@@ -319,9 +318,9 @@ class GenericFileSystem(AsyncFileSystem):
         path2: list[str],
         recursive: bool = False,
         on_error: str = "ignore",
-        maxdepth: Optional[int] = None,
-        batch_size: Optional[int] = None,
-        tempdir: Optional[str] = None,
+        maxdepth: int | None = None,
+        batch_size: int | None = None,
+        tempdir: str | None = None,
         **kwargs,
     ):
         # TODO: special case for one FS being local, which can use get/put

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/gui.py

@@ -3,7 +3,8 @@ import contextlib
 import logging
 import os
 import re
-from typing import ClassVar, Sequence
+from collections.abc import Sequence
+from typing import ClassVar
 
 import panel as pn
 

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/implementations/asyn_wrapper.py

@@ -82,7 +82,7 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
                 continue
 
             method = getattr(self.sync_fs, method_name)
-            if callable(method) and not asyncio.iscoroutinefunction(method):
+            if callable(method) and not inspect.iscoroutinefunction(method):
                 async_method = async_wrapper(method, obj=self)
                 setattr(self, f"_{method_name}", async_method)
 

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/implementations/cache_metadata.py

@@ -14,13 +14,14 @@ except ImportError:
         import json
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Iterator, Literal
+    from collections.abc import Iterator
+    from typing import Any, Literal
 
     from typing_extensions import TypeAlias
 
     from .cached import CachingFileSystem
 
-    Detail: TypeAlias = Dict[str, Any]
+    Detail: TypeAlias = dict[str, Any]
 
 
 class CacheMetadata:

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/implementations/cached.py

@@ -16,6 +16,7 @@ from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
 from fsspec.implementations.cache_mapper import create_cache_mapper
 from fsspec.implementations.cache_metadata import CacheMetadata
+from fsspec.implementations.local import LocalFileSystem
 from fsspec.spec import AbstractBufferedFile
 from fsspec.transaction import Transaction
 from fsspec.utils import infer_compression
@@ -338,7 +339,7 @@ class CachingFileSystem(AbstractFileSystem):
         # explicitly submitting the size to the open call will avoid extra
         # operations when opening. This is particularly relevant
         # for any file that is read over a network, e.g. S3.
-        size = detail.get("size", None)
+        size = detail.get("size")
 
         # call target filesystems open
         self._mkcache()
@@ -433,7 +434,9 @@ class CachingFileSystem(AbstractFileSystem):
             "open",
             "cat",
             "cat_file",
+            "_cat_file",
             "cat_ranges",
+            "_cat_ranges",
             "get",
             "read_block",
             "tail",
@@ -835,14 +838,55 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         else:
             raise ValueError("path must be str or dict")
 
+    async def _cat_file(self, path, start=None, end=None, **kwargs):
+        logger.debug("async cat_file %s", path)
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+        fn = self._check_file(path)
+
+        if not fn:
+            fn = os.path.join(self.storage[-1], sha)
+            await self.fs._get_file(path, fn, **kwargs)
+
+        with open(fn, "rb") as f:  # noqa ASYNC230
+            if start:
+                f.seek(start)
+            size = -1 if end is None else end - f.tell()
+            return f.read(size)
+
+    async def _cat_ranges(
+        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
+    ):
+        logger.debug("async cat ranges %s", paths)
+        lpaths = []
+        rset = set()
+        download = []
+        rpaths = []
+        for p in paths:
+            fn = self._check_file(p)
+            if fn is None and p not in rset:
+                sha = self._mapper(p)
+                fn = os.path.join(self.storage[-1], sha)
+                download.append(fn)
+                rset.add(p)
+                rpaths.append(p)
+            lpaths.append(fn)
+        if download:
+            await self.fs._get(rpaths, download, on_error=on_error)
+
+        return LocalFileSystem().cat_ranges(
+            lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
+        )
+
     def cat_ranges(
         self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
     ):
+        logger.debug("cat ranges %s", paths)
         lpaths = [self._check_file(p) for p in paths]
         rpaths = [p for l, p in zip(lpaths, paths) if l is False]
         lpaths = [l for l, p in zip(lpaths, paths) if l is False]
         self.fs.get(rpaths, lpaths)
-        return super().cat_ranges(
+        return LocalFileSystem().cat_ranges(
             paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
         )
 
@@ -940,7 +984,7 @@ class LocalTempFile:
 
     def commit(self):
         self.fs.put(self.fn, self.path, **self.kwargs)
-        # we do not delete local copy - it's still in the cache
+        # we do not delete the local copy, it's still in the cache.
 
     @property
     def name(self):

{fsspec-2025.5.0 → fsspec-2025.7.0}/fsspec/implementations/ftp.py

@@ -1,7 +1,5 @@
 import os
-import sys
 import uuid
-import warnings
 from ftplib import FTP, FTP_TLS, Error, error_perm
 from typing import Any
 
@@ -81,13 +79,7 @@ class FTPFileSystem(AbstractFileSystem):
             ftp_cls = FTP_TLS
         else:
             ftp_cls = FTP
-        if sys.version_info >= (3, 9):
-            self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
-        elif self.encoding:
-            warnings.warn("`encoding` not supported for python<3.9, ignoring")
-            self.ftp = ftp_cls(timeout=self.timeout)
-        else:
-            self.ftp = ftp_cls(timeout=self.timeout)
+        self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
         self.ftp.connect(self.host, self.port)
         self.ftp.login(*self.cred)