fsspec 2024.12.0__tar.gz → 2025.2.0__tar.gz

{fsspec-2024.12.0 → fsspec-2025.2.0}/.pre-commit-config.yaml

@@ -14,10 +14,14 @@ repos:
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.6.9
+    rev: v0.9.2
     hooks:
       # Run the linter.
       - id: ruff
         args: [ --fix, "--show-fixes"]
       - id: ruff-format
         types_or: [python]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.0
+    hooks:
+      - id: codespell

{fsspec-2024.12.0 → fsspec-2025.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2024.12.0
+Version: 2025.2.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -131,7 +131,6 @@ Requires-Dist: pytest-rerunfailures; extra == 'test'
 Requires-Dist: requests; extra == 'test'
 Provides-Extra: test-downstream
 Requires-Dist: aiobotocore<3.0.0,>=2.5.4; extra == 'test-downstream'
-Requires-Dist: dask-expr; extra == 'test-downstream'
 Requires-Dist: dask[dataframe,test]; extra == 'test-downstream'
 Requires-Dist: moto[server]<5,>4; extra == 'test-downstream'
 Requires-Dist: pytest-timeout; extra == 'test-downstream'

{fsspec-2024.12.0 → fsspec-2025.2.0}/ci/environment-downstream.yml

@@ -5,4 +5,3 @@ dependencies:
   - python=3.11
   - pip:
       - git+https://github.com/dask/dask
-      - git+https://github.com/dask/dask-expr

{fsspec-2024.12.0 → fsspec-2025.2.0}/docs/source/changelog.rst

@@ -1,6 +1,25 @@
 Changelog
 =========
 
+2025.2.0
+--------
+
+Enhancements
+
+- add open() to referenceFS (#1778)
+
+Fixes
+
+- don't make async open() in async-wrapper (#1769)
+- fix CI following dask-expr upstream change (#1781)
+- cope with zarr3 "Buffer" objects in referenceFS (#1784)
+
+Other
+
+- use itemgetter in archiveFS (#1764)
+- document that newline is included in readline(s) (#1770)
+- format/spelling (#1774, 1779, 1780)
+
 2024.12.0
 ---------
 

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.12.0'
-__version_tuple__ = version_tuple = (2024, 12, 0)
+__version__ = version = '2025.2.0'
+__version_tuple__ = version_tuple = (2025, 2, 0)

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/archive.py

@@ -1,3 +1,5 @@
+import operator
+
 from fsspec import AbstractFileSystem
 from fsspec.utils import tokenize
 
@@ -67,7 +69,7 @@ class AbstractArchiveFileSystem(AbstractFileSystem):
                     out = {"name": ppath, "size": 0, "type": "directory"}
                     paths[ppath] = out
         if detail:
-            out = sorted(paths.values(), key=lambda _: _["name"])
+            out = sorted(paths.values(), key=operator.itemgetter("name"))
             return out
         else:
             return sorted(paths)

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/implementations/asyn_wrapper.py

@@ -57,8 +57,9 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         """
         Wrap all synchronous methods of the underlying filesystem with asynchronous versions.
         """
+        excluded_methods = {"open"}
         for method_name in dir(self.sync_fs):
-            if method_name.startswith("_"):
+            if method_name.startswith("_") or method_name in excluded_methods:
                 continue
 
             attr = inspect.getattr_static(self.sync_fs, method_name)

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/implementations/dbfs.py

@@ -412,9 +412,9 @@ class DatabricksFile(AbstractBufferedFile):
         if block_size is None or block_size == "default":
             block_size = self.DEFAULT_BLOCK_SIZE
 
-        assert (
-            block_size == self.DEFAULT_BLOCK_SIZE
-        ), f"Only the default block size is allowed, not {block_size}"
+        assert block_size == self.DEFAULT_BLOCK_SIZE, (
+            f"Only the default block size is allowed, not {block_size}"
+        )
 
         super().__init__(
             fs,

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/implementations/reference.py

@@ -10,6 +10,7 @@ from itertools import chain
 from typing import TYPE_CHECKING, Literal
 
 import fsspec.core
+from fsspec.spec import AbstractBufferedFile
 
 try:
     import ujson as json
@@ -394,10 +395,14 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
                 self.write(field, record)
         else:
             # metadata or top-level
-            self._items[key] = value
-            new_value = json.loads(
-                value.decode() if isinstance(value, bytes) else value
-            )
+            if hasattr(value, "to_bytes"):
+                val = value.to_bytes().decode()
+            elif isinstance(value, bytes):
+                val = value.decode()
+            else:
+                val = value
+            self._items[key] = val
+            new_value = json.loads(val)
             self.zmetadata[key] = {**self.zmetadata.get(key, {}), **new_value}
 
     @staticmethod
@@ -595,8 +600,7 @@ class ReferenceFileSystem(AsyncFileSystem):
     async, and must allow start and end args in _cat_file. Later versions
     may allow multiple arbitrary URLs for the targets.
     This FileSystem is read-only. It is designed to be used with async
-    targets (for now). This FileSystem only allows whole-file access, no
-    ``open``. We do not get original file details from the target FS.
+    targets (for now). We do not get original file details from the target FS.
     Configuration is by passing a dict of references at init, or a URL to
     a JSON file containing the same; this dict
     can also contain concrete data for some set of paths.
@@ -606,6 +610,7 @@ class ReferenceFileSystem(AsyncFileSystem):
     """
 
     protocol = "reference"
+    cachable = False
 
     def __init__(
         self,
@@ -762,6 +767,11 @@ class ReferenceFileSystem(AsyncFileSystem):
         for k, f in self.fss.items():
             if not f.async_impl:
                 self.fss[k] = AsyncFileSystemWrapper(f)
+            elif self.asynchronous ^ f.asynchronous:
+                raise ValueError(
+                    "Reference-FS's target filesystem must have same value"
+                    "of asynchronous"
+                )
 
     def _cat_common(self, path, start=None, end=None):
         path = self._strip_protocol(path)
@@ -772,6 +782,8 @@ class ReferenceFileSystem(AsyncFileSystem):
             raise FileNotFoundError(path) from exc
         if isinstance(part, str):
             part = part.encode()
+        if hasattr(part, "to_bytes"):
+            part = part.to_bytes()
         if isinstance(part, bytes):
             logger.debug(f"Reference: {path}, type bytes")
             if part.startswith(b"base64:"):
@@ -1073,7 +1085,7 @@ class ReferenceFileSystem(AsyncFileSystem):
         self.dircache = {"": []}
         it = self.references.items()
         for path, part in it:
-            if isinstance(part, (bytes, str)):
+            if isinstance(part, (bytes, str)) or hasattr(part, "to_bytes"):
                 size = len(part)
             elif len(part) == 1:
                 size = None
@@ -1100,10 +1112,33 @@ class ReferenceFileSystem(AsyncFileSystem):
             self.dircache[par].append({"name": path, "type": "file", "size": size})
 
     def _open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs):
-        data = self.cat_file(path)  # load whole chunk into memory
-        return io.BytesIO(data)
+        part_or_url, start0, end0 = self._cat_common(path)
+        # This logic is kept outside `ReferenceFile` to avoid unnecessary redirection.
+        # That does mean `_cat_common` gets called twice if it eventually reaches `ReferenceFile`.
+        if isinstance(part_or_url, bytes):
+            return io.BytesIO(part_or_url[start0:end0])
+
+        protocol, _ = split_protocol(part_or_url)
+        if start0 is None and end0 is None:
+            return self.fss[protocol]._open(
+                part_or_url,
+                mode,
+                block_size=block_size,
+                cache_options=cache_options,
+                **kwargs,
+            )
+
+        return ReferenceFile(
+            self,
+            path,
+            mode,
+            block_size=block_size,
+            cache_options=cache_options,
+            **kwargs,
+        )
 
     def ls(self, path, detail=True, **kwargs):
+        logger.debug("list %s", path)
         path = self._strip_protocol(path)
         if isinstance(self.references, LazyReferenceMapper):
             try:
@@ -1214,3 +1249,58 @@ class ReferenceFileSystem(AsyncFileSystem):
                 out[k] = v
         with fsspec.open(url, "wb", **storage_options) as f:
             f.write(json.dumps({"version": 1, "refs": out}).encode())
+
+
+class ReferenceFile(AbstractBufferedFile):
+    def __init__(
+        self,
+        fs,
+        path,
+        mode="rb",
+        block_size="default",
+        autocommit=True,
+        cache_type="readahead",
+        cache_options=None,
+        size=None,
+        **kwargs,
+    ):
+        super().__init__(
+            fs,
+            path,
+            mode=mode,
+            block_size=block_size,
+            autocommit=autocommit,
+            size=size,
+            cache_type=cache_type,
+            cache_options=cache_options,
+            **kwargs,
+        )
+        part_or_url, self.start, self.end = self.fs._cat_common(self.path)
+        protocol, _ = split_protocol(part_or_url)
+        self.src_fs = self.fs.fss[protocol]
+        self.src_path = part_or_url
+        self._f = None
+
+    @property
+    def f(self):
+        if self._f is None or self._f.closed:
+            self._f = self.src_fs._open(
+                self.src_path,
+                mode=self.mode,
+                block_size=self.blocksize,
+                autocommit=self.autocommit,
+                cache_type="none",
+                **self.kwargs,
+            )
+        return self._f
+
+    def close(self):
+        if self._f is not None:
+            self._f.close()
+        return super().close()
+
+    def _fetch_range(self, start, end):
+        start = start + self.start
+        end = min(end + self.start, self.end)
+        self.f.seek(start)
+        return self.f.read(end - start)

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/registry.py

@@ -218,9 +218,9 @@ known_implementations = {
     "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"},
 }
 
-assert list(known_implementations) == sorted(
-    known_implementations
-), "Not in alphabetical order"
+assert list(known_implementations) == sorted(known_implementations), (
+    "Not in alphabetical order"
+)
 
 
 def get_filesystem_class(protocol):

{fsspec-2024.12.0 → fsspec-2025.2.0}/fsspec/spec.py

@@ -382,7 +382,7 @@ class AbstractFileSystem(metaclass=_Cached):
             pass
 
     def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs):
-        """Return all files belows path
+        """Return all files under the given path.
 
         List all files, recursing into subdirectories; output is iterator-style,
         like ``os.walk()``. For a simple list of files, ``find()`` is available.
@@ -2131,7 +2131,7 @@ class AbstractBufferedFile(io.IOBase):
         return b"".join(out)
 
     def readline(self):
-        """Read until first occurrence of newline character
+        """Read until and including the first occurrence of newline character
 
         Note that, because of character encoding, this is not necessarily a
         true line ending.
@@ -2148,7 +2148,7 @@ class AbstractBufferedFile(io.IOBase):
         return self
 
     def readlines(self):
-        """Return all data, split by the newline character"""
+        """Return all data, split by the newline character, including the newline character"""
         data = self.read()
         lines = data.split(b"\n")
         out = [l + b"\n" for l in lines[:-1]]

{fsspec-2024.12.0 → fsspec-2025.2.0}/pyproject.toml

@@ -122,7 +122,6 @@ test_full = [
 ]
 test_downstream = [
     "dask[dataframe,test]",
-    "dask-expr",
     "moto[server]>4,<5",
     "pytest-timeout",
     "xarray",
@@ -215,8 +214,6 @@ ignore = [
     "UP011",
     "UP015",
     "UP018",
-    # deprecated
-    "UP027",
     "SIM102",
     "SIM105",
     "SIM108",
@@ -238,8 +235,6 @@ ignore = [
     "Q003",
     "COM812",
     "COM819",
-    "ISC001",
-    "ISC002",
 ]
 
 [tool.pytest.ini_options]

fsspec-2024.12.0/.github/workflows/codespell.yml

@@ -1,19 +0,0 @@
----
-name: Codespell
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-jobs:
-  codespell:
-    name: Check for spelling errors
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Codespell
-        uses: codespell-project/actions-codespell@v1