fsspec 2025.12.0__py3-none-any.whl → 2026.2.0__py3-none-any.whl

fsspec/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2025.12.0'
-__version_tuple__ = version_tuple = (2025, 12, 0)
+__version__ = version = '2026.2.0'
+__version_tuple__ = version_tuple = (2026, 2, 0)
 
 __commit_id__ = commit_id = None

fsspec/caching.py

@@ -25,7 +25,7 @@ else:
 T = TypeVar("T")
 
 
-logger = logging.getLogger("fsspec")
+logger = logging.getLogger("fsspec.caching")
 
 Fetcher = Callable[[int, int], bytes]  # Maps (start, end) to bytes
 MultiFetcher = Callable[[list[int, int]], bytes]  # Maps [(start, end)] to bytes
@@ -214,7 +214,7 @@ class MMapCache(BaseCache):
         if self.multi_fetcher:
             logger.debug(f"MMap get blocks {ranges}")
             for idx, r in enumerate(self.multi_fetcher(ranges)):
-                (sstart, send) = ranges[idx]
+                sstart, send = ranges[idx]
                 logger.debug(f"MMap copy block ({sstart}-{send}")
                 self.cache[sstart:send] = r
         else:
@@ -391,19 +391,8 @@ class BlockCache(BaseCache):
         if start >= self.size or start >= end:
             return b""
 
-        # byte position -> block numbers
-        start_block_number = start // self.blocksize
-        end_block_number = end // self.blocksize
-
-        # these are cached, so safe to do multiple calls for the same start and end.
-        for block_number in range(start_block_number, end_block_number + 1):
-            self._fetch_block_cached(block_number)
-
         return self._read_cache(
-            start,
-            end,
-            start_block_number=start_block_number,
-            end_block_number=end_block_number,
+            start, end, start // self.blocksize, (end - 1) // self.blocksize
         )
 
     def _fetch_block(self, block_number: int) -> bytes:
@@ -439,6 +428,8 @@ class BlockCache(BaseCache):
         """
         start_pos = start % self.blocksize
         end_pos = end % self.blocksize
+        if end_pos == 0:
+            end_pos = self.blocksize
 
         self.hit_count += 1
         if start_block_number == end_block_number:
@@ -662,12 +653,12 @@ class KnownPartsOfAFile(BaseCache):
         pass
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
+        logger.debug("Known parts request %s %s", start, stop)
         if start is None:
             start = 0
         if stop is None:
             stop = self.size
         self.total_requested_bytes += stop - start
-
         out = b""
         started = False
         loc_old = 0
@@ -698,11 +689,13 @@ class KnownPartsOfAFile(BaseCache):
             elif loc0 <= stop <= loc1:
                 # end block
                 self.hit_count += 1
-                return out + self.data[(loc0, loc1)][: stop - loc0]
+                out = out + self.data[(loc0, loc1)][: stop - loc0]
+                return out
             loc_old = loc1
         self.miss_count += 1
         if started and not self.strict:
-            return out + b"\x00" * (stop - loc_old)
+            out = out + b"\x00" * (stop - loc_old)
+            return out
         raise ValueError
 
 

fsspec/compression.py

@@ -1,5 +1,6 @@
 """Helper functions for a standard streaming compression API"""
 
+import sys
 from zipfile import ZipFile
 
 import fsspec.utils
@@ -155,11 +156,12 @@ except ImportError:
     pass
 
 try:
-    # zstd in the standard library for python >= 3.14
-    from compression.zstd import ZstdFile
-
-    register_compression("zstd", ZstdFile, "zst")
+    if sys.version_info >= (3, 14):
+        from compression import zstd
+    else:
+        from backports import zstd
 
+    register_compression("zstd", zstd.ZstdFile, "zst")
 except ImportError:
     try:
         import zstandard as zstd
@@ -175,6 +177,7 @@ except ImportError:
         register_compression("zstd", zstandard_file, "zst")
     except ImportError:
         pass
+    pass
 
 
 def available_compressions():

fsspec/implementations/arrow.py

@@ -242,7 +242,9 @@ class ArrowFile(io.IOBase):
 
     @property
     def size(self):
-        return self.stream.size()
+        if self.stream.seekable():
+            return self.stream.size()
+        return None
 
     def __exit__(self, *args):
         return self.close()

fsspec/implementations/cached.py

@@ -427,6 +427,7 @@ class CachingFileSystem(ChainedFileSystem):
     def __getattribute__(self, item):
         if item in {
             "load_cache",
+            "_get_cached_file_before_open",
             "_open",
             "save_cache",
             "close_and_update",
@@ -678,46 +679,12 @@ class WholeFileCacheFileSystem(CachingFileSystem):
             out = out[paths[0]]
         return out
 
-    def _open(self, path, mode="rb", **kwargs):
-        path = self._strip_protocol(path)
-        if "r" not in mode:
-            hash = self._mapper(path)
-            fn = os.path.join(self.storage[-1], hash)
-            user_specified_kwargs = {
-                k: v
-                for k, v in kwargs.items()
-                # those kwargs were added by open(), we don't want them
-                if k not in ["autocommit", "block_size", "cache_options"]
-            }
-            return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
-        detail = self._check_file(path)
-        if detail:
-            detail, fn = detail
-            _, blocks = detail["fn"], detail["blocks"]
-            if blocks is True:
-                logger.debug("Opening local copy of %s", path)
-
-                # In order to support downstream filesystems to be able to
-                # infer the compression from the original filename, like
-                # the `TarFileSystem`, let's extend the `io.BufferedReader`
-                # fileobject protocol by adding a dedicated attribute
-                # `original`.
-                f = open(fn, mode)
-                f.original = detail.get("original")
-                return f
-            else:
-                raise ValueError(
-                    f"Attempt to open partially cached file {path}"
-                    f" as a wholly cached file"
-                )
-        else:
-            fn = self._make_local_details(path)
-        kwargs["mode"] = mode
-
+    def _get_cached_file_before_open(self, path, **kwargs):
+        fn = self._make_local_details(path)
         # call target filesystems open
         self._mkcache()
         if self.compression:
-            with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
+            with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):
                     # want no type of caching if just downloading whole thing
                     f.cache = BaseCache(0, f.cache.fetcher, f.size)
@@ -735,7 +702,47 @@ class WholeFileCacheFileSystem(CachingFileSystem):
         else:
             self.fs.get_file(path, fn)
         self.save_cache()
-        return self._open(path, mode)
+
+    def _open(self, path, mode="rb", **kwargs):
+        path = self._strip_protocol(path)
+        # For read (or append), (try) download from remote
+        if "r" in mode or "a" in mode:
+            if not self._check_file(path):
+                if self.fs.exists(path):
+                    self._get_cached_file_before_open(path, **kwargs)
+                elif "r" in mode:
+                    raise FileNotFoundError(path)
+
+            detail, fn = self._check_file(path)
+            _, blocks = detail["fn"], detail["blocks"]
+            if blocks is True:
+                logger.debug("Opening local copy of %s", path)
+            else:
+                raise ValueError(
+                    f"Attempt to open partially cached file {path}"
+                    f" as a wholly cached file"
+                )
+
+        # Just reading does not need special file handling
+        if "r" in mode and "+" not in mode:
+            # In order to support downstream filesystems to be able to
+            # infer the compression from the original filename, like
+            # the `TarFileSystem`, let's extend the `io.BufferedReader`
+            # fileobject protocol by adding a dedicated attribute
+            # `original`.
+            f = open(fn, mode)
+            f.original = detail.get("original")
+            return f
+
+        hash = self._mapper(path)
+        fn = os.path.join(self.storage[-1], hash)
+        user_specified_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            # those kwargs were added by open(), we don't want them
+            if k not in ["autocommit", "block_size", "cache_options"]
+        }
+        return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs)
 
 
 class SimpleCacheFileSystem(WholeFileCacheFileSystem):
@@ -894,37 +901,16 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
             paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
         )
 
-    def _open(self, path, mode="rb", **kwargs):
-        path = self._strip_protocol(path)
+    def _get_cached_file_before_open(self, path, **kwargs):
         sha = self._mapper(path)
-
-        if "r" not in mode:
-            fn = os.path.join(self.storage[-1], sha)
-            user_specified_kwargs = {
-                k: v
-                for k, v in kwargs.items()
-                if k not in ["autocommit", "block_size", "cache_options"]
-            }  # those were added by open()
-            return LocalTempFile(
-                self,
-                path,
-                mode=mode,
-                autocommit=not self._intrans,
-                fn=fn,
-                **user_specified_kwargs,
-            )
-        fn = self._check_file(path)
-        if fn:
-            return open(fn, mode)
-
         fn = os.path.join(self.storage[-1], sha)
         logger.debug("Copying %s to local cache", path)
-        kwargs["mode"] = mode
 
         self._mkcache()
         self._cache_size = None
+
         if self.compression:
-            with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
+            with self.fs._open(path, mode="rb", **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):
                     # want no type of caching if just downloading whole thing
                     f.cache = BaseCache(0, f.cache.fetcher, f.size)
@@ -941,7 +927,39 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
                     f2.write(data)
         else:
             self.fs.get_file(path, fn)
-        return self._open(path, mode)
+
+    def _open(self, path, mode="rb", **kwargs):
+        path = self._strip_protocol(path)
+        sha = self._mapper(path)
+
+        # For read (or append), (try) download from remote
+        if "r" in mode or "a" in mode:
+            if not self._check_file(path):
+                # append does not require an existing file but read does
+                if self.fs.exists(path):
+                    self._get_cached_file_before_open(path, **kwargs)
+                elif "r" in mode:
+                    raise FileNotFoundError(path)
+
+        fn = self._check_file(path)
+        # Just reading does not need special file handling
+        if "r" in mode and "+" not in mode:
+            return open(fn, mode)
+
+        fn = os.path.join(self.storage[-1], sha)
+        user_specified_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            if k not in ["autocommit", "block_size", "cache_options"]
+        }  # those were added by open()
+        return LocalTempFile(
+            self,
+            path,
+            mode=mode,
+            autocommit=not self._intrans,
+            fn=fn,
+            **user_specified_kwargs,
+        )
 
 
 class LocalTempFile:

fsspec/implementations/ftp.py

@@ -1,4 +1,5 @@
 import os
+import ssl
 import uuid
 from ftplib import FTP, FTP_TLS, Error, error_perm
 from typing import Any
@@ -6,6 +7,37 @@ from typing import Any
 from ..spec import AbstractBufferedFile, AbstractFileSystem
 from ..utils import infer_storage_options, isfilelike
 
+SECURITY_PROTOCOL_MAP = {
+    "tls": ssl.PROTOCOL_TLS,
+    "tlsv1": ssl.PROTOCOL_TLSv1,
+    "tlsv1_1": ssl.PROTOCOL_TLSv1_1,
+    "tlsv1_2": ssl.PROTOCOL_TLSv1_2,
+    "sslv23": ssl.PROTOCOL_SSLv23,
+}
+
+
+class ImplicitFTPTLS(FTP_TLS):
+    """
+    FTP_TLS subclass that automatically wraps sockets in SSL
+    to support implicit FTPS.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._sock = None
+
+    @property
+    def sock(self):
+        """Return the socket."""
+        return self._sock
+
+    @sock.setter
+    def sock(self, value):
+        """When modifying the socket, ensure that it is ssl wrapped."""
+        if value is not None and not isinstance(value, ssl.SSLSocket):
+            value = self.context.wrap_socket(value)
+        self._sock = value
+
 
 class FTPFileSystem(AbstractFileSystem):
     """A filesystem over classic FTP"""
@@ -55,8 +87,14 @@ class FTPFileSystem(AbstractFileSystem):
             Timeout of the ftp connection in seconds
         encoding: str
             Encoding to use for directories and filenames in FTP connection
-        tls: bool
-            Use FTP-TLS, by default False
+        tls: bool or str
+            Enable FTP-TLS for secure connections:
+                - False: Plain FTP (default)
+                - True: Explicit TLS (FTPS with AUTH TLS command)
+                - "tls": Auto-negotiate highest protocol
+                - "tlsv1": TLS v1.0
+                - "tlsv1_1": TLS v1.1
+                - "tlsv1_2": TLS v1.2
         """
         super().__init__(**kwargs)
         self.host = host
@@ -71,15 +109,27 @@ class FTPFileSystem(AbstractFileSystem):
             self.blocksize = 2**16
         self.tls = tls
         self._connect()
-        if self.tls:
+        if isinstance(self.tls, bool) and self.tls:
             self.ftp.prot_p()
 
     def _connect(self):
+        security = None
         if self.tls:
-            ftp_cls = FTP_TLS
+            if isinstance(self.tls, str):
+                ftp_cls = ImplicitFTPTLS
+                security = SECURITY_PROTOCOL_MAP.get(
+                    self.tls,
+                    f"Not supported {self.tls} protocol",
+                )
+                if isinstance(security, str):
+                    raise ValueError(security)
+            else:
+                ftp_cls = FTP_TLS
         else:
             ftp_cls = FTP
         self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding)
+        if security:
+            self.ftp.ssl_version = security
         self.ftp.connect(self.host, self.port)
         self.ftp.login(*self.cred)
 

fsspec/implementations/local.py

@@ -166,6 +166,10 @@ class LocalFileSystem(AbstractFileSystem):
         """
         path1 = self._strip_protocol(path1)
         path2 = self._strip_protocol(path2)
+
+        if self.auto_mkdir:
+            self.makedirs(self._parent(path2), exist_ok=True)
+
         shutil.move(path1, path2)
 
     def link(self, src, dst, **kwargs):

fsspec/implementations/webhdfs.py

@@ -7,6 +7,7 @@ import shutil
 import tempfile
 import uuid
 from contextlib import suppress
+from datetime import datetime
 from urllib.parse import quote
 
 import requests
@@ -268,6 +269,23 @@ class WebHDFS(AbstractFileSystem):
         info["name"] = path
         return self._process_info(info)
 
+    def created(self, path):
+        """Return the created timestamp of a file as a datetime.datetime"""
+        # The API does not provide creation time, so we use modification time
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve creation time (modification time).")
+
+    def modified(self, path):
+        """Return the modified timestamp of a file as a datetime.datetime"""
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve modification time.")
+
     def ls(self, path, detail=False, **kwargs):
         out = self._call("LISTSTATUS", path=path)
         infos = out.json()["FileStatuses"]["FileStatus"]

fsspec/implementations/zip.py

@@ -138,14 +138,17 @@ class ZipFileSystem(AbstractArchiveFileSystem):
         if maxdepth is not None and maxdepth < 1:
             raise ValueError("maxdepth must be at least 1")
 
+        def to_parts(_path: str):
+            return list(filter(None, _path.replace("\\", "/").split("/")))
+
+        if not isinstance(path, str):
+            path = str(path)
+
         # Remove the leading slash, as the zip file paths are always
         # given without a leading slash
         path = path.lstrip("/")
-        path_parts = list(filter(lambda s: bool(s), path.split("/")))
-
-        def _matching_starts(file_path):
-            file_parts = filter(lambda s: bool(s), file_path.split("/"))
-            return all(a == b for a, b in zip(path_parts, file_parts))
+        path_parts = to_parts(path)
+        path_depth = len(path_parts)
 
         self._get_dirs()
 
@@ -157,21 +160,22 @@ class ZipFileSystem(AbstractArchiveFileSystem):
             return result if detail else [path]
 
         for file_path, file_info in self.dir_cache.items():
-            if not (path == "" or _matching_starts(file_path)):
+            if len(file_parts := to_parts(file_path)) < path_depth or any(
+                a != b for a, b in zip(path_parts, file_parts)
+            ):
+                # skip parent folders and mismatching paths
                 continue
 
             if file_info["type"] == "directory":
-                if withdirs:
-                    if file_path not in result:
-                        result[file_path.strip("/")] = file_info
+                if withdirs and file_path not in result:
+                    result[file_path.strip("/")] = file_info
                 continue
 
             if file_path not in result:
                 result[file_path] = file_info if detail else None
 
         if maxdepth:
-            path_depth = path.count("/")
             result = {
-                k: v for k, v in result.items() if k.count("/") - path_depth < maxdepth
+                k: v for k, v in result.items() if k.count("/") < maxdepth + path_depth
             }
         return result if detail else sorted(result)

fsspec/parquet.py

@@ -1,7 +1,6 @@
 import io
 import json
 import warnings
-from typing import Literal
 
 import fsspec
 
@@ -25,7 +24,6 @@ class AlreadyBufferedFile(AbstractBufferedFile):
 
 def open_parquet_files(
     path: list[str],
-    mode: Literal["rb"] = "rb",
     fs: None | fsspec.AbstractFileSystem = None,
     metadata=None,
     columns: None | list[str] = None,
@@ -54,8 +52,6 @@ def open_parquet_files(
     ----------
     path: str
         Target file path.
-    mode: str, optional
-        Mode option to be passed through to `fs.open`. Default is "rb".
     metadata: Any, optional
         Parquet metadata object. Object type must be supported
         by the backend parquet engine. For now, only the "fastparquet"
@@ -150,16 +146,16 @@ def open_parquet_files(
         AlreadyBufferedFile(
             fs=None,
             path=fn,
-            mode=mode,
+            mode="rb",
             cache_type="parts",
             cache_options={
                 **options,
-                "data": data.get(fn, {}),
+                "data": ranges,
             },
-            size=max(_[1] for _ in data.get(fn, {})),
+            size=max(_[1] for _ in ranges),
             **kwargs,
         )
-        for fn in data
+        for fn, ranges in data.items()
     ]
 
 
@@ -197,7 +193,7 @@ def _get_parquet_byte_ranges(
     if isinstance(engine, str):
         engine = _set_engine(engine)
 
-    # Pass to specialized function if metadata is defined
+    # Pass to a specialized function if metadata is defined
     if metadata is not None:
         # Use the provided parquet metadata object
         # to avoid transferring/parsing footer metadata
@@ -212,63 +208,54 @@ def _get_parquet_byte_ranges(
             filters=filters,
         )
 
-    # Get file sizes asynchronously
-    file_sizes = fs.sizes(paths)
-
     # Populate global paths, starts, & ends
-    result = {}
-    data_paths = []
-    data_starts = []
-    data_ends = []
-    add_header_magic = True
     if columns is None and row_groups is None and filters is None:
         # We are NOT selecting specific columns or row-groups.
         #
         # We can avoid sampling the footers, and just transfer
         # all file data with cat_ranges
-        for i, path in enumerate(paths):
-            result[path] = {}
-            data_paths.append(path)
-            data_starts.append(0)
-            data_ends.append(file_sizes[i])
-        add_header_magic = False  # "Magic" should already be included
+        result = {path: {(0, len(data)): data} for path, data in fs.cat(paths).items()}
     else:
         # We ARE selecting specific columns or row-groups.
         #
+        # Get file sizes asynchronously
+        file_sizes = fs.sizes(paths)
+        data_paths = []
+        data_starts = []
+        data_ends = []
         # Gather file footers.
         # We just take the last `footer_sample_size` bytes of each
         # file (or the entire file if it is smaller than that)
-        footer_starts = []
-        footer_ends = []
-        for i, path in enumerate(paths):
-            footer_ends.append(file_sizes[i])
-            sample_size = max(0, file_sizes[i] - footer_sample_size)
-            footer_starts.append(sample_size)
-        footer_samples = fs.cat_ranges(paths, footer_starts, footer_ends)
+        footer_starts = [
+            max(0, file_size - footer_sample_size) for file_size in file_sizes
+        ]
+        footer_samples = fs.cat_ranges(paths, footer_starts, file_sizes)
 
         # Check our footer samples and re-sample if necessary.
-        missing_footer_starts = footer_starts.copy()
-        large_footer = 0
+        large_footer = []
         for i, path in enumerate(paths):
             footer_size = int.from_bytes(footer_samples[i][-8:-4], "little")
             real_footer_start = file_sizes[i] - (footer_size + 8)
             if real_footer_start < footer_starts[i]:
-                missing_footer_starts[i] = real_footer_start
-                large_footer = max(large_footer, (footer_size + 8))
+                large_footer.append((i, real_footer_start))
         if large_footer:
             warnings.warn(
                 f"Not enough data was used to sample the parquet footer. "
                 f"Try setting footer_sample_size >= {large_footer}."
             )
-            for i, block in enumerate(
-                fs.cat_ranges(
-                    paths,
-                    missing_footer_starts,
-                    footer_starts,
-                )
-            ):
+            path0 = [paths[i] for i, _ in large_footer]
+            starts = [_[1] for _ in large_footer]
+            ends = [file_sizes[i] - footer_sample_size for i, _ in large_footer]
+            data = fs.cat_ranges(path0, starts, ends)
+            for i, (path, start, block) in enumerate(zip(path0, starts, data)):
                 footer_samples[i] = block + footer_samples[i]
-                footer_starts[i] = missing_footer_starts[i]
+                footer_starts[i] = start
+        result = {
+            path: {(start, size): data}
+            for path, start, size, data in zip(
+                paths, footer_starts, file_sizes, footer_samples
+            )
+        }
 
         # Calculate required byte ranges for each path
         for i, path in enumerate(paths):
@@ -284,9 +271,6 @@ def _get_parquet_byte_ranges(
             data_paths += [path] * len(path_data_starts)
             data_starts += path_data_starts
             data_ends += path_data_ends
-            result.setdefault(path, {})[(footer_starts[i], file_sizes[i])] = (
-                footer_samples[i]
-            )
 
         # Merge adjacent offset ranges
         data_paths, data_starts, data_ends = merge_offset_ranges(
@@ -295,19 +279,14 @@ def _get_parquet_byte_ranges(
             data_ends,
             max_gap=max_gap,
             max_block=max_block,
-            sort=False,  # Should already be sorted
+            sort=True,
         )
 
-        # Start by populating `result` with footer samples
-        for i, path in enumerate(paths):
-            result[path] = {(footer_starts[i], footer_ends[i]): footer_samples[i]}
-
-    # Transfer the data byte-ranges into local memory
-    _transfer_ranges(fs, result, data_paths, data_starts, data_ends)
+        # Transfer the data byte-ranges into local memory
+        _transfer_ranges(fs, result, data_paths, data_starts, data_ends)
 
-    # Add b"PAR1" to header if necessary
-    if add_header_magic:
-        _add_header_magic(result)
+    # Add b"PAR1" to headers
+    _add_header_magic(result)
 
     return result
 
@@ -362,7 +341,7 @@ def _transfer_ranges(fs, blocks, paths, starts, ends):
 
 def _add_header_magic(data):
     # Add b"PAR1" to file headers
-    for path in list(data.keys()):
+    for path in list(data):
         add_magic = True
         for k in data[path]:
             if k[0] == 0 and k[1] >= 4:
@@ -419,9 +398,6 @@ class FastparquetEngine:
 
         self.fp = fp
 
-    def _row_group_filename(self, row_group, pf):
-        return pf.row_group_filename(row_group)
-
     def _parquet_byte_ranges(
         self,
         columns,
@@ -465,6 +441,10 @@ class FastparquetEngine:
             # Input row_groups contains row-group indices
             row_group_indices = row_groups
             row_groups = pf.row_groups
+        if column_set is not None:
+            column_set = [
+                _ if isinstance(_, list) else _.split(".") for _ in column_set
+            ]
 
         # Loop through column chunks to add required byte ranges
         for r, row_group in enumerate(row_groups):
@@ -472,13 +452,12 @@ class FastparquetEngine:
             # specific row-groups
             if row_group_indices is None or r in row_group_indices:
                 # Find the target parquet-file path for `row_group`
-                fn = self._row_group_filename(row_group, pf)
+                fn = pf.row_group_filename(row_group)
 
                 for column in row_group.columns:
-                    name = column.meta_data.path_in_schema[0]
-                    # Skip this column if we are targeting a
-                    # specific columns
-                    if column_set is None or name in column_set:
+                    name = column.meta_data.path_in_schema
+                    # Skip this column if we are targeting specific columns
+                    if column_set is None or _cmp(name, column_set):
                         file_offset0 = column.meta_data.dictionary_page_offset
                         if file_offset0 is None:
                             file_offset0 = column.meta_data.data_page_offset
@@ -512,9 +491,6 @@ class PyarrowEngine:
 
         self.pq = pq
 
-    def _row_group_filename(self, row_group, metadata):
-        raise NotImplementedError
-
     def _parquet_byte_ranges(
         self,
         columns,
@@ -527,6 +503,7 @@ class PyarrowEngine:
         if metadata is not None:
             raise ValueError("metadata input not supported for PyarrowEngine")
         if filters:
+            # there must be a way!
             raise NotImplementedError
 
         data_starts, data_ends = [], []
@@ -550,6 +527,10 @@ class PyarrowEngine:
                     if not isinstance(ind, dict)
                 ]
                 column_set |= set(md_index)
+        if column_set is not None:
+            column_set = [
+                _[:1] if isinstance(_, list) else _.split(".")[:1] for _ in column_set
+            ]
 
         # Loop through column chunks to add required byte ranges
         for r in range(md.num_row_groups):
@@ -559,22 +540,33 @@ class PyarrowEngine:
                 row_group = md.row_group(r)
                 for c in range(row_group.num_columns):
                     column = row_group.column(c)
-                    name = column.path_in_schema
-                    # Skip this column if we are targeting a
-                    # specific columns
-                    split_name = name.split(".")[0]
-                    if (
-                        column_set is None
-                        or name in column_set
-                        or split_name in column_set
-                    ):
-                        file_offset0 = column.dictionary_page_offset
-                        if file_offset0 is None:
-                            file_offset0 = column.data_page_offset
-                        num_bytes = column.total_compressed_size
+                    name = column.path_in_schema.split(".")
+                    # Skip this column if we are targeting specific columns
+                    if column_set is None or _cmp(name, column_set):
+                        meta = column.to_dict()
+                        # Any offset could be the first one
+                        file_offset0 = min(
+                            _
+                            for _ in [
+                                meta.get("dictionary_page_offset"),
+                                meta.get("data_page_offset"),
+                                meta.get("index_page_offset"),
+                            ]
+                            if _ is not None
+                        )
                         if file_offset0 < footer_start:
                             data_starts.append(file_offset0)
                             data_ends.append(
-                                min(file_offset0 + num_bytes, footer_start)
+                                min(
+                                    meta["total_compressed_size"] + file_offset0,
+                                    footer_start,
+                                )
                             )
+
+        data_starts.append(footer_start)
+        data_ends.append(footer_start + len(footer))
         return data_starts, data_ends
+
+
+def _cmp(name, column_set):
+    return any(all(a == b for a, b in zip(name, _)) for _ in column_set)

fsspec/registry.py

@@ -189,7 +189,7 @@ known_implementations = {
     },
     "pyscript": {
         "class": "pyscript_fsspec_client.client.PyscriptFileSystem",
-        "err": "Install requests (cpython) or run in pyscript",
+        "err": "This only runs in a pyscript context",
     },
     "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"},
     "root": {

fsspec/utils.py

@@ -566,6 +566,16 @@ def merge_offset_ranges(
                 )
             )
         )
+    remove = []
+    for i, (path, start, end) in enumerate(zip(paths, starts, ends)):
+        if any(
+            e is not None and p == path and start >= s and end <= e and i != i2
+            for i2, (p, s, e) in enumerate(zip(paths, starts, ends))
+        ):
+            remove.append(i)
+    paths = [p for i, p in enumerate(paths) if i not in remove]
+    starts = [s for i, s in enumerate(starts) if i not in remove]
+    ends = [e for i, e in enumerate(ends) if i not in remove]
 
     if paths:
         # Loop through the coupled `paths`, `starts`, and
@@ -587,7 +597,7 @@ def merge_offset_ranges(
                 new_starts.append(starts[i])
                 new_ends.append(ends[i])
             else:
-                # Merge with previous block by updating the
+                # Merge with the previous block by updating the
                 # last element of `ends`
                 new_ends[-1] = ends[i]
         return new_paths, new_starts, new_ends

{fsspec-2025.12.0.dist-info → fsspec-2026.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.12.0
+Version: 2026.2.0
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/
@@ -49,7 +49,7 @@ Requires-Dist: distributed; extra == 'full'
 Requires-Dist: dropbox; extra == 'full'
 Requires-Dist: dropboxdrivefs; extra == 'full'
 Requires-Dist: fusepy; extra == 'full'
-Requires-Dist: gcsfs; extra == 'full'
+Requires-Dist: gcsfs>2024.2.0; extra == 'full'
 Requires-Dist: libarchive-c; extra == 'full'
 Requires-Dist: ocifs; extra == 'full'
 Requires-Dist: panel; extra == 'full'
@@ -57,13 +57,13 @@ Requires-Dist: paramiko; extra == 'full'
 Requires-Dist: pyarrow>=1; extra == 'full'
 Requires-Dist: pygit2; extra == 'full'
 Requires-Dist: requests; extra == 'full'
-Requires-Dist: s3fs; extra == 'full'
+Requires-Dist: s3fs>2024.2.0; extra == 'full'
 Requires-Dist: smbprotocol; extra == 'full'
 Requires-Dist: tqdm; extra == 'full'
 Provides-Extra: fuse
 Requires-Dist: fusepy; extra == 'fuse'
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == 'gcs'
+Requires-Dist: gcsfs>2024.2.0; extra == 'gcs'
 Provides-Extra: git
 Requires-Dist: pygit2; extra == 'git'
 Provides-Extra: github
@@ -81,7 +81,7 @@ Requires-Dist: libarchive-c; extra == 'libarchive'
 Provides-Extra: oci
 Requires-Dist: ocifs; extra == 'oci'
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == 's3'
+Requires-Dist: s3fs>2024.2.0; extra == 's3'
 Provides-Extra: sftp
 Requires-Dist: paramiko; extra == 'sftp'
 Provides-Extra: smb
@@ -108,6 +108,7 @@ Requires-Dist: xarray; extra == 'test-downstream'
 Provides-Extra: test-full
 Requires-Dist: adlfs; extra == 'test-full'
 Requires-Dist: aiohttp!=4.0.0a0,!=4.0.0a1; extra == 'test-full'
+Requires-Dist: backports-zstd; (python_version < '3.14') and extra == 'test-full'
 Requires-Dist: cloudpickle; extra == 'test-full'
 Requires-Dist: dask; extra == 'test-full'
 Requires-Dist: distributed; extra == 'test-full'
@@ -123,7 +124,7 @@ Requires-Dist: lz4; extra == 'test-full'
 Requires-Dist: notebook; extra == 'test-full'
 Requires-Dist: numpy; extra == 'test-full'
 Requires-Dist: ocifs; extra == 'test-full'
-Requires-Dist: pandas; extra == 'test-full'
+Requires-Dist: pandas<3.0.0; extra == 'test-full'
 Requires-Dist: panel; extra == 'test-full'
 Requires-Dist: paramiko; extra == 'test-full'
 Requires-Dist: pyarrow; extra == 'test-full'

{fsspec-2025.12.0.dist-info → fsspec-2026.2.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 fsspec/__init__.py,sha256=L7qwNBU1iMNQd8Of87HYSNFT9gWlNMSESaJC8fY0AaQ,2053
-fsspec/_version.py,sha256=u_WCUBrOC78o9bqe-21i1S8FKT43ei4dqyL4oKUga9g,712
+fsspec/_version.py,sha256=AEamMn8IHx_wGXrogRYVBycv6M5u2_UCmFedn7R8hYI,710
 fsspec/archive.py,sha256=vM6t_lgV6lBWbBYwpm3S4ofBQFQxUPr5KkDQrrQcQro,2411
 fsspec/asyn.py,sha256=LP_OicTWXmKHe31wBoYs2MrrNf8rmlhjVeGg5AqvVy8,36630
-fsspec/caching.py,sha256=B2xeDz9-VDgr_dDeVOTNRq3vaS9zVUe0nxtOBgsrjUk,34260
+fsspec/caching.py,sha256=8IJ4rgcWnvq_b_DqlcMGJ-K59d4Db5O9Gz8PkATAgHo,34023
 fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210
-fsspec/compression.py,sha256=gBK2MV_oTFVW2XDq8bZVbYQKYrl6JDUou6_-kyvmxuk,5086
+fsspec/compression.py,sha256=3v_Fe39gzRRWfaeXpzNjAGPqgTzmETYRCo3qHVqD3po,5132
 fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
 fsspec/conftest.py,sha256=uWfm_Qs5alPRxOhRpDfQ0-1jqSJ54pni4y96IxOREXM,3446
 fsspec/core.py,sha256=lc7XSnZU6_C6xljp7Z_xEGN3V7704hbeQLkxvPP0wds,24173
@@ -15,23 +15,23 @@ fsspec/generic.py,sha256=9QHQYMNb-8w8-eYuIqShcTjO_LeHXFoQTyt8J5oEq5Q,13482
 fsspec/gui.py,sha256=CQ7QsrTpaDlWSLNOpwNoJc7khOcYXIZxmrAJN9bHWQU,14002
 fsspec/json.py,sha256=4EBZ-xOmRiyxmIqPIwxmDImosRQ7io7qBM2xjJPsEE4,3768
 fsspec/mapping.py,sha256=m2ndB_gtRBXYmNJg0Ie1-BVR75TFleHmIQBzC-yWhjU,8343
-fsspec/parquet.py,sha256=vpOuoxg0y0iS2yoiXeAUcWB0n-wct2x2L6Vmn_O-hRQ,20668
-fsspec/registry.py,sha256=Kvv7NEqvIDhI2PXoYmwi4Z9RHAjrLMhGRu3m492801s,12157
+fsspec/parquet.py,sha256=xGW3xfd9js7hrre7qN85XpSM0A1FObqkTcAv_H2xSwY,20505
+fsspec/registry.py,sha256=o7EGl8TEaLkcwN53X_103arzuzJeeOoVaNUWnPiXgf0,12148
 fsspec/spec.py,sha256=Ym-Ust6LRjHgbhrmvNqwOBZxoVnaw3g3xHXMZGHx_xg,77692
 fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398
-fsspec/utils.py,sha256=5DOxB_eE-wNHwrc6zp3h-oMp5mCLw4tsQZqqPTjLDmM,23136
+fsspec/utils.py,sha256=E24ji0XLWC6n3bw2sHA28OYxrGU9Wy_al2XydsRgrRk,23623
 fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fsspec/implementations/arrow.py,sha256=6BaSEBZ4nb8UuY6NsyFevGzXcdJWamt3qEHjMe2S-W8,8831
+fsspec/implementations/arrow.py,sha256=8FhvcvOYLZNMMegCYFFCEHgEqig8AkOU7Ehb8XfcgnA,8890
 fsspec/implementations/asyn_wrapper.py,sha256=3lfJkGs6D_AwRBdxTSYlL-RCVdaXBZ9Itys2P5o5Si0,3738
 fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421
 fsspec/implementations/cache_metadata.py,sha256=ipIe4S8nlU_M9oRJkvTqr-b0tcbXVZsxH3GxaelaNOY,8502
-fsspec/implementations/cached.py,sha256=gp1eaM2X7ix2eGRDaC8rtTuO0icK6hbz3yDXh9YdB0E,35392
+fsspec/implementations/cached.py,sha256=67ipbj-3o8O1zMGR11rZ_IWCi_7h-VRYpEAowFXqrvA,36175
 fsspec/implementations/chained.py,sha256=iGivpNaHUFjB_ea0-HAPhcmm6CL8qnDf270PSj7JwuE,680
 fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
 fsspec/implementations/data.py,sha256=IhOGDkacYp5gkl9jhEu4msQfZPb0gS5Q_ml7Mbr6dgQ,1627
 fsspec/implementations/dbfs.py,sha256=1cvvC6KBWOb8pBVpc01xavVbEPXO1xsgZvPD7H73M9k,16217
 fsspec/implementations/dirfs.py,sha256=VNj6gPMfmmLPK4wxbtxt7mUqW7xkh2XDgMmEmSK_E1c,12166
-fsspec/implementations/ftp.py,sha256=bzL_TgH77nMMtTMewRGkbq4iObSHGu7YoMRCXBH4nrc,11639
+fsspec/implementations/ftp.py,sha256=fJhaMIKq2RvzYlLwG3bewy2jq4iRqjVt1aIpwtUIRwI,13235
 fsspec/implementations/gist.py,sha256=Y6jTDrE-wuTwvpPyAQDuuOMBGxlajafKWoB1_yX6jdY,8528
 fsspec/implementations/git.py,sha256=qBDWMz5LNllPqVjr5jf_1FuNha4P5lyQI3IlhYg-wUE,3731
 fsspec/implementations/github.py,sha256=aCsZL8UvXZgdkcB1RUs3DdLeNrjLKcFsFYeQFDWbBFo,11653
@@ -39,14 +39,14 @@ fsspec/implementations/http.py,sha256=-AV5qeNpBWqnsmgnIO9Ily9B6--SR4sQJ7G4cBHarG
 fsspec/implementations/http_sync.py,sha256=UmBqd938ebwVjYgVtzg-ysG3ZoGhIJw0wFtQAfxV3Aw,30332
 fsspec/implementations/jupyter.py,sha256=q1PlQ66AAswGFyr8MFKWyobaV2YekMWRtqENBDQtD28,4002
 fsspec/implementations/libarchive.py,sha256=SpIA1F-zf7kb2-VYUVuhMrXTBOhBxUXKgEW1RaAdDoA,7098
-fsspec/implementations/local.py,sha256=DQeK7jRGv4_mJAweLKALO5WzIIkjXxZ_jRvwQ_xadSA,16936
+fsspec/implementations/local.py,sha256=ERDUdXdRI8AvRX06icXaDKwO-hcQgivc7EorqnayFFM,17028
 fsspec/implementations/memory.py,sha256=TDdLtSPWXxZKrrVGwmc3uS3oK_2mlcVTk2BiqR8IeII,10507
 fsspec/implementations/reference.py,sha256=xSUpB8o_QFAZiVJE2dt78QZMCUMLo5TaJ27e5DwDAfg,48814
 fsspec/implementations/sftp.py,sha256=L9pZOa6eLUWfJNtxkxeG2YI96SQwrM5Hj6ocyUZXUbg,5923
 fsspec/implementations/smb.py,sha256=5fhu8h06nOLBPh2c48aT7WBRqh9cEcbIwtyu06wTjec,15236
 fsspec/implementations/tar.py,sha256=dam78Tp_CozybNqCY2JYgGBS3Uc9FuJUAT9oB0lolOs,4111
-fsspec/implementations/webhdfs.py,sha256=PUgZM9HbVPAeW4u4B-rWl8wTcKKpPhwZO7xcINDmTNQ,16779
-fsspec/implementations/zip.py,sha256=9LBMHPft2OutJl2Ft-r9u_z3GptLkc2n91ur2A3bCbg,6072
+fsspec/implementations/webhdfs.py,sha256=osF2m0nhDil6sbMzYW_4DZzhxF4ygtb59XDiybd9Fyg,17589
+fsspec/implementations/zip.py,sha256=6f3z0s12tDbz1RMx7iDc3JDx730IAaKDdx7tf_XYDp0,6151
 fsspec/tests/abstract/__init__.py,sha256=4xUJrv7gDgc85xAOz1p-V_K1hrsdMWTSa0rviALlJk8,10181
 fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973
 fsspec/tests/abstract/copy.py,sha256=gU5-d97U3RSde35Vp4RxPY4rWwL744HiSrJ8IBOp9-8,19967
@@ -55,7 +55,7 @@ fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1
 fsspec/tests/abstract/open.py,sha256=Fi2PBPYLbRqysF8cFm0rwnB41kMdQVYjq8cGyDXp3BU,329
 fsspec/tests/abstract/pipe.py,sha256=LFzIrLCB5GLXf9rzFKJmE8AdG7LQ_h4bJo70r8FLPqM,402
 fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201
-fsspec-2025.12.0.dist-info/METADATA,sha256=Fm2bIMio0NYy1EdMfxgGq4OE2kE5nt1qZDaDYbOuC0M,10401
-fsspec-2025.12.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-fsspec-2025.12.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
-fsspec-2025.12.0.dist-info/RECORD,,
+fsspec-2026.2.0.dist-info/METADATA,sha256=Pw6QhbyXeg-elb0hpWPiQRpAeMD_ApJ9vdNFoYCnrPs,10524
+fsspec-2026.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+fsspec-2026.2.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
+fsspec-2026.2.0.dist-info/RECORD,,