fsspec 2025.2.0__tar.gz → 2025.3.1__tar.gz

{fsspec-2025.2.0 → fsspec-2025.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.2.0
+Version: 2025.3.1
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/

fsspec-2025.3.1/Untitled.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

fsspec-2025.3.1/correct_permissions.bin

@@ -0,0 +1 @@
+hello

{fsspec-2025.2.0 → fsspec-2025.3.1}/docs/source/changelog.rst

@@ -1,6 +1,39 @@
 Changelog
 =========
 
+2025.3.1
+--------
+
+Enhancements
+
+- LFS support in github: (#1810)
+
+Fixes
+
+- json should be a method fo requests shim (#1814)
+- don't raise if known_implementation has no given error string (#1804)
+
+Other
+
+- rename protocols for sync-http (#1810)
+
+
+2025.3.0
+--------
+
+Enhancements
+
+- add pipe_file to HTTP (#1799, 1801)
+- add sync http for pyodide (#1177)
+- ls performance for local and detail=False (#1789)
+
+Fixes
+
+- dir/info consistency in dirfs (#1798)
+- referenceFS async consistency (#1794, 1795)
+- CI (#1793)
+
+
 2025.2.0
 --------
 

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2025.2.0'
-__version_tuple__ = version_tuple = (2025, 2, 0)
+__version__ = version = '2025.3.1'
+__version_tuple__ = version_tuple = (2025, 3, 1)

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/asyn.py

@@ -151,6 +151,18 @@ def get_loop():
     return loop[0]
 
 
+def reset_after_fork():
+    global lock
+    loop[0] = None
+    iothread[0] = None
+    lock = None
+
+
+if hasattr(os, "register_at_fork"):
+    # should be posix; this will do nothing for spawn or forkserver subprocesses
+    os.register_at_fork(after_in_child=reset_after_fork)
+
+
 if TYPE_CHECKING:
     import resource
 

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/caching.py

@@ -37,6 +37,7 @@ T = TypeVar("T")
 logger = logging.getLogger("fsspec")
 
 Fetcher = Callable[[int, int], bytes]  # Maps (start, end) to bytes
+MultiFetcher = Callable[list[[int, int]], bytes]  # Maps [(start, end)] to bytes
 
 
 class BaseCache:
@@ -109,6 +110,26 @@ class MMapCache(BaseCache):
     Ensure there is enough disc space in the temporary location.
 
     This cache method might only work on posix
+
+    Parameters
+    ----------
+    blocksize: int
+        How far to read ahead in numbers of bytes
+    fetcher: Fetcher
+        Function of the form f(start, end) which gets bytes from remote as
+        specified
+    size: int
+        How big this file is
+    location: str
+        Where to create the temporary file. If None, a temporary file is
+        created using tempfile.TemporaryFile().
+    blocks: set[int]
+        Set of block numbers that have already been fetched. If None, an empty
+        set is created.
+    multi_fetcher: MultiFetcher
+        Function of the form f([(start, end)]) which gets bytes from remote
+        as specified. This function is used to fetch multiple blocks at once.
+        If not specified, the fetcher function is used instead.
     """
 
     name = "mmap"
@@ -120,10 +141,12 @@ class MMapCache(BaseCache):
         size: int,
         location: str | None = None,
         blocks: set[int] | None = None,
+        multi_fetcher: MultiFetcher | None = None,
     ) -> None:
         super().__init__(blocksize, fetcher, size)
         self.blocks = set() if blocks is None else blocks
         self.location = location
+        self.multi_fetcher = multi_fetcher
         self.cache = self._makefile()
 
     def _makefile(self) -> mmap.mmap | bytearray:
@@ -164,6 +187,8 @@ class MMapCache(BaseCache):
         # Count the number of blocks already cached
         self.hit_count += sum(1 for i in block_range if i in self.blocks)
 
+        ranges = []
+
         # Consolidate needed blocks.
         # Algorithm adapted from Python 2.x itertools documentation.
         # We are grouping an enumerated sequence of blocks. By comparing when the difference
@@ -185,13 +210,27 @@ class MMapCache(BaseCache):
             logger.debug(
                 f"MMap get blocks {_blocks[0]}-{_blocks[-1]} ({sstart}-{send})"
             )
-            self.cache[sstart:send] = self.fetcher(sstart, send)
+            ranges.append((sstart, send))
 
             # Update set of cached blocks
             self.blocks.update(_blocks)
             # Update cache statistics with number of blocks we had to cache
             self.miss_count += len(_blocks)
 
+        if not ranges:
+            return self.cache[start:end]
+
+        if self.multi_fetcher:
+            logger.debug(f"MMap get blocks {ranges}")
+            for idx, r in enumerate(self.multi_fetcher(ranges)):
+                (sstart, send) = ranges[idx]
+                logger.debug(f"MMap copy block ({sstart}-{send}")
+                self.cache[sstart:send] = r
+        else:
+            for sstart, send in ranges:
+                logger.debug(f"MMap get block ({sstart}-{send}")
+                self.cache[sstart:send] = self.fetcher(sstart, send)
+
         return self.cache[start:end]
 
     def __getstate__(self) -> dict[str, Any]:

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/core.py

@@ -452,7 +452,7 @@ def open(
     newline: bytes or None
         Used for line terminator in text mode. If None, uses system default;
         if blank, uses no translation.
-    expand: bool or Nonw
+    expand: bool or None
         Whether to regard file paths containing special glob characters as needing
         expansion (finding the first match) or absolute. Setting False allows using
         paths which do embed such characters. If None (default), this argument

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/implementations/asyn_wrapper.py

@@ -2,7 +2,7 @@ import asyncio
 import functools
 import inspect
 
-from fsspec.asyn import AsyncFileSystem
+from fsspec.asyn import AsyncFileSystem, running_async
 
 
 def async_wrapper(func, obj=None):
@@ -42,10 +42,14 @@ class AsyncFileSystemWrapper(AsyncFileSystem):
         The synchronous filesystem instance to wrap.
     """
 
-    def __init__(self, sync_fs, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.asynchronous = True
-        self.sync_fs = sync_fs
+    protocol = "async_wrapper"
+    cachable = False
+
+    def __init__(self, fs, *args, asynchronous=None, **kwargs):
+        if asynchronous is None:
+            asynchronous = running_async()
+        super().__init__(*args, asynchronous=asynchronous, **kwargs)
+        self.sync_fs = fs
         self.protocol = self.sync_fs.protocol
         self._wrap_all_sync_methods()
 

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/implementations/cached.py

@@ -362,7 +362,19 @@ class CachingFileSystem(AbstractFileSystem):
                 )
         else:
             detail["blocksize"] = f.blocksize
-        f.cache = MMapCache(f.blocksize, f._fetch_range, f.size, fn, blocks)
+
+        def _fetch_ranges(ranges):
+            return self.fs.cat_ranges(
+                [path] * len(ranges),
+                [r[0] for r in ranges],
+                [r[1] for r in ranges],
+                **kwargs,
+            )
+
+        multi_fetcher = None if self.compression else _fetch_ranges
+        f.cache = MMapCache(
+            f.blocksize, f._fetch_range, f.size, fn, blocks, multi_fetcher=multi_fetcher
+        )
         close = f.close
         f.close = lambda: self.close_and_update(f, close)
         self.save_cache()

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/implementations/dirfs.py

@@ -36,8 +36,6 @@ class DirFileSystem(AsyncFileSystem):
         super().__init__(**storage_options)
         if fs is None:
             fs = filesystem(protocol=target_protocol, **(target_options or {}))
-        if (path is not None) ^ (fo is not None) is False:
-            raise ValueError("Provide path or fo, not both")
         path = path or fo
 
         if self.asynchronous and not fs.async_impl:
@@ -233,10 +231,16 @@ class DirFileSystem(AsyncFileSystem):
         return self.fs.exists(self._join(path))
 
     async def _info(self, path, **kwargs):
-        return await self.fs._info(self._join(path), **kwargs)
+        info = await self.fs._info(self._join(path), **kwargs)
+        info = info.copy()
+        info["name"] = self._relpath(info["name"])
+        return info
 
     def info(self, path, **kwargs):
-        return self.fs.info(self._join(path), **kwargs)
+        info = self.fs.info(self._join(path), **kwargs)
+        info = info.copy()
+        info["name"] = self._relpath(info["name"])
+        return info
 
     async def _ls(self, path, detail=True, **kwargs):
         ret = (await self.fs._ls(self._join(path), detail=detail, **kwargs)).copy()

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/implementations/github.py

@@ -1,6 +1,6 @@
-import requests
+import base64
 
-import fsspec
+import requests
 
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
@@ -16,8 +16,10 @@ class GithubFileSystem(AbstractFileSystem):
     repository. You may specify a point in the repos history, by SHA, branch
     or tag (default is current master).
 
-    Given that code files tend to be small, and that github does not support
-    retrieving partial content, we always fetch whole files.
+    For files less than 1 MB in size, file content is returned directly in a
+    MemoryFile. For larger files, or for files tracked by git-lfs, file content
+    is returned as an HTTPFile wrapping the ``download_url`` provided by the
+    GitHub API.
 
     When using fsspec.open, allows URIs of the form:
 
@@ -36,7 +38,7 @@ class GithubFileSystem(AbstractFileSystem):
     """
 
     url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}"
-    rurl = "https://raw.githubusercontent.com/{org}/{repo}/{sha}/{path}"
+    content_url = "https://api.github.com/repos/{org}/{repo}/contents/{path}?ref={sha}"
     protocol = "github"
     timeout = (60, 60)  # connect, read timeouts
 
@@ -63,6 +65,12 @@ class GithubFileSystem(AbstractFileSystem):
 
         self.root = sha
         self.ls("")
+        try:
+            from .http import HTTPFileSystem
+
+            self.http_fs = HTTPFileSystem(**kwargs)
+        except ImportError:
+            self.http_fs = None
 
     @property
     def kw(self):
@@ -212,28 +220,48 @@ class GithubFileSystem(AbstractFileSystem):
         path,
         mode="rb",
         block_size=None,
-        autocommit=True,
         cache_options=None,
         sha=None,
         **kwargs,
     ):
         if mode != "rb":
             raise NotImplementedError
-        url = self.rurl.format(
+
+        # construct a url to hit the GitHub API's repo contents API
+        url = self.content_url.format(
             org=self.org, repo=self.repo, path=path, sha=sha or self.root
         )
+
+        # make a request to this API, and parse the response as JSON
         r = requests.get(url, timeout=self.timeout, **self.kw)
         if r.status_code == 404:
             raise FileNotFoundError(path)
         r.raise_for_status()
-        return MemoryFile(None, None, r.content)
-
-    def cat(self, path, recursive=False, on_error="raise", **kwargs):
-        paths = self.expand_path(path, recursive=recursive)
-        urls = [
-            self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root)
-            for u, sh in paths
-        ]
-        fs = fsspec.filesystem("http")
-        data = fs.cat(urls, on_error="return")
-        return {u: v for ((k, v), u) in zip(data.items(), urls)}
+        content_json = r.json()
+
+        # if the response's content key is not empty, try to parse it as base64
+        if content_json["content"]:
+            content = base64.b64decode(content_json["content"])
+
+            # as long as the content does not start with the string
+            # "version https://git-lfs.github.com/"
+            # then it is probably not a git-lfs pointer and we can just return
+            # the content directly
+            if not content.startswith(b"version https://git-lfs.github.com/"):
+                return MemoryFile(None, None, content)
+
+        # we land here if the content was not present in the first response
+        # (regular file over 1MB or git-lfs tracked file)
+        # in this case, we get let the HTTPFileSystem handle the download
+        if self.http_fs is None:
+            raise ImportError(
+                "Please install fsspec[http] to access github files >1 MB "
+                "or git-lfs tracked files."
+            )
+        return self.http_fs.open(
+            content_json["download_url"],
+            mode=mode,
+            block_size=block_size,
+            cache_options=cache_options,
+            **kwargs,
+        )

{fsspec-2025.2.0 → fsspec-2025.3.1}/fsspec/implementations/http.py

@@ -522,6 +522,30 @@ class HTTPFileSystem(AsyncFileSystem):
         except (FileNotFoundError, ValueError):
             return False
 
+    async def _pipe_file(self, path, value, mode="overwrite", **kwargs):
+        """
+        Write bytes to a remote file over HTTP.
+
+        Parameters
+        ----------
+        path : str
+            Target URL where the data should be written
+        value : bytes
+            Data to be written
+        mode : str
+            How to write to the file - 'overwrite' or 'append'
+        **kwargs : dict
+            Additional parameters to pass to the HTTP request
+        """
+        url = self._strip_protocol(path)
+        headers = kwargs.pop("headers", {})
+        headers["Content-Length"] = str(len(value))
+
+        session = await self.set_session()
+
+        async with session.put(url, data=value, headers=headers, **kwargs) as r:
+            r.raise_for_status()
+
 
 class HTTPFile(AbstractBufferedFile):
     """