fsspec 2025.3.0__py3-none-any.whl → 2025.3.1__py3-none-any.whl

fsspec/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2025.3.0'
-__version_tuple__ = version_tuple = (2025, 3, 0)
+__version__ = version = '2025.3.1'
+__version_tuple__ = version_tuple = (2025, 3, 1)

fsspec/asyn.py

@@ -151,6 +151,18 @@ def get_loop():
     return loop[0]
 
 
+def reset_after_fork():
+    global lock
+    loop[0] = None
+    iothread[0] = None
+    lock = None
+
+
+if hasattr(os, "register_at_fork"):
+    # should be posix; this will do nothing for spawn or forkserver subprocesses
+    os.register_at_fork(after_in_child=reset_after_fork)
+
+
 if TYPE_CHECKING:
     import resource
 

fsspec/caching.py

@@ -37,6 +37,7 @@ T = TypeVar("T")
 logger = logging.getLogger("fsspec")
 
 Fetcher = Callable[[int, int], bytes]  # Maps (start, end) to bytes
+MultiFetcher = Callable[list[[int, int]], bytes]  # Maps [(start, end)] to bytes
 
 
 class BaseCache:
@@ -109,6 +110,26 @@ class MMapCache(BaseCache):
     Ensure there is enough disc space in the temporary location.
 
     This cache method might only work on posix
+
+    Parameters
+    ----------
+    blocksize: int
+        How far to read ahead in numbers of bytes
+    fetcher: Fetcher
+        Function of the form f(start, end) which gets bytes from remote as
+        specified
+    size: int
+        How big this file is
+    location: str
+        Where to create the temporary file. If None, a temporary file is
+        created using tempfile.TemporaryFile().
+    blocks: set[int]
+        Set of block numbers that have already been fetched. If None, an empty
+        set is created.
+    multi_fetcher: MultiFetcher
+        Function of the form f([(start, end)]) which gets bytes from remote
+        as specified. This function is used to fetch multiple blocks at once.
+        If not specified, the fetcher function is used instead.
     """
 
     name = "mmap"
@@ -120,10 +141,12 @@ class MMapCache(BaseCache):
         size: int,
         location: str | None = None,
         blocks: set[int] | None = None,
+        multi_fetcher: MultiFetcher | None = None,
     ) -> None:
         super().__init__(blocksize, fetcher, size)
         self.blocks = set() if blocks is None else blocks
         self.location = location
+        self.multi_fetcher = multi_fetcher
         self.cache = self._makefile()
 
     def _makefile(self) -> mmap.mmap | bytearray:
@@ -164,6 +187,8 @@ class MMapCache(BaseCache):
         # Count the number of blocks already cached
         self.hit_count += sum(1 for i in block_range if i in self.blocks)
 
+        ranges = []
+
         # Consolidate needed blocks.
         # Algorithm adapted from Python 2.x itertools documentation.
         # We are grouping an enumerated sequence of blocks. By comparing when the difference
@@ -185,13 +210,27 @@ class MMapCache(BaseCache):
             logger.debug(
                 f"MMap get blocks {_blocks[0]}-{_blocks[-1]} ({sstart}-{send})"
             )
-            self.cache[sstart:send] = self.fetcher(sstart, send)
+            ranges.append((sstart, send))
 
             # Update set of cached blocks
             self.blocks.update(_blocks)
             # Update cache statistics with number of blocks we had to cache
             self.miss_count += len(_blocks)
 
+        if not ranges:
+            return self.cache[start:end]
+
+        if self.multi_fetcher:
+            logger.debug(f"MMap get blocks {ranges}")
+            for idx, r in enumerate(self.multi_fetcher(ranges)):
+                (sstart, send) = ranges[idx]
+                logger.debug(f"MMap copy block ({sstart}-{send}")
+                self.cache[sstart:send] = r
+        else:
+            for sstart, send in ranges:
+                logger.debug(f"MMap get block ({sstart}-{send}")
+                self.cache[sstart:send] = self.fetcher(sstart, send)
+
         return self.cache[start:end]
 
     def __getstate__(self) -> dict[str, Any]:

fsspec/implementations/cached.py

@@ -362,7 +362,19 @@ class CachingFileSystem(AbstractFileSystem):
                 )
         else:
             detail["blocksize"] = f.blocksize
-        f.cache = MMapCache(f.blocksize, f._fetch_range, f.size, fn, blocks)
+
+        def _fetch_ranges(ranges):
+            return self.fs.cat_ranges(
+                [path] * len(ranges),
+                [r[0] for r in ranges],
+                [r[1] for r in ranges],
+                **kwargs,
+            )
+
+        multi_fetcher = None if self.compression else _fetch_ranges
+        f.cache = MMapCache(
+            f.blocksize, f._fetch_range, f.size, fn, blocks, multi_fetcher=multi_fetcher
+        )
         close = f.close
         f.close = lambda: self.close_and_update(f, close)
         self.save_cache()

fsspec/implementations/github.py

@@ -1,6 +1,6 @@
-import requests
+import base64
 
-import fsspec
+import requests
 
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
@@ -16,8 +16,10 @@ class GithubFileSystem(AbstractFileSystem):
     repository. You may specify a point in the repos history, by SHA, branch
     or tag (default is current master).
 
-    Given that code files tend to be small, and that github does not support
-    retrieving partial content, we always fetch whole files.
+    For files less than 1 MB in size, file content is returned directly in a
+    MemoryFile. For larger files, or for files tracked by git-lfs, file content
+    is returned as an HTTPFile wrapping the ``download_url`` provided by the
+    GitHub API.
 
     When using fsspec.open, allows URIs of the form:
 
@@ -36,7 +38,7 @@ class GithubFileSystem(AbstractFileSystem):
     """
 
     url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}"
-    rurl = "https://raw.githubusercontent.com/{org}/{repo}/{sha}/{path}"
+    content_url = "https://api.github.com/repos/{org}/{repo}/contents/{path}?ref={sha}"
     protocol = "github"
     timeout = (60, 60)  # connect, read timeouts
 
@@ -63,6 +65,12 @@ class GithubFileSystem(AbstractFileSystem):
 
         self.root = sha
         self.ls("")
+        try:
+            from .http import HTTPFileSystem
+
+            self.http_fs = HTTPFileSystem(**kwargs)
+        except ImportError:
+            self.http_fs = None
 
     @property
     def kw(self):
@@ -212,28 +220,48 @@ class GithubFileSystem(AbstractFileSystem):
         path,
         mode="rb",
         block_size=None,
-        autocommit=True,
         cache_options=None,
         sha=None,
         **kwargs,
     ):
         if mode != "rb":
             raise NotImplementedError
-        url = self.rurl.format(
+
+        # construct a url to hit the GitHub API's repo contents API
+        url = self.content_url.format(
             org=self.org, repo=self.repo, path=path, sha=sha or self.root
         )
+
+        # make a request to this API, and parse the response as JSON
         r = requests.get(url, timeout=self.timeout, **self.kw)
         if r.status_code == 404:
             raise FileNotFoundError(path)
         r.raise_for_status()
-        return MemoryFile(None, None, r.content)
-
-    def cat(self, path, recursive=False, on_error="raise", **kwargs):
-        paths = self.expand_path(path, recursive=recursive)
-        urls = [
-            self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root)
-            for u, sh in paths
-        ]
-        fs = fsspec.filesystem("http")
-        data = fs.cat(urls, on_error="return")
-        return {u: v for ((k, v), u) in zip(data.items(), urls)}
+        content_json = r.json()
+
+        # if the response's content key is not empty, try to parse it as base64
+        if content_json["content"]:
+            content = base64.b64decode(content_json["content"])
+
+            # as long as the content does not start with the string
+            # "version https://git-lfs.github.com/"
+            # then it is probably not a git-lfs pointer and we can just return
+            # the content directly
+            if not content.startswith(b"version https://git-lfs.github.com/"):
+                return MemoryFile(None, None, content)
+
+        # we land here if the content was not present in the first response
+        # (regular file over 1MB or git-lfs tracked file)
+        # in this case, we get let the HTTPFileSystem handle the download
+        if self.http_fs is None:
+            raise ImportError(
+                "Please install fsspec[http] to access github files >1 MB "
+                "or git-lfs tracked files."
+            )
+        return self.http_fs.open(
+            content_json["download_url"],
+            mode=mode,
+            block_size=block_size,
+            cache_options=cache_options,
+            **kwargs,
+        )

fsspec/implementations/http_sync.py

@@ -110,7 +110,6 @@ class ResponseProxy:
         self.stream = False
         return self.raw
 
-    @property
     def json(self):
         return loads(self.text)
 
@@ -201,7 +200,7 @@ class HTTPFileSystem(AbstractFileSystem):
     you are testing pyodide/pyscript integration***
     """
 
-    protocol = ("http", "https", "sync_http", "sync_https")
+    protocol = ("http", "https", "sync-http", "sync-https")
     sep = "/"
 
     def __init__(
@@ -269,7 +268,7 @@ class HTTPFileSystem(AbstractFileSystem):
 
     @property
     def fsid(self):
-        return "http_sync"
+        return "sync-http"
 
     def encode_url(self, url):
         if yarl:
@@ -279,8 +278,8 @@ class HTTPFileSystem(AbstractFileSystem):
     @classmethod
     def _strip_protocol(cls, path: str) -> str:
         """For HTTP, we always want to keep the full URL"""
-        path = path.replace("http_sync://", "http://").replace(
-            "https_sync://", "https://"
+        path = path.replace("sync-http://", "http://").replace(
+            "sync-https://", "https://"
         )
         return path
 
@@ -918,8 +917,8 @@ def _file_info(url, session, size_policy="head", **kwargs):
 def register():
     register_implementation("http", HTTPFileSystem, clobber=True)
     register_implementation("https", HTTPFileSystem, clobber=True)
-    register_implementation("sync_http", HTTPFileSystem, clobber=True)
-    register_implementation("sync_https", HTTPFileSystem, clobber=True)
+    register_implementation("sync-http", HTTPFileSystem, clobber=True)
+    register_implementation("sync-https", HTTPFileSystem, clobber=True)
 
 
 register()

fsspec/registry.py

@@ -73,7 +73,7 @@ known_implementations = {
         "err": "pyarrow and local java libraries required for HDFS",
     },
     "async_wrapper": {
-        "class": "morefs.asyn_wrapper.AsyncWrapperFileSystem",
+        "class": "fsspec.asyn_wrapper.AsyncWrapperFileSystem",
     },
     "asynclocal": {
         "class": "morefs.asyn_local.AsyncLocalFileSystem",
@@ -248,7 +248,7 @@ def get_filesystem_class(protocol):
         try:
             register_implementation(protocol, _import_class(bit["class"]))
         except ImportError as e:
-            raise ImportError(bit["err"]) from e
+            raise ImportError(bit.get("err")) from e
     cls = registry[protocol]
     if getattr(cls, "protocol", None) in ("abstract", None):
         cls.protocol = protocol

{fsspec-2025.3.0.dist-info → fsspec-2025.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec
-Version: 2025.3.0
+Version: 2025.3.1
 Summary: File-system specification
 Project-URL: Changelog, https://filesystem-spec.readthedocs.io/en/latest/changelog.html
 Project-URL: Documentation, https://filesystem-spec.readthedocs.io/en/latest/

{fsspec-2025.3.0.dist-info → fsspec-2025.3.1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 fsspec/__init__.py,sha256=l9MJaNNV2d4wKpCtMvXDr55n92DkdrAayGy3F9ICjzk,1998
-fsspec/_version.py,sha256=mWdw-j-kCQGfmn6901GQSyjFXi2pK3h-NrKTuLRVqCw,517
+fsspec/_version.py,sha256=UTo0k75fb91X4ciJgm5Nvk4otinCDkfq8764_G1IYzQ,517
 fsspec/archive.py,sha256=vM6t_lgV6lBWbBYwpm3S4ofBQFQxUPr5KkDQrrQcQro,2411
-fsspec/asyn.py,sha256=rsnCsFUmBZmKJqg9m-IDWInoQtE4wV0rGDZEXZwuU3c,36500
-fsspec/caching.py,sha256=oHVy9zpy4Oqk5f1t3-Q31bbw0tsmfddGGKLJs__OdKA,32790
+fsspec/asyn.py,sha256=VJ2jBdYgUjV4_dETpKeCp2wF1XHAdeUET95d2HqNZck,36776
+fsspec/caching.py,sha256=pGEBXy2hUh7YBZdZAcCT-38LFmJFDWU9gbmQi05AvFY,34292
 fsspec/callbacks.py,sha256=BDIwLzK6rr_0V5ch557fSzsivCElpdqhXr5dZ9Te-EE,9210
 fsspec/compression.py,sha256=jCSUMJu-zSNyrusnHT0wKXgOd1tTJR6vM126i5SR5Zc,4865
 fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
@@ -16,7 +16,7 @@ fsspec/gui.py,sha256=xBnHL2-r0LVwhDAtnHoPpXts7jd4Z32peawCJiI-7lI,13975
 fsspec/json.py,sha256=65sQ0Y7mTj33u_Y4IId5up4abQ3bAel4E4QzbKMiQSg,3826
 fsspec/mapping.py,sha256=m2ndB_gtRBXYmNJg0Ie1-BVR75TFleHmIQBzC-yWhjU,8343
 fsspec/parquet.py,sha256=6ibAmG527L5JNFS0VO8BDNlxHdA3bVYqdByeiFgpUVM,19448
-fsspec/registry.py,sha256=Bbur6KJilN62hx_lSRgCx9HlKrv91pLpdEPG7Vzme1M,11566
+fsspec/registry.py,sha256=5kz-61fkb29lgSLzXWvk67ft6ooeYJR24Hs1583rD8w,11570
 fsspec/spec.py,sha256=l7ZEbgLsnrFuS-yrGl9re6ia1Yts1_10RqGV_mT-5P8,76032
 fsspec/transaction.py,sha256=xliRG6U2Zf3khG4xcw9WiB-yAoqJSHEGK_VjHOdtgo0,2398
 fsspec/utils.py,sha256=A11t25RnpiQ30RO6xeR0Qqlu3fGj8bnc40jg08tlYSI,22980
@@ -25,16 +25,16 @@ fsspec/implementations/arrow.py,sha256=721Dikne_lV_0tlgk9jyKmHL6W-5MT0h2LKGvOYQT
 fsspec/implementations/asyn_wrapper.py,sha256=PNkYdHiLVWwk-GJok5O6dTnhPwDaSU9QTtBTE9CIRec,3082
 fsspec/implementations/cache_mapper.py,sha256=W4wlxyPxZbSp9ItJ0pYRVBMh6bw9eFypgP6kUYuuiI4,2421
 fsspec/implementations/cache_metadata.py,sha256=pcOJYcBQY5OaC7Yhw0F3wjg08QLYApGmoISCrbs59ks,8511
-fsspec/implementations/cached.py,sha256=KA6c4jqrGeeg8WNPLsh8FkL3KeRAQtGLzKw18vSF1CI,32820
+fsspec/implementations/cached.py,sha256=2EqeHZi6TKZzr222ZyXHjuYBnx_1g_HWXj1CJBFsXbc,33173
 fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
 fsspec/implementations/data.py,sha256=LDLczxRh8h7x39Zjrd-GgzdQHr78yYxDlrv2C9Uxb5E,1658
 fsspec/implementations/dbfs.py,sha256=XwpotuS_ncz3XK1dkUteww9GnTja7HoY91c0m4GUfwI,15092
 fsspec/implementations/dirfs.py,sha256=f1sGnQ9Vf0xTxrXo4jDeBy4Qfq3RTqAEemqBSeb0hwY,12108
 fsspec/implementations/ftp.py,sha256=sorsczLp_2J3ukONsbZY-11sRZP6H5a3V7XXf6o6ip0,11936
 fsspec/implementations/git.py,sha256=4SElW9U5d3k3_ITlvUAx59Yk7XLNRTqkGa2C3hCUkWM,3754
-fsspec/implementations/github.py,sha256=eAn1kJ7VeWR6gVoVRLBYclF_rQDXSJU-xzMXpvPQWqs,8002
+fsspec/implementations/github.py,sha256=tvLepeDtBsFHYocsD7XS1W6r8wxaoK57Us3kHGisnjU,9146
 fsspec/implementations/http.py,sha256=_gLt0yGbVOYWvE9pK81WCC-3TgbOMOKJYllBU72ALo8,30138
-fsspec/implementations/http_sync.py,sha256=vHf2_O9RRlaW6k-R4hsS-91BkqyPIlc0zwo_ENNGU4U,30147
+fsspec/implementations/http_sync.py,sha256=UydDqSdUBdhiJ1KufzV8rKGrTftFR4QmNV0safILb8g,30133
 fsspec/implementations/jupyter.py,sha256=B2uj7OEm7yIk-vRSsO37_ND0t0EBvn4B-Su43ibN4Pg,3811
 fsspec/implementations/libarchive.py,sha256=5_I2DiLXwQ1JC8x-K7jXu-tBwhO9dj7tFLnb0bTnVMQ,7102
 fsspec/implementations/local.py,sha256=g2iK8uWPGkSiI6bwmnIRXhJMQvTegCmXZ8Kb8ojhvAo,15543
@@ -53,7 +53,7 @@ fsspec/tests/abstract/mv.py,sha256=k8eUEBIrRrGMsBY5OOaDXdGnQUKGwDIfQyduB6YD3Ns,1
 fsspec/tests/abstract/open.py,sha256=Fi2PBPYLbRqysF8cFm0rwnB41kMdQVYjq8cGyDXp3BU,329
 fsspec/tests/abstract/pipe.py,sha256=LFzIrLCB5GLXf9rzFKJmE8AdG7LQ_h4bJo70r8FLPqM,402
 fsspec/tests/abstract/put.py,sha256=7aih17OKB_IZZh1Mkq1eBDIjobhtMQmI8x-Pw-S_aZk,21201
-fsspec-2025.3.0.dist-info/METADATA,sha256=L2OjjikjNWpjvp8i5_ec515Cd09eUi8qZ_3peQmWG8M,11747
-fsspec-2025.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-fsspec-2025.3.0.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
-fsspec-2025.3.0.dist-info/RECORD,,
+fsspec-2025.3.1.dist-info/METADATA,sha256=CJaBlMGwxuW44faR38hNdBi2I1sydTS-KgC_hp6HzWc,11747
+fsspec-2025.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+fsspec-2025.3.1.dist-info/licenses/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
+fsspec-2025.3.1.dist-info/RECORD,,