fsspec 2024.3.1__py3-none-any.whl → 2024.6.0__py3-none-any.whl

fsspec/__init__.py

@@ -1,6 +1,7 @@
 from importlib.metadata import entry_points
 
-from . import _version, caching
+from . import caching
+from ._version import __version__  # noqa: F401
 from .callbacks import Callback
 from .compression import available_compressions
 from .core import get_fs_token_paths, open, open_files, open_local, url_to_fs
@@ -15,8 +16,6 @@ from .registry import (
 )
 from .spec import AbstractFileSystem
 
-__version__ = _version.get_versions()["version"]
-
 __all__ = [
     "AbstractFileSystem",
     "FSTimeoutError",

fsspec/_version.py

@@ -1,21 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
 
-# This file was generated by 'versioneer.py' (0.29) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
 
-import json
-
-version_json = '''
-{
- "date": "2024-03-18T15:33:58-0400",
- "dirty": false,
- "error": null,
- "full-revisionid": "47b445ae4c284a82dd15e0287b1ffc410e8fc470",
- "version": "2024.3.1"
-}
-'''  # END VERSION_JSON
-
-
-def get_versions():
-    return json.loads(version_json)
+__version__ = version = '2024.6.0'
+__version_tuple__ = version_tuple = (2024, 6, 0)

fsspec/caching.py

@@ -15,6 +15,7 @@ from typing import (
     ClassVar,
     Generic,
     NamedTuple,
+    Optional,
     OrderedDict,
     TypeVar,
 )
@@ -56,8 +57,13 @@ class BaseCache:
 
     def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None:
         self.blocksize = blocksize
+        self.nblocks = 0
         self.fetcher = fetcher
         self.size = size
+        self.hit_count = 0
+        self.miss_count = 0
+        # the bytes that we actually requested
+        self.total_requested_bytes = 0
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:
@@ -68,6 +74,36 @@ class BaseCache:
             return b""
         return self.fetcher(start, stop)
 
+    def _reset_stats(self) -> None:
+        """Reset hit and miss counts for a more ganular report e.g. by file."""
+        self.hit_count = 0
+        self.miss_count = 0
+        self.total_requested_bytes = 0
+
+    def _log_stats(self) -> str:
+        """Return a formatted string of the cache statistics."""
+        if self.hit_count == 0 and self.miss_count == 0:
+            # a cache that does nothing, this is for logs only
+            return ""
+        return " , %s: %d hits, %d misses, %d total requested bytes" % (
+            self.name,
+            self.hit_count,
+            self.miss_count,
+            self.total_requested_bytes,
+        )
+
+    def __repr__(self) -> str:
+        # TODO: use rich for better formatting
+        return f"""
+        <{self.__class__.__name__}:
+            block size  :   {self.blocksize}
+            block count :   {self.nblocks}
+            file size   :   {self.size}
+            cache hits  :   {self.hit_count}
+            cache misses:   {self.miss_count}
+            total requested bytes: {self.total_requested_bytes}>
+        """
+
 
 class MMapCache(BaseCache):
     """memory-mapped sparse file cache
@@ -126,13 +162,18 @@ class MMapCache(BaseCache):
         start_block = start // self.blocksize
         end_block = end // self.blocksize
         need = [i for i in range(start_block, end_block + 1) if i not in self.blocks]
+        hits = [i for i in range(start_block, end_block + 1) if i in self.blocks]
+        self.miss_count += len(need)
+        self.hit_count += len(hits)
         while need:
             # TODO: not a for loop so we can consolidate blocks later to
             # make fewer fetch calls; this could be parallel
             i = need.pop(0)
+
             sstart = i * self.blocksize
             send = min(sstart + self.blocksize, self.size)
-            logger.debug(f"MMap get block #{i} ({sstart}-{send}")
+            self.total_requested_bytes += send - sstart
+            logger.debug(f"MMap get block #{i} ({sstart}-{send})")
             self.cache[sstart:send] = self.fetcher(sstart, send)
             self.blocks.add(i)
 
@@ -176,16 +217,20 @@ class ReadAheadCache(BaseCache):
         l = end - start
         if start >= self.start and end <= self.end:
             # cache hit
+            self.hit_count += 1
             return self.cache[start - self.start : end - self.start]
         elif self.start <= start < self.end:
             # partial hit
+            self.miss_count += 1
             part = self.cache[start - self.start :]
             l -= len(part)
             start = self.end
         else:
             # miss
+            self.miss_count += 1
             part = b""
         end = min(self.size, end + self.blocksize)
+        self.total_requested_bytes += end - start
         self.cache = self.fetcher(start, end)  # new block replaces old
         self.start = start
         self.end = self.start + len(self.cache)
@@ -202,24 +247,39 @@ class FirstChunkCache(BaseCache):
     name = "first"
 
     def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None:
+        if blocksize > size:
+            # this will buffer the whole thing
+            blocksize = size
         super().__init__(blocksize, fetcher, size)
         self.cache: bytes | None = None
 
     def _fetch(self, start: int | None, end: int | None) -> bytes:
         start = start or 0
-        end = end or self.size
+        if start > self.size:
+            logger.debug("FirstChunkCache: requested start > file size")
+            return b""
+
+        end = min(end, self.size)
+
         if start < self.blocksize:
             if self.cache is None:
+                self.miss_count += 1
                 if end > self.blocksize:
+                    self.total_requested_bytes += end
                     data = self.fetcher(0, end)
                     self.cache = data[: self.blocksize]
                     return data[start:]
                 self.cache = self.fetcher(0, self.blocksize)
+                self.total_requested_bytes += self.blocksize
             part = self.cache[start:end]
             if end > self.blocksize:
+                self.total_requested_bytes += end - self.blocksize
                 part += self.fetcher(self.blocksize, end)
+            self.hit_count += 1
             return part
         else:
+            self.miss_count += 1
+            self.total_requested_bytes += end - start
             return self.fetcher(start, end)
 
 
@@ -256,12 +316,6 @@ class BlockCache(BaseCache):
         self.maxblocks = maxblocks
         self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block)
 
-    def __repr__(self) -> str:
-        return (
-            f"<BlockCache blocksize={self.blocksize}, "
-            f"size={self.size}, nblocks={self.nblocks}>"
-        )
-
     def cache_info(self):
         """
         The statistics on the block cache.
@@ -319,6 +373,8 @@ class BlockCache(BaseCache):
 
         start = block_number * self.blocksize
         end = start + self.blocksize
+        self.total_requested_bytes += end - start
+        self.miss_count += 1
         logger.info("BlockCache fetching block %d", block_number)
         block_contents = super()._fetch(start, end)
         return block_contents
@@ -339,6 +395,7 @@ class BlockCache(BaseCache):
         start_pos = start % self.blocksize
         end_pos = end % self.blocksize
 
+        self.hit_count += 1
         if start_block_number == end_block_number:
             block: bytes = self._fetch_block_cached(start_block_number)
             return block[start_pos:end_pos]
@@ -404,6 +461,7 @@ class BytesCache(BaseCache):
         ):
             # cache hit: we have all the required data
             offset = start - self.start
+            self.hit_count += 1
             return self.cache[offset : offset + end - start]
 
         if self.blocksize:
@@ -418,17 +476,22 @@ class BytesCache(BaseCache):
             self.end is None or end > self.end
         ):
             # First read, or extending both before and after
+            self.total_requested_bytes += bend - start
+            self.miss_count += 1
             self.cache = self.fetcher(start, bend)
             self.start = start
         else:
             assert self.start is not None
             assert self.end is not None
+            self.miss_count += 1
 
             if start < self.start:
                 if self.end is None or self.end - end > self.blocksize:
+                    self.total_requested_bytes += bend - start
                     self.cache = self.fetcher(start, bend)
                     self.start = start
                 else:
+                    self.total_requested_bytes += self.start - start
                     new = self.fetcher(start, self.start)
                     self.start = start
                     self.cache = new + self.cache
@@ -436,9 +499,11 @@ class BytesCache(BaseCache):
                 if self.end > self.size:
                     pass
                 elif end - self.end > self.blocksize:
+                    self.total_requested_bytes += bend - start
                     self.cache = self.fetcher(start, bend)
                     self.start = start
                 else:
+                    self.total_requested_bytes += bend - self.end
                     new = self.fetcher(self.end, bend)
                     self.cache = self.cache + new
 
@@ -470,10 +535,13 @@ class AllBytes(BaseCache):
     ) -> None:
         super().__init__(blocksize, fetcher, size)  # type: ignore[arg-type]
         if data is None:
+            self.miss_count += 1
+            self.total_requested_bytes += self.size
             data = self.fetcher(0, self.size)
         self.data = data
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
+        self.hit_count += 1
         return self.data[start:stop]
 
 
@@ -507,7 +575,7 @@ class KnownPartsOfAFile(BaseCache):
         blocksize: int,
         fetcher: Fetcher,
         size: int,
-        data: dict[tuple[int, int], bytes] = {},
+        data: Optional[dict[tuple[int, int], bytes]] = None,
         strict: bool = True,
         **_: Any,
     ):
@@ -530,7 +598,7 @@ class KnownPartsOfAFile(BaseCache):
 
             self.data = dict(zip(offsets, blocks))
         else:
-            self.data = data
+            self.data = {}
 
     def _fetch(self, start: int | None, stop: int | None) -> bytes:
         if start is None:
@@ -551,6 +619,7 @@ class KnownPartsOfAFile(BaseCache):
                     # are allowed to pad reads beyond the
                     # buffer with zero
                     out += b"\x00" * (stop - start - len(out))
+                    self.hit_count += 1
                     return out
                 else:
                     # The request ends outside a known range,
@@ -572,6 +641,8 @@ class KnownPartsOfAFile(BaseCache):
             f"IO/caching performance may be poor!"
         )
         logger.debug(f"KnownPartsOfAFile cache fetching {start}-{stop}")
+        self.total_requested_bytes += stop - start
+        self.miss_count += 1
         return out + super()._fetch(start, stop)
 
 
@@ -676,12 +747,6 @@ class BackgroundBlockCache(BaseCache):
         self._fetch_future: Future[bytes] | None = None
         self._fetch_future_lock = threading.Lock()
 
-    def __repr__(self) -> str:
-        return (
-            f"<BackgroundBlockCache blocksize={self.blocksize}, "
-            f"size={self.size}, nblocks={self.nblocks}>"
-        )
-
     def cache_info(self) -> UpdatableLRU.CacheInfo:
         """
         The statistics on the block cache.
@@ -799,6 +864,8 @@ class BackgroundBlockCache(BaseCache):
         start = block_number * self.blocksize
         end = start + self.blocksize
         logger.info("BlockCache fetching block (%s) %d", log_info, block_number)
+        self.total_requested_bytes += end - start
+        self.miss_count += 1
         block_contents = super()._fetch(start, end)
         return block_contents
 
@@ -818,6 +885,9 @@ class BackgroundBlockCache(BaseCache):
         start_pos = start % self.blocksize
         end_pos = end % self.blocksize
 
+        # kind of pointless to count this as a hit, but it is
+        self.hit_count += 1
+
         if start_block_number == end_block_number:
             block = self._fetch_block_cached(start_block_number)
             return block[start_pos:end_pos]

fsspec/compression.py

@@ -1,4 +1,5 @@
 """Helper functions for a standard streaming compression API"""
+
 from zipfile import ZipFile
 
 import fsspec.utils
@@ -138,7 +139,7 @@ class SnappyFile(AbstractBufferedFile):
 try:
     import snappy
 
-    snappy.compress
+    snappy.compress(b"")
     # Snappy may use the .sz file extension, but this is not part of the
     # standard implementation.
     register_compression("snappy", SnappyFile, [])

fsspec/core.py

@@ -8,7 +8,7 @@ from glob import has_magic
 from pathlib import Path
 
 # for backwards compat, we export cache things from here too
-from .caching import (  # noqa: F401
+from fsspec.caching import (  # noqa: F401
     BaseCache,
     BlockCache,
     BytesCache,
@@ -16,9 +16,10 @@ from .caching import (  # noqa: F401
     ReadAheadCache,
     caches,
 )
-from .compression import compr
-from .registry import filesystem, get_filesystem_class
-from .utils import (
+from fsspec.compression import compr
+from fsspec.config import conf
+from fsspec.registry import filesystem, get_filesystem_class
+from fsspec.utils import (
     _unstrip_protocol,
     build_name_function,
     infer_compression,
@@ -100,7 +101,18 @@ class OpenFile:
     def __enter__(self):
         mode = self.mode.replace("t", "").replace("b", "") + "b"
 
-        f = self.fs.open(self.path, mode=mode)
+        try:
+            f = self.fs.open(self.path, mode=mode)
+        except FileNotFoundError as e:
+            if has_magic(self.path):
+                raise FileNotFoundError(
+                    "%s not found. The URL contains glob characters: you maybe needed\n"
+                    "to pass expand=True in fsspec.open() or the storage_options of \n"
+                    "your library. You can also set the config value 'open_expand'\n"
+                    "before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.",
+                    self.path,
+                ) from e
+            raise
 
         self.fobjects = [f]
 
@@ -367,6 +379,7 @@ def url_to_fs(url, **kwargs):
     urlpath : str
         The file-systems-specific URL for ``url``.
     """
+    url = stringify_path(url)
     # non-FS arguments that appear in fsspec.open()
     # inspect could keep this in sync with open()'s signature
     known_kwargs = {
@@ -396,6 +409,9 @@ def url_to_fs(url, **kwargs):
     return fs, urlpath
 
 
+DEFAULT_EXPAND = conf.get("open_expand", False)
+
+
 def open(
     urlpath,
     mode="rb",
@@ -404,6 +420,7 @@ def open(
     errors=None,
     protocol=None,
     newline=None,
+    expand=None,
     **kwargs,
 ):
     """Given a path or paths, return one ``OpenFile`` object.
@@ -428,6 +445,13 @@ def open(
     newline: bytes or None
         Used for line terminator in text mode. If None, uses system default;
         if blank, uses no translation.
+    expand: bool or Nonw
+        Whether to regard file paths containing special glob characters as needing
+        expansion (finding the first match) or absolute. Setting False allows using
+        paths which do embed such characters. If None (default), this argument
+        takes its value from the DEFAULT_EXPAND module variable, which takes
+        its initial value from the "open_expand" config value at startup, which will
+        be False if not set.
     **kwargs: dict
         Extra options that make sense to a particular storage connection, e.g.
         host, port, username, password, etc.
@@ -456,8 +480,7 @@ def open(
     - For implementations in separate packages see
       https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations
     """
-    kw = {"expand": False}
-    kw.update(kwargs)
+    expand = DEFAULT_EXPAND if expand is None else expand
     out = open_files(
         urlpath=[urlpath],
         mode=mode,
@@ -466,7 +489,8 @@ def open(
         errors=errors,
         protocol=protocol,
         newline=newline,
-        **kw,
+        expand=expand,
+        **kwargs,
     )
     if not out:
         raise FileNotFoundError(urlpath)

fsspec/exceptions.py

@@ -1,6 +1,7 @@
 """
 fsspec user-defined exception classes
 """
+
 import asyncio
 
 

fsspec/generic.py

@@ -139,7 +139,7 @@ def rsync(
         source_files, target_files = zip(*allfiles.items())
         fs.cp(source_files, target_files, **kwargs)
     logger.debug(f"{len(to_delete)} files to delete")
-    if delete_missing:
+    if delete_missing and to_delete:
         fs.rm(to_delete)
     return allfiles
 

fsspec/gui.py

@@ -94,7 +94,7 @@ class SigSlot:
         try:
             return self.panel._repr_mimebundle_(*args, **kwargs)
         except (ValueError, AttributeError):
-            raise NotImplementedError("Panel does not seem to be set " "up properly")
+            raise NotImplementedError("Panel does not seem to be set up properly")
 
     def connect(self, signal, slot):
         """Associate call back with given event

fsspec/implementations/arrow.py

@@ -139,8 +139,6 @@ class ArrowFSWrapper(AbstractFileSystem):
         path2 = self._strip_protocol(path2).rstrip("/")
         self.fs.move(path1, path2)
 
-    mv_file = mv
-
     @wrap_exceptions
     def rm_file(self, path):
         path = self._strip_protocol(path)

fsspec/implementations/cache_mapper.py

@@ -12,8 +12,7 @@ class AbstractCacheMapper(abc.ABC):
     """
 
     @abc.abstractmethod
-    def __call__(self, path: str) -> str:
-        ...
+    def __call__(self, path: str) -> str: ...
 
     def __eq__(self, other: object) -> bool:
         # Identity only depends on class. When derived classes have attributes

fsspec/implementations/cache_metadata.py

@@ -57,10 +57,14 @@ class CacheMetadata:
         """Low-level function to load metadata from specific file"""
         try:
             with open(fn, "r") as f:
-                return json.load(f)
+                loaded = json.load(f)
         except ValueError:
             with open(fn, "rb") as f:
-                return pickle.load(f)
+                loaded = pickle.load(f)
+        for c in loaded.values():
+            if isinstance(c.get("blocks"), list):
+                c["blocks"] = set(c["blocks"])
+        return loaded
 
     def _save(self, metadata_to_save: Detail, fn: str) -> None:
         """Low-level function to save metadata to specific file"""
@@ -152,11 +156,7 @@ class CacheMetadata:
         for fn, _, _ in self._scan_locations():
             if os.path.exists(fn):
                 # TODO: consolidate blocks here
-                loaded_cached_files = self._load(fn)
-                for c in loaded_cached_files.values():
-                    if isinstance(c["blocks"], list):
-                        c["blocks"] = set(c["blocks"])
-                cached_files.append(loaded_cached_files)
+                cached_files.append(self._load(fn))
             else:
                 cached_files.append({})
         self.cached_files = cached_files or [{}]

fsspec/implementations/cached.py

@@ -425,7 +425,6 @@ class CachingFileSystem(AbstractFileSystem):
             "clear_cache",
             "clear_expired_cache",
             "pop_from_cache",
-            "_mkcache",
             "local_file",
             "_paths_from_path",
             "get_mapper",
@@ -435,12 +434,10 @@ class CachingFileSystem(AbstractFileSystem):
             "__hash__",
             "__eq__",
             "to_json",
+            "to_dict",
             "cache_size",
             "pipe_file",
             "pipe",
-            "isdir",
-            "isfile",
-            "exists",
             "start_transaction",
             "end_transaction",
         }:
@@ -510,15 +507,6 @@ class CachingFileSystem(AbstractFileSystem):
             ^ hash(self.target_protocol)
         )
 
-    def to_json(self):
-        """Calculate JSON representation.
-
-        Not implemented yet for CachingFileSystem.
-        """
-        raise NotImplementedError(
-            "CachingFileSystem JSON representation not implemented"
-        )
-
 
 class WholeFileCacheFileSystem(CachingFileSystem):
     """Caches whole remote files on first access

fsspec/implementations/dirfs.py

@@ -329,8 +329,8 @@ class DirFileSystem(AsyncFileSystem):
     def rmdir(self, path):
         return self.fs.rmdir(self._join(path))
 
-    def mv_file(self, path1, path2, **kwargs):
-        return self.fs.mv_file(
+    def mv(self, path1, path2, **kwargs):
+        return self.fs.mv(
             self._join(path1),
             self._join(path2),
             **kwargs,

fsspec/implementations/github.py

@@ -1,5 +1,7 @@
 import requests
 
+import fsspec
+
 from ..spec import AbstractFileSystem
 from ..utils import infer_storage_options
 from .memory import MemoryFile
@@ -225,3 +227,13 @@ class GithubFileSystem(AbstractFileSystem):
             raise FileNotFoundError(path)
         r.raise_for_status()
         return MemoryFile(None, None, r.content)
+
+    def cat(self, path, recursive=False, on_error="raise", **kwargs):
+        paths = self.expand_path(path, recursive=recursive)
+        urls = [
+            self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root)
+            for u, sh in paths
+        ]
+        fs = fsspec.filesystem("http")
+        data = fs.cat(urls, on_error="return")
+        return {u: v for ((k, v), u) in zip(data.items(), urls)}

fsspec/implementations/http.py

@@ -451,7 +451,7 @@ class HTTPFileSystem(AsyncFileSystem):
 
         ends_with_slash = path.endswith("/")  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
-        append_slash_to_dirname = ends_with_slash or path.endswith("/**")
+        append_slash_to_dirname = ends_with_slash or path.endswith(("/**", "/*"))
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
 
@@ -494,15 +494,15 @@ class HTTPFileSystem(AsyncFileSystem):
         pattern = re.compile(pattern)
 
         out = {
-            p: info
+            (
+                p.rstrip("/")
+                if not append_slash_to_dirname
+                and info["type"] == "directory"
+                and p.endswith("/")
+                else p
+            ): info
             for p, info in sorted(allpaths.items())
-            if pattern.match(
-                (
-                    p + "/"
-                    if append_slash_to_dirname and info["type"] == "directory"
-                    else p
-                )
-            )
+            if pattern.match(p.rstrip("/"))
         }
 
         if detail: