fsspec 2023.10.0__py3-none-any.whl → 2023.12.0__py3-none-any.whl

fsspec/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-21T13:35:51-0400",
+ "date": "2023-12-02T20:51:30-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "e20f626b87b5bb87d223495a56aefd768272a7ca",
- "version": "2023.10.0"
+ "full-revisionid": "5cf9cd952c5d276835d3caef9c32fcf69d55b10c",
+ "version": "2023.12.0"
 }
 '''  # END VERSION_JSON
 

fsspec/asyn.py

@@ -15,7 +15,7 @@ from .callbacks import _DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
 from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep
 from .spec import AbstractBufferedFile, AbstractFileSystem
-from .utils import is_exception, other_paths
+from .utils import glob_translate, is_exception, other_paths
 
 private = re.compile("_[^_]")
 iothread = [None]  # dedicated fsspec IO thread
@@ -106,7 +106,7 @@ def sync(loop, func, *args, timeout=None, **kwargs):
 
 
 def sync_wrapper(func, obj=None):
-    """Given a function, make so can be called in async or blocking contexts
+    """Given a function, make so can be called in blocking contexts
 
     Leave obj=None if defining within a class. Pass the instance if attaching
     as an attribute of the instance.
@@ -467,6 +467,16 @@ class AsyncFileSystem(AbstractFileSystem):
         on_error="return",
         **kwargs,
     ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
         # TODO: on_error
         if max_gap is not None:
             # use utils.merge_offset_ranges
@@ -476,7 +486,7 @@ class AsyncFileSystem(AbstractFileSystem):
         if not isinstance(starts, Iterable):
             starts = [starts] * len(paths)
         if not isinstance(ends, Iterable):
-            ends = [starts] * len(paths)
+            ends = [ends] * len(paths)
         if len(starts) != len(paths) or len(ends) != len(paths):
             raise ValueError
         coros = [
@@ -662,9 +672,9 @@ class AsyncFileSystem(AbstractFileSystem):
             [self._size(p) for p in paths], batch_size=batch_size
         )
 
-    async def _exists(self, path):
+    async def _exists(self, path, **kwargs):
         try:
-            await self._info(path)
+            await self._info(path, **kwargs)
             return True
         except FileNotFoundError:
             return False
@@ -735,8 +745,12 @@ class AsyncFileSystem(AbstractFileSystem):
 
         import re
 
-        ends = path.endswith("/")
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
@@ -746,11 +760,11 @@ class AsyncFileSystem(AbstractFileSystem):
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            if await self._exists(path):
+            if await self._exists(path, **kwargs):
                 if not detail:
                     return [path]
                 else:
-                    return {path: await self._info(path)}
+                    return {path: await self._info(path, **kwargs)}
             else:
                 if not detail:
                     return []  # glob of non-existent returns empty
@@ -775,46 +789,22 @@ class AsyncFileSystem(AbstractFileSystem):
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=True, detail=True, **kwargs
         )
-        # Escape characters special to python regex, leaving our supported
-        # special characters in place.
-        # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html
-        # for shell globbing details.
-        pattern = (
-            "^"
-            + (
-                path.replace("\\", r"\\")
-                .replace(".", r"\.")
-                .replace("+", r"\+")
-                .replace("//", "/")
-                .replace("(", r"\(")
-                .replace(")", r"\)")
-                .replace("|", r"\|")
-                .replace("^", r"\^")
-                .replace("$", r"\$")
-                .replace("{", r"\{")
-                .replace("}", r"\}")
-                .rstrip("/")
-                .replace("?", ".")
-            )
-            + "$"
-        )
-        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
-        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)
+
         out = {
-            p: allpaths[p]
-            for p in sorted(allpaths)
-            if pattern.match(p.replace("//", "/").rstrip("/"))
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
         }
 
-        # Return directories only when the glob end by a slash
-        # This is needed for posix glob compliance
-        if ends:
-            out = {k: v for k, v in out.items() if v["type"] == "directory"}
-
         if detail:
             return out
         else:

fsspec/caching.py

@@ -111,7 +111,7 @@ class MMapCache(BaseCache):
             fd.write(b"1")
             fd.flush()
         else:
-            fd = open(self.location, "rb+")
+            fd = open(self.location, "r+b")
 
         return mmap.mmap(fd.fileno(), self.size)
 

fsspec/core.py

@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import io
 import logging
 import os
 import re
 from glob import has_magic
+from pathlib import Path
 
 # for backwards compat, we export cache things from here too
 from .caching import (  # noqa: F401
@@ -290,7 +293,11 @@ def open_files(
         fs.auto_mkdir = auto_mkdir
     elif "r" not in mode and auto_mkdir:
         parents = {fs._parent(path) for path in paths}
-        [fs.makedirs(parent, exist_ok=True) for parent in parents]
+        for parent in parents:
+            try:
+                fs.makedirs(parent, exist_ok=True)
+            except PermissionError:
+                pass
     return OpenFiles(
         [
             OpenFile(
@@ -465,7 +472,11 @@ def open(
     return out[0]
 
 
-def open_local(url, mode="rb", **storage_options):
+def open_local(
+    url: str | list[str] | Path | list[Path],
+    mode: str = "rb",
+    **storage_options: dict,
+) -> str | list[str]:
     """Open file(s) which can be resolved to local
 
     For files which either are local, or get downloaded upon open
@@ -489,7 +500,7 @@ def open_local(url, mode="rb", **storage_options):
         )
     with of as files:
         paths = [f.name for f in files]
-    if isinstance(url, str) and not has_magic(url):
+    if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path):
         return paths[0]
     return paths
 
@@ -510,6 +521,8 @@ def split_protocol(urlpath):
         if len(protocol) > 1:
             # excludes Windows paths
             return protocol, path
+    if ":" in urlpath and urlpath.find(":") > 1:
+        return urlpath.split(":", 1)
     return None, urlpath
 
 

fsspec/generic.py

@@ -171,6 +171,10 @@ class GenericFileSystem(AsyncFileSystem):
         self.method = default_method
         super().__init__(**kwargs)
 
+    def _parent(self, path):
+        fs = _resolve_fs(path, self.method)
+        return fs.unstrip_protocol(fs._parent(path))
+
     def _strip_protocol(self, path):
         # normalization only
         fs = _resolve_fs(path, self.method)

fsspec/gui.py

@@ -242,7 +242,7 @@ class FileSelector(SigSlot):
         else:
             self.init_protocol, url = "file", os.getcwd()
         self.init_url = url
-        self.init_kwargs = kwargs or "{}"
+        self.init_kwargs = (kwargs if isinstance(kwargs, str) else str(kwargs)) or "{}"
         self.filters = filters
         self.ignore = [re.compile(i) for i in ignore or []]
         self._fs = None

fsspec/implementations/cached.py

@@ -17,6 +17,7 @@ from fsspec.exceptions import BlocksizeMismatchError
 from fsspec.implementations.cache_mapper import create_cache_mapper
 from fsspec.implementations.cache_metadata import CacheMetadata
 from fsspec.spec import AbstractBufferedFile
+from fsspec.transaction import Transaction
 from fsspec.utils import infer_compression
 
 if TYPE_CHECKING:
@@ -25,6 +26,16 @@ if TYPE_CHECKING:
 logger = logging.getLogger("fsspec.cached")
 
 
+class WriteCachedTransaction(Transaction):
+    def complete(self, commit=True):
+        rpaths = [f.path for f in self.files]
+        lpaths = [f.fn for f in self.files]
+        if commit:
+            self.fs.put(lpaths, rpaths)
+        # else remove?
+        self.fs._intrans = False
+
+
 class CachingFileSystem(AbstractFileSystem):
     """Locally caching filesystem, layer over any other FS
 
@@ -355,6 +366,9 @@ class CachingFileSystem(AbstractFileSystem):
         self.save_cache()
         return f
 
+    def _parent(self, path):
+        return self.fs._parent(path)
+
     def hash_name(self, path: str, *args: Any) -> str:
         # Kept for backward compatibility with downstream libraries.
         # Ignores extra arguments, previously same_name boolean.
@@ -390,6 +404,7 @@ class CachingFileSystem(AbstractFileSystem):
             "open",
             "cat",
             "cat_file",
+            "cat_ranges",
             "get",
             "read_block",
             "tail",
@@ -411,6 +426,10 @@ class CachingFileSystem(AbstractFileSystem):
             "__eq__",
             "to_json",
             "cache_size",
+            "pipe_file",
+            "pipe",
+            "start_transaction",
+            "end_transaction",
         ]:
             # all the methods defined in this class. Note `open` here, since
             # it calls `_open`, but is actually in superclass
@@ -419,7 +438,10 @@ class CachingFileSystem(AbstractFileSystem):
             )
         if item in ["__reduce_ex__"]:
             raise AttributeError
-        if item in ["_cache"]:
+        if item in ["transaction"]:
+            # property
+            return type(self).transaction.__get__(self)
+        if item in ["_cache", "transaction_type"]:
             # class attributes
             return getattr(type(self), item)
         if item == "__class__":
@@ -508,7 +530,13 @@ class WholeFileCacheFileSystem(CachingFileSystem):
             self._mkcache()
         else:
             return [
-                LocalTempFile(self.fs, path, mode=open_files.mode) for path in paths
+                LocalTempFile(
+                    self.fs,
+                    path,
+                    mode=open_files.mode,
+                    fn=os.path.join(self.storage[-1], self._mapper(path)),
+                )
+                for path in paths
             ]
 
         if self.compression:
@@ -621,7 +649,8 @@ class WholeFileCacheFileSystem(CachingFileSystem):
     def _open(self, path, mode="rb", **kwargs):
         path = self._strip_protocol(path)
         if "r" not in mode:
-            return LocalTempFile(self, path, mode=mode)
+            fn = self._make_local_details(path)
+            return LocalTempFile(self, path, mode=mode, fn=fn)
         detail = self._check_file(path)
         if detail:
             detail, fn = detail
@@ -688,6 +717,7 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
 
     protocol = "simplecache"
     local_file = True
+    transaction_type = WriteCachedTransaction
 
     def __init__(self, **kwargs):
         kw = kwargs.copy()
@@ -712,16 +742,46 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
     def load_cache(self):
         pass
 
+    def pipe_file(self, path, value=None, **kwargs):
+        if self._intrans:
+            with self.open(path, "wb") as f:
+                f.write(value)
+        else:
+            super().pipe_file(path, value)
+
+    def pipe(self, path, value=None, **kwargs):
+        if isinstance(path, str):
+            self.pipe_file(self._strip_protocol(path), value, **kwargs)
+        elif isinstance(path, dict):
+            for k, v in path.items():
+                self.pipe_file(self._strip_protocol(k), v, **kwargs)
+        else:
+            raise ValueError("path must be str or dict")
+
+    def cat_ranges(
+        self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
+    ):
+        lpaths = [self._check_file(p) for p in paths]
+        rpaths = [p for l, p in zip(lpaths, paths) if l is False]
+        lpaths = [l for l, p in zip(lpaths, paths) if l is False]
+        self.fs.get(rpaths, lpaths)
+        return super().cat_ranges(
+            paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
+        )
+
     def _open(self, path, mode="rb", **kwargs):
         path = self._strip_protocol(path)
+        sha = self._mapper(path)
 
         if "r" not in mode:
-            return LocalTempFile(self, path, mode=mode)
+            fn = os.path.join(self.storage[-1], sha)
+            return LocalTempFile(
+                self, path, mode=mode, autocommit=not self._intrans, fn=fn
+            )
         fn = self._check_file(path)
         if fn:
             return open(fn, mode)
 
-        sha = self._mapper(path)
         fn = os.path.join(self.storage[-1], sha)
         logger.debug("Copying %s to local cache", path)
         kwargs["mode"] = mode
@@ -752,13 +812,9 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
 class LocalTempFile:
     """A temporary local file, which will be uploaded on commit"""
 
-    def __init__(self, fs, path, fn=None, mode="wb", autocommit=True, seek=0):
-        if fn:
-            self.fn = fn
-            self.fh = open(fn, mode)
-        else:
-            fd, self.fn = tempfile.mkstemp()
-            self.fh = open(fd, mode)
+    def __init__(self, fs, path, fn, mode="wb", autocommit=True, seek=0):
+        self.fn = fn
+        self.fh = open(fn, mode)
         self.mode = mode
         if seek:
             self.fh.seek(seek)
@@ -768,10 +824,10 @@ class LocalTempFile:
         self.autocommit = autocommit
 
     def __reduce__(self):
-        # always open in rb+ to allow continuing writing at a location
+        # always open in r+b to allow continuing writing at a location
         return (
             LocalTempFile,
-            (self.fs, self.path, self.fn, "rb+", self.autocommit, self.tell()),
+            (self.fs, self.path, self.fn, "r+b", self.autocommit, self.tell()),
         )
 
     def __enter__(self):

fsspec/implementations/data.py

@@ -0,0 +1,48 @@
+import base64
+import io
+from urllib.parse import unquote
+
+from fsspec import AbstractFileSystem
+
+
+class DataFileSystem(AbstractFileSystem):
+    """A handy decoder for data-URLs
+
+    Example
+    -------
+    >>> with fsspec.open("data:,Hello%2C%20World%21") as f:
+    ...     print(f.read())
+    b"Hello, World!"
+
+    """
+
+    protocol = "data"
+
+    def __init__(self, **kwargs):
+        """No parameters for this filesystem"""
+        super().__init__(**kwargs)
+
+    def cat_file(self, path, start=None, end=None, **kwargs):
+        pref, data = path.split(",", 1)
+        if pref.endswith("base64"):
+            return base64.b64decode(data)[start:end]
+        return unquote(data).encode()[start:end]
+
+    def info(self, path, **kwargs):
+        pref, name = path.split(",", 1)
+        data = self.cat_file(path)
+        mime = pref.split(":", 1)[1].split(";", 1)[0]
+        return {"name": name, "size": len(data), "type": "file", "mimetype": mime}
+
+    def _open(
+        self,
+        path,
+        mode="rb",
+        block_size=None,
+        autocommit=True,
+        cache_options=None,
+        **kwargs,
+    ):
+        if "r" not in mode:
+            raise ValueError("Read only filesystem")
+        return io.BytesIO(self.cat_file(path))

fsspec/implementations/http.py

@@ -14,7 +14,13 @@ from fsspec.asyn import AbstractAsyncStreamedFile, AsyncFileSystem, sync, sync_w
 from fsspec.callbacks import _DEFAULT_CALLBACK
 from fsspec.exceptions import FSTimeoutError
 from fsspec.spec import AbstractBufferedFile
-from fsspec.utils import DEFAULT_BLOCK_SIZE, isfilelike, nullcontext, tokenize
+from fsspec.utils import (
+    DEFAULT_BLOCK_SIZE,
+    glob_translate,
+    isfilelike,
+    nullcontext,
+    tokenize,
+)
 
 from ..caching import AllBytes
 
@@ -441,8 +447,9 @@ class HTTPFileSystem(AsyncFileSystem):
             raise ValueError("maxdepth must be at least 1")
         import re
 
-        ends = path.endswith("/")
+        ends_with_slash = path.endswith("/")  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_slash or path.endswith("/**")
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
 
@@ -451,11 +458,11 @@ class HTTPFileSystem(AsyncFileSystem):
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            if await self._exists(path):
+            if await self._exists(path, **kwargs):
                 if not detail:
                     return [path]
                 else:
-                    return {path: await self._info(path)}
+                    return {path: await self._info(path, **kwargs)}
             else:
                 if not detail:
                     return []  # glob of non-existent returns empty
@@ -480,45 +487,22 @@ class HTTPFileSystem(AsyncFileSystem):
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=True, detail=True, **kwargs
         )
-        # Escape characters special to python regex, leaving our supported
-        # special characters in place.
-        # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html
-        # for shell globbing details.
-        pattern = (
-            "^"
-            + (
-                path.replace("\\", r"\\")
-                .replace(".", r"\.")
-                .replace("+", r"\+")
-                .replace("//", "/")
-                .replace("(", r"\(")
-                .replace(")", r"\)")
-                .replace("|", r"\|")
-                .replace("^", r"\^")
-                .replace("$", r"\$")
-                .replace("{", r"\{")
-                .replace("}", r"\}")
-                .rstrip("/")
-            )
-            + "$"
-        )
-        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
-        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+
+        pattern = glob_translate(path + ("/" if ends_with_slash else ""))
         pattern = re.compile(pattern)
+
         out = {
-            p: allpaths[p]
-            for p in sorted(allpaths)
-            if pattern.match(p.replace("//", "/").rstrip("/"))
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
         }
 
-        # Return directories only when the glob end by a slash
-        # This is needed for posix glob compliance
-        if ends:
-            out = {k: v for k, v in out.items() if v["type"] == "directory"}
-
         if detail:
             return out
         else:
@@ -818,7 +802,7 @@ async def get_range(session, url, start, end, file=None, **kwargs):
     async with r:
         out = await r.read()
     if file:
-        with open(file, "rb+") as f:
+        with open(file, "r+b") as f:
             f.seek(start)
             f.write(out)
     else:
@@ -855,7 +839,10 @@ async def _file_info(url, session, size_policy="head", **kwargs):
         if "Content-Length" in r.headers:
             # Some servers may choose to ignore Accept-Encoding and return
             # compressed content, in which case the returned size is unreliable.
-            if r.headers.get("Content-Encoding", "identity") == "identity":
+            if "Content-Encoding" not in r.headers or r.headers["Content-Encoding"] in [
+                "identity",
+                "",
+            ]:
                 info["size"] = int(r.headers["Content-Length"])
         elif "Content-Range" in r.headers:
             info["size"] = int(r.headers["Content-Range"].split("/")[1])

fsspec/implementations/memory.py

@@ -175,7 +175,7 @@ class MemoryFileSystem(AbstractFileSystem):
             parent = self._parent(parent)
             if self.isfile(parent):
                 raise FileExistsError(parent)
-        if mode in ["rb", "ab", "rb+"]:
+        if mode in ["rb", "ab", "r+b"]:
             if path in self.store:
                 f = self.store[path]
                 if mode == "ab":

fsspec/implementations/reference.py

@@ -150,8 +150,29 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
         self.open_refs = open_refs
 
     @staticmethod
-    def create(record_size, root, fs, **kwargs):
+    def create(root, storage_options=None, fs=None, record_size=10000, **kwargs):
+        """Make empty parquet reference set
+
+        Parameters
+        ----------
+        root: str
+            Directory to contain the output; will be created
+        storage_options: dict | None
+            For making the filesystem to use for writing is fs is None
+        fs: FileSystem | None
+            Filesystem for writing
+        record_size: int
+            Number of references per parquet file
+        kwargs: passed to __init__
+
+        Returns
+        -------
+        LazyReferenceMapper instance
+        """
         met = {"metadata": {}, "record_size": record_size}
+        if fs is None:
+            fs, root = fsspec.core.url_to_fs(root, **(storage_options or {}))
+        fs.makedirs(root, exist_ok=True)
         fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
         return LazyReferenceMapper(root, fs, **kwargs)
 
@@ -292,7 +313,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
     def _generate_record(self, field, record):
         """The references for a given parquet file of a given field"""
         refs = self.open_refs(field, record)
-        it = iter(zip(refs.values()))
+        it = iter(zip(*refs.values()))
         if len(refs) == 3:
             # All urls
             return (list(t) for t in it)
@@ -603,7 +624,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                 **(ref_storage_args or target_options or {}), protocol=target_protocol
             )
             ref_fs, fo2 = fsspec.core.url_to_fs(fo, **dic)
-            if ref_fs.isfile(fo):
+            if ref_fs.isfile(fo2):
                 # text JSON
                 with fsspec.open(fo, "rb", **dic) as f:
                     logger.info("Read reference from URL %s", fo)
@@ -650,6 +671,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                     self.fss[protocol] = fs
         if remote_protocol is None:
             # get single protocol from references
+            # TODO: warning here, since this can be very expensive?
             for ref in self.references.values():
                 if callable(ref):
                     ref = ref()
@@ -772,24 +794,27 @@ class ReferenceFileSystem(AsyncFileSystem):
             raise NotImplementedError
         if isinstance(path, list) and (recursive or any("*" in p for p in path)):
             raise NotImplementedError
+        # TODO: if references is lazy, pre-fetch all paths in batch before access
         proto_dict = _protocol_groups(path, self.references)
         out = {}
         for proto, paths in proto_dict.items():
             fs = self.fss[proto]
-            urls, starts, ends = [], [], []
+            urls, starts, ends, valid_paths = [], [], [], []
             for p in paths:
                 # find references or label not-found. Early exit if any not
                 # found and on_error is "raise"
                 try:
                     u, s, e = self._cat_common(p)
-                    urls.append(u)
-                    starts.append(s)
-                    ends.append(e)
                 except FileNotFoundError as err:
                     if on_error == "raise":
                         raise
                     if on_error != "omit":
                         out[p] = err
+                else:
+                    urls.append(u)
+                    starts.append(s)
+                    ends.append(e)
+                    valid_paths.append(p)
 
             # process references into form for merging
             urls2 = []
@@ -797,7 +822,7 @@ class ReferenceFileSystem(AsyncFileSystem):
             ends2 = []
             paths2 = []
             whole_files = set()
-            for u, s, e, p in zip(urls, starts, ends, paths):
+            for u, s, e, p in zip(urls, starts, ends, valid_paths):
                 if isinstance(u, bytes):
                     # data
                     out[p] = u
@@ -809,7 +834,7 @@ class ReferenceFileSystem(AsyncFileSystem):
                     starts2.append(s)
                     ends2.append(e)
                     paths2.append(p)
-            for u, s, e, p in zip(urls, starts, ends, paths):
+            for u, s, e, p in zip(urls, starts, ends, valid_paths):
                 # second run to account for files that are to be loaded whole
                 if s is not None and u not in whole_files:
                     urls2.append(u)
@@ -829,7 +854,7 @@ class ReferenceFileSystem(AsyncFileSystem):
             bytes_out = fs.cat_ranges(new_paths, new_starts, new_ends)
 
             # unbundle from merged bytes - simple approach
-            for u, s, e, p in zip(urls, starts, ends, paths):
+            for u, s, e, p in zip(urls, starts, ends, valid_paths):
                 if p in out:
                     continue  # was bytes, already handled
                 for np, ns, ne, b in zip(new_paths, new_starts, new_ends, bytes_out):
@@ -963,16 +988,24 @@ class ReferenceFileSystem(AsyncFileSystem):
             elif len(part) == 1:
                 size = None
             else:
-                _, start, size = part
+                _, _, size = part
             par = path.rsplit("/", 1)[0] if "/" in path else ""
             par0 = par
+            subdirs = [par0]
             while par0 and par0 not in self.dircache:
-                # build parent directories
-                self.dircache[par0] = []
-                self.dircache.setdefault(
-                    par0.rsplit("/", 1)[0] if "/" in par0 else "", []
-                ).append({"name": par0, "type": "directory", "size": 0})
+                # collect parent directories
                 par0 = self._parent(par0)
+                subdirs.append(par0)
+
+            subdirs = subdirs[::-1]
+            for parent, child in zip(subdirs, subdirs[1:]):
+                # register newly discovered directories
+                assert child not in self.dircache
+                assert parent in self.dircache
+                self.dircache[parent].append(
+                    {"name": child, "type": "directory", "size": 0}
+                )
+                self.dircache[child] = []
 
             self.dircache[par].append({"name": path, "type": "file", "size": size})
 

fsspec/implementations/smb.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 This module contains SMBFileSystem class responsible for handling access to
 Windows Samba network shares by using package smbprotocol

fsspec/implementations/webhdfs.py

@@ -21,7 +21,7 @@ class WebHDFS(AbstractFileSystem):
     """
     Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways.
 
-    Three auth mechanisms are supported:
+    Four auth mechanisms are supported:
 
     insecure: no auth is done, and the user is assumed to be whoever they
         say they are (parameter ``user``), or a predefined value such as
@@ -34,6 +34,8 @@ class WebHDFS(AbstractFileSystem):
         service. Indeed, this client can also generate such tokens when
         not insecure. Note that tokens expire, but can be renewed (by a
         previously specified user) and may allow for proxying.
+    basic-auth: used when both parameter ``user`` and parameter ``password``
+        are provided.
 
     """
 
@@ -47,6 +49,7 @@ class WebHDFS(AbstractFileSystem):
         kerberos=False,
         token=None,
         user=None,
+        password=None,
         proxy_to=None,
         kerb_kwargs=None,
         data_proxy=None,
@@ -68,6 +71,9 @@ class WebHDFS(AbstractFileSystem):
             given
         user: str or None
             If given, assert the user name to connect with
+        password: str or None
+            If given, assert the password to use for basic auth. If password
+            is provided, user must be provided also
         proxy_to: str or None
             If given, the user has the authority to proxy, and this value is
             the user in who's name actions are taken
@@ -102,8 +108,19 @@ class WebHDFS(AbstractFileSystem):
                     " token"
                 )
             self.pars["delegation"] = token
-        if user is not None:
-            self.pars["user.name"] = user
+        self.user = user
+        self.password = password
+
+        if password is not None:
+            if user is None:
+                raise ValueError(
+                    "If passing a password, the user must also be"
+                    "set in order to set up the basic-auth"
+                )
+        else:
+            if user is not None:
+                self.pars["user.name"] = user
+
         if proxy_to is not None:
             self.pars["doas"] = proxy_to
         if kerberos and user is not None:
@@ -126,8 +143,13 @@ class WebHDFS(AbstractFileSystem):
 
             self.session.auth = HTTPKerberosAuth(**self.kerb_kwargs)
 
+        if self.user is not None and self.password is not None:
+            from requests.auth import HTTPBasicAuth
+
+            self.session.auth = HTTPBasicAuth(self.user, self.password)
+
     def _call(self, op, method="get", path=None, data=None, redirect=True, **kwargs):
-        url = self.url + quote(path or "")
+        url = self._apply_proxy(self.url + quote(path or "", safe="/="))
         args = kwargs.copy()
         args.update(self.pars)
         args["op"] = op.upper()

fsspec/registry.py

@@ -57,9 +57,10 @@ def register_implementation(name, cls, clobber=False, errtxt=None):
             _registry[name] = cls
 
 
-# protocols mapped to the class which implements them. This dict can
+# protocols mapped to the class which implements them. This dict can be
 # updated with register_implementation
 known_implementations = {
+    "data": {"class": "fsspec.implementations.data.DataFileSystem"},
     "file": {"class": "fsspec.implementations.local.LocalFileSystem"},
     "local": {"class": "fsspec.implementations.local.LocalFileSystem"},
     "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"},

fsspec/spec.py

@@ -17,6 +17,7 @@ from .dircache import DirCache
 from .transaction import Transaction
 from .utils import (
     _unstrip_protocol,
+    glob_translate,
     isfilelike,
     other_paths,
     read_block,
@@ -109,6 +110,7 @@ class AbstractFileSystem(metaclass=_Cached):
     async_impl = False
     mirror_sync_methods = False
     root_marker = ""  # For some FSs, may require leading '/' or other character
+    transaction_type = Transaction
 
     #: Extra *class attributes* that should be considered when hashing.
     _extra_tokenize_attributes = ()
@@ -235,20 +237,20 @@ class AbstractFileSystem(metaclass=_Cached):
         for the normal and exception cases.
         """
         if self._transaction is None:
-            self._transaction = Transaction(self)
+            self._transaction = self.transaction_type(self)
         return self._transaction
 
     def start_transaction(self):
         """Begin write transaction for deferring files, non-context version"""
         self._intrans = True
-        self._transaction = Transaction(self)
+        self._transaction = self.transaction_type(self)
         return self.transaction
 
     def end_transaction(self):
         """Finish write transaction, non-context version"""
         self.transaction.complete()
         self._transaction = None
-        # The invalid cache must be cleared after the transcation is completed.
+        # The invalid cache must be cleared after the transaction is completed.
         for path in self._invalidated_caches_in_transaction:
             self.invalidate_cache(path)
         self._invalidated_caches_in_transaction.clear()
@@ -551,10 +553,6 @@ class AbstractFileSystem(metaclass=_Cached):
 
         The `maxdepth` option is applied on the first `**` found in the path.
 
-        Search path names that contain embedded characters special to this
-        implementation of glob may not produce expected results;
-        e.g., ``foo/bar/*starredfilename*``.
-
         kwargs are passed to ``ls``.
         """
         if maxdepth is not None and maxdepth < 1:
@@ -562,8 +560,12 @@ class AbstractFileSystem(metaclass=_Cached):
 
         import re
 
-        ends = path.endswith("/")
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
@@ -573,11 +575,11 @@ class AbstractFileSystem(metaclass=_Cached):
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            if self.exists(path):
+            if self.exists(path, **kwargs):
                 if not detail:
                     return [path]
                 else:
-                    return {path: self.info(path)}
+                    return {path: self.info(path, **kwargs)}
             else:
                 if not detail:
                     return []  # glob of non-existent returns empty
@@ -600,47 +602,22 @@ class AbstractFileSystem(metaclass=_Cached):
                 depth = None
 
         allpaths = self.find(root, maxdepth=depth, withdirs=True, detail=True, **kwargs)
-        # Escape characters special to python regex, leaving our supported
-        # special characters in place.
-        # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html
-        # for shell globbing details.
-        pattern = (
-            "^"
-            + (
-                path.replace("\\", r"\\")
-                .replace(".", r"\.")
-                .replace("+", r"\+")
-                .replace("//", "/")
-                .replace("(", r"\(")
-                .replace(")", r"\)")
-                .replace("|", r"\|")
-                .replace("^", r"\^")
-                .replace("$", r"\$")
-                .replace("{", r"\{")
-                .replace("}", r"\}")
-                .rstrip("/")
-                .replace("?", ".")
-            )
-            + "$"
-        )
-        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
-        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)
 
         out = {
-            p: allpaths[p]
-            for p in sorted(allpaths)
-            if pattern.match(p.replace("//", "/").rstrip("/"))
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
         }
 
-        # Return directories only when the glob end by a slash
-        # This is needed for posix glob compliance
-        if ends:
-            out = {k: v for k, v in out.items() if v["type"] == "directory"}
-
         if detail:
             return out
         else:
@@ -828,6 +805,16 @@ class AbstractFileSystem(metaclass=_Cached):
     def cat_ranges(
         self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
     ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
         if max_gap is not None:
             raise NotImplementedError
         if not isinstance(paths, list):
@@ -835,7 +822,7 @@ class AbstractFileSystem(metaclass=_Cached):
         if not isinstance(starts, list):
             starts = [starts] * len(paths)
         if not isinstance(ends, list):
-            ends = [starts] * len(paths)
+            ends = [ends] * len(paths)
         if len(starts) != len(paths) or len(ends) != len(paths):
             raise ValueError
         out = []
@@ -901,7 +888,8 @@ class AbstractFileSystem(metaclass=_Cached):
             os.makedirs(lpath, exist_ok=True)
             return None
 
-        LocalFileSystem(auto_mkdir=True).makedirs(self._parent(lpath), exist_ok=True)
+        fs = LocalFileSystem(auto_mkdir=True)
+        fs.makedirs(fs._parent(lpath), exist_ok=True)
 
         with self.open(rpath, "rb", **kwargs) as f1:
             if outfile is None:

fsspec/tests/abstract/common.py

@@ -107,9 +107,9 @@ GLOB_EDGE_CASES_TESTS = {
                 "subdir1/subfile2",
             ],
         ),
-        ("**1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
+        ("**/*1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
         (
-            "**1",
+            "**/*1",
             True,
             None,
             [
@@ -120,14 +120,14 @@ GLOB_EDGE_CASES_TESTS = {
                 "subdir1/nesteddir/nestedfile",
             ],
         ),
-        ("**1", True, 1, ["file1"]),
+        ("**/*1", True, 1, ["file1"]),
         (
-            "**1",
+            "**/*1",
             True,
             2,
             ["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"],
         ),
-        ("**1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
+        ("**/*1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
         ("**/subdir0", False, None, []),
         ("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]),
         ("**/subdir0/nested*", False, 2, []),

fsspec/transaction.py

@@ -1,3 +1,6 @@
+from collections import deque
+
+
 class Transaction:
     """Filesystem transaction write context
 
@@ -13,10 +16,11 @@ class Transaction:
         fs: FileSystem instance
         """
         self.fs = fs
-        self.files = []
+        self.files = deque()
 
     def __enter__(self):
         self.start()
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """End transaction and commit, if exit is not due to exception"""
@@ -27,17 +31,17 @@ class Transaction:
 
     def start(self):
         """Start a transaction on this FileSystem"""
-        self.files = []  # clean up after previous failed completions
+        self.files = deque()  # clean up after previous failed completions
         self.fs._intrans = True
 
     def complete(self, commit=True):
         """Finish transaction: commit or discard all deferred files"""
-        for f in self.files:
+        while self.files:
+            f = self.files.popleft()
             if commit:
                 f.commit()
             else:
                 f.discard()
-        self.files = []
         self.fs._intrans = False
 
 

fsspec/utils.py

@@ -320,7 +320,7 @@ def tokenize(*args: Any, **kwargs: Any) -> str:
         h = md5(str(args).encode())
     except ValueError:
         # FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380
-        h = md5(str(args).encode(), usedforsecurity=False)  # type: ignore[call-arg]
+        h = md5(str(args).encode(), usedforsecurity=False)
     return h.hexdigest()
 
 
@@ -436,6 +436,7 @@ def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
 
 
 def get_protocol(url: str) -> str:
+    url = stringify_path(url)
     parts = re.split(r"(\:\:|\://)", url, 1)
     if len(parts) > 1:
         return parts[0]
@@ -627,3 +628,115 @@ def atomic_write(path: str, mode: str = "wb"):
         raise
     else:
         os.replace(fn, path)
+
+
+def _translate(pat, STAR, QUESTION_MARK):
+    # Copied from: https://github.com/python/cpython/pull/106703.
+    res: list[str] = []
+    add = res.append
+    i, n = 0, len(pat)
+    while i < n:
+        c = pat[i]
+        i = i + 1
+        if c == "*":
+            # compress consecutive `*` into one
+            if (not res) or res[-1] is not STAR:
+                add(STAR)
+        elif c == "?":
+            add(QUESTION_MARK)
+        elif c == "[":
+            j = i
+            if j < n and pat[j] == "!":
+                j = j + 1
+            if j < n and pat[j] == "]":
+                j = j + 1
+            while j < n and pat[j] != "]":
+                j = j + 1
+            if j >= n:
+                add("\\[")
+            else:
+                stuff = pat[i:j]
+                if "-" not in stuff:
+                    stuff = stuff.replace("\\", r"\\")
+                else:
+                    chunks = []
+                    k = i + 2 if pat[i] == "!" else i + 1
+                    while True:
+                        k = pat.find("-", k, j)
+                        if k < 0:
+                            break
+                        chunks.append(pat[i:k])
+                        i = k + 1
+                        k = k + 3
+                    chunk = pat[i:j]
+                    if chunk:
+                        chunks.append(chunk)
+                    else:
+                        chunks[-1] += "-"
+                    # Remove empty ranges -- invalid in RE.
+                    for k in range(len(chunks) - 1, 0, -1):
+                        if chunks[k - 1][-1] > chunks[k][0]:
+                            chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:]
+                            del chunks[k]
+                    # Escape backslashes and hyphens for set difference (--).
+                    # Hyphens that create ranges shouldn't be escaped.
+                    stuff = "-".join(
+                        s.replace("\\", r"\\").replace("-", r"\-") for s in chunks
+                    )
+                # Escape set operations (&&, ~~ and ||).
+                stuff = re.sub(r"([&~|])", r"\\\1", stuff)
+                i = j + 1
+                if not stuff:
+                    # Empty range: never match.
+                    add("(?!)")
+                elif stuff == "!":
+                    # Negated empty range: match any character.
+                    add(".")
+                else:
+                    if stuff[0] == "!":
+                        stuff = "^" + stuff[1:]
+                    elif stuff[0] in ("^", "["):
+                        stuff = "\\" + stuff
+                    add(f"[{stuff}]")
+        else:
+            add(re.escape(c))
+    assert i == n
+    return res
+
+
+def glob_translate(pat):
+    # Copied from: https://github.com/python/cpython/pull/106703.
+    # The keyword parameters' values are fixed to:
+    # recursive=True, include_hidden=True, seps=None
+    """Translate a pathname with shell wildcards to a regular expression."""
+    if os.path.altsep:
+        seps = os.path.sep + os.path.altsep
+    else:
+        seps = os.path.sep
+    escaped_seps = "".join(map(re.escape, seps))
+    any_sep = f"[{escaped_seps}]" if len(seps) > 1 else escaped_seps
+    not_sep = f"[^{escaped_seps}]"
+    one_last_segment = f"{not_sep}+"
+    one_segment = f"{one_last_segment}{any_sep}"
+    any_segments = f"(?:.+{any_sep})?"
+    any_last_segments = ".*"
+    results = []
+    parts = re.split(any_sep, pat)
+    last_part_idx = len(parts) - 1
+    for idx, part in enumerate(parts):
+        if part == "*":
+            results.append(one_segment if idx < last_part_idx else one_last_segment)
+            continue
+        if part == "**":
+            results.append(any_segments if idx < last_part_idx else any_last_segments)
+            continue
+        elif "**" in part:
+            raise ValueError(
+                "Invalid pattern: '**' can only be an entire path component"
+            )
+        if part:
+            results.extend(_translate(part, f"{not_sep}*", not_sep))
+        if idx < last_part_idx:
+            results.append(any_sep)
+    res = "".join(results)
+    return rf"(?s:{res})\Z"

{fsspec-2023.10.0.dist-info → fsspec-2023.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fsspec
-Version: 2023.10.0
+Version: 2023.12.0
 Summary: File-system specification
 Home-page: https://github.com/fsspec/filesystem_spec
 Maintainer: Martin Durant

{fsspec-2023.10.0.dist-info → fsspec-2023.12.0.dist-info}/RECORD

@@ -1,53 +1,54 @@
 fsspec/__init__.py,sha256=2kT62GfFK-AjgS-LgwSsCo_VA2IePvsyv8Ash5oiaFA,1982
-fsspec/_version.py,sha256=tXkjc9ntL7xecBDIEGncLHqDTApc-4eHEprCgBTMc2Y,501
+fsspec/_version.py,sha256=jAwgcB8zkKMbpk-J9sbW492ytNhl4CO0dj1nip8cBPg,501
 fsspec/archive.py,sha256=42f8FEAyP0LIvAhdzS1djyxEyCvlWlnzPo7igWFflYM,2400
-fsspec/asyn.py,sha256=T0lbd5U7mK8HVTq4uhbRaeOfs8S2pGKp8PLMGTGh8tY,36587
-fsspec/caching.py,sha256=jaHAKBUczmynooHbDQZHrmeyOz-groI_GlfRJG7fI9Q,28781
+fsspec/asyn.py,sha256=wx6vr5eBJYdW7a2cyv-LkfWu5dCDCcAjcDKjp3ylgR0,36154
+fsspec/caching.py,sha256=N45pzJdD4w5FOX_sxGvHWirggPNB66JTGP1HH6fpSck,28781
 fsspec/callbacks.py,sha256=qmD1v-WWxWmTmcUkEadq-_F_n3OGp9JYarjupUq_j3o,6358
 fsspec/compression.py,sha256=Zrbbb_m2SCF427BMJRYbDKMuSZIIV2YqteoS7AdR8Sc,4867
 fsspec/config.py,sha256=LF4Zmu1vhJW7Je9Q-cwkRc3xP7Rhyy7Xnwj26Z6sv2g,4279
 fsspec/conftest.py,sha256=fVfx-NLrH_OZS1TIpYNoPzM7efEcMoL62reHOdYeFCA,1245
-fsspec/core.py,sha256=iCEdBN9cL3LV9RbaCUjV8P4QOR_kCtdD0tcu5vNBSqg,22029
+fsspec/core.py,sha256=9oaIXN4PbbCRo4RZ-r7ZZ6LLm4vMyDwqv-x53kBfOmQ,22376
 fsspec/dircache.py,sha256=YzogWJrhEastHU7vWz-cJiJ7sdtLXFXhEpInGKd4EcM,2717
 fsspec/exceptions.py,sha256=s5eA2wIwzj-aeV0i_KDXsBaIhJJRKzmMGUGwuBHTnS4,348
 fsspec/fuse.py,sha256=66amOa6wdIbS0DMhhfAPUoOB37HPorfXD1izV0prmTY,10145
-fsspec/generic.py,sha256=GpuEKtlU--YLNrCEzq3wb9EnfG2rsEGfAzrHyL_0Cfo,13246
-fsspec/gui.py,sha256=w0OnzIkzCu0VofSP6t0ajIBl-GkkzI6WbeidxEg9tJg,13867
+fsspec/generic.py,sha256=StTDGXR-r9DJ7YEformhxfoexfewt_JkEUWN0oFRfbA,13373
+fsspec/gui.py,sha256=BEVFplRsQyakNeCWU-vyZBD-16x_flEe0XiDxXparEU,13913
 fsspec/mapping.py,sha256=WFEXRWxujQwfzzkRP5tpdIE0265okAtlP97qFZGvV1k,8165
 fsspec/parquet.py,sha256=i4H3EU3K1Q6jp8sqjFji6a6gKnlOEZufaa7DRNE5X-4,19516
-fsspec/registry.py,sha256=mLdGDQYC3LqphVdBKUL4Q-QS_9wV--pEDjBZvUdyeXw,11073
-fsspec/spec.py,sha256=9HR1IWe8jWhtq8w9IYBW-waXMaCrQp3Rxe90joixGBU,66712
-fsspec/transaction.py,sha256=gNcyHtN1mzdazsGyhDadWsR_E9WyCr3S9OypBQGpp2s,2179
-fsspec/utils.py,sha256=1sedrZZfN5EE1_CggJSQBiZYGvwaXgRy2m6RubY_bIw,18904
+fsspec/registry.py,sha256=-dl7sh2tsfhMA2uxz5KQDsPFehQTgMJIbVjNq6QLoKU,11145
+fsspec/spec.py,sha256=kfZpvKoh-fftKG6cOkOi2k0PJJwRqV4ZX_NElCBdcB8,66154
+fsspec/transaction.py,sha256=jeexB-H6Aw_gN6Z7hoKKe6v8zizITq39-gyTgpipIKE,2251
+fsspec/utils.py,sha256=_VX_0VwDtoAFSjMYrxvJvnPNX9FMoHO5BlFHXJ0bHFI,23053
 fsspec/implementations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fsspec/implementations/arrow.py,sha256=1d-c5KceQJxm8QXML8fFXHvQx0wstG-tNJNsrgMX_CI,8240
 fsspec/implementations/cache_mapper.py,sha256=nE_sY3vw-jJbeBcAP6NGtacP3jHW_7EcG3yUSf0A-4Y,2502
 fsspec/implementations/cache_metadata.py,sha256=ZvyA7Y3KK-5Ct4E5pELzD6mH_5T03XqaKVT96qYDADU,8576
-fsspec/implementations/cached.py,sha256=DTheLWDihvaq4WI4ZuJXP7kjnof3XhfiuOEGidyZGFE,28271
+fsspec/implementations/cached.py,sha256=jCQSAIiO7M8OOmwG4cCYn4LGvMVCbldC9j7GeonwoEc,30238
 fsspec/implementations/dask.py,sha256=CXZbJzIVOhKV8ILcxuy3bTvcacCueAbyQxmvAkbPkrk,4466
+fsspec/implementations/data.py,sha256=Oti0dKzyeadnVIedo3s8CADoh9bNM-96_6viTEYr4lo,1245
 fsspec/implementations/dbfs.py,sha256=0ndCE2OQqrWv6Y8ETufxOQ9ymIIO2JA_Q82bnilqTaw,14660
 fsspec/implementations/dirfs.py,sha256=8EEgKin5JgFBqzHaKig7ipiFAZJvbChUX_vpC_jagoY,11136
 fsspec/implementations/ftp.py,sha256=FzcHeieyda-ai_D8w4YKCzvI4gshuFYlBACBuEIx2Nk,11419
 fsspec/implementations/git.py,sha256=vKGI-Vd5q4H2RrvhebkPc9NwlfkZ980OUGhebeCw-M0,4034
 fsspec/implementations/github.py,sha256=hCisC1vXzZ9kP1UnyGz2Ba8c9cS2JmSGFHtgHG_2Gqw,7190
-fsspec/implementations/http.py,sha256=MI2_MyG4cgvN_abUNvof1NcO80SaNtFS271uvXxbAhk,30042
+fsspec/implementations/http.py,sha256=G3mBiMLDoYddwwzFg5B2zagmDX0r_2jFktrL3BqLU34,29248
 fsspec/implementations/jupyter.py,sha256=B2uj7OEm7yIk-vRSsO37_ND0t0EBvn4B-Su43ibN4Pg,3811
 fsspec/implementations/libarchive.py,sha256=YYZoHefBQItg5lsyJiSy2qPEdUDnfO9IbQWm8dwsGtY,7150
 fsspec/implementations/local.py,sha256=iqXA60ICWEftzJd6LO2nkMH-vy1AoGe8BD4Wwlw94dw,13291
-fsspec/implementations/memory.py,sha256=4llvIMyQre_Pr63HFLOYliEJavRWCpHevMKVHyXzPpw,9698
-fsspec/implementations/reference.py,sha256=wtdxuIlGMucNQly8P64ya8qAvqb-5lBlXeyjUDgdvWY,41223
+fsspec/implementations/memory.py,sha256=-a-NR66T-sGj9xTInUsu8KsEiqd156bF8Ui9BuXfmEA,9698
+fsspec/implementations/reference.py,sha256=BHhvx8LIYyBk5OVBWw-PmZsAs_OCaLvF1p8656bwVJE,42438
 fsspec/implementations/sftp.py,sha256=kZRsE8lwMOSyX-wlVBOGVVrc3cxZBcYbsoX0L28w6sQ,5524
-fsspec/implementations/smb.py,sha256=dbLMWboZjSd5pxgxaGRc5qnwNNmZv1AyDzjgNO7OMlM,10615
+fsspec/implementations/smb.py,sha256=k3RtzW97lJtYuw_QpP1rJRFnUBmSsw9twFjUCex0a5U,10591
 fsspec/implementations/tar.py,sha256=5ZpUp4E3SYbqrwAX2ezvZJqUoZO74Pjb9FpF8o1YBGs,4071
-fsspec/implementations/webhdfs.py,sha256=pjRmcQpDR-PLp-dhzXoCqT8oseOpgPr0eiJml1l7a-Y,15322
+fsspec/implementations/webhdfs.py,sha256=C5T96C_p66pUf2cQda-7HIZ9fKYwfCkupf2LN_7n7Dw,16145
 fsspec/implementations/zip.py,sha256=SJQihbLqUaMvN8kQiLmTOnc6Mdy3lozQp3MxmrJXqJw,4150
 fsspec/tests/abstract/__init__.py,sha256=i1wcFixV6QhOwdoB24c8oXjzobISNqiKVz9kl2DvAY8,10028
-fsspec/tests/abstract/common.py,sha256=X1ijH_pdMc9uVpZtgGj1P-2Zj9VIY-Y0tG3u1vTGcpE,4963
+fsspec/tests/abstract/common.py,sha256=1GQwNo5AONzAnzZj0fWgn8NJPLXALehbsuGxS3FzWVU,4973
 fsspec/tests/abstract/copy.py,sha256=nyCp1Q9apHzti2_UPDh3HzVhRmV7dciD-3dq-wM7JuU,19643
 fsspec/tests/abstract/get.py,sha256=vNR4HztvTR7Cj56AMo7_tx7TeYz1Jgr_2Wb8Lv-UiBY,20755
 fsspec/tests/abstract/put.py,sha256=hEf-yuMWBOT7B6eWcck3tMyJWzdVXtxkY-O6LUt1KAE,20877
-fsspec-2023.10.0.dist-info/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
-fsspec-2023.10.0.dist-info/METADATA,sha256=5dTihW4k5zH2F6aQqi3VTIVxTWCe5qSbTXsDr-d0MmM,6829
-fsspec-2023.10.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-fsspec-2023.10.0.dist-info/top_level.txt,sha256=blt2pDrQDwN3Gklcw13CSPLQRd6aaOgJ8AxqrW395MI,7
-fsspec-2023.10.0.dist-info/RECORD,,
+fsspec-2023.12.0.dist-info/LICENSE,sha256=LcNUls5TpzB5FcAIqESq1T53K0mzTN0ARFBnaRQH7JQ,1513
+fsspec-2023.12.0.dist-info/METADATA,sha256=0PDf5Q5_GcLo-r7fJruRt98YZFONP9hX2NbmPfSllrM,6829
+fsspec-2023.12.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+fsspec-2023.12.0.dist-info/top_level.txt,sha256=blt2pDrQDwN3Gklcw13CSPLQRd6aaOgJ8AxqrW395MI,7
+fsspec-2023.12.0.dist-info/RECORD,,

{fsspec-2023.10.0.dist-info → fsspec-2023.12.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any