fsspec 2023.10.0__py3-none-any.whl → 2024.2.0__py3-none-any.whl

fsspec/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-21T13:35:51-0400",
+ "date": "2024-02-04T20:21:42-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "e20f626b87b5bb87d223495a56aefd768272a7ca",
- "version": "2023.10.0"
+ "full-revisionid": "5dc364e13b63609717d77b7361e80cfa64e3b8fd",
+ "version": "2024.2.0"
 }
 '''  # END VERSION_JSON
 

fsspec/archive.py

@@ -38,7 +38,7 @@ class AbstractArchiveFileSystem(AbstractFileSystem):
         self._get_dirs()
         path = self._strip_protocol(path)
         if path in {"", "/"} and self.dir_cache:
-            return {"name": "/", "type": "directory", "size": 0}
+            return {"name": "", "type": "directory", "size": 0}
         if path in self.dir_cache:
             return self.dir_cache[path]
         elif path + "/" in self.dir_cache:
@@ -64,10 +64,10 @@ class AbstractArchiveFileSystem(AbstractFileSystem):
                 # root directory entry
                 ppath = p.rstrip("/").split("/", 1)[0]
                 if ppath not in paths:
-                    out = {"name": ppath + "/", "size": 0, "type": "directory"}
+                    out = {"name": ppath, "size": 0, "type": "directory"}
                     paths[ppath] = out
-        out = sorted(paths.values(), key=lambda _: _["name"])
         if detail:
+            out = sorted(paths.values(), key=lambda _: _["name"])
             return out
         else:
-            return [f["name"] for f in out]
+            return sorted(paths)

fsspec/asyn.py

@@ -11,11 +11,11 @@ from contextlib import contextmanager
 from glob import has_magic
 from typing import TYPE_CHECKING, Iterable
 
-from .callbacks import _DEFAULT_CALLBACK
+from .callbacks import DEFAULT_CALLBACK
 from .exceptions import FSTimeoutError
 from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep
 from .spec import AbstractBufferedFile, AbstractFileSystem
-from .utils import is_exception, other_paths
+from .utils import glob_translate, is_exception, other_paths
 
 private = re.compile("_[^_]")
 iothread = [None]  # dedicated fsspec IO thread
@@ -106,7 +106,7 @@ def sync(loop, func, *args, timeout=None, **kwargs):
 
 
 def sync_wrapper(func, obj=None):
-    """Given a function, make so can be called in async or blocking contexts
+    """Given a function, make so can be called in blocking contexts
 
     Leave obj=None if defining within a class. Pass the instance if attaching
     as an attribute of the instance.
@@ -205,7 +205,7 @@ def running_async() -> bool:
 async def _run_coros_in_chunks(
     coros,
     batch_size=None,
-    callback=_DEFAULT_CALLBACK,
+    callback=DEFAULT_CALLBACK,
     timeout=None,
     return_exceptions=False,
     nofiles=False,
@@ -245,7 +245,7 @@ async def _run_coros_in_chunks(
             asyncio.Task(asyncio.wait_for(c, timeout=timeout))
             for c in coros[start : start + batch_size]
         ]
-        if callback is not _DEFAULT_CALLBACK:
+        if callback is not DEFAULT_CALLBACK:
             [
                 t.add_done_callback(lambda *_, **__: callback.relative_update(1))
                 for t in chunk
@@ -467,6 +467,16 @@ class AsyncFileSystem(AbstractFileSystem):
         on_error="return",
         **kwargs,
     ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
         # TODO: on_error
         if max_gap is not None:
             # use utils.merge_offset_ranges
@@ -476,7 +486,7 @@ class AsyncFileSystem(AbstractFileSystem):
         if not isinstance(starts, Iterable):
             starts = [starts] * len(paths)
         if not isinstance(ends, Iterable):
-            ends = [starts] * len(paths)
+            ends = [ends] * len(paths)
         if len(starts) != len(paths) or len(ends) != len(paths):
             raise ValueError
         coros = [
@@ -496,7 +506,7 @@ class AsyncFileSystem(AbstractFileSystem):
         lpath,
         rpath,
         recursive=False,
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         batch_size=None,
         maxdepth=None,
         **kwargs,
@@ -558,8 +568,8 @@ class AsyncFileSystem(AbstractFileSystem):
         coros = []
         callback.set_size(len(file_pairs))
         for lfile, rfile in file_pairs:
-            callback.branch(lfile, rfile, kwargs)
-            coros.append(self._put_file(lfile, rfile, **kwargs))
+            put_file = callback.branch_coro(self._put_file)
+            coros.append(put_file(lfile, rfile, **kwargs))
 
         return await _run_coros_in_chunks(
             coros, batch_size=batch_size, callback=callback
@@ -573,7 +583,7 @@ class AsyncFileSystem(AbstractFileSystem):
         rpath,
         lpath,
         recursive=False,
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         maxdepth=None,
         **kwargs,
     ):
@@ -635,8 +645,8 @@ class AsyncFileSystem(AbstractFileSystem):
         coros = []
         callback.set_size(len(lpaths))
         for lpath, rpath in zip(lpaths, rpaths):
-            callback.branch(rpath, lpath, kwargs)
-            coros.append(self._get_file(rpath, lpath, **kwargs))
+            get_file = callback.branch_coro(self._get_file)
+            coros.append(get_file(rpath, lpath, **kwargs))
         return await _run_coros_in_chunks(
             coros, batch_size=batch_size, callback=callback
         )
@@ -662,9 +672,9 @@ class AsyncFileSystem(AbstractFileSystem):
             [self._size(p) for p in paths], batch_size=batch_size
         )
 
-    async def _exists(self, path):
+    async def _exists(self, path, **kwargs):
         try:
-            await self._info(path)
+            await self._info(path, **kwargs)
             return True
         except FileNotFoundError:
             return False
@@ -735,8 +745,12 @@ class AsyncFileSystem(AbstractFileSystem):
 
         import re
 
-        ends = path.endswith("/")
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
@@ -746,11 +760,11 @@ class AsyncFileSystem(AbstractFileSystem):
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            if await self._exists(path):
+            if await self._exists(path, **kwargs):
                 if not detail:
                     return [path]
                 else:
-                    return {path: await self._info(path)}
+                    return {path: await self._info(path, **kwargs)}
             else:
                 if not detail:
                     return []  # glob of non-existent returns empty
@@ -775,46 +789,22 @@ class AsyncFileSystem(AbstractFileSystem):
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=True, detail=True, **kwargs
         )
-        # Escape characters special to python regex, leaving our supported
-        # special characters in place.
-        # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html
-        # for shell globbing details.
-        pattern = (
-            "^"
-            + (
-                path.replace("\\", r"\\")
-                .replace(".", r"\.")
-                .replace("+", r"\+")
-                .replace("//", "/")
-                .replace("(", r"\(")
-                .replace(")", r"\)")
-                .replace("|", r"\|")
-                .replace("^", r"\^")
-                .replace("$", r"\$")
-                .replace("{", r"\{")
-                .replace("}", r"\}")
-                .rstrip("/")
-                .replace("?", ".")
-            )
-            + "$"
-        )
-        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
-        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)
+
         out = {
-            p: allpaths[p]
-            for p in sorted(allpaths)
-            if pattern.match(p.replace("//", "/").rstrip("/"))
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
         }
 
-        # Return directories only when the glob end by a slash
-        # This is needed for posix glob compliance
-        if ends:
-            out = {k: v for k, v in out.items() if v["type"] == "directory"}
-
         if detail:
             return out
         else:

fsspec/caching.py

@@ -111,7 +111,7 @@ class MMapCache(BaseCache):
             fd.write(b"1")
             fd.flush()
         else:
-            fd = open(self.location, "rb+")
+            fd = open(self.location, "r+b")
 
         return mmap.mmap(fd.fileno(), self.size)
 

fsspec/callbacks.py

@@ -1,3 +1,6 @@
+from functools import wraps
+
+
 class Callback:
     """
     Base class and interface for callback mechanism
@@ -25,6 +28,60 @@ class Callback:
         self.hooks = hooks or {}
         self.kw = kwargs
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_args):
+        self.close()
+
+    def close(self):
+        """Close callback."""
+
+    def branched(self, path_1, path_2, **kwargs):
+        """
+        Return callback for child transfers
+
+        If this callback is operating at a higher level, e.g., put, which may
+        trigger transfers that can also be monitored. The function returns a callback
+        that has to be passed to the child method, e.g., put_file,
+        as `callback=` argument.
+
+        The implementation uses `callback.branch` for compatibility.
+        When implementing callbacks, it is recommended to override this function instead
+        of `branch` and avoid calling `super().branched(...)`.
+
+        Prefer using this function over `branch`.
+
+        Parameters
+        ----------
+        path_1: str
+            Child's source path
+        path_2: str
+            Child's destination path
+        **kwargs:
+            Arbitrary keyword arguments
+
+        Returns
+        -------
+        callback: Callback
+            A callback instance to be passed to the child method
+        """
+        self.branch(path_1, path_2, kwargs)
+        # mutate kwargs so that we can force the caller to pass "callback=" explicitly
+        return kwargs.pop("callback", DEFAULT_CALLBACK)
+
+    def branch_coro(self, fn):
+        """
+        Wraps a coroutine, and pass a new child callback to it.
+        """
+
+        @wraps(fn)
+        async def func(path1, path2: str, **kwargs):
+            with self.branched(path1, path2, **kwargs) as child:
+                return await fn(path1, path2, callback=child, **kwargs)
+
+        return func
+
     def set_size(self, size):
         """
         Set the internal maximum size attribute
@@ -140,10 +197,10 @@ class Callback:
 
         For the special value of ``None``, return the global instance of
         ``NoOpCallback``. This is an alternative to including
-        ``callback=_DEFAULT_CALLBACK`` directly in a method signature.
+        ``callback=DEFAULT_CALLBACK`` directly in a method signature.
         """
         if maybe_callback is None:
-            return _DEFAULT_CALLBACK
+            return DEFAULT_CALLBACK
         return maybe_callback
 
 
@@ -186,7 +243,9 @@ class TqdmCallback(Callback):
     tqdm_kwargs : dict, (optional)
         Any argument accepted by the tqdm constructor.
         See the `tqdm doc <https://tqdm.github.io/docs/tqdm/#__init__>`_.
-        Will be forwarded to tqdm.
+        Will be forwarded to `tqdm_cls`.
+    tqdm_cls: (optional)
+        subclass of `tqdm.tqdm`. If not passed, it will default to `tqdm.tqdm`.
 
     Examples
     --------
@@ -209,30 +268,57 @@ class TqdmCallback(Callback):
             recursive=True,
             callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}),
         )
+
+    You can also customize the progress bar by passing a subclass of `tqdm`.
+
+    .. code-block:: python
+
+        class TqdmFormat(tqdm):
+            '''Provides a `total_time` format parameter'''
+            @property
+            def format_dict(self):
+                d = super().format_dict
+                total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1)
+                d.update(total_time=self.format_interval(total_time) + " in total")
+                return d
+
+    >>> with TqdmCallback(
+            tqdm_kwargs={
+                "desc": "desc",
+                "bar_format": "{total_time}: {percentage:.0f}%|{bar}{r_bar}",
+            },
+            tqdm_cls=TqdmFormat,
+        ) as callback:
+            fs.upload(".", path2distant_data, recursive=True, callback=callback)
     """
 
     def __init__(self, tqdm_kwargs=None, *args, **kwargs):
         try:
-            import tqdm
+            from tqdm import tqdm
 
-            self._tqdm = tqdm
         except ImportError as exce:
             raise ImportError(
                 "Using TqdmCallback requires tqdm to be installed"
             ) from exce
 
+        self._tqdm_cls = kwargs.pop("tqdm_cls", tqdm)
         self._tqdm_kwargs = tqdm_kwargs or {}
+        self.tqdm = None
         super().__init__(*args, **kwargs)
 
-    def set_size(self, size):
-        self.tqdm = self._tqdm.tqdm(total=size, **self._tqdm_kwargs)
+    def call(self, *args, **kwargs):
+        if self.tqdm is None:
+            self.tqdm = self._tqdm_cls(total=self.size, **self._tqdm_kwargs)
+        self.tqdm.total = self.size
+        self.tqdm.update(self.value - self.tqdm.n)
 
-    def relative_update(self, inc=1):
-        self.tqdm.update(inc)
+    def close(self):
+        if self.tqdm is not None:
+            self.tqdm.close()
+            self.tqdm = None
 
     def __del__(self):
-        self.tqdm.close()
-        self.tqdm = None
+        return self.close()
 
 
-_DEFAULT_CALLBACK = NoOpCallback()
+DEFAULT_CALLBACK = _DEFAULT_CALLBACK = NoOpCallback()

fsspec/compression.py

@@ -90,15 +90,15 @@ except ImportError:
 try:
     from lzma import LZMAFile
 
-    register_compression("lzma", LZMAFile, "xz")
-    register_compression("xz", LZMAFile, "xz", force=True)
+    register_compression("lzma", LZMAFile, "lzma")
+    register_compression("xz", LZMAFile, "xz")
 except ImportError:
     pass
 
 try:
     import lzmaffi
 
-    register_compression("lzma", lzmaffi.LZMAFile, "xz", force=True)
+    register_compression("lzma", lzmaffi.LZMAFile, "lzma", force=True)
     register_compression("xz", lzmaffi.LZMAFile, "xz", force=True)
 except ImportError:
     pass

fsspec/core.py

@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import io
 import logging
 import os
 import re
 from glob import has_magic
+from pathlib import Path
 
 # for backwards compat, we export cache things from here too
 from .caching import (  # noqa: F401
@@ -290,7 +293,11 @@ def open_files(
         fs.auto_mkdir = auto_mkdir
     elif "r" not in mode and auto_mkdir:
         parents = {fs._parent(path) for path in paths}
-        [fs.makedirs(parent, exist_ok=True) for parent in parents]
+        for parent in parents:
+            try:
+                fs.makedirs(parent, exist_ok=True)
+            except PermissionError:
+                pass
     return OpenFiles(
         [
             OpenFile(
@@ -465,7 +472,11 @@ def open(
     return out[0]
 
 
-def open_local(url, mode="rb", **storage_options):
+def open_local(
+    url: str | list[str] | Path | list[Path],
+    mode: str = "rb",
+    **storage_options: dict,
+) -> str | list[str]:
     """Open file(s) which can be resolved to local
 
     For files which either are local, or get downloaded upon open
@@ -489,7 +500,7 @@ def open_local(url, mode="rb", **storage_options):
         )
     with of as files:
         paths = [f.name for f in files]
-    if isinstance(url, str) and not has_magic(url):
+    if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path):
         return paths[0]
     return paths
 
@@ -510,6 +521,8 @@ def split_protocol(urlpath):
         if len(protocol) > 1:
             # excludes Windows paths
             return protocol, path
+    if urlpath.startswith("data:"):
+        return urlpath.split(":", 1)
     return None, urlpath
 
 

fsspec/exceptions.py

@@ -10,12 +10,8 @@ class BlocksizeMismatchError(ValueError):
     written with
     """
 
-    ...
-
 
 class FSTimeoutError(asyncio.TimeoutError):
     """
     Raised when a fsspec function timed out occurs
     """
-
-    ...

fsspec/generic.py

@@ -8,7 +8,7 @@ import uuid
 from typing import Optional
 
 from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper
-from .callbacks import _DEFAULT_CALLBACK
+from .callbacks import DEFAULT_CALLBACK
 from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs
 
 _generic_fs = {}
@@ -171,6 +171,10 @@ class GenericFileSystem(AsyncFileSystem):
         self.method = default_method
         super().__init__(**kwargs)
 
+    def _parent(self, path):
+        fs = _resolve_fs(path, self.method)
+        return fs.unstrip_protocol(fs._parent(path))
+
     def _strip_protocol(self, path):
         # normalization only
         fs = _resolve_fs(path, self.method)
@@ -246,9 +250,12 @@ class GenericFileSystem(AsyncFileSystem):
             return fs.pipe_file(path, value, **kwargs)
 
     async def _rm(self, url, **kwargs):
-        fs = _resolve_fs(url, self.method)
+        urls = url
+        if isinstance(urls, str):
+            urls = [urls]
+        fs = _resolve_fs(urls[0], self.method)
         if fs.async_impl:
-            await fs._rm(url, **kwargs)
+            await fs._rm(urls, **kwargs)
         else:
             fs.rm(url, **kwargs)
 
@@ -272,7 +279,7 @@ class GenericFileSystem(AsyncFileSystem):
         url,
         url2,
         blocksize=2**20,
-        callback=_DEFAULT_CALLBACK,
+        callback=DEFAULT_CALLBACK,
         **kwargs,
     ):
         fs = _resolve_fs(url, self.method)

fsspec/gui.py

@@ -153,8 +153,9 @@ class SigSlot:
                         break
                 except Exception as e:
                     logger.exception(
-                        "Exception (%s) while executing callback for signal: %s"
-                        "" % (e, sig)
+                        "Exception (%s) while executing callback for signal: %s",
+                        e,
+                        sig,
                     )
 
     def show(self, threads=False):
@@ -242,7 +243,7 @@ class FileSelector(SigSlot):
         else:
             self.init_protocol, url = "file", os.getcwd()
         self.init_url = url
-        self.init_kwargs = kwargs or "{}"
+        self.init_kwargs = (kwargs if isinstance(kwargs, str) else str(kwargs)) or "{}"
         self.filters = filters
         self.ignore = [re.compile(i) for i in ignore or []]
         self._fs = None

fsspec/implementations/arrow.py

@@ -5,6 +5,7 @@ import secrets
 import shutil
 from contextlib import suppress
 from functools import cached_property, wraps
+from urllib.parse import parse_qs
 
 from fsspec.spec import AbstractFileSystem
 from fsspec.utils import (
@@ -255,6 +256,7 @@ class HadoopFileSystem(ArrowFSWrapper):
         port=0,
         user=None,
         kerb_ticket=None,
+        replication=3,
         extra_conf=None,
         **kwargs,
     ):
@@ -270,6 +272,8 @@ class HadoopFileSystem(ArrowFSWrapper):
             If given, connect as this username
         kerb_ticket: str or None
             If given, use this ticket for authentication
+        replication: int
+            set replication factor of file for write operations. default value is 3.
         extra_conf: None or dict
             Passed on to HadoopFileSystem
         """
@@ -280,6 +284,7 @@ class HadoopFileSystem(ArrowFSWrapper):
             port=port,
             user=user,
             kerb_ticket=kerb_ticket,
+            replication=replication,
             extra_conf=extra_conf,
         )
         super().__init__(fs=fs, **kwargs)
@@ -294,4 +299,8 @@ class HadoopFileSystem(ArrowFSWrapper):
             out["user"] = ops["username"]
         if ops.get("port", None):
             out["port"] = ops["port"]
+        if ops.get("url_query", None):
+            queries = parse_qs(ops["url_query"])
+            if queries.get("replication", None):
+                out["replication"] = int(queries["replication"][0])
         return out

fsspec/implementations/cache_mapper.py

@@ -2,13 +2,9 @@ from __future__ import annotations
 
 import abc
 import hashlib
-from typing import TYPE_CHECKING
 
 from fsspec.implementations.local import make_path_posix
 
-if TYPE_CHECKING:
-    from typing import Any
-
 
 class AbstractCacheMapper(abc.ABC):
     """Abstract super-class for mappers from remote URLs to local cached
@@ -19,7 +15,7 @@ class AbstractCacheMapper(abc.ABC):
     def __call__(self, path: str) -> str:
         ...
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         # Identity only depends on class. When derived classes have attributes
         # they will need to be included.
         return isinstance(other, type(self))
@@ -56,7 +52,7 @@ class BasenameCacheMapper(AbstractCacheMapper):
         else:
             return prefix  # No separator found, simple filename
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         return super().__eq__(other) and self.directory_levels == other.directory_levels
 
     def __hash__(self) -> int: