fsspec 2023.9.2__py3-none-any.whl → 2023.12.0__py3-none-any.whl

fsspec/spec.py

@@ -17,6 +17,7 @@ from .dircache import DirCache
 from .transaction import Transaction
 from .utils import (
     _unstrip_protocol,
+    glob_translate,
     isfilelike,
     other_paths,
     read_block,
@@ -109,6 +110,7 @@ class AbstractFileSystem(metaclass=_Cached):
     async_impl = False
     mirror_sync_methods = False
     root_marker = ""  # For some FSs, may require leading '/' or other character
+    transaction_type = Transaction
 
     #: Extra *class attributes* that should be considered when hashing.
     _extra_tokenize_attributes = ()
@@ -196,7 +198,7 @@ class AbstractFileSystem(metaclass=_Cached):
         # use of root_marker to make minimum required path, e.g., "/"
         return path or cls.root_marker
 
-    def unstrip_protocol(self, name):
+    def unstrip_protocol(self, name: str) -> str:
         """Format FS-specific path to generic, including protocol"""
         protos = (self.protocol,) if isinstance(self.protocol, str) else self.protocol
         for protocol in protos:
@@ -235,20 +237,20 @@ class AbstractFileSystem(metaclass=_Cached):
         for the normal and exception cases.
         """
         if self._transaction is None:
-            self._transaction = Transaction(self)
+            self._transaction = self.transaction_type(self)
         return self._transaction
 
     def start_transaction(self):
         """Begin write transaction for deferring files, non-context version"""
         self._intrans = True
-        self._transaction = Transaction(self)
+        self._transaction = self.transaction_type(self)
         return self.transaction
 
     def end_transaction(self):
         """Finish write transaction, non-context version"""
         self.transaction.complete()
         self._transaction = None
-        # The invalid cache must be cleared after the transcation is completed.
+        # The invalid cache must be cleared after the transaction is completed.
         for path in self._invalidated_caches_in_transaction:
             self.invalidate_cache(path)
         self._invalidated_caches_in_transaction.clear()
@@ -551,10 +553,6 @@ class AbstractFileSystem(metaclass=_Cached):
 
         The `maxdepth` option is applied on the first `**` found in the path.
 
-        Search path names that contain embedded characters special to this
-        implementation of glob may not produce expected results;
-        e.g., 'foo/bar/*starredfilename*'.
-
         kwargs are passed to ``ls``.
         """
         if maxdepth is not None and maxdepth < 1:
@@ -562,8 +560,12 @@ class AbstractFileSystem(metaclass=_Cached):
 
         import re
 
-        ends = path.endswith("/")
+        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
+        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
+        append_slash_to_dirname = ends_with_sep or path.endswith(
+            tuple(sep + "**" for sep in seps)
+        )
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_qmark = path.find("?") if path.find("?") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
@@ -573,11 +575,11 @@ class AbstractFileSystem(metaclass=_Cached):
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            if self.exists(path):
+            if self.exists(path, **kwargs):
                 if not detail:
                     return [path]
                 else:
-                    return {path: self.info(path)}
+                    return {path: self.info(path, **kwargs)}
             else:
                 if not detail:
                     return []  # glob of non-existent returns empty
@@ -600,47 +602,22 @@ class AbstractFileSystem(metaclass=_Cached):
                 depth = None
 
         allpaths = self.find(root, maxdepth=depth, withdirs=True, detail=True, **kwargs)
-        # Escape characters special to python regex, leaving our supported
-        # special characters in place.
-        # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html
-        # for shell globbing details.
-        pattern = (
-            "^"
-            + (
-                path.replace("\\", r"\\")
-                .replace(".", r"\.")
-                .replace("+", r"\+")
-                .replace("//", "/")
-                .replace("(", r"\(")
-                .replace(")", r"\)")
-                .replace("|", r"\|")
-                .replace("^", r"\^")
-                .replace("$", r"\$")
-                .replace("{", r"\{")
-                .replace("}", r"\}")
-                .rstrip("/")
-                .replace("?", ".")
-            )
-            + "$"
-        )
-        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
-        pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
-        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+
+        pattern = glob_translate(path + ("/" if ends_with_sep else ""))
         pattern = re.compile(pattern)
 
         out = {
-            p: allpaths[p]
-            for p in sorted(allpaths)
-            if pattern.match(p.replace("//", "/").rstrip("/"))
+            p: info
+            for p, info in sorted(allpaths.items())
+            if pattern.match(
+                (
+                    p + "/"
+                    if append_slash_to_dirname and info["type"] == "directory"
+                    else p
+                )
+            )
         }
 
-        # Return directories only when the glob end by a slash
-        # This is needed for posix glob compliance
-        if ends:
-            out = {k: v for k, v in out.items() if v["type"] == "directory"}
-
         if detail:
             return out
         else:
@@ -828,6 +805,16 @@ class AbstractFileSystem(metaclass=_Cached):
     def cat_ranges(
         self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
     ):
+        """Get the contents of byte ranges from one or more files
+
+        Parameters
+        ----------
+        paths: list
+            A list of of filepaths on this filesystems
+        starts, ends: int or list
+            Bytes limits of the read. If using a single int, the same value will be
+            used to read all the specified files.
+        """
         if max_gap is not None:
             raise NotImplementedError
         if not isinstance(paths, list):
@@ -835,7 +822,7 @@ class AbstractFileSystem(metaclass=_Cached):
         if not isinstance(starts, list):
             starts = [starts] * len(paths)
         if not isinstance(ends, list):
-            ends = [starts] * len(paths)
+            ends = [ends] * len(paths)
         if len(starts) != len(paths) or len(ends) != len(paths):
             raise ValueError
         out = []
@@ -901,7 +888,8 @@ class AbstractFileSystem(metaclass=_Cached):
             os.makedirs(lpath, exist_ok=True)
             return None
 
-        LocalFileSystem(auto_mkdir=True).makedirs(self._parent(lpath), exist_ok=True)
+        fs = LocalFileSystem(auto_mkdir=True)
+        fs.makedirs(fs._parent(lpath), exist_ok=True)
 
         with self.open(rpath, "rb", **kwargs) as f1:
             if outfile is None:
@@ -1187,9 +1175,7 @@ class AbstractFileSystem(metaclass=_Cached):
     def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
         """Move file(s) from one location to another"""
         if path1 == path2:
-            logger.debug(
-                "%s mv: The paths are the same, so no files were moved." % (self)
-            )
+            logger.debug("%s mv: The paths are the same, so no files were moved.", self)
         else:
             self.copy(path1, path2, recursive=recursive, maxdepth=maxdepth)
             self.rm(path1, recursive=recursive)
@@ -1744,7 +1730,7 @@ class AbstractBufferedFile(io.IOBase):
         elif whence == 2:
             nloc = self.size + loc
         else:
-            raise ValueError("invalid whence (%s, should be 0, 1 or 2)" % whence)
+            raise ValueError(f"invalid whence ({whence}, should be 0, 1 or 2)")
         if nloc < 0:
             raise ValueError("Seek before start of file")
         self.loc = nloc
@@ -1851,7 +1837,7 @@ class AbstractBufferedFile(io.IOBase):
             length = self.size - self.loc
         if self.closed:
             raise ValueError("I/O operation on closed file.")
-        logger.debug("%s read: %i - %i" % (self, self.loc, self.loc + length))
+        logger.debug("%s read: %i - %i", self, self.loc, self.loc + length)
         if length == 0:
             # don't even bother calling fetch
             return b""
@@ -1966,7 +1952,7 @@ class AbstractBufferedFile(io.IOBase):
             self.close()
 
     def __str__(self):
-        return "<File-like object %s, %s>" % (type(self.fs).__name__, self.path)
+        return f"<File-like object {type(self.fs).__name__}, {self.path}>"
 
     __repr__ = __str__
 

fsspec/tests/abstract/common.py

@@ -107,9 +107,9 @@ GLOB_EDGE_CASES_TESTS = {
                 "subdir1/subfile2",
             ],
         ),
-        ("**1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
+        ("**/*1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
         (
-            "**1",
+            "**/*1",
             True,
             None,
             [
@@ -120,14 +120,14 @@ GLOB_EDGE_CASES_TESTS = {
                 "subdir1/nesteddir/nestedfile",
             ],
         ),
-        ("**1", True, 1, ["file1"]),
+        ("**/*1", True, 1, ["file1"]),
         (
-            "**1",
+            "**/*1",
             True,
             2,
             ["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"],
         ),
-        ("**1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
+        ("**/*1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
         ("**/subdir0", False, None, []),
         ("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]),
         ("**/subdir0/nested*", False, 2, []),

fsspec/transaction.py

@@ -1,3 +1,6 @@
+from collections import deque
+
+
 class Transaction:
     """Filesystem transaction write context
 
@@ -13,10 +16,11 @@ class Transaction:
         fs: FileSystem instance
         """
         self.fs = fs
-        self.files = []
+        self.files = deque()
 
     def __enter__(self):
         self.start()
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """End transaction and commit, if exit is not due to exception"""
@@ -27,17 +31,17 @@ class Transaction:
 
     def start(self):
         """Start a transaction on this FileSystem"""
-        self.files = []  # clean up after previous failed completions
+        self.files = deque()  # clean up after previous failed completions
         self.fs._intrans = True
 
     def complete(self, commit=True):
         """Finish transaction: commit or discard all deferred files"""
-        for f in self.files:
+        while self.files:
+            f = self.files.popleft()
             if commit:
                 f.commit()
             else:
                 f.discard()
-        self.files = []
         self.fs._intrans = False