fsspec 2023.6.0__py3-none-any.whl → 2023.9.1__py3-none-any.whl

fsspec/implementations/cached.py

@@ -1,24 +1,27 @@
 from __future__ import annotations
 
-import contextlib
-import hashlib
 import inspect
 import logging
 import os
-import pickle
 import tempfile
 import time
+import weakref
 from shutil import rmtree
-from typing import ClassVar
+from typing import TYPE_CHECKING, Any, Callable, ClassVar
 
 from fsspec import AbstractFileSystem, filesystem
 from fsspec.callbacks import _DEFAULT_CALLBACK
 from fsspec.compression import compr
 from fsspec.core import BaseCache, MMapCache
 from fsspec.exceptions import BlocksizeMismatchError
+from fsspec.implementations.cache_mapper import create_cache_mapper
+from fsspec.implementations.cache_metadata import CacheMetadata
 from fsspec.spec import AbstractBufferedFile
 from fsspec.utils import infer_compression
 
+if TYPE_CHECKING:
+    from fsspec.implementations.cache_mapper import AbstractCacheMapper
+
 logger = logging.getLogger("fsspec.cached")
 
 
@@ -53,8 +56,9 @@ class CachingFileSystem(AbstractFileSystem):
         expiry_time=604800,
         target_options=None,
         fs=None,
-        same_names=False,
+        same_names: bool | None = None,
         compression=None,
+        cache_mapper: AbstractCacheMapper | None = None,
         **kwargs,
     ):
         """
@@ -84,13 +88,19 @@ class CachingFileSystem(AbstractFileSystem):
         fs: filesystem instance
             The target filesystem to run against. Provide this or ``protocol``.
         same_names: bool (optional)
-            By default, target URLs are hashed, so that files from different backends
-            with the same basename do not conflict. If this is true, the original
-            basename is used.
+            By default, target URLs are hashed using a ``HashCacheMapper`` so
+            that files from different backends with the same basename do not
+            conflict. If this argument is ``true``, a ``BasenameCacheMapper``
+            is used instead. Other cache mapper options are available by using
+            the ``cache_mapper`` keyword argument. Only one of this and
+            ``cache_mapper`` should be specified.
         compression: str (optional)
             To decompress on download. Can be 'infer' (guess from the URL name),
             one of the entries in ``fsspec.compression.compr``, or None for no
             decompression.
+        cache_mapper: AbstractCacheMapper (optional)
+            The object use to map from original filenames to cached filenames.
+            Only one of this and ``same_names`` should be specified.
         """
         super().__init__(**kwargs)
         if fs is None and target_protocol is None:
@@ -102,7 +112,9 @@ class CachingFileSystem(AbstractFileSystem):
                 "Both filesystems (fs) and target_protocol may not be both given."
             )
         if cache_storage == "TMP":
-            storage = [tempfile.mkdtemp()]
+            tempdir = tempfile.mkdtemp()
+            storage = [tempdir]
+            weakref.finalize(self, self._remove_tempdir, tempdir)
         else:
             if isinstance(cache_storage, str):
                 storage = [cache_storage]
@@ -115,14 +127,25 @@ class CachingFileSystem(AbstractFileSystem):
         self.check_files = check_files
         self.expiry = expiry_time
         self.compression = compression
-        # TODO: same_names should allow for variable prefix, not only
-        #  to keep the basename
-        self.same_names = same_names
+
+        if same_names is not None and cache_mapper is not None:
+            raise ValueError(
+                "Cannot specify both same_names and cache_mapper in "
+                "CachingFileSystem.__init__"
+            )
+        if cache_mapper is not None:
+            self._mapper = cache_mapper
+        else:
+            self._mapper = create_cache_mapper(
+                same_names if same_names is not None else False
+            )
+
         self.target_protocol = (
             target_protocol
             if isinstance(target_protocol, str)
             else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
         )
+        self._metadata = CacheMetadata(self.storage)
         self.load_cache()
         self.fs = fs if fs is not None else filesystem(target_protocol, **self.kwargs)
 
@@ -130,68 +153,28 @@ class CachingFileSystem(AbstractFileSystem):
             # acts as a method, since each instance has a difference target
             return self.fs._strip_protocol(type(self)._strip_protocol(path))
 
-        self._strip_protocol = _strip_protocol
+        self._strip_protocol: Callable = _strip_protocol
+
+    @staticmethod
+    def _remove_tempdir(tempdir):
+        try:
+            rmtree(tempdir)
+        except Exception:
+            pass
 
     def _mkcache(self):
         os.makedirs(self.storage[-1], exist_ok=True)
 
     def load_cache(self):
         """Read set of stored blocks from file"""
-        cached_files = []
-        for storage in self.storage:
-            fn = os.path.join(storage, "cache")
-            if os.path.exists(fn):
-                with open(fn, "rb") as f:
-                    # TODO: consolidate blocks here
-                    loaded_cached_files = pickle.load(f)
-                    for c in loaded_cached_files.values():
-                        if isinstance(c["blocks"], list):
-                            c["blocks"] = set(c["blocks"])
-                    cached_files.append(loaded_cached_files)
-            else:
-                cached_files.append({})
+        self._metadata.load()
         self._mkcache()
-        self.cached_files = cached_files or [{}]
         self.last_cache = time.time()
 
     def save_cache(self):
         """Save set of stored blocks from file"""
-        fn = os.path.join(self.storage[-1], "cache")
-        # TODO: a file lock could be used to ensure file does not change
-        #  between re-read and write; but occasional duplicated reads ok.
-        cache = self.cached_files[-1]
-        if os.path.exists(fn):
-            with open(fn, "rb") as f:
-                cached_files = pickle.load(f)
-            for k, c in cached_files.items():
-                if k in cache:
-                    if c["blocks"] is True or cache[k]["blocks"] is True:
-                        c["blocks"] = True
-                    else:
-                        # self.cached_files[*][*]["blocks"] must continue to
-                        # point to the same set object so that updates
-                        # performed by MMapCache are propagated back to
-                        # self.cached_files.
-                        blocks = cache[k]["blocks"]
-                        blocks.update(c["blocks"])
-                        c["blocks"] = blocks
-                    c["time"] = max(c["time"], cache[k]["time"])
-                    c["uid"] = cache[k]["uid"]
-
-            # Files can be added to cache after it was written once
-            for k, c in cache.items():
-                if k not in cached_files:
-                    cached_files[k] = c
-        else:
-            cached_files = cache
-        cache = {k: v.copy() for k, v in cached_files.items()}
-        for c in cache.values():
-            if isinstance(c["blocks"], set):
-                c["blocks"] = list(c["blocks"])
         self._mkcache()
-        with atomic_write(fn) as f:
-            pickle.dump(cache, f)
-        self.cached_files[-1] = cached_files
+        self._metadata.save()
         self.last_cache = time.time()
 
     def _check_cache(self):
@@ -208,25 +191,11 @@ class CachingFileSystem(AbstractFileSystem):
     def _check_file(self, path):
         """Is path in cache and still valid"""
         path = self._strip_protocol(path)
-
         self._check_cache()
-        for storage, cache in zip(self.storage, self.cached_files):
-            if path not in cache:
-                continue
-            detail = cache[path].copy()
-            if self.check_files:
-                if detail["uid"] != self.fs.ukey(path):
-                    continue
-            if self.expiry:
-                if time.time() - detail["time"] > self.expiry:
-                    continue
-            fn = os.path.join(storage, detail["fn"])
-            if os.path.exists(fn):
-                return detail, fn
-        return False
+        return self._metadata.check_file(path, self)
 
     def clear_cache(self):
-        """Remove all files and metadat from the cache
+        """Remove all files and metadata from the cache
 
         In the case of multiple cache locations, this clears only the last one,
         which is assumed to be the read/write one.
@@ -253,22 +222,12 @@ class CachingFileSystem(AbstractFileSystem):
 
         self._check_cache()
 
-        for path, detail in self.cached_files[-1].copy().items():
-            if time.time() - detail["time"] > expiry_time:
-                if self.same_names:
-                    basename = os.path.basename(detail["original"])
-                    fn = os.path.join(self.storage[-1], basename)
-                else:
-                    fn = os.path.join(self.storage[-1], detail["fn"])
-                if os.path.exists(fn):
-                    os.remove(fn)
-                    self.cached_files[-1].pop(path)
-
-        if self.cached_files[-1]:
-            cache_path = os.path.join(self.storage[-1], "cache")
-            with atomic_write(cache_path) as fc:
-                pickle.dump(self.cached_files[-1], fc)
-        else:
+        expired_files, writable_cache_empty = self._metadata.clear_expired(expiry_time)
+        for fn in expired_files:
+            if os.path.exists(fn):
+                os.remove(fn)
+
+        if writable_cache_empty:
             rmtree(self.storage[-1])
             self.load_cache()
 
@@ -280,19 +239,9 @@ class CachingFileSystem(AbstractFileSystem):
         raises PermissionError
         """
         path = self._strip_protocol(path)
-        details = self._check_file(path)
-        if not details:
-            return
-        _, fn = details
-        if fn.startswith(self.storage[-1]):
-            # is in in writable cache
+        fn = self._metadata.pop_file(path)
+        if fn is not None:
             os.remove(fn)
-            self.cached_files[-1].pop(path)
-            self.save_cache()
-        else:
-            raise PermissionError(
-                "Can only delete cached file in last, writable cache location"
-            )
 
     def _open(
         self,
@@ -339,7 +288,7 @@ class CachingFileSystem(AbstractFileSystem):
             # TODO: action where partial file exists in read-only cache
             logger.debug("Opening partially cached copy of %s" % path)
         else:
-            hash = self.hash_name(path, self.same_names)
+            hash = self._mapper(path)
             fn = os.path.join(self.storage[-1], hash)
             blocks = set()
             detail = {
@@ -349,7 +298,7 @@ class CachingFileSystem(AbstractFileSystem):
                 "time": time.time(),
                 "uid": self.fs.ukey(path),
             }
-            self.cached_files[-1][path] = detail
+            self._metadata.update_file(path, detail)
             logger.debug("Creating local sparse file for %s" % path)
 
         # call target filesystems open
@@ -385,18 +334,17 @@ class CachingFileSystem(AbstractFileSystem):
         self.save_cache()
         return f
 
-    def hash_name(self, path, same_name):
-        return hash_name(path, same_name=same_name)
+    def hash_name(self, path: str, *args: Any) -> str:
+        # Kept for backward compatibility with downstream libraries.
+        # Ignores extra arguments, previously same_name boolean.
+        return self._mapper(path)
 
     def close_and_update(self, f, close):
         """Called when a file is closing, so store the set of blocks"""
         if f.closed:
             return
         path = self._strip_protocol(f.path)
-
-        c = self.cached_files[-1][path]
-        if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size:
-            c["blocks"] = True
+        self._metadata.on_close_cached_file(f, path)
         try:
             logger.debug("going to save")
             self.save_cache()
@@ -488,7 +436,7 @@ class CachingFileSystem(AbstractFileSystem):
             and self.check_files == other.check_files
             and self.expiry == other.expiry
             and self.compression == other.compression
-            and self.same_names == other.same_names
+            and self._mapper == other._mapper
             and self.target_protocol == other.target_protocol
         )
 
@@ -501,7 +449,7 @@ class CachingFileSystem(AbstractFileSystem):
             ^ hash(self.check_files)
             ^ hash(self.expiry)
             ^ hash(self.compression)
-            ^ hash(self.same_names)
+            ^ hash(self._mapper)
             ^ hash(self.target_protocol)
         )
 
@@ -546,7 +494,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
         details = [self._check_file(sp) for sp in paths]
         downpath = [p for p, d in zip(paths, details) if not d]
         downfn0 = [
-            os.path.join(self.storage[-1], self.hash_name(p, self.same_names))
+            os.path.join(self.storage[-1], self._mapper(p))
             for p, d in zip(paths, details)
         ]  # keep these path names for opening later
         downfn = [fn for fn, d in zip(downfn0, details) if not d]
@@ -558,16 +506,15 @@ class WholeFileCacheFileSystem(CachingFileSystem):
             newdetail = [
                 {
                     "original": path,
-                    "fn": self.hash_name(path, self.same_names),
+                    "fn": self._mapper(path),
                     "blocks": True,
                     "time": time.time(),
                     "uid": self.fs.ukey(path),
                 }
                 for path in downpath
             ]
-            self.cached_files[-1].update(
-                {path: detail for path, detail in zip(downpath, newdetail)}
-            )
+            for path, detail in zip(downpath, newdetail):
+                self._metadata.update_file(path, detail)
             self.save_cache()
 
         def firstpart(fn):
@@ -590,7 +537,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
                 pass
 
     def _make_local_details(self, path):
-        hash = self.hash_name(path, self.same_names)
+        hash = self._mapper(path)
         fn = os.path.join(self.storage[-1], hash)
         detail = {
             "original": path,
@@ -599,7 +546,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
             "time": time.time(),
             "uid": self.fs.ukey(path),
         }
-        self.cached_files[-1][path] = detail
+        self._metadata.update_file(path, detail)
         logger.debug("Copying %s to local cache" % path)
         return fn
 
@@ -695,7 +642,7 @@ class WholeFileCacheFileSystem(CachingFileSystem):
                     data = f.read(block)
                     f2.write(data)
         else:
-            self.fs.get(path, fn)
+            self.fs.get_file(path, fn)
         self.save_cache()
         return self._open(path, mode)
 
@@ -727,11 +674,10 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         for storage in self.storage:
             if not os.path.exists(storage):
                 os.makedirs(storage, exist_ok=True)
-        self.cached_files = [{}]
 
     def _check_file(self, path):
         self._check_cache()
-        sha = self.hash_name(path, self.same_names)
+        sha = self._mapper(path)
         for storage in self.storage:
             fn = os.path.join(storage, sha)
             if os.path.exists(fn):
@@ -752,7 +698,7 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
         if fn:
             return open(fn, mode)
 
-        sha = self.hash_name(path, self.same_names)
+        sha = self._mapper(path)
         fn = os.path.join(self.storage[-1], sha)
         logger.debug("Copying %s to local cache" % path)
         kwargs["mode"] = mode
@@ -775,7 +721,7 @@ class SimpleCacheFileSystem(WholeFileCacheFileSystem):
                     data = f.read(block)
                     f2.write(data)
         else:
-            self.fs.get(path, fn)
+            self.fs.get_file(path, fn)
         return self._open(path, mode)
 
 
@@ -836,32 +782,3 @@ class LocalTempFile:
 
     def __getattr__(self, item):
         return getattr(self.fh, item)
-
-
-def hash_name(path, same_name):
-    if same_name:
-        hash = os.path.basename(path)
-    else:
-        hash = hashlib.sha256(path.encode()).hexdigest()
-    return hash
-
-
-@contextlib.contextmanager
-def atomic_write(path, mode="wb"):
-    """
-    A context manager that opens a temporary file next to `path` and, on exit,
-    replaces `path` with the temporary file, thereby updating `path`
-    atomically.
-    """
-    fd, fn = tempfile.mkstemp(
-        dir=os.path.dirname(path), prefix=os.path.basename(path) + "-"
-    )
-    try:
-        with open(fd, mode) as fp:
-            yield fp
-    except BaseException:
-        with contextlib.suppress(FileNotFoundError):
-            os.unlink(fn)
-        raise
-    else:
-        os.replace(fn, path)

fsspec/implementations/dirfs.py

@@ -10,6 +10,8 @@ class DirFileSystem(AsyncFileSystem):
     delegates everything to the wrapped filesystem.
     """
 
+    protocol = "dir"
+
     def __init__(
         self,
         path=None,
@@ -53,7 +55,7 @@ class DirFileSystem(AsyncFileSystem):
                 return path
             if not path:
                 return self.path
-            return self.fs.sep.join((self.path, path))
+            return self.fs.sep.join((self.path, self._strip_protocol(path)))
         return [self._join(_path) for _path in path]
 
     def _relpath(self, path):

fsspec/implementations/http.py

@@ -192,10 +192,9 @@ class HTTPFileSystem(AsyncFileSystem):
                 for u in out
             ]
         else:
-            return list(sorted(out))
+            return sorted(out)
 
     async def _ls(self, url, detail=True, **kwargs):
-
         if self.use_listings_cache and url in self.dircache:
             out = self.dircache[url]
         else:
@@ -432,7 +431,7 @@ class HTTPFileSystem(AsyncFileSystem):
 
         return {"name": url, "size": None, **info, "type": "file"}
 
-    async def _glob(self, path, **kwargs):
+    async def _glob(self, path, maxdepth=None, **kwargs):
         """
         Find files by glob-matching.
 
@@ -440,23 +439,21 @@ class HTTPFileSystem(AsyncFileSystem):
         but "?" is not considered as a character for globbing, because it is
         so common in URLs, often identifying the "query" part.
         """
+        if maxdepth is not None and maxdepth < 1:
+            raise ValueError("maxdepth must be at least 1")
         import re
 
         ends = path.endswith("/")
         path = self._strip_protocol(path)
-        indstar = path.find("*") if path.find("*") >= 0 else len(path)
-        indbrace = path.find("[") if path.find("[") >= 0 else len(path)
+        idx_star = path.find("*") if path.find("*") >= 0 else len(path)
+        idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
 
-        ind = min(indstar, indbrace)
+        min_idx = min(idx_star, idx_brace)
 
         detail = kwargs.pop("detail", False)
 
         if not has_magic(path):
-            root = path
-            depth = 1
-            if ends:
-                path += "/*"
-            elif await self._exists(path):
+            if await self._exists(path):
                 if not detail:
                     return [path]
                 else:
@@ -466,13 +463,21 @@ class HTTPFileSystem(AsyncFileSystem):
                     return []  # glob of non-existent returns empty
                 else:
                     return {}
-        elif "/" in path[:ind]:
-            ind2 = path[:ind].rindex("/")
-            root = path[: ind2 + 1]
-            depth = None if "**" in path else path[ind2 + 1 :].count("/") + 1
+        elif "/" in path[:min_idx]:
+            min_idx = path[:min_idx].rindex("/")
+            root = path[: min_idx + 1]
+            depth = path[min_idx + 1 :].count("/") + 1
         else:
             root = ""
-            depth = None if "**" in path else path[ind + 1 :].count("/") + 1
+            depth = path[min_idx + 1 :].count("/") + 1
+
+        if "**" in path:
+            if maxdepth is not None:
+                idx_double_stars = path.find("**")
+                depth_double_stars = path[idx_double_stars:].count("/") + 1
+                depth = depth - depth_double_stars + maxdepth
+            else:
+                depth = None
 
         allpaths = await self._find(
             root, maxdepth=depth, withdirs=True, detail=True, **kwargs
@@ -499,14 +504,23 @@ class HTTPFileSystem(AsyncFileSystem):
             )
             + "$"
         )
-        pattern = re.sub("[*]{2}", "=PLACEHOLDER=", pattern)
+        pattern = re.sub("/[*]{2}", "=SLASH_DOUBLE_STARS=", pattern)
+        pattern = re.sub("[*]{2}/?", "=DOUBLE_STARS=", pattern)
         pattern = re.sub("[*]", "[^/]*", pattern)
-        pattern = re.compile(pattern.replace("=PLACEHOLDER=", ".*"))
+        pattern = re.sub("=SLASH_DOUBLE_STARS=", "(|/.*)", pattern)
+        pattern = re.sub("=DOUBLE_STARS=", ".*", pattern)
+        pattern = re.compile(pattern)
         out = {
             p: allpaths[p]
             for p in sorted(allpaths)
             if pattern.match(p.replace("//", "/").rstrip("/"))
         }
+
+        # Return directories only when the glob end by a slash
+        # This is needed for posix glob compliance
+        if ends:
+            out = {k: v for k, v in out.items() if v["type"] == "directory"}
+
         if detail:
             return out
         else:
@@ -841,7 +855,10 @@ async def _file_info(url, session, size_policy="head", **kwargs):
         #                 or 'Accept-Ranges': 'none' (not 'bytes')
         #  to mean streaming only, no random access => return None
         if "Content-Length" in r.headers:
-            info["size"] = int(r.headers["Content-Length"])
+            # Some servers may choose to ignore Accept-Encoding and return
+            # compressed content, in which case the returned size is unreliable.
+            if r.headers.get("Content-Encoding", "identity") == "identity":
+                info["size"] = int(r.headers["Content-Length"])
         elif "Content-Range" in r.headers:
             info["size"] = int(r.headers["Content-Range"].split("/")[1])
 

fsspec/implementations/local.py

@@ -65,10 +65,6 @@ class LocalFileSystem(AbstractFileSystem):
         else:
             return [posixpath.join(path, f) for f in os.listdir(path)]
 
-    def glob(self, path, **kwargs):
-        path = self._strip_protocol(path)
-        return super().glob(path, **kwargs)
-
     def info(self, path, **kwargs):
         if isinstance(path, os.DirEntry):
             # scandir DirEntry
@@ -196,11 +192,13 @@ class LocalFileSystem(AbstractFileSystem):
 
     def created(self, path):
         info = self.info(path=path)
-        return datetime.datetime.utcfromtimestamp(info["created"])
+        return datetime.datetime.fromtimestamp(
+            info["created"], tz=datetime.timezone.utc
+        )
 
     def modified(self, path):
         info = self.info(path=path)
-        return datetime.datetime.utcfromtimestamp(info["mtime"])
+        return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc)
 
     @classmethod
     def _parent(cls, path):
@@ -285,21 +283,6 @@ def trailing_sep(path):
     return path.endswith(os.sep) or (os.altsep is not None and path.endswith(os.altsep))
 
 
-def trailing_sep_maybe_asterisk(path):
-    """Return True if the path ends with a path separator and optionally an
-    asterisk.
-
-    A forward slash is always considered a path separator, even on Operating
-    Systems that normally use a backslash.
-    """
-    # TODO: if all incoming paths were posix-compliant then separator would
-    # always be a forward slash, simplifying this function.
-    # See https://github.com/fsspec/filesystem_spec/pull/1250
-    return path.endswith((os.sep, os.sep + "*")) or (
-        os.altsep is not None and path.endswith((os.altsep, os.altsep + "*"))
-    )
-
-
 class LocalFileOpener(io.IOBase):
     def __init__(
         self, path, mode, autocommit=True, fs=None, compression=None, **kwargs

fsspec/implementations/memory.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import, annotations, division, print_function
 
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from errno import ENOTEMPTY
 from io import BytesIO
 from typing import Any, ClassVar
@@ -137,10 +137,6 @@ class MemoryFileSystem(AbstractFileSystem):
         else:
             raise FileNotFoundError(path)
 
-    def exists(self, path, **kwargs):
-        path = self._strip_protocol(path)
-        return path in self.store or path in self.pseudo_dirs
-
     def info(self, path, **kwargs):
         path = self._strip_protocol(path)
         if path in self.pseudo_dirs or any(
@@ -191,11 +187,14 @@ class MemoryFileSystem(AbstractFileSystem):
                 return f
             else:
                 raise FileNotFoundError(path)
-        if mode == "wb":
+        elif mode == "wb":
             m = MemoryFile(self, path, kwargs.get("data"))
             if not self._intrans:
                 m.commit()
             return m
+        else:
+            name = self.__class__.__name__
+            raise ValueError(f"unsupported file mode for {name}: {mode!r}")
 
     def cp_file(self, path1, path2, **kwargs):
         path1 = self._strip_protocol(path1)
@@ -269,8 +268,8 @@ class MemoryFile(BytesIO):
         logger.debug("open file %s", path)
         self.fs = fs
         self.path = path
-        self.created = datetime.utcnow()
-        self.modified = datetime.utcnow()
+        self.created = datetime.now(tz=timezone.utc)
+        self.modified = datetime.now(tz=timezone.utc)
         if data:
             super().__init__(data)
             self.seek(0)
@@ -290,4 +289,4 @@ class MemoryFile(BytesIO):
 
     def commit(self):
         self.fs.store[self.path] = self
-        self.modified = datetime.utcnow()
+        self.modified = datetime.now(tz=timezone.utc)